Merge branch 'master' of bitbucket.org:bluetradesys/spark

The compute_synthetic_book method (also to be imported
from spartan) depends on a couple of them.

.gitignore

@@ -34,5 +34,7 @@ trace_data/
 /doc/user/public/*
 .ionide/*
 /.ignore
+*.rej
 /build
 .clangd/
+*CMakeDeps.cmake

.vscode/launch.json

@@ -8,13 +8,15 @@
             "program": "${workspaceRoot}/build/ninja_debug/server/sparkd",
             "args": [
                 "run",
-                // "test",
-                // "-a",
-                // "[ilink3]"
+                "test",
+                "-a",
+                "OutrightVanillaOrder"
             ],
             "stopAtEntry": false,
             "cwd": "${workspaceRoot}/build/ninja_debug/server/",
-            "environment": [],
+            "environment": [
+                { "name": "TZ", "value": "America/Chicago" }
+            ],
             "externalConsole": false,
             "MIMode": "gdb",
             "setupCommands": [
@@ -31,7 +33,9 @@
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/server/sparkd",
             "args": [
-                "run"
+                "test",
+                "-a",
+                "[ilink3]"
             ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server/",

CMakeLists.txt

@@ -41,6 +41,7 @@ else ()
                                -Wno-missing-braces -Wno-ignored-qualifiers \
                                --std=c++17 -fdiagnostics-color")
   set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
+  set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
   set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wno-undef")
   add_subdirectory(src/server/sparkd)
   add_subdirectory(src/lib)

doc/user/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
+set(sources /home/pfisher/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

src/client/spark/AlgoProvider/AlgoProviderFactory.cs

@@ -0,0 +1,193 @@
+using bts;
+using bts.api;
+using bts.api.core;
+using mbts.ui.utility;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.ComponentModel.Composition;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text.RegularExpressions;
+using AlgoType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
+
+namespace mbts.spark {
+   class AlgoProviderFactory {
+      [ImportingConstructor]
+      AlgoProviderFactory(IAlgoStudioModel studio_model, DynamicOrderHelper dynamic_helper, ISecurityAuxiliaryDataModel security_auxiliary_model) {
+         var from_conf = config.Configuration.get<bts.config.AlgoTypeRuleSet>();
+         if (from_conf.rules.Count == 0) {
+            // There seems to be a config race bug, where the returned config values are empty, despite having entries in spark.conf.
+            // [John]I've only noticed it happen when attached to the debugger,
+            // but best to have a contingency plan in place anyways, to prevent any chance of regression.
+            // The fallback sets below mimic the old hardcoded algo type arrays.
+            var full_algo_set = new AlgoTypeSetImmutable(
+               ImmutableHashSet.CreateRange(EnumHelper.get_values_except(
+                  // non-factory algos
+                  AlgoType.None,
+                  AlgoType.Dynamic,
+                  AlgoType.Composite,
+                  // obsolete algos
+                  AlgoType.Oco,
+                  AlgoType.TimeBox,
+                  AlgoType.Pricer,
+                  AlgoType.Tick,
+                  AlgoType.Trigger,
+                  AlgoType.QueueHolder)),
+               ImmutableHashSet.CreateRange(dynamic_helper.dynamic_types));
+            var rules = new[] {
+               // Glbx: Futures/Spreads
+               new SecurityAlgoTypeSetImmutable(
+                  new SecurityCriteriaClause(
+                     exchanges: new[] { Venue.Glbx },
+                     security_types: new[] { SecurityType.Future, SecurityType.Spread }),
+                  new AlgoTypeSetImmutable(
+                     full_algo_set.factory_types.Except(new[] { AlgoType.Spreader }),
+                     full_algo_set.dynamic_types)),
+               // Gen: Futures/Spreads
+               new SecurityAlgoTypeSetImmutable(
+                  new SecurityCriteriaClause(
+                     exchanges: EnumHelper.get_values_except(Venue.Glbx, Venue.Unset).ToArray(),
+                     security_types: new[] { SecurityType.Future, SecurityType.Spread }),
+                  new AlgoTypeSetImmutable(
+                     full_algo_set.factory_types.Except(new[] { AlgoType.Spreader, AlgoType.GlbxIceberg }),
+                     full_algo_set.dynamic_types)),
+               // Gen: Synthetics
+               new SecurityAlgoTypeSetImmutable(
+                  new SecurityCriteriaClause(
+                     security_types: new[] { SecurityType.Synthetic }),
+                  new AlgoTypeSetImmutable(
+                     ImmutableHashSet.CreateRange(new[] {
+                        AlgoType.Spreader,
+                        AlgoType.Iceberg,
+                        AlgoType.Lean,
+                        AlgoType.Join,
+                        AlgoType.Cover,
+                        AlgoType.Offset,
+                        AlgoType.Top,
+                        AlgoType.Splitter,
+                        AlgoType.Scratch,
+                        AlgoType.Prorata
+                     }),
+                  full_algo_set.dynamic_types))
+             };
+            // attachable
+            var attachable_algos =
+               ImmutableHashSet.CreateRange(new[] {
+                  AlgoType.Iceberg,
+                  AlgoType.Lean,
+                  AlgoType.Join,
+                  AlgoType.Cover,
+                  AlgoType.Top,
+                  AlgoType.Offset,
+                  AlgoType.Scratch,
+                  AlgoType.Prorata
+               });
+            var attachable_rules = Array.ConvertAll(rules,
+               satsi => new SecurityAlgoTypeSetImmutable(
+                  satsi.security_criteria,
+                  new AlgoTypeSetImmutable(
+                     satsi.algo_type_set.factory_types.Intersect(attachable_algos),
+                     satsi.algo_type_set.dynamic_types)));
+
+            Log.Config.warn($"Failed to load AlgoTypes config, falling back to the following algo set rules:" +
+               "\n\tStandard\n" +
+               rules.fmt_diagnostic("\n\t\t") +
+               "\n\n\tAttachable\n" +
+               attachable_rules.fmt_diagnostic("\n\t\t"));
+
+            security_algo_provider = new AlgoProvider(rules, studio_model, dynamic_helper, security_auxiliary_model);
+            attachable_algo_provider = new AttachableAlgoProvider(attachable_rules, studio_model, dynamic_helper, security_auxiliary_model);
+         } else {
+            var algo_type_data =
+               EnumHelper.get_values_except(AlgoType.Dynamic, AlgoType.Composite, AlgoType.None)
+              .Select(t => _get_type_data(t))
+              .Concat(dynamic_helper.dynamic_types.Select(dt => _get_type_data(AlgoType.Dynamic, dt)))
+              .ToListLazy();
+            var conf_sets = from_conf.rules.Select(r => _to_security_algo_set(r, dynamic_helper, algo_type_data)).ToList();
+            security_algo_provider = new AlgoProvider(conf_sets.Select(tup => tup.standard).ToList(), studio_model, dynamic_helper, security_auxiliary_model);
+            attachable_algo_provider = new AttachableAlgoProvider(conf_sets.Select(tup => tup.attachable).ToList(), studio_model, dynamic_helper, security_auxiliary_model);
+         }
+      }
+
+      [Export]
+      public IAlgoProvider security_algo_provider { get; }
+      [Export]
+      public IAttachableAlgoProvider attachable_algo_provider { get; }
+
+      static (string name, AlgoType factory, string dynamic) _get_type_data(AlgoType type, string dynamic_type = null) {
+         string type_name = type.ToString();
+         return (type_name, type, dynamic_type ?? type_name);
+      }
+
+      static SecurityCriteriaClause _to_clause(bts.config.AlgoTypeContext context) =>
+         context == null ?
+            new SecurityCriteriaClause() :
+            new SecurityCriteriaClause(
+               exchanges: context.exchanges?.Select(EnumHelper.parse<Venue>)?.ToArray(),
+               security_types: context.security_types?.Select(EnumHelper.parse<Security.ContractOneofCase>)?.Select(t => t.security_type())?.ToArray(),
+               symbols: context.symbols?.ToArray());
+
+      static (SecurityAlgoTypeSetImmutable standard, SecurityAlgoTypeSetImmutable attachable) _to_security_algo_set(
+         bts.config.AlgoTypeRule rule,
+         DynamicOrderHelper dynamic_helper,
+         IEnumerable<(string name, AlgoType factory, string dynamic)> algo_type_data) {
+         //
+         var factory_types = ImmutableHashSet<AlgoType>.Empty.ToBuilder();
+         var dynamic_types = ImmutableHashSet<string>.Empty.ToBuilder();
+         var attachable_factory_types = ImmutableHashSet<AlgoType>.Empty.ToBuilder();
+         var attachable_dynamic_types = ImmutableHashSet<string>.Empty.ToBuilder();
+         foreach (var a in rule.algos) {
+            if (Enum.TryParse(a.name, out AlgoType sys_algo) && sys_algo != AlgoType.Composite && sys_algo != AlgoType.Dynamic) {
+               on_sys_match(sys_algo, a.is_attachable);
+            }
+            else if (dynamic_helper.try_parse(a.name, out var dyn_type)) {
+               on_dyn_match(dyn_type, a.is_attachable);
+            } else {
+               foreach (var (name, factory, dynamic) in algo_type_data) {
+                  if (Regex.Match(name, a.name).Success) {
+                     switch (factory) {
+                        case AlgoType.None:
+                        case AlgoType.Composite:
+                           // unexpected, but no need to crash
+                           continue;
+                        case AlgoType.Dynamic:
+                           on_dyn_match(dynamic, a.is_attachable);
+                           break;
+                        default:
+                           on_sys_match(factory, a.is_attachable);
+                           break;
+                     }
+                  }
+               }
+            }
+         }
+         return (
+            // standard
+            new SecurityAlgoTypeSetImmutable(
+               _to_clause(rule.context),
+               new AlgoTypeSetImmutable(
+                  factory_types.ToImmutable(),
+                  dynamic_types.ToImmutable())),
+            // attachable
+            new SecurityAlgoTypeSetImmutable(
+               _to_clause(rule.context),
+            new AlgoTypeSetImmutable(
+               attachable_factory_types.ToImmutable(),
+               attachable_dynamic_types.ToImmutable())));
+
+         // local methods
+         void on_sys_match(AlgoType sys_algo, bool is_attachable) {
+            factory_types.Add(sys_algo);
+            if (is_attachable)
+               attachable_factory_types.Add(sys_algo);
+         }
+         void on_dyn_match(string dyn_algo, bool is_attachable) {
+            dynamic_types.Add(dyn_algo);
+            if (is_attachable)
+               attachable_dynamic_types.Add(dyn_algo);
+         }
+      }
+   }
+}

src/client/spark/AlgoProvider/AlgoTypeSet.cs

@@ -0,0 +1,132 @@
+using bts.api.core;
+using bts.api.orderman;
+using mbts.spark.utilities;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using AlgoType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
+
+namespace mbts.spark {
+   readonly struct AlgoTypeSetImmutable {
+      public static readonly AlgoTypeSetImmutable empty = new AlgoTypeSetImmutable(ImmutableHashSet<AlgoType>.Empty, ImmutableHashSet<string>.Empty);
+      public readonly ImmutableHashSet<AlgoType> factory_types;
+      public readonly ImmutableHashSet<string> dynamic_types;
+
+      public AlgoTypeSetImmutable(
+         ImmutableHashSet<AlgoType> factory_types,
+         ImmutableHashSet<string> dynamic_types) {
+         this.factory_types = factory_types;
+         this.dynamic_types = dynamic_types;
+      }
+   }
+
+   readonly struct SecurityAlgoTypeSetImmutable {
+      public readonly SecurityCriteriaClause security_criteria;
+      public readonly AlgoTypeSetImmutable algo_type_set;
+
+      public SecurityAlgoTypeSetImmutable(SecurityCriteriaClause security_criteria, AlgoTypeSetImmutable algo_type_set, CustomOrderDefinitionImmutable? default_algo = null) {
+         this.security_criteria = security_criteria;
+         this.algo_type_set = algo_type_set;
+      }
+   }
+
+   static class AlgoTypeSetExtensions {
+      public static string fmt_diagnostic(this IEnumerable<SecurityAlgoTypeSetImmutable> rules, string line_delim = "\n") {
+         var sb = new StringBuilder();
+         int rule_count = 1;
+         using (var en = rules.GetEnumerator()) {
+            if (en.MoveNext()) {
+               string line_delim_tab1 = line_delim + "\t";
+               string line_delim_tab2 = line_delim + "\t\t";
+               append_rule(en.Current);
+               while (en.MoveNext()) {
+                  sb.AppendLine();
+                  append_rule(en.Current);
+               }
+               //
+               void append_rule(SecurityAlgoTypeSetImmutable r) {
+                  sb.Append(line_delim + $"**RULE {rule_count++}**");
+                  sb.Append(line_delim_tab1);
+                  sb.Append("Security Criteria: ");
+                  sb.Append(r.security_criteria.is_defined ? string.Join("; ", r.security_criteria.fmt_diagnostic_msgs()) : "(Undefined)");
+                  sb.Append(line_delim_tab1);
+                  // Factory
+                  sb.Append("Factory Types:");
+                  sb.Append(line_delim_tab2);
+                  sb.Append(string.Join(line_delim_tab2, r.algo_type_set.factory_types));
+                  sb.Append(line_delim_tab1);
+                  // Dynamic
+                  sb.Append("Dynamic Types:");
+                  sb.Append(line_delim_tab2);
+                  sb.Append(string.Join(line_delim_tab2, r.algo_type_set.dynamic_types));
+               }
+            }
+         }
+         return sb.ToString();
+      }
+
+      public static AlgoTypeSetImmutable intersect_with(this AlgoTypeSetImmutable a, AlgoTypeSetImmutable b) =>
+         new AlgoTypeSetImmutable(a.factory_types.Intersect(b.factory_types), a.dynamic_types.Intersect(b.dynamic_types));
+
+      public static AlgoTypeSetImmutable union_with(this AlgoTypeSetImmutable a, AlgoTypeSetImmutable b) =>
+         new AlgoTypeSetImmutable(a.factory_types.Union(b.factory_types), a.dynamic_types.Union(b.dynamic_types));
+
+      public static IEnumerable<AlgoData> define_algos(
+         this AlgoTypeSetImmutable algo_set,
+         DynamicOrderHelper dynamic_helper,
+         Func<CustomOrderDefinitionImmutable, CustomOrderDefinitionImmutable> fit_to_type = null,
+         uint display_qty_divisor = 1) =>
+         //
+         algo_set.factory_types
+         .Select(algo_type => {
+            var def = OrderDataHelper.Defaults.CustomOrderDefinitionImmutable(algo_type);
+            if (fit_to_type != null)
+               def = fit_to_type(def);
+            // multiply quantity props by display qty divisor to get appropriate actual value 
+            if (display_qty_divisor > 1) {
+               switch (def.CustomCase) {
+                  case AlgoType.Iceberg:
+                     var mdef = def.CloneToMutable();
+                     mdef.Iceberg.ShowQuantity *= display_qty_divisor;
+                     def = new CustomOrderDefinitionImmutable(mdef);
+                     break;
+               }
+            }
+            return def;
+         })
+         .Concat(algo_set.dynamic_types.Select(dynamic_helper.create_custom_defi))
+         .Select(AlgoData.from_custom_def);
+
+      public static ImmutableDictionary<AlgoKey, AlgoData> match_algos(
+         this IEnumerable<SecurityAlgoTypeSetImmutable> security_algo_sets,
+         SecurityCriteriaClause clause,
+         DynamicOrderHelper dynamic_helper,
+         Func<CustomOrderDefinitionImmutable, CustomOrderDefinitionImmutable> fit_to_type = null,
+         uint display_qty_divisor = 1) {
+         //
+         ImmutableDictionary<AlgoKey, AlgoData> rv = null;
+         foreach (var set in security_algo_sets) {
+            if (!clause.is_subset_of(set.security_criteria)) {
+               continue;
+            } else if (rv == null) {
+               rv = ImmutableDictionary.CreateRange(
+                  set.algo_type_set.define_algos(dynamic_helper, fit_to_type, display_qty_divisor)
+                  .Select(a => new KeyValuePair<AlgoKey, AlgoData>(a.key, a)));
+            } else {
+               foreach (var kv in rv) {
+                  if (kv.Value.value.CustomCase == AlgoType.Dynamic) {
+                     if (!set.algo_type_set.dynamic_types.Contains(kv.Value.value.Dynamic.DynamicOrderType))
+                        rv = rv.Remove(kv.Key);
+                  } else if (!set.algo_type_set.factory_types.Contains(kv.Value.value.CustomCase))
+                     rv = rv.Remove(kv.Key);
+               }
+            }
+         }
+         return rv ?? ImmutableDictionary<AlgoKey, AlgoData>.Empty;
+      }
+   }
+}

src/client/spark/AlgoProvider/IAlgoProvider.cs

@@ -0,0 +1,234 @@
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.utils;
+using mbts.spark.utilities;
+using mbts.ui.utility;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reactive.Linq;
+
+namespace mbts.spark {
+   interface IAlgoProvider {
+      AlgoTypeSetImmutable get_permitted_algos(SecurityImmutable security);
+      AlgoTypeSetImmutable get_permitted_algos(SecurityCriteriaClause clause);
+      IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos();
+      IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos(SecurityImmutable security);
+      IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos(SecurityCriteriaClause clause);
+   }
+
+   // This interface is used to distinguish the attachable algo provider
+   // from the standard one wrt MEF export/import.
+   interface IAttachableAlgoProvider : IAlgoProvider { }
+
+   class AlgoProvider : IAlgoProvider {
+      readonly IEnumerable<SecurityAlgoTypeSetImmutable> _security_algo_types;
+      readonly IAlgoStudioModel _studio_model;
+      readonly ISecurityAuxiliaryDataModel _security_auxiliary_model;
+      readonly IObservable<TableUpdate<AlgoData, AlgoKey>> _studio_algos_source;
+      readonly DynamicOrderHelper _dynamic_helper;
+      readonly AlgoTypeSetImmutable _full_set;
+
+      public AlgoProvider(IEnumerable<SecurityAlgoTypeSetImmutable> algo_sets, IAlgoStudioModel studio_model, DynamicOrderHelper dynamic_helper, ISecurityAuxiliaryDataModel security_auxiliary_model) {
+         _security_algo_types = algo_sets;
+         _studio_model = studio_model;
+         _dynamic_helper = dynamic_helper;
+         _full_set = new AlgoTypeSetImmutable(
+            ImmutableHashSet.CreateRange(
+               EnumHelper.get_values_except(CustomOrderDefinition.CustomOneofCase.None, CustomOrderDefinition.CustomOneofCase.Dynamic, CustomOrderDefinition.CustomOneofCase.Composite)),
+            ImmutableHashSet.CreateRange(dynamic_helper.dynamic_types));
+         _security_auxiliary_model = security_auxiliary_model;
+         /// initialize <see cref="_studio_algos_source"/> with an empty table, so the
+         /// <see cref="UpdateExtensions.AppendTable{T, TKey}(IObservable{TableUpdate{T, TKey}}, IEnumerable{T})"/>
+         /// calls made by the 'subscribe_algos' methods will fire at least once, even if the underlying
+         /// studio algos stream is empty have been created (this would be the case if the user had not yet created any studio algos).
+         /// <see cref="subscribe_algos"/>,
+         /// <see cref="subscribe_algos(SecurityCriteriaClause)"/>,
+         /// <see cref="subscribe_algos(SecurityImmutable)"/>
+         _studio_algos_source = studio_model.definitions
+            .StartWith(TableUpdate<AlgoData, AlgoKey>.mk_empty(AlgoData.key_fn))
+            .Cache();
+      }
+
+      public AlgoTypeSetImmutable get_permitted_algos(SecurityImmutable security) =>
+         _security_algo_types.Where(p => p.security_criteria.includes(security))
+            .Select(sec_algos => sec_algos.algo_type_set)
+            .Aggregate(_full_set, (result, algos) => result.intersect_with(algos));
+
+      public AlgoTypeSetImmutable get_permitted_algos(SecurityCriteriaClause clause) =>
+         _security_algo_types.Where(p => p.security_criteria.is_subset_of(clause))
+            .Select(sec_algos => sec_algos.algo_type_set)
+            .Aggregate(_full_set, (result, algos) => result.intersect_with(algos));
+
+      public IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos() =>
+         _studio_algos_source
+         .AppendTable(_security_algo_types
+            .Select(s => s.algo_type_set)
+            .Aggregate(AlgoTypeSetImmutable.empty, (a, b) => a.union_with(b))
+            .define_algos(_dynamic_helper));
+
+      public IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos(SecurityCriteriaClause security_clause) {
+         var system_algos = _security_algo_types.match_algos(
+            security_clause,
+            _dynamic_helper,
+            def => _fit_to_type(def, security_clause, _security_algo_types));
+
+         return _studio_algos_source.Select(
+            a => (algo: a, fitted_def: _fit_to_type(a.value, security_clause, _security_algo_types)),
+            tup => tup.algo.key)
+         .Select(tup => AlgoData.create(tup.fitted_def, tup.algo.name, tup.algo.key), a => a.key)
+         .AppendTable(system_algos.Values);
+      }
+
+      public IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos(SecurityImmutable security) {
+         var clause = new SecurityCriteriaClause(
+            exchanges: new[] { security.Exchange },
+            security_types: new[] { security.security_type() });
+         // i.e., non-studio algos
+         var system_algos = _security_algo_types.match_algos(
+            clause,
+            _dynamic_helper,
+            def => _fit_to_type(def, security, _security_algo_types),
+            _security_auxiliary_model.get_display_quantity_divisor(security));
+
+         return _studio_algos_source.Select(
+            a => (algo: a, fitted_def: _fit_to_type(a.value, security, _security_algo_types)),
+            tup => tup.algo.key)
+         // filter out algos that use an unsupported root, because this would have the effect of duplicating the default algo
+         // (e.g., GlbxIceberg orders would be converted to Limit/Spreader orders for non-Glbx securities).
+         .Filter(tup => tup.fitted_def.actual().CustomCase == tup.algo.value.actual().CustomCase, tup => tup.algo.key)
+         .Select(tup => AlgoData.create(tup.fitted_def, tup.algo.name, tup.algo.key), AlgoData.key_fn)
+         .AppendTable(system_algos.Values);
+      }
+
+      protected virtual void assign_fallback_algo(CustomOrderDefinition mdef, SecurityCriteriaClause security_clause) {
+         if (security_clause.security_types.empty_if_null().Contains(Security.ContractOneofCase.Synthetic.security_type())) {
+            mdef.Spreader = new SpreaderOrderDefinition();
+         } else {
+            mdef.Limit = new LimitOrderDefinition();
+         }
+      }
+
+      protected virtual void assign_fallback_algo(CustomOrderDefinition mdef, SecurityImmutable security) {
+         if (security.ContractCase == Security.ContractOneofCase.Synthetic) {
+            mdef.Spreader = new SpreaderOrderDefinition {
+               Security = security.CloneToMutable()
+            };
+         } else {
+            mdef.Limit = new LimitOrderDefinition();
+         }
+      }
+
+      CustomOrderDefinitionImmutable _fit_to_type(CustomOrderDefinitionImmutable defi, SecurityCriteriaClause security_clause, IEnumerable<SecurityAlgoTypeSetImmutable> algo_sets) {
+         var mroot = defi.CloneToMutable();
+         if (_fit_to_type(mroot, security_clause, algo_sets)) {
+            defi = new CustomOrderDefinitionImmutable(mroot);
+         }
+         return defi;
+      }
+
+      CustomOrderDefinitionImmutable _fit_to_type(CustomOrderDefinitionImmutable defi, SecurityImmutable security, IEnumerable<SecurityAlgoTypeSetImmutable> algo_sets) {
+         var mroot = defi.CloneToMutable();
+         if (_fit_to_type(mroot, security, algo_sets)) {
+            defi = new CustomOrderDefinitionImmutable(mroot);
+         }
+         return defi;
+      }
+
+      bool _fit_to_type(CustomOrderDefinition mdef, SecurityCriteriaClause security_clause, IEnumerable<SecurityAlgoTypeSetImmutable> algo_sets) {
+         while (mdef.CustomCase == CustomOrderDefinition.CustomOneofCase.Composite)
+            mdef = mdef.Composite.Definition;
+         var sec_sets = algo_sets.Where(s => security_clause.is_subset_of(s.security_criteria));
+         if (mdef.CustomCase == CustomOrderDefinition.CustomOneofCase.Dynamic) {
+            if (!algo_sets.All(s => s.algo_type_set.dynamic_types.Contains(mdef.Dynamic.DynamicOrderType))) {
+               assign_fallback_algo(mdef, security_clause);
+               return true;
+            }
+         } else if (!sec_sets.All(s => s.algo_type_set.factory_types.Contains(mdef.CustomCase))) {
+            assign_fallback_algo(mdef, security_clause);
+            return true;
+         }
+         return OrderDataHelper.get_child_algo_data(mdef).Count(cad => _fit_to_type(cad.get_child(mdef), security_clause, algo_sets)) > 0;
+      }
+
+      bool _fit_to_type(CustomOrderDefinition mdef, SecurityImmutable security, IEnumerable<SecurityAlgoTypeSetImmutable> algo_sets) {
+         while (mdef.CustomCase == CustomOrderDefinition.CustomOneofCase.Composite)
+            mdef = mdef.Composite.Definition;
+         var sec_sets = algo_sets.Where(s => s.security_criteria.includes(security));
+         if (mdef.CustomCase == CustomOrderDefinition.CustomOneofCase.Dynamic) {
+            if (!sec_sets.All(s => s.algo_type_set.dynamic_types.Contains(mdef.Dynamic.DynamicOrderType))) {
+               assign_fallback_algo(mdef, security);
+               return true;
+            }
+         } else if (!sec_sets.All(s => s.algo_type_set.factory_types.Contains(mdef.CustomCase))) {
+            assign_fallback_algo(mdef, security);
+            return true;
+         }
+         return OrderDataHelper.get_child_algo_data(mdef).Count(cad => _fit_to_type(cad.get_child(mdef), security, algo_sets)) > 0;
+      }
+   }
+
+   class AttachableAlgoProvider : AlgoProvider, IAttachableAlgoProvider {
+      public AttachableAlgoProvider(IEnumerable<SecurityAlgoTypeSetImmutable> algo_sets, IAlgoStudioModel studio_model, DynamicOrderHelper dynamic_helper, ISecurityAuxiliaryDataModel security_auxiliary_model)
+         : base(algo_sets, studio_model, dynamic_helper, security_auxiliary_model) { }
+
+      // arbitrarily assign LimitOrderDefinition (this is just a placeholder; the real child is the attach target algo).
+      // Note: although 'None' might seem approrpriate, the server requires a valid definition (perhaps this will change in the future).
+      protected override void assign_fallback_algo(CustomOrderDefinition mdef, SecurityCriteriaClause security_clause) =>
+         mdef.Limit = new LimitOrderDefinition();
+
+      // arbitrarily assign LimitOrderDefinition (this is just a placeholder; the real child is the attach target algo).
+      // Note: although 'None' might seem approrpriate, the server requires a valid definition (perhaps this will change in the future).
+      protected override void assign_fallback_algo(CustomOrderDefinition mdef, SecurityImmutable security) =>
+         mdef.Limit = new LimitOrderDefinition();
+   }
+
+   static class AlgoProviderExtensions {
+      // This stream filters out composite algos that are not subsets of the non-composite algo types in the table
+      // (normally, such composites are 'auto-fit,' meaning the algo nodes that are not part of the set are replaced with ones that are).
+      // For example, consider a composite algo the following structure: Offset -> GlbxIceberg.
+      // If an update arrives with these noncomposite types: "Iceberg, Limit, Market," that composite algo is not a subset of the non-composite algo types,
+      // because GlbxIceberg is not in the table, so it would only be added to the table if/when GlbxIceberg arrives.
+      public static IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_complete_algos(this IAlgoProvider algo_provider, bool allow_undefined_child = false) =>
+         subscribe_complete_algos(algo_provider.subscribe_algos(), allow_undefined_child);
+
+      public static IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_complete_algos(IObservable<TableUpdate<AlgoData, AlgoKey>> source, bool allow_undefined_child = false) =>
+         Observable.Create<TableUpdate<AlgoData, AlgoKey>>(o => {
+            var last_out = TableUpdate<AlgoData, AlgoKey>.mk_empty(AlgoData.key_fn);
+            var incomplete_composites = ImmutableHashSet<AlgoData>.Empty;
+            return source.Subscribe(table => {
+               var bd = last_out.to_builder();
+               foreach (var algo in incomplete_composites.Where(a => _validate_algo_structure(a.value, table.state, allow_undefined_child))) {
+                  incomplete_composites = incomplete_composites.Remove(algo);
+                  bd.add_or_update(algo);
+               }
+               foreach (var a in table.removed.Concat(table.changes.Old()))
+                  bd.remove(a.key);
+               foreach (var a in table.added.Concat(table.changed)) {
+                  if (_validate_algo_structure(a.value, table.state, allow_undefined_child)) {
+                     bd.add_or_update(a);
+                  } else {
+                     incomplete_composites = incomplete_composites.Add(a);
+                  }
+               }
+               o.OnNext(last_out = bd.to_table());
+            });
+         });
+
+      static bool _validate_algo_structure(CustomOrderDefinitionImmutable custom_def, IReadOnlyDictionary<AlgoKey, AlgoData> allowed_algo_parts, bool allow_undefined_child) =>
+         OrderDataHelper.enumerate_algo_structure(custom_def)
+         .Select((algo_node, index) =>
+            // None is an acceptable algo type as long as it's not the root algo (i.e., index > 0)
+            (index > 0 && allow_undefined_child && algo_node.type == CustomOrderDefinition.CustomOneofCase.None) || _is_part_contained(algo_node, allowed_algo_parts))
+         .All();
+
+      static bool _is_part_contained((CustomOrderDefinition.CustomOneofCase type, string dyn, uint studio_id, uint spreader_sec) structure, IReadOnlyDictionary<AlgoKey, AlgoData> state) =>
+         structure.type == CustomOrderDefinition.CustomOneofCase.Composite ||
+         structure.type == CustomOrderDefinition.CustomOneofCase.Dynamic ?
+            state.ContainsKey(AlgoKey.from_dynamic_type(structure.dyn)) :
+            state.ContainsKey(AlgoKey.from_known_type(structure.type));
+   }
+}

src/client/spark/Modules/Spreader/ShowSyntheticEditorHandler.cs

@@ -46,7 +46,7 @@ namespace bts.shell {
 
       async void _show_synthetic_create(SecurityCriteria security_criteria) {
          var found_securities = await security_criteria.securities(_bus);
-         var respect_clause_security_ordering_is_possible = security_criteria.clauses.Any(clause => clause.security_keys != null && clause.security_keys.Count > 0);
+         var respect_clause_security_ordering_is_possible = security_criteria.clauses.Any(clause => clause.security_keys != null && clause.security_keys.Length > 0);
          if (!respect_clause_security_ordering_is_possible) {
             _show_synthetic_create(found_securities.Values.ToList());
          } else {

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -85,9 +85,7 @@ namespace mbts.spark {
          icon = Icons.Synthetics;
          _has_working_orders = backing(nameof(has_working_orders), false);
          contract_picker_model = new ContractPickerModel(bus, false, null, new SecurityCriteria(new[] {
-            new SecurityCriteriaClause() {
-               exchanges = EnumHelper.get_values_except(Venue.Bts).ToArray()
-            }
+            new SecurityCriteriaClause(exchanges: EnumHelper.get_values_except(Venue.Bts).ToArray())
          }));
          contract_picker_model.cancel_action += _close_new_leg_popup;
          contract_picker_model.ok_action += () => {

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoData.cs

@@ -9,6 +9,8 @@ namespace mbts.spark {
    // This is a core data type on the front end, so data validation is strict, as bad values may yield weird/difficult to track bugs.
    // Descriptive factory methods are favored over public constructors, as they are less likely to be misused.
    struct AlgoData : IEquatable<AlgoData> {
+      public static readonly Func<AlgoData, AlgoKey> key_fn = a => a.key;
+
       public readonly CustomOrderDefinitionImmutable value;
       public readonly string name;
       public readonly AlgoKey key;

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoKey.cs

@@ -128,6 +128,7 @@ namespace mbts.spark {
       }
 
       public bool is_persisted => mode == Mode.Persisted;
+      public bool is_composite => _composite_key.HasValue || _unpersisted.HasValue;
 
       public uint persisted => _composite_key.Value;
       public Guid unpersisted => _unpersisted.Value;
@@ -199,6 +200,15 @@ namespace mbts.spark {
          }
          return EqualityComparer.phase_hash(hash, phase_mult);
       }
+      public override string ToString() {
+         switch (mode) {
+            case Mode.Dynamic: return $"Dynamic: {dynamic_type}";
+            case Mode.Factory: return $"Factory: {factory}";
+            case Mode.Persisted: return $"Composite: {persisted}";
+            case Mode.Unpersisted: return $"(Composite): {unpersisted}";
+            default: return mode.ToString();
+         }
+      }
 
       public static bool operator ==(AlgoKey a, AlgoKey b) => a.Equals(b);
       public static bool operator !=(AlgoKey a, AlgoKey b) => !a.Equals(b);

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoStudioFlatDefinitionModel.cs

@@ -12,20 +12,20 @@ namespace mbts.spark {
    [Export(typeof(IAlgoStudioModel))]
    class AlgoStudioFlatDefinitionModel : IAlgoStudioModel, IDisposable {
       readonly AlgoDefinitionService _service;
-      readonly AdminModel _admin_model;
+      readonly UserTuple _user;
       readonly ErrorReporter _error_reporter;
       IReadOnlyDictionary<AlgoKey, AlgoData> _algos = DictionaryHelper.empty<AlgoKey, AlgoData>();
       IDisposable _dispose;
 
       [ImportingConstructor]
-      public AlgoStudioFlatDefinitionModel(AlgoDefinitionService service, IAlgoDefinitionMetadataService metadata_service, AdminModel admin_model, ErrorReporter error_reporter) {
+      public AlgoStudioFlatDefinitionModel(AlgoDefinitionService service, IAlgoDefinitionMetadataService metadata_service, UserTuple user, ErrorReporter error_reporter) {
          _service = service;
          this.metadata_service = metadata_service;
-         _admin_model = admin_model;
+         _user = user;
          _error_reporter = error_reporter;
          definitions =
             service.subscribei_algo_definitions()
-            .Filter(r => r.Metadata.IsActive && _admin_model.user.can_access_user_data(r.Metadata.CreatorUserId))
+            .Filter(r => r.Metadata.IsActive && _user.can_access_user_data(r.Metadata.CreatorUserId))
             .SelectKeyValue(_definition_data, d => d.key)
             .Table()
             .Cache();
@@ -37,7 +37,7 @@ namespace mbts.spark {
 
       public async Task<AlgoData?> save_definition(AlgoData definition_data) {
          var (value, success) = await _error_reporter.safe_execute(() => _service.save_algo_definition_async(
-            _with_key(new AlgoDefinitionUpdateRequest { UserId = _admin_model.user.id, Algo = _api_definition(definition_data) }, definition_data.key)),
+            _with_key(new AlgoDefinitionUpdateRequest { UserId = _user.id, Algo = _api_definition(definition_data) }, definition_data.key)),
             r => r.Success, r => r.ErrorMessage, "saving definition");
          if (success)
             return _definition_data(new AlgoDefinitionRecordImmutable(value.Record));
@@ -93,4 +93,4 @@ namespace mbts.spark {
          }
       }
    }
-}
+}

src/client/spark/Modules/Trading/AlgoStudio/Model/Metadata/AlgoDefinitionMetadata.cs

@@ -11,6 +11,8 @@ namespace mbts.spark {
    /// decouple the client-side key type from the server-side one, so both key types may evolve independently moving forward.
    /// </summary>
    struct AlgoDefinitionMetadata : IEquatable<AlgoDefinitionMetadata> {
+      public static readonly Func<AlgoDefinitionMetadata, AlgoKey> key_fn = m => m.id;
+
       public readonly AlgoKey id, default_template;
       readonly AlgoKey[] _favorite_templates;
       public readonly bool is_favorite, is_hidden;

src/client/spark/Modules/Trading/AlgoStudio/Utils/AlgoCollectionHelper.cs

@@ -35,34 +35,21 @@ namespace mbts.spark {
                Comparer<IAlgoVM>.Create(_CompareUtils.compare_names)
             }).to_comparer().to_safe();
 
-      public static void update_persisted_definitions(ObservableKeyedCollection<AlgoDefinitionVM, AlgoKey> target, TableUpdate<AlgoData, AlgoKey> update, Func<AlgoData, AlgoDefinitionVM> factory) {
-         foreach (var d in update.removed)
-            target.Remove(d.key);
-         foreach (var d in update.changed)
+      public static void update_persisted_definitions(ObservableKeyedCollection<AlgoDefinitionVM, AlgoKey> target, TableUpdate<AlgoData, AlgoKey> update, Func<AlgoData, AlgoDefinitionVM> factory) =>
+         update_persisted_definitions(target, update.added, update.changed, update.removed.Select(AlgoData.key_fn), factory);
+
+      public static void update_persisted_definitions(ObservableKeyedCollection<AlgoDefinitionVM, AlgoKey> target, IEnumerable<AlgoData> added, IEnumerable<AlgoData> changed, IEnumerable<AlgoKey> removed, Func<AlgoData, AlgoDefinitionVM> factory) {
+         foreach (var k in removed)
+            target.Remove(k);
+         foreach (var d in changed)
             target[d.key].load(d);
-         foreach (var d in update.added) {
+         foreach (var d in added) {
             if (target.TryGetValue(d.key, out AlgoDefinitionVM def)) {
                def.load(d);
             } else target.Add(factory(d));
          }
       }
 
-      public static void update_persisted_definition_wrappers(IDictionary<AlgoKey, IAlgoVM> target, TableUpdate<AlgoData, AlgoKey> update, Func<AlgoData, AlgoDefinitionVM> factory) {
-         foreach (var d in update.removed)
-            target.Remove(d.key);
-         foreach (var d in update.changed) {
-            var vm = target[d.key];
-            if (vm is AlgoVMWrapper wrapper) {
-               wrapper.value.load(d);
-            } else vm.load_from(d.value);
-         }
-         foreach (var d in update.added) {
-            if (target.TryGetValue(d.key, out IAlgoVM def)) {
-               def.load_from(d.value);
-            } else target.Add(d.key, new AlgoVMWrapper(factory(d)));
-         }
-      }
-
       public static void commit_all(this IEnumerable<AlgoParamRowVM> algo_param_rows) =>
          algo_param_rows.Select(r => r.param).commit_all();
 

src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoDefinitionVM.cs

@@ -42,21 +42,21 @@ namespace mbts.spark {
       readonly Property<string> _name, _abbreviation;
       readonly ObservableKeyedCollection<IAlgoVM, AlgoKey> _algo_roots = new ObservableKeyedCollection<IAlgoVM, AlgoKey>(vm => vm.id);
       readonly DeferrableActionWrapper _refresh_on_local_changes;
-      readonly Property<AlgoFormats> _selected_format;
       readonly EnumMaskInt32<AlgoEditingPermissions> _permissions_mask = new EnumMaskInt32<AlgoEditingPermissions>();
 
       int? _favorite_index;
       Brush _badge_fill;
       Snapshot? _committed;
       AlgoDefinitionMetadata _metadata;
+      IDisposable _loaded_data_hook;
 
-      public static AlgoDefinitionVM create(OrdersModel model, AlgoData data, AlgoFormats init_format = AlgoFormats.Exchange, AlgoEditingPermissions editing_permissions = (AlgoEditingPermissions)int.MaxValue) =>
-         new AlgoDefinitionVM(model, data, init_format, editing_permissions);
+      public static AlgoDefinitionVM create(OrdersModel model, AlgoData data, AlgoEditingPermissions editing_permissions = (AlgoEditingPermissions)int.MaxValue) =>
+         new AlgoDefinitionVM(model, data, editing_permissions);
 
-      public static AlgoDefinitionVM create_placeholder(OrdersModel model, CustomOrderDefinitionImmutable def, string name, AlgoFormats init_format) =>
-         new AlgoDefinitionVM(model, AlgoData.create_unpersisted(def, name), init_format, (AlgoEditingPermissions)int.MaxValue);
+      public static AlgoDefinitionVM create_placeholder(OrdersModel model, CustomOrderDefinitionImmutable def, string name) =>
+         new AlgoDefinitionVM(model, AlgoData.create_unpersisted(def, name), (AlgoEditingPermissions)int.MaxValue);
 
-      AlgoDefinitionVM(OrdersModel model, AlgoData def_data, AlgoFormats init_format, AlgoEditingPermissions init_permissions) {
+      AlgoDefinitionVM(OrdersModel model, AlgoData def_data, AlgoEditingPermissions init_permissions) {
          // init core data
          _model = model;
          _permissions_mask.mask = init_permissions;
@@ -80,7 +80,6 @@ namespace mbts.spark {
          // init backing props
          _name = backing(nameof(name), def_data.name);
          _abbreviation = backing(nameof(abbreviation), string.Empty);
-         _selected_format = backing(nameof(selected_format), init_format);
          _is_favorite = backing(nameof(is_favorite), () => _favorite_index.HasValue);
          _is_expanded = backing(nameof(is_expanded), true);
          _is_selected = backing(nameof(is_selected), true);
@@ -133,17 +132,6 @@ namespace mbts.spark {
                   break;
             }
             var permission_src = _permissions_mask.observe_result();
-            resources.Add(_selected_format.observe_current(true).CombineLatest(permission_src, (fmt, perms) => (fmt, perms))
-               .SubscribeEach(t => {
-                  if ((t.perms & AlgoEditingPermissions.AllowEditStructure) != 0) {
-                     using (_refresh_on_local_changes.defer_execution()) {
-                        var algo_data = model.factory.default_algos(t.fmt);
-                        var default_algo = AlgoKey.from_known_type(OrderDataHelper.get_default_algo(t.fmt));
-                        return this.load_algo_data_recursive(algo_data, default_algo);
-                     }
-                  }
-                  return Disposable.empty;
-               }));
             resources.Add(permission_src.Subscribe(_ => RaisePropertyChanged(nameof(editing_permissions))));
             resources.Add(this.connect_data(permission_src));
             resources.Add(this.connect_data(Observable.Return(model)));
@@ -202,9 +190,6 @@ namespace mbts.spark {
       public EnabledValueVM<Color> custom_badge_color { get; } = new EnabledValueVM<Color>();
       public ObservableCollection<ColorPaletteVM> custom_badge_color_palettes { get; } = new ObservableCollection<ColorPaletteVM>();
       public Brush badge_fill => _badge_fill;
-      // format
-      public AlgoFormats selected_format { get => _selected_format.Value; set => _selected_format.set(value); }
-      public ObservableCollection<AlgoFormats> formats { get; } = new ObservableCollection<AlgoFormats> { AlgoFormats.Exchange, AlgoFormats.Spreader, AlgoFormats.Exchange | AlgoFormats.Spreader };
       // visual states
       public ObservableCollection<string> edit_pane_visual_states => _edit_pane_vs_model.states;
       public ObservableCollection<string> tree_view_visual_states => _tree_view_vs_model.states;
@@ -270,17 +255,14 @@ namespace mbts.spark {
       void IObserver<(IReadOnlyDictionary<AlgoKey, AlgoData> algo_data, AlgoKey? default_algo)>.OnError(Exception error) { }
       #endregion
 
-      void _load_algo_data(IReadOnlyDictionary<AlgoKey, AlgoData> algo_data, AlgoKey? default_algo) {
-         var orig_algo = algo_root?.id;
-         _algo_roots.SyncWithDictionary(algo_data, d => OrderTypeVMFactory.create(d, is_expanded: true));
-         if (!try_select(orig_algo) && !try_select(default_algo))
-            algo_root = algo_roots.FirstOrDefault();
-         //
-         bool try_select(AlgoKey? algo_key) {
-            if (algo_key.HasValue && _algo_roots.TryGetValue(algo_key.Value, out IAlgoVM match)) {
+      void _load_algo_data(IReadOnlyDictionary<AlgoKey, AlgoData> data, AlgoKey? default_algo) {
+         Disposable.dispose(ref _loaded_data_hook);
+         _algo_roots.SyncWithDictionary(data, d => OrderTypeVMFactory.create(d, is_expanded: true));
+         _loaded_data_hook = ObservableAutoWrapper.connect_item_hooks(algo_roots, vm =>
+            ObservableAutoWrapper.connect_item_actions(vm.child_algo_selectors, s => s.load_algo_data(data, null)));
+         if (algo_root == null || !_algo_roots.ContainsKey(algo_root.id)) {
+            if (default_algo.HasValue && _algo_roots.TryGetValue(default_algo.Value, out IAlgoVM match))
                algo_root = match;
-               return true;
-            } else return false;
          }
       }
 
@@ -346,7 +328,7 @@ namespace mbts.spark {
 
       // pass a copy of the persisted state - rather than use local values - because if something goes wrong, it is better to be in sync with data fresh from the server/api.
       public AlgoDefinitionVM to_persisted(AlgoData def) =>
-         new AlgoDefinitionVM(_model, def, selected_format, editing_permissions_mask);
+         new AlgoDefinitionVM(_model, def, editing_permissions_mask);
 
       public AlgoData create_algo_def() =>
          AlgoData.create(algo_root == null ? new CustomOrderDefinitionImmutable(new CustomOrderDefinition()) : algo_root.definition, name, key);
@@ -368,7 +350,6 @@ namespace mbts.spark {
 
       void _assign_from_data(Snapshot state) {
          _assign_from_data(state.data);
-         _selected_format.set(state.format);
          _abbreviation.set(state.abbreviation);
          custom_badge_color.set_nullable(state.custom_badge_color);
       }
@@ -429,41 +410,36 @@ namespace mbts.spark {
       }
 
       Snapshot _snapshot() =>
-         new Snapshot(create_algo_def(), abbreviation, custom_badge_color.get_nullable(), selected_format);
+         new Snapshot(create_algo_def(), abbreviation, custom_badge_color.get_nullable());
 
       public struct Snapshot : IEquatable<Snapshot> {
          public readonly AlgoData data;
          public readonly string abbreviation;
          public readonly Color? custom_badge_color;
-         public readonly AlgoFormats format;
 
-         public Snapshot(AlgoData data, string abbreviation, Color? custom_badge_color, AlgoFormats format) {
+         public Snapshot(AlgoData data, string abbreviation, Color? custom_badge_color) {
             this.data = data;
             this.abbreviation = abbreviation;
             this.custom_badge_color = custom_badge_color;
-            this.format = format;
          }
 
          public bool Equals(Snapshot other) =>
             data == other.data &&
             abbreviation == other.abbreviation &&
-            custom_badge_color == other.custom_badge_color &&
-            format == other.format;
+            custom_badge_color == other.custom_badge_color;
 
          public override bool Equals(object obj) => EqualityComparer.equals_obj(this, obj);
          public override int GetHashCode() =>
             EqualityComparer.aggregate_hash(
                data.GetHashCode(),
                abbreviation?.GetHashCode() ?? 0,
-               custom_badge_color?.GetHashCode() ?? 0,
-               (int)format);
+               custom_badge_color?.GetHashCode() ?? 0);
 #if DEBUG
          public override string ToString() {
             var sb = new System.Text.StringBuilder();
             sb.Append($"name: {data.name}; ");
             sb.Append($"abbrev.: {abbreviation}; ");
             sb.Append($"badge: {custom_badge_color} ");
-            sb.Append($"fmt: {format} ");
             sb.Append($"def: {data.value}");
             return sb.ToString();
          }
@@ -477,7 +453,6 @@ namespace mbts.spark {
          public bool Equals(Snapshot x, Snapshot y) =>
             x.abbreviation == y.abbreviation &&
             x.custom_badge_color == y.custom_badge_color &&
-            x.format == y.format &&
             x.data.key == y.data.key &&
             x.data.name == y.data.name &&
             x.data.value.Equals(y.data.value);

src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoStudioVM.cs

@@ -1,7 +1,11 @@
 using bts;
+using bts.api;
+using bts.api.core;
 using bts.api.orderman;
+using bts.collections;
 using bts.utils;
 using mbts.collections;
+using mbts.spark.utilities;
 using mbts.ui;
 using mbts.ui.shell;
 using mbts.ui.utility;
@@ -37,7 +41,6 @@ namespace mbts.spark {
    }
 
    class AlgoStudioVM : DisposableViewModelBase {
-      static readonly AlgoFormats _studio_formats = AlgoFormats.Exchange;
       readonly OrdersModel _model;
       readonly Property<AlgoDefinitionVM> _selected_definition;
       readonly Property<IAlgoVM> _editable_content;
@@ -70,12 +73,24 @@ namespace mbts.spark {
          _revert_cmd_tooltip = backing(nameof(revert_cmd_tooltip), get_revert_tooltip);
          //
          _dispose = new CompositeDisposable(
-            _model.algo_model.definitions.Subscribe(_definitions_update_buffer),
-            _definitions_update_buffer.OnDispatcher().Subscribe(u => {
-               var orig_selected_def = selected_definition;
-               AlgoCollectionHelper.update_persisted_definitions(_definitions, u, d => AlgoDefinitionVM.create(model, d, _studio_formats));
+            _model.subscribe_complete_algos(allow_undefined_child: true).OnDispatcher().Subscribe(_definitions_update_buffer),
+            _definitions_update_buffer.OnDispatcher()
+            .SubscribeEach(u => {
+               u.FilterIncremental(a => a.key.is_composite, out var added, out var changes, out var removed);
+               AlgoCollectionHelper.update_persisted_definitions(
+                  _definitions, added, changes.New(), removed.Select(AlgoData.key_fn), algo => AlgoDefinitionVM.create(_model, algo));
                if (selected_definition == null)
                   selected_definition = definitions.FirstOrDefault();
+               // Load the algo building blocks (i.e., factory and dynamic algos):
+               //    - Composite algos cannot reference other composite algos - that would create awkward dependency issues as composite algos can be modified/deleted.
+               //    - Spreader algos are filtered out as well, because that don't actually mean anything to the algo structure.
+               //      Synthetic securities in our system are defined by a set of contract legs, and a [single]strategy.
+               //      Note: synthetic strategy templates, however, could be introduced at a later date (no reason why 1 synthetic security couldn't have several ExecutionStrategies).
+               var algo_building_blocks = new FilteredDictionary<AlgoKey, AlgoData>(
+                  u.state,
+                  a => a.key.mode == AlgoKey.Mode.Dynamic || (a.key.mode == AlgoKey.Mode.Factory && a.key.factory != CustomOrderDefinition.CustomOneofCase.Spreader));
+               return ObservableAutoWrapper.connect_item_actions(_definitions.AsCollection(),
+                  d => ((IAlgoSelector)d).load_algo_data(algo_building_blocks, null));
             }),
             _model.algo_model.metadata_service.subscribe_metadata().OnDispatcher().Subscribe(meta => {
                _persisted_meta = meta;
@@ -125,7 +140,7 @@ namespace mbts.spark {
          }
 
          void add_definition_common(CustomOrderDefinitionImmutable def, string name) {
-            var new_def = AlgoDefinitionVM.create_placeholder(_model, def, name, _studio_formats);
+            var new_def = AlgoDefinitionVM.create_placeholder(_model, def, name);
             _definitions.Add(new_def);
             selected_definition = new_def;
             request_view_focus_name?.Invoke(new_def);

src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoVMWrapper.cs

@@ -1,214 +0,0 @@
-using bts.api;
-using bts.api.orderman;
-using bts.collections;
-using bts.spark;
-using mbts.spark.utilities;
-using mbts.ui;
-using mbts.ui.controls;
-using mbts.ui.shell;
-using mbts.utils;
-using System;
-using System.Collections.Generic;
-using System.Collections.ObjectModel;
-using System.Collections.Specialized;
-using System.ComponentModel;
-using System.Reactive.Disposables;
-using System.Reactive.Linq;
-using System.Threading.Tasks;
-using System.Windows.Input;
-
-namespace mbts.spark {
-   class AlgoVMWrapper : DisposableViewModelBase, IAlgoVM,
-      IObserver<SecurityMD?>,
-      IObserver<SecurityImmutable?>,
-      IObserver<OrdersModel>,
-      IObserver<AlgoEditingPermissions> {
-      static readonly HashSet<string>
-         _algo_root_echo_props = new HashSet<string> {
-            nameof(is_expanded),
-            nameof(editing_permissions),
-            nameof(definition),
-            nameof(display_name)
-         };
-      static readonly IReadOnlyDictionary<string, string>
-         _value_echo_props = new Dictionary<string, string> {
-            { nameof(AlgoDefinitionVM.name), nameof(display_name) },
-            { nameof(AlgoDefinitionVM.abbreviation), nameof(abbreviation) },
-            { nameof(AlgoDefinitionVM.badge_fill), nameof(meta) }
-         };
-      readonly ObservableAggregateCollection<IAlgoSelector> _child_algo_selectors = new ObservableAggregateCollection<IAlgoSelector>();
-      readonly ObservableAggregateCollection<IAlgoParamVM> _algo_params = new ObservableAggregateCollection<IAlgoParamVM>();
-      readonly DisposableContainer _value_hooks = new DisposableContainer();
-      Action _definition_changed = () => { };
-      _ObservedData? _data_cache;
-      AlgoDefinitionVM _value;
-
-      public AlgoVMWrapper(AlgoDefinitionVM algo_definition_vm) {
-         value = algo_definition_vm;
-         _dispose = new CompositeDisposable(_value_hooks) {
-            _child_algo_selectors.clear_collections,
-            _algo_params.clear_collections,
-            () => _value?.Dispose()
-         };
-      }
-
-      public event Action definition_changed { add => _definition_changed += value; remove => _definition_changed -= value; }
-
-      public AlgoDefinitionVM value {
-         get => _value;
-         set {
-            if (_value != value) {
-               _value?.Dispose();
-               _value = value;
-               _value_hooks.replace(_set_hooks(value));
-               RaisePropertyChanged(nameof(value));
-            }
-         }
-      }
-      public AlgoKey id => _value.key;
-      public CustomOrderDefinition.CustomOneofCase order_type => _algo_root?.order_type ?? CustomOrderDefinition.CustomOneofCase.None;
-      public OrderMetadata meta {
-         get {
-            if (_value != null) {
-               if (_value.custom_badge_color.enabled)
-                  return new OrderMetadata { Color = value.custom_badge_color.value.rgb_color(), KeyBinding = value.abbreviation.if_null_or_empty(value.name) };
-               else if (_value.algo_root != null)
-                  return _value.algo_root.meta;
-            }
-            return OrderDataHelper.get_metadata(order_type);
-         }
-      }
-      public AlgoEditingPermissions editing_permissions => _value?.editing_permissions ?? AlgoEditingPermissions.None;
-      public bool can_expand => _algo_root?.can_expand ?? false;
-      public bool is_expanded {
-         get => _value.algo_root?.is_expanded ?? false;
-         set {
-            if (_value != null)
-               _value.is_expanded = value;
-         }
-      }
-      public string display_name => _value?.name ?? string.Empty;
-      public string abbreviation => _value?.abbreviation;
-      public AlgoDisplayGroup group => _value?.group ?? AlgoDisplayGroup.BTS;
-      public CustomOrderDefinitionImmutable definition => _algo_root?.definition ?? new CustomOrderDefinitionImmutable(new CustomOrderDefinition());
-      public ObservableCollection<IAlgoSelector> child_algo_selectors => _child_algo_selectors;
-      [EditorBrowsable(EditorBrowsableState.Never)]
-      public ObservableCollection<IAlgoSelector> child_algo_selectors_view {
-         get {
-            _value?.algo_root?.ensure_init();
-            return _child_algo_selectors;
-         }
-      }
-      public IReadOnlyObservableCollection<IAlgoParamVM> algo_params => _algo_params;
-      [EditorBrowsable(EditorBrowsableState.Never)]
-      public dynamic children_vm => _algo_root?.children_vm;
-
-      IAlgoVM _algo_root => _value?.algo_root;
-
-      public void assign_to(CustomOrderDefinition value) => _algo_root?.assign_to(value);
-      public bool key_match(KeyEventArgs e) => _algo_root?.key_match(e) ?? false;
-      public void load_from(CustomOrderDefinitionImmutable value) => _algo_root?.load_from(value);
-      public void load_many(IEnumerable<CustomOrderDefinitionImmutable> values) => _algo_root?.load_many(values);
-      void IAlgoVM.ensure_init() => _value?.algo_root?.ensure_init();
-
-      IDisposable _set_hooks(AlgoDefinitionVM vm) {
-         var algo_root_subs = new DisposableContainer();
-         vm.PropertyChanged += on_value_prop_changed;
-         return new CompositeDisposable(
-            vm.observe_algo_root().Subscribe(r => {
-               algo_root_subs.replace(null);
-               _child_algo_selectors.clear_collections();
-               _algo_params.clear_collections();
-               if (r != null) {
-                  if (_data_cache.HasValue) {
-                     _data_cache.Value.forward_to(r);
-                  }
-                  _child_algo_selectors.add_collection(r.child_algo_selectors);
-                  _algo_params.add_collection(r.algo_params);
-                  ((INotifyPropertyChanged)r).PropertyChanged += on_algo_root_prop_changed;
-                  algo_root_subs.replace(new CompositeDisposable(
-                     r.subscribe_definition().Subscribe(_ => _definition_changed?.Invoke())) {
-                     () => ((INotifyPropertyChanged)r).PropertyChanged -= on_algo_root_prop_changed
-                  });
-               }
-               RaisePropertyChanged(nameof(group));
-            }),
-            algo_root_subs) {
-            () => vm.PropertyChanged -= on_value_prop_changed
-         };
-         void on_algo_root_prop_changed(object sender, PropertyChangedEventArgs e) {
-            if (_algo_root_echo_props.Contains(e.PropertyName))
-               RaisePropertyChanged(e.PropertyName);
-         }
-         void on_value_prop_changed(object sender, PropertyChangedEventArgs e) {
-            if (_value_echo_props.TryGetValue(e.PropertyName, out string wrapper_prop))
-               RaisePropertyChanged(wrapper_prop);
-         }
-      }
-
-      // permissions
-      void IObserver<AlgoEditingPermissions>.OnNext(AlgoEditingPermissions value) {
-         _data_cache = _data_cache?.with(value) ?? new _ObservedData(value, default, default, default);
-         (_algo_root as IObserver<AlgoEditingPermissions>)?.OnNext(value);
-      }
-      void IObserver<AlgoEditingPermissions>.OnError(Exception error) { }
-      void IObserver<AlgoEditingPermissions>.OnCompleted() { }
-      // model
-      void IObserver<OrdersModel>.OnNext(OrdersModel value) {
-         _data_cache = _data_cache?.with(value) ?? new _ObservedData(default, value, default, default);
-         (_algo_root as IObserver<OrdersModel>)?.OnNext(value);
-      }
-      void IObserver<OrdersModel>.OnError(Exception error) { }
-      void IObserver<OrdersModel>.OnCompleted() { }
-      // security
-      void IObserver<SecurityImmutable?>.OnNext(SecurityImmutable? value) {
-         _data_cache = _data_cache?.with(value) ?? new _ObservedData(default, default, value, default);
-         (_algo_root as IObserver<SecurityImmutable?>)?.OnNext(value);
-      }
-      void IObserver<SecurityImmutable?>.OnError(Exception error) { }
-      void IObserver<SecurityImmutable?>.OnCompleted() { }
-      // security md
-      void IObserver<SecurityMD?>.OnNext(SecurityMD? value) {
-         _data_cache = _data_cache?.with(value) ?? new _ObservedData(default, default, default, value);
-         (_algo_root as IObserver<SecurityMD?>)?.OnNext(value);
-      }
-      void IObserver<SecurityMD?>.OnError(Exception error) { }
-      void IObserver<SecurityMD?>.OnCompleted() { }
-
-      struct _ObservedData {
-         public readonly AlgoEditingPermissions permissions;
-         public readonly OrdersModel model;
-         public readonly SecurityImmutable? security;
-         public readonly SecurityMD? security_md;
-
-         public _ObservedData(AlgoEditingPermissions permissions, OrdersModel model, SecurityImmutable? security, SecurityMD? security_md) {
-            this.permissions = permissions;
-            this.model = model;
-            this.security = security;
-            this.security_md = security_md;
-         }
-
-         public void forward_to(IAlgoVM recipient) {
-            (recipient as IObserver<AlgoEditingPermissions>)?.OnNext(permissions);
-            if (model != null)
-               (recipient as IObserver<OrdersModel>)?.OnNext(model);
-            if (security_md.HasValue)
-               (recipient as IObserver<SecurityMD?>)?.OnNext(security_md);
-            if (security.HasValue)
-               (recipient as IObserver<SecurityImmutable?>)?.OnNext(security);
-         }
-
-         public _ObservedData with(AlgoEditingPermissions? permissions) =>
-            new _ObservedData(permissions ?? this.permissions, model, security, security_md);
-
-         public _ObservedData with(OrdersModel model) =>
-            new _ObservedData(permissions, model, security, security_md);
-
-         public _ObservedData with(SecurityImmutable? security) =>
-            new _ObservedData(permissions, model, security, security_md);
-
-         public _ObservedData with(SecurityMD? security_md) =>
-            new _ObservedData(permissions, model, security, security_md);
-      }
-   }
-}

src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoEditorGridView.xaml

@@ -302,7 +302,7 @@
         <!--Double-->
         <usp:AlgoParamTemplateSelector.DoubleTemplate>
           <DataTemplate>
-            <TextBlock Text="{Binding param.value, Converter={StaticResource DoubleTextConverter}}"/>
+            <TextBlock Text="{Binding param.display_value, Converter={StaticResource DoubleTextConverter}}"/>
           </DataTemplate>
         </usp:AlgoParamTemplateSelector.DoubleTemplate>
         <!--Enum-->
@@ -406,7 +406,7 @@
         <!--Double-->
         <usp:AlgoParamTemplateSelector.DoubleTemplate>
           <DataTemplate>
-            <c:TickUpDownDouble x:Name="tick_up_down" Focusable="False" HorizontalAlignment="Stretch" Value="{Binding param.value, UpdateSourceTrigger=PropertyChanged}" ArrowKeyTickModifiers="Ctrl">
+            <c:TickUpDownDouble x:Name="tick_up_down" Focusable="False" HorizontalAlignment="Stretch" Value="{Binding param.display_value, UpdateSourceTrigger=PropertyChanged}" ArrowKeyTickModifiers="Ctrl">
               <c:TickUpDownDouble.CommandBindings>
                 <CommandBinding Command="{x:Static u:NavigationHelper.MoveFocusCommand}" Executed="{x:Static u:NavigationHelper.KeyboardMoveFocusExecuted}"/>
               </c:TickUpDownDouble.CommandBindings>

src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoParamRowsHelper.cs

@@ -39,31 +39,6 @@ namespace mbts.spark {
          }
       }
 
-      static IDisposable _wrap_rows(AlgoDefinitionVM algo_def, ObservableAggregateCollection<AlgoParamRowVM> rows, bool include_algo_type_rows) {
-         AlgoParamRowVM root = null;
-         ObservableAggregateCollection<AlgoParamRowVM> algo_params;
-         if (include_algo_type_rows) {
-            // while normal params live and die with the selected algo-type (when the algo-type changes, they are removed, and replaced with params for the new algo type),
-            // the algo-type row is constant (it is the row with the algo-type dropdown, which the user may use to select different algos).
-            algo_params = new ObservableAggregateCollection<AlgoParamRowVM>();
-            rows.add_collection(new ObservableAppendCollection<AlgoParamRowVM>(algo_params, root = new AlgoSelectorParamRowVM(new AlgoVMWrapper(algo_def), algo_def, null)).AsReadOnly());
-         } else {
-            algo_params = rows;
-         }
-         return algo_def.observe_algo_root().SubscribeEach(a => {
-            algo_params.clear_collections();
-            if (a == null)
-               return Disposable.empty;
-            var child_rows = new ObservableAggregateCollection<AlgoParamRowVM>();
-            /// Note: the order of operations here matters
-            ///   <see cref="_wrap_rows(IAlgoVM, AlgoParamRowVM, ObservableAggregateCollection{AlgoParamRowVM}, bool)" /> has the potential to trigger a lot of CollectionChanged events (depending on how deep the algo-tree is).
-            ///   There is no need to burden the aggregate 'rows' with echoing all of these events, if the child-row collection is finished changing when it is added to the aggregate.
-            var hook = _wrap_rows(a, root, child_rows, include_algo_type_rows);
-            algo_params.add_collection(child_rows.AsReadOnly());
-            return hook;
-         });
-      }
-
       static IDisposable _wrap_rows(IAlgoVM algo_root, AlgoParamRowVM parent, ObservableAggregateCollection<AlgoParamRowVM> all_rows, bool include_algo_type_rows) {
          var wrapped_params = new ObservableAutoWrapper<AlgoParamRowVM, IAlgoParamVM>(algo_root.algo_params, pvm => new AlgoParamRowVM(algo_root, pvm, parent));
          var selector_params = new ObservableCollection<AlgoParamRowVM>();
@@ -85,4 +60,4 @@ namespace mbts.spark {
          }
       }
    }
-}
+}

src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorView.xaml

@@ -106,7 +106,7 @@
     </GroupBox.Header>
     <ContentControl
       Grid.IsSharedSizeScope="True"
-      Content="{Binding algo_root, Converter={x:Static usp:AlgoContentTemplateSelector.DataContextConverter}}"
+      Content="{Binding algo_root}"
       ContentTemplateSelector="{StaticResource AlgoTemplateSelector}"
       Margin="{StaticResource Margin.TP1}"
       Visibility="{Binding algo_root.can_expand, Converter={StaticResource BooleanToVisibilityConverter}}"

src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs

@@ -144,7 +144,7 @@ namespace mbts.spark {
             string candidate = string.Empty;
             if (a.HasValue) {
                var algo_name = a.Value.name;
-               var words = algo_name.Split(delim);
+               var words = algo_name.Split(delim, StringSplitOptions.RemoveEmptyEntries);
                if (words.Length == 1) {
                   var single_word = words[0];
                   if (single_word.Length > 1) {
@@ -153,7 +153,7 @@ namespace mbts.spark {
                      candidate = single_word;
                   }
                } else {
-                  candidate = words.Take(2).Select(w => w.Substring(0, 1)).list_string(string.Empty);
+                  candidate = string.Join(string.Empty, words.Take(2).Select(w => w.Substring(0, 1)));
                }
             }
             return candidate.ToUpper();

src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs

@@ -7,6 +7,7 @@ using bts.proto;
 using bts.shell.orderbook;
 using bts.utils;
 using mbts.collections;
+using mbts.spark.utilities;
 using mbts.ui;
 using mbts.ui.controls;
 using mbts.ui.controls.color_picker;
@@ -115,7 +116,16 @@ namespace mbts.spark {
          foreach (var c in _observable_components)
             c.PropertyChanged += on_prop_changed;
          favorite_algos_vm.items.CollectionChanged += on_favorite_algo_items_CollectionChanged;
-         _algo_definitions.AddRange(_model.factory.default_algos(security).Values.Select(algo_factory));
+         _algo_definitions.AddRange(_model.get_permitted_algos(security).factory_types
+            .Select(OrderDataHelper.Defaults.CustomOrderDefinitionImmutable)
+            .Select(AlgoData.from_custom_def)
+            .Select(algo_factory));
+         _algo_definitions.AddRange(_model.get_permitted_algos(security).dynamic_types
+            .Select(dyn_type => new CustomOrderDefinitionImmutable(new CustomOrderDefinition {
+               Dynamic = _model.dynamic_helper.create_def(dyn_type)
+            }))
+            .Select(AlgoData.from_custom_def)
+            .Select(algo_factory));
          _dispose = new CompositeDisposable(
             _model.factory.subscribe_accounts().OnDispatcher().Subscribe(acc_map => {
                _accounts.SyncWithDictionary(MappedDictionary.create(acc_map, account_factory));
@@ -144,7 +154,7 @@ namespace mbts.spark {
                c.PropertyChanged -= on_prop_changed;
          }
          AccountTuple? account_factory(uint key, string name) => new AccountTuple(key, name);
-         AlgoDefinitionVM algo_factory(AlgoData data) => AlgoDefinitionVM.create(_model, data, is_synthetic ? AlgoFormats.Spreader : AlgoFormats.Exchange);
+         AlgoDefinitionVM algo_factory(AlgoData data) => AlgoDefinitionVM.create(_model, data);
       }
 
       public event Action result_changed;

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -398,7 +398,8 @@ namespace mbts.spark {
       public uint last_column_width { get => _last_column_width.Value; set => _last_column_width.set(value); }
       public uint piq_column_width { get => _piq_column_width.Value; set => _piq_column_width.set(value); }
       public uint volume_column_width { get => _volume_column_width.Value; set => _volume_column_width.set(value); }
-      public uint alternate_order_size_from_settings => current_settings.alternate_order_size;
+      public uint actual_alternate_order_size =>
+         (current_settings?.alternate_order_size ?? 0) * Math.Max(1, display_quantity_divisor);
       public bool show_warn_on_cross_from_settings => current_settings.show_warn_on_cross;
       public bool enforce_max_clip_from_settings => current_settings.enforce_max_clip;
       public uint max_clip_from_settings => current_settings.max_clip;
@@ -489,11 +490,11 @@ namespace mbts.spark {
       public Task send_order_async((double price, Side? side, MouseButton button) args) {
          if (order_properties_vm != null && args.button != MouseButton.Middle) {
             var size_from_alternate = args.button == MouseButton.Right;
-            if (!DoubleHelper.IsNaNOrInfinity(args.price) && args.side.HasValue && !HasErrors && !order_properties_vm.HasErrors && (!size_from_alternate || current_settings.alternate_order_size > 0)) {
+            if (!DoubleHelper.IsNaNOrInfinity(args.price) && args.side.HasValue && !HasErrors && !order_properties_vm.HasErrors && (!size_from_alternate || actual_alternate_order_size > 0)) {
                if (!size_from_alternate && (!size_calculator.value.HasValue || size_calculator.value.Value == 0)) {
                   size_calculator.flash_error();
                } else {
-                  uint size = size_from_alternate ? current_settings.alternate_order_size : (uint)Math.Abs(size_calculator.value ?? 0);
+                  uint size = size_from_alternate ? actual_alternate_order_size : (uint)Math.Abs(size_calculator.value ?? 0);
                   // enforce max clip
                   if (enforce_max_clip_from_settings && size > max_clip_from_settings) {
                      size = max_clip_from_settings;
@@ -530,7 +531,7 @@ namespace mbts.spark {
          if (fav_algo != null) {
             if (_favorite_algo_map.TryGetValue(fav_algo.key, out var d)) {
                if (!d.flags.HasFlag(LadderFavoriteAlgoFlags.Sticky) && !order_properties_vm.try_pop_selected_algo_definition()) {
-                  var key = new AlgoKey(OrderDataHelper.get_default_algo(OrderDataHelper.get_algo_format(security.ContractCase)));
+                  var key = new AlgoKey(OrderDataHelper.get_default_algo(security.ContractCase));
                   if (order_properties_vm.try_get_algo_definition(key, out var r)) {
                      order_properties_vm.selected_algo_definition = r;
                   }

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -353,7 +353,9 @@ namespace mbts.spark {
                return;
             }
          } else if (_vm != null) {
-            uint size = e.ChangedButton == MouseButton.Right ? _vm.alternate_order_size_from_settings : _vm.size_calculator.value ?? 0;
+            uint size = e.ChangedButton == MouseButton.Right ?
+               _vm.actual_alternate_order_size :
+               _vm.size_calculator.value ?? 0;
             
             // does the size violate the local max clip
             if (_vm.enforce_max_clip_from_settings && size > _vm.max_clip_from_settings) {

src/client/spark/Modules/Trading/ManualFills/ManualFillRow.cs

@@ -26,7 +26,7 @@ namespace bts.shell.manual_fills {
       readonly Property<bool> _is_dirty;
       readonly Property<EditableItemState> _editable_item_state;
       readonly DelegateProperty<bool> _is_valid;
-      readonly DisplayQuantityMultiplier _size = new DisplayQuantityMultiplier();
+      readonly DisplayQuantityMultiplier<uint> _size = DisplayQuantityMultiplier.create_uint();
       readonly Func<SecurityImmutable?, uint> _get_display_qty_divisor;
       SecurityImmutable? _security;
       AccountTuple? _account;

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -29,7 +29,7 @@ namespace mbts.spark {
       int _depth_level;
       SecurityImmutable? _security => order_tuple.order.security;
       // for mini-ob 
-      DisplayQuantityMultiplier _prospective_qty_translator_backing;
+      DisplayQuantityMultiplier<uint> _prospective_qty_translator_backing;
       uint _display_qty_divisor_backing;
       bool _display_qty_divisor_dirty = true;
       double _prospective_price;
@@ -239,10 +239,9 @@ namespace mbts.spark {
       IDisplayTranslator<uint> _prospective_qty_translator {
          get {
             if (_prospective_qty_translator_backing == null) {
-               _prospective_qty_translator_backing = new DisplayQuantityMultiplier {
-                  conversion_data = new QuantityConversionData(_display_qty_divisor, QuantityConversionData.OpType.DivideActual),
-                  actual_value = quantity
-               };
+               _prospective_qty_translator_backing = DisplayQuantityMultiplier.create_uint();
+               _prospective_qty_translator_backing.conversion_data = new QuantityConversionData(_display_qty_divisor, QuantityConversionData.OpType.DivideActual);
+               _prospective_qty_translator_backing.actual_value = quantity;
                _prospective_qty_translator.value_changed += () =>
                   RaisePropertyChanged(nameof(is_qty_dirty));
             } else {

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -86,7 +86,7 @@ namespace mbts.spark {
          _model = model;
          _selected_params_mon = new OrderParamsAggregator(model.dynamic_helper, _selection_change_throttle, k => _cached_orders.TryGetValue(k, out var o) ? o.order : (OrderSecurityAlgo?)null);
          ladder_vm = new LadderVMContainer(bus, model);
-         order_mod_vm = new OrderParamsVM(this, AlgoFormats.Exchange, AlgoEditingPermissions.AllowEditValues);
+         order_mod_vm = new OrderParamsVM(this, AlgoEditingPermissions.AllowEditValues);
          order_mod_vm.algo_vm.editing_permissions_mask = AlgoEditingPermissions.None;
          order_mod_vm.algo_vm.algo_root = null;
          _show_order_params = backing(nameof(show_order_params), () => (_right_panel.visible_components & OrderBookRightPanelSettings.Components.Params) != 0);
@@ -99,7 +99,7 @@ namespace mbts.spark {
          };
          _can_attach_selected_orders = backing(nameof(can_attach_selected_orders), () => _attachable_selected_orders.Count == selected_rows.Count && selected_rows.Count > 0);
          _can_detach_selected_orders = backing(nameof(can_detach_selected_orders), () => _detachable_selected_orders.Count == selected_rows.Count && selected_rows.Count > 0);
-         attach_vm = new OrderParamsVM(this, AlgoFormats.Attachable, AlgoEditingPermissions.AllowEditStructure | AlgoEditingPermissions.AllowEditValues);
+         attach_vm = new OrderParamsVM(this, AlgoEditingPermissions.AllowEditStructure | AlgoEditingPermissions.AllowEditValues);
          _settings_manager = settings_manager;
          _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(x => (x.added.Values, x.removed.Values)));
          // we use an ObservableAppendCollection b/c the mini-ob needs a cancel column at the front
@@ -244,7 +244,22 @@ namespace mbts.spark {
             }),
             grid_settings_manager,
             order_mod_vm.algo_vm.connect_data(Observable.Return(model)),
-            order_mod_vm.algo_vm.connect_data(_selected_params_mon.subscribe_spreader_security()),
+            order_mod_vm.algo_vm.connect_data(
+               /// This is not the same security source used to populate <see cref="OrderParamsVM.security_md"/>.
+               /// The latter is always synced with the root of the selected row(s),
+               /// while this source is synced with that row with one exception:
+               ///   If the root row's algo has a spreader algo (possibly nested),
+               ///   the synthetic security is taken instead of the root security.
+               ///   This has the affect of passing the synthetic security to the spreader algo viewmodel,
+               ///   which then uses the security to populate the synthetic params ui.
+               /// TODO (maybe): clean this design up; the different security sources make it precarious.
+               /// One idea is to pass a special data type to the algo vm (which contains the spreader
+               /// security and execution strategy), which is only observed by the spreader algo vm.
+               _selected_params_mon.subscribe_security().StartWith((SecurityImmutable?)null).CombineLatest(
+               _selected_params_mon.subscribe_spreader_security().StartWith((SecurityImmutable?)null),
+               (root_sec, spreader_sec) => (root_sec, spreader_sec))
+               .Select(tup => tup.spreader_sec ?? tup.root_sec)
+               .OnDispatcher()),
             order_mod_vm,
             attach_vm,
             _show_fills.observe_current(true)
@@ -329,32 +344,32 @@ namespace mbts.spark {
          }
 
          void on_begin_attaching() {
-            Disposable.replace(ref attach_vm_hooks,
-               Disposable.Create(
-                  connect_common_selected_params(attach_vm),
-                  attach_vm.algo_vm.observe_algo_root().Subscribe(algo_root => {
-                     bool show_price, show_size;
-                     if (algo_root == null) {
-                        show_price = false;
-                        show_size = false;
-                     } else {
-                        show_price = AttachConf.show_price(algo_root.order_type);
-                        show_size = AttachConf.show_size(algo_root.order_type);
-                     }
-                     attach_vm.price_permissions = bool_to_perm(show_price);
-                     attach_vm.size_permissions = bool_to_perm(show_size);
-                     AlgoParamPermissions bool_to_perm(bool b) =>
-                        b ? AlgoParamPermissions.ReadWrite : AlgoParamPermissions.None;
-                  }),
-                  // custom templates (created by user)
-                  _model.algo_model.definitions.OnDispatcher().Subscribe(u =>
-                     AlgoCollectionHelper.update_persisted_definition_wrappers(attach_vm.algo_vm.algo_roots_index, u, mk_algo_def_vm))));
-            // vm factory
-            AlgoDefinitionVM mk_algo_def_vm(AlgoData fd) {
-               var vm = AlgoDefinitionVM.create(_model, fd, OrderDataHelper.get_algo_format(_selected_params_mon.security?.ContractCase), AlgoEditingPermissions.AllowEditValues);
-               vm.selected_format = OrderDataHelper.get_algo_format(_selected_params_mon.security?.ContractCase);
-               return vm;
-            }
+            Disposable.dispose(ref attach_vm_hooks);
+            attach_vm_hooks = Disposable.Create(
+               _selected_security_mon.SubscribeThrottled(_selection_change_throttle).to_security_md(bus).OnDispatcher().SubscribeEach(newval => {
+                  attach_vm.security_md = newval;
+                  if (newval.HasValue) {
+                     return _model.subscribe_algos(newval.Value.security, true)
+                     .OnDispatcher()
+                     .Subscribe(t => ((IAlgoSelector)attach_vm.algo_vm).load_algo_data(t.state, null));
+                  }
+                  return null;
+               }),
+               connect_common_selected_params(attach_vm),
+               attach_vm.algo_vm.observe_algo_root().Subscribe(algo_root => {
+                  bool show_price, show_size;
+                  if (algo_root == null) {
+                     show_price = false;
+                     show_size = false;
+                  } else {
+                     show_price = AttachConf.show_price(algo_root.order_type);
+                     show_size = AttachConf.show_size(algo_root.order_type);
+                  }
+                  attach_vm.price_permissions = bool_to_perm(show_price);
+                  attach_vm.size_permissions = bool_to_perm(show_size);
+                  AlgoParamPermissions bool_to_perm(bool b) =>
+                     b ? AlgoParamPermissions.ReadWrite : AlgoParamPermissions.None;
+               }));
          }
          void on_end_attaching() => Disposable.dispose(ref attach_vm_hooks);
 
@@ -365,11 +380,11 @@ namespace mbts.spark {
                   modify_selected_orders_async_cmd.RaiseCanExecuteChanged();
                   revert_selected_orders_cmd.RaiseCanExecuteChanged();
                }),
+               _selected_params_mon.subscribe_security().to_security_md(bus).OnDispatcher().Subscribe(newval => order_mod_vm.security_md = newval),
                connect_common_selected_params(order_mod_vm));
 
          IDisposable connect_common_selected_params(OrderParamsVM vm) =>
             Disposable.Create(
-               _selected_params_mon.subscribe_security().OnDispatcher().to_security_md(bus).OnDispatcher().Subscribe(newval => vm.security_md = newval),
                _selected_params_mon.subscribe_price().OnDispatcher().Subscribe(vm.on_common_price),
                _selected_params_mon.subscribe_size().OnDispatcher().Subscribe(vm.on_common_size),
                _selected_params_mon.subscribe_account().OnDispatcher().Subscribe(newval => vm.selected_account = newval));
@@ -807,12 +822,11 @@ namespace mbts.spark {
             _size_permissions = AlgoParamPermissions.ReadOnly | AlgoParamPermissions.Write;
          SecurityMD? _security_md;
 
-         public OrderParamsVM(OrderBookVM owner, AlgoFormats init_format, AlgoEditingPermissions init_permissions) {
+         public OrderParamsVM(OrderBookVM owner, AlgoEditingPermissions init_permissions) {
             _owner = owner;
             algo_vm = AlgoDefinitionVM.create(
                owner._model,
                AlgoData.create_unpersisted(OrderDataHelper.Defaults.CustomOrderDefinitionImmutable(CustomOrderDefinition.CustomOneofCase.Iceberg), CustomOrderDefinition.CustomOneofCase.Iceberg.display_name()),
-               init_format: init_format,
                editing_permissions: init_permissions);
             _accounts = backing(
                nameof(selected_account),
@@ -879,6 +893,7 @@ namespace mbts.spark {
          public ReadOnlyObservableCollection<AccountTuple> accounts => _accounts.items;
          public AccountTuple? selected_account { get => _accounts.selected_item; set => _accounts.selected_item = value; }
          public SecurityImmutable? security => _owner._selected_params_mon.security;
+         /// <see cref="security_md"/> should be synced with the root of the selected row(s),
          public SecurityMD? security_md {
             get => _security_md;
             set {

src/client/spark/Modules/Trading/PNL/Data/PNLData.cs

@@ -622,9 +622,7 @@ namespace bts.shell.pnl {
 
    static class PNLDataExtensions {
       public static string pnl_symbol(this SecurityImmutable security) {
-         if(security.HasIce)
-            return PNLICEHelper.get_symbol(security);
-         return security.Symbol;
+         return bts.api.ExchangeSpecificUitlities.get_pnl_symbol(security);
       }
 
       // while override_cpp could use double.NaN to indicate the lack of an override,
@@ -636,8 +634,8 @@ namespace bts.shell.pnl {
 
       public static (double default_cpp, double? override_cpp) pnl_cpp_tuple(this PNLUnitImmutable pnl_unit, SecurityImmutable security) {
          // TODO: actual impl
-         double calced_pnl = PNLICEHelper.get_cpp(pnl_unit,security);
-         return (calced_pnl, /*pnl_unit.HasCppOverride ? pnl_unit.CppOverride.Value : */(double?)null);
+         double calced_pnl = bts.api.ExchangeSpecificUitlities.get_cpp(pnl_unit,security);
+         return (calced_pnl, pnl_unit.HasCppOverride ? pnl_unit.CppOverride.Value : (double?)null);
       }
    }
 }

src/client/spark/Modules/Trading/PNL/Data/PNLICEHelper.cs

@@ -1,77 +0,0 @@
-using bts.api;
-using bts.api.orderman;
-using bts.api.position;
-using bts.utils;
-using mbts;
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-
-
-namespace bts.shell.pnl {
-
-static class PNLICEHelper{   
-
-   public static bool is_spread(SecurityImmutable sec){
-      return sec.HasSpread && sec.Ice.StripName.Contains("/");
-   }
-
-   public static bool is_strip(SecurityImmutable sec){
-      return sec.HasSpread && !is_spread(sec);
-   }
-
-   public static bool is_ice_lots(SecurityImmutable sec){
-      return sec.Ice.MarketDescription.Contains("ICE Lots");
-   }   
-
-   public static double get_future_cpp (SecurityImmutable sec){
-      if(sec.Symbol == "HHD")
-         return 1;
-      if(is_ice_lots(sec))
-         return 2500;
-      else{
-         var days_in_month = DateTime.DaysInMonth(Convert.ToInt32(sec.Future.ExpirationYear),
-                                                  Convert.ToInt32(sec.Future.ExpirationMonth));
-         return days_in_month; // because the size is multiplied by 2500 already
-      }
-   }
-
-   public static double get_spread_cpp(SecurityImmutable sec){
-      return 0;
-   }
-
-   public static double get_strip_cpp(PNLUnitImmutable pnl,
-                                      SecurityImmutable sec){
-      double rv = 0;
-      foreach (var l in sec.Spread.Legs)
-         if(l.Security.HasFuture)
-            rv += get_cpp(pnl,l.Security);
-      return rv;
-   }
-
-   public static string get_symbol (SecurityImmutable sec){
-      var rv = String.Format("{0}{1}{2}",
-                            sec.Symbol,
-                            is_ice_lots(sec)? " ICE LOTS" : String.Empty,
-                            is_spread(sec)? " Spread" : String.Empty);
-      return rv;
-   }
-
-   public static double get_cpp (PNLUnitImmutable pnl,
-                                 SecurityImmutable sec){
-      if(sec.Symbol == "HHD")
-         Console.WriteLine("yes!");
-      if (sec.HasGlbx)
-         return pnl.CashValue;
-      else if (sec.HasIce){
-         if(sec.HasFuture)
-            return get_future_cpp(sec);
-         else if(is_spread(sec))
-            return get_spread_cpp(sec);
-         else if(is_strip(sec))
-            return get_strip_cpp(pnl,sec);
-      }
-      return Double.NaN;
-   }
-
-}}

src/client/spark/Orders/OrderDataHelper.cs

@@ -1,5 +1,6 @@
 using bts;
 using bts.api;
+using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
 using bts.spark;
@@ -26,11 +27,13 @@ namespace mbts.spark.utilities {
       public readonly Func<CustomOrderDefinition, CustomOrderDefinition> get_child;
       public readonly Action<CustomOrderDefinition, CustomOrderDefinition> assign_child;
 
-      public ChildAlgoData(string name, Func<IMessage, CustomOrderDefinition> get_child_from_impl, 
-                           Action<IMessage, CustomOrderDefinition> assign_child_to_impl, 
-                           Func<CustomOrderDefinition, CustomOrderDefinition> get_child, 
-                           Action<CustomOrderDefinition, CustomOrderDefinition> assign_child) 
-      {
+      public ChildAlgoData(
+         string name,
+         Func<IMessage, CustomOrderDefinition> get_child_from_impl,
+         Action<IMessage, CustomOrderDefinition> assign_child_to_impl,
+         Func<CustomOrderDefinition, CustomOrderDefinition> get_child,
+         Action<CustomOrderDefinition, CustomOrderDefinition> assign_child) {
+         //
          this.name = name;
          this.get_child = get_child;
          this.assign_child = assign_child;
@@ -46,13 +49,19 @@ namespace mbts.spark.utilities {
       static readonly OrderMetadata _default_order_metadata = new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order };
       readonly AdminModel _admin_model;
       readonly DynamicOrderHelper _dynamic_helper;
+      readonly IAlgoProvider _security_algo_provider;
       readonly IReadOnlyDictionary<OneOf<CustomOrderDefinition.CustomOneofCase, string>, string> _algo_abbreviation_index;
+      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
       IDisposable _dispose;
 
       [ImportingConstructor]
-      public OrderFactory(AdminModel admin_model, DynamicOrderHelper dynamic_helper) {
+      public OrderFactory(OrderService service, AdminModel admin_model, DynamicOrderHelper dynamic_helper, IAlgoProvider security_algo_provider) {
          _admin_model = admin_model;
          _dynamic_helper = dynamic_helper;
+         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
+            .DynamicOrderDesc
+            .ToDictionary(d => d.DynamicOrderType);
+         _security_algo_provider = security_algo_provider;
          var algo_name_index = new HashSet<OneOf<CustomOrderDefinition.CustomOneofCase, string>>(OneOf<CustomOrderDefinition.CustomOneofCase, string>.ValueEqualityComparer);
          // add standard types
          algo_name_index.AddRange(
@@ -86,66 +95,13 @@ namespace mbts.spark.utilities {
       public IEnumerable<CustomOrderDefinition.CustomOneofCase> default_order_types() =>
          OrderDataHelper.exchange_order_types;
 
-      public IEnumerable<CustomOrderDefinition.CustomOneofCase> default_order_types(SecurityImmutable? security) =>
-         security.HasValue ? default_order_types(security.Value) : OrderDataHelper.exchange_order_types;
-
-      public IEnumerable<CustomOrderDefinition.CustomOneofCase> default_order_types(SecurityImmutable security) {
-         switch (security.ContractCase) {
-            case Security.ContractOneofCase.Synthetic: return OrderDataHelper.spreader_order_types;
-            default: return OrderDataHelper.exchange_order_types;
-         }
-      }
-
-      public IEnumerable<CustomOrderDefinition.CustomOneofCase> default_order_types(SecurityImmutable? security, bool is_child = false) =>
-         security.HasValue ? default_order_types(security.Value, is_child) : default_order_types();
-
-      public IEnumerable<TimeInForce> default_time_in_forces() =>
-         EnumHelper.get_values_except(TimeInForce.Undefined);
-
-      public IEnumerable<TimeInForce> default_time_in_forces(SecurityImmutable security, bool is_child = false) =>
-         default_time_in_forces();
-
       public IEnumerable<TimeInForce> default_time_in_forces(SecurityImmutable security, CustomOrderDefinition.CustomOneofCase order_type, bool is_child = false) {
-         var tifs = default_time_in_forces(security, is_child);
+         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
          if (order_type != CustomOrderDefinition.CustomOneofCase.Limit) {
             tifs = tifs.Except(new[] { TimeInForce.Ioc });
          }
          return tifs;
       }
-
-      public IEnumerable<TimeInForce> default_time_in_forces(SecurityImmutable? security, bool is_child = false) =>
-         security.HasValue ? default_time_in_forces(security.Value, is_child) : default_time_in_forces();
-
-      public LadderSettings default_ladder_settings(SecurityImmutable security) {
-         var settings = LadderSettings.Default;
-         if (security.ContractCase == Security.ContractOneofCase.Synthetic)
-            settings.default_order_type = CustomOrderDefinition.CustomOneofCase.Spreader;
-         return settings;
-      }
-
-      public Dictionary<AlgoKey, AlgoData> default_algos() =>
-         default_algos((SecurityImmutable?)null);
-
-      public Dictionary<AlgoKey, AlgoData> default_algos(SecurityImmutable? security) =>
-         default_algos(default_order_types(security));
-
-      public Dictionary<AlgoKey, AlgoData> default_algos(AlgoFormats algo_format) =>
-         default_algos(OrderDataHelper.get_format_algo_types(algo_format));
-
-      public Dictionary<AlgoKey, AlgoData> default_algos(IEnumerable<CustomOrderDefinition.CustomOneofCase> types) {
-         var def_map = new Dictionary<AlgoKey, AlgoData>();
-         foreach (var ot in types.Except(new[] { CustomOrderDefinition.CustomOneofCase.Dynamic, CustomOrderDefinition.CustomOneofCase.Composite })) {
-            var data = AlgoData.from_custom_def(OrderDataHelper.Defaults.CustomOrderDefinitionImmutable(ot));
-            def_map[data.key] = data;
-         }
-         var dynamic_defs = new List<CustomOrderDefinitionImmutable>();
-         _dynamic_helper.add_dynamic_order_defs(dynamic_defs, custom_order);
-         foreach (var dd in dynamic_defs) {
-            var data = AlgoData.from_custom_def(dd);
-            def_map[data.key] = data;
-         }
-         return def_map;
-      }
       #endregion
 
       public OrderDefinitionImmutable order(SecurityImmutable security) =>
@@ -156,7 +112,7 @@ namespace mbts.spark.utilities {
             SecurityKey = security.Id,
             Side = Side.Buy,
             TimeInForce = TimeInForce.Day,
-            CustomDef = custom_order(),
+            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
             CtiCode = CTICode.CtiCode1,
             CustomerOrFirm = CustomerOrFirm.Firm,
             // this will be the tag 50 on CME orders
@@ -173,9 +129,6 @@ namespace mbts.spark.utilities {
          return rv;
       }
 
-      public string abbreviate_algo(CustomOrderDefinitionImmutable defi) =>
-         abbreviate_algo(_simple_algo_id(defi));
-
       public string abbreviate_algo(OneOf<CustomOrderDefinition.CustomOneofCase, string> id) {
          if (id == null)
             throw new ArgumentNullException(nameof(id));
@@ -187,26 +140,6 @@ namespace mbts.spark.utilities {
       }
 
       // mutable definitions
-      public CustomOrderDefinition custom_order() => OrderDataHelper.Defaults.CustomOrderDefinition();
-      public MarketOrderDefinition market_order() => OrderDataHelper.Defaults.MarketOrderDefinition();
-      public LimitOrderDefinition limit_order() => OrderDataHelper.Defaults.LimitOrderDefinition();
-      public StopOrderDefinition stop_order() => OrderDataHelper.Defaults.StopOrderDefinition();
-      public StopLimitOrderDefinition stop_limit_order() => OrderDataHelper.Defaults.StopLimitOrderDefinition();
-      public GLBXIcebergOrderDefinition glbx_iceberg_order() => OrderDataHelper.Defaults.GLBXIcebergOrderDefinition();
-      public IcebergOrderDefinition iceberg_order() => OrderDataHelper.Defaults.IcebergOrderDefinition();
-      public TimeBoxOrderDefinition time_box_order() => OrderDataHelper.Defaults.TimeBoxOrderDefinition();
-      public TickOrderDefinition tick_order() => OrderDataHelper.Defaults.TickOrderDefinition();
-      public LeanOrderDefinition lean_order() => OrderDataHelper.Defaults.LeanOrderDefinition();
-      public JoinOrderDefinition join_order() => OrderDataHelper.Defaults.JoinOrderDefinition();
-      public TopOrderDefinition top_order() => OrderDataHelper.Defaults.TopOrderDefinition();
-      public CoverOrderDefinition cover_order() => OrderDataHelper.Defaults.CoverOrderDefinition();
-      public OCOOrderDefinition oco_order() => OrderDataHelper.Defaults.OCOOrderDefinition();
-      public PricerOrderDefinition pricer_order() => OrderDataHelper.Defaults.PricerOrderDefinition();
-      public OffsetOrderDefinition offset_order() => OrderDataHelper.Defaults.OffsetOrderDefinition();
-      public TriggerOrderDefinition trigger_order() => OrderDataHelper.Defaults.TriggerOrderDefinition();
-      public DynamicOrderDefinition dynamic_order() => OrderDataHelper.Defaults.DynamicOrderDefinition();
-      public SpreaderOrderDefinition spreader_order() => OrderDataHelper.Defaults.SpreaderOrderDefinition();
-      public SplitterOrderDefinition splitter_order() => OrderDataHelper.Defaults.SplitterOrderDefinition();
       public SpreaderOrderDefinition spreader_order(SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
          new SpreaderOrderDefinition {
             Security = security.CloneToMutable(),
@@ -214,40 +147,8 @@ namespace mbts.spark.utilities {
          };
 
       // immutable definitions
-      public CustomOrderDefinitionImmutable custom_orderi(CustomOrderDefinition.CustomOneofCase order_type) => new CustomOrderDefinitionImmutable(custom_order(order_type));
       public SpreaderOrderDefinitionImmutable spreader_orderi(SecurityImmutable security, ExecutionStrategyImmutable strategy) => new SpreaderOrderDefinitionImmutable(spreader_order(security, strategy));
 
-      public CustomOrderDefinition custom_order(CustomOrderDefinition.CustomOneofCase order_type) {
-         var rv = new CustomOrderDefinition();
-         switch (order_type) {
-            case CustomOrderDefinition.CustomOneofCase.Market: rv.Market = market_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Limit: rv.Limit = limit_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Stop: rv.Stop = stop_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.StopLimit: rv.StopLimit = stop_limit_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.GlbxIceberg: rv.GlbxIceberg = glbx_iceberg_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Iceberg: rv.Iceberg = iceberg_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.TimeBox: rv.TimeBox = time_box_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Tick: rv.Tick = tick_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Lean: rv.Lean = lean_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Join: rv.Join = join_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Top: rv.Top = top_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Cover: rv.Cover = cover_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Oco: rv.Oco = oco_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Pricer: rv.Pricer = pricer_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Offset: rv.Offset = offset_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Trigger: rv.Trigger = trigger_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = spreader_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Splitter: rv.Splitter = splitter_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Dynamic: rv.Dynamic = dynamic_order(); break;
-               //case CustomOrderDefinition.CustomOneofCase.QueueHolder: rv.QueueHolder = queue_holder_order(); break;
-            default:
-               OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, order_type);
-               break;            
-         }
-         return rv;
-      }
-
-      public OrderMetadata get_metadata(IMessage order_type_def) => get_metadata(OrderDataHelper.get_order_type(order_type_def));
       public OrderMetadata get_metadata(OrderRecordImmutable order) => get_metadata(order.Def.CustomDef.CustomCase);
       public OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) {
          switch (order_type) {
@@ -278,14 +179,6 @@ namespace mbts.spark.utilities {
 
       public void Dispose() =>
          Disposable.dispose(ref _dispose);
-
-      static OneOf<CustomOrderDefinition.CustomOneofCase, string> _simple_algo_id(CustomOrderDefinitionImmutable defi) {
-         switch (defi.CustomCase) {
-            case CustomOrderDefinition.CustomOneofCase.Dynamic:
-               return new OneOf<CustomOrderDefinition.CustomOneofCase, string>(defi.Dynamic.DynamicOrderType);
-            default: return new OneOf<CustomOrderDefinition.CustomOneofCase, string>(defi.CustomCase);
-         }
-      }
    }
 
    static class OrderDataHelper {
@@ -301,8 +194,6 @@ namespace mbts.spark.utilities {
          }
       }
 
-      public static OneofDescriptor oneof_descriptor => _oneof_desc;
-
       const double _price_error_margin = 0.0000001;
       const decimal _price_error_margin_decimal = 0.0000001m;
 
@@ -320,10 +211,10 @@ namespace mbts.spark.utilities {
       });
 
       public static readonly IComparer<decimal> PriceComparerDecimal = Comparer<decimal>.Create((a, b) => {
-            decimal diff = a - b;
-            if (diff <= _price_error_margin_decimal && diff >= -_price_error_margin_decimal)
-               return 0;
-            return diff > 0 ? 1 : -1;
+         decimal diff = a - b;
+         if (diff <= _price_error_margin_decimal && diff >= -_price_error_margin_decimal)
+            return 0;
+         return diff > 0 ? 1 : -1;
       });
 
       const string _primary_child_field = "child";
@@ -378,7 +269,7 @@ namespace mbts.spark.utilities {
             switch (cd.CustomCase) {
                case CustomOrderDefinition.CustomOneofCase.Composite: studio = cd.Composite.AlgoTemplateKey; break;
                case CustomOrderDefinition.CustomOneofCase.Dynamic: dyn = cd.Dynamic.DynamicOrderType; break;
-               case CustomOrderDefinition.CustomOneofCase.Spreader: spreader_sec = cd.Spreader.Security.Id; break;
+               case CustomOrderDefinition.CustomOneofCase.Spreader: spreader_sec = cd.Spreader.Security?.Id ?? 0; break;
             }
             yield return (cd.CustomCase, dyn, studio, spreader_sec);
          }
@@ -466,22 +357,6 @@ namespace mbts.spark.utilities {
             child_data.TryGetFirst(out best_match) ? best_match : (ChildAlgoData?)null;
       }
 
-      public static CustomOrderDefinition create_algo(CustomOrderDefinition.CustomOneofCase algo_type) {
-         var algo = new CustomOrderDefinition();
-         if (_algo_type_case_descriptor_data.TryGetValue(algo_type, out AlgoTypeCaseDescriptorData desc))
-            desc.type_case_descriptor.Accessor.SetValue(algo, ProtoHelper.mutable_factory(ProtoHelper.get_field_type(desc.type_case_descriptor)));
-         return algo;
-      }
-
-      public static bool append_algo_node(CustomOrderDefinition target, CustomOrderDefinition.CustomOneofCase algo_type) {
-         if (_algo_type_case_descriptor_data.TryGetValue(algo_type, out var desc)) {
-            var node = ProtoHelper.mutable_factory(ProtoHelper.get_field_type(desc.type_case_descriptor));
-            desc.type_case_descriptor.Accessor.SetValue(target, node);
-            return true;
-         }
-         return false;
-      }
-
       public static IReadOnlyList<ChildAlgoData> get_child_algo_data(CustomOrderDefinition.CustomOneofCase algo_type) =>
          _algo_type_case_descriptor_data.TryGetValue(algo_type, out AlgoTypeCaseDescriptorData d) ? d.child_algo_data : Array.Empty<ChildAlgoData>();
       #endregion
@@ -535,122 +410,10 @@ namespace mbts.spark.utilities {
          }
       }
 
-      public static AlgoFormats get_algo_format(Security.ContractOneofCase? security_type) =>
-         security_type.HasValue ? get_algo_format(security_type.Value) : AlgoFormats.Exchange;
-
-      public static AlgoFormats get_algo_format(Security.ContractOneofCase security_type) =>
-         security_type == Security.ContractOneofCase.Synthetic ? AlgoFormats.Spreader : AlgoFormats.Exchange;
-
-      public static HashSet<CustomOrderDefinition.CustomOneofCase> get_format_algo_types(AlgoFormats f) {
-         var rv = new HashSet<CustomOrderDefinition.CustomOneofCase>();
-         if ((f & AlgoFormats.Exchange) == AlgoFormats.Exchange) {
-            rv.AddRange(exchange_order_types);
-            if ((f & AlgoFormats.Spreader) == AlgoFormats.Spreader) {
-               rv.IntersectWith(spreader_order_types);
-            }
-         } else if ((f & AlgoFormats.Spreader) == AlgoFormats.Spreader) {
-            rv.AddRange(spreader_order_types);
-         }
-         if ((f & AlgoFormats.Attachable) == AlgoFormats.Attachable)
-            rv.AddRange(attachable_order_types);
-         if ((f & AlgoFormats.DefaultPlaceholder) == AlgoFormats.DefaultPlaceholder)
-            rv.Add(CustomOrderDefinition.CustomOneofCase.None);
-         return rv;
-      }
-
-      public static CustomOrderDefinition.CustomOneofCase get_default_algo(AlgoFormats f) {
-         switch (f) {
-            case AlgoFormats.Exchange: return CustomOrderDefinition.CustomOneofCase.Limit;
-            case AlgoFormats.Spreader: return CustomOrderDefinition.CustomOneofCase.Spreader;
-            default: return CustomOrderDefinition.CustomOneofCase.None;
-         }
-      }
-
-      public static IReadOnlyDictionary<AlgoKey, AlgoData> get_format_algo_data(AlgoFormats f) {
-         var algo_types = get_format_algo_types(f);
-         var algo_dict = algo_types.Select(at => AlgoData.from_custom_def(Defaults.CustomOrderDefinitionImmutable(at))).ToDictionaryCrushed(a => a.key);
-         return algo_dict;
-      }
-
-      public static HashSet<CustomOrderDefinition.CustomOneofCase> get_algo_types_present(CustomOrderDefinitionImmutable def) {
-         return new HashSet<CustomOrderDefinition.CustomOneofCase>(impl(def.CloneToMutable()));
-         // hide mutable-def implementation
-         IEnumerable<CustomOrderDefinition.CustomOneofCase> impl(CustomOrderDefinition mdef) {
-            yield return mdef.CustomCase;
-            foreach (var sd in get_child_algo_data(mdef)) {
-               var child = sd.get_child(mdef);
-               if (child != null) {
-                  yield return child.CustomCase;
-                  foreach (var t in impl(child))
-                     yield return t;
-               }
-            }
-         }
-      }
-
-      public static CustomOrderDefinitionImmutable? fit_to_type(CustomOrderDefinitionImmutable defi, Security.ContractOneofCase security_type, bool fit_all_to_spreader = false) {
-         var mdef = defi.CloneToMutable();
+      public static CustomOrderDefinition.CustomOneofCase get_default_algo(Security.ContractOneofCase security_type) {
          switch (security_type) {
-            case Security.ContractOneofCase.Synthetic:
-               mdef = spreader_impl(mdef);
-               break;
-            default:
-               mdef = outright_impl(mdef);
-               break;
-         }
-         if (mdef == null)
-            return null;
-         return new CustomOrderDefinitionImmutable(mdef);
-         // local methods
-         CustomOrderDefinition spreader_impl(CustomOrderDefinition def) {
-            if (def == null) {
-               def = new CustomOrderDefinition { Spreader = new SpreaderOrderDefinition() };
-            } else {
-               switch (def.CustomCase) {
-                  case CustomOrderDefinition.CustomOneofCase.Limit:
-                  case CustomOrderDefinition.CustomOneofCase.None:
-                     def = new CustomOrderDefinition { Spreader = new SpreaderOrderDefinition() };
-                     break;
-                  case CustomOrderDefinition.CustomOneofCase.Stop:
-                  case CustomOrderDefinition.CustomOneofCase.StopLimit:
-                  case CustomOrderDefinition.CustomOneofCase.Market:
-                  case CustomOrderDefinition.CustomOneofCase.GlbxIceberg:
-                     def = fit_all_to_spreader ? new CustomOrderDefinition { Spreader = new SpreaderOrderDefinition() } : null;
-                     break;
-                  default:
-                     var desc_data = _algo_type_case_descriptor_data[def.CustomCase];
-                     foreach (var cad in desc_data.child_algo_data) {
-                        var child = spreader_impl(cad.get_child(def));
-                        if (child == null)
-                           return null;
-                        var type_def = (IMessage)desc_data.type_case_descriptor.Accessor.GetValue(def);
-                        cad.assign_child_to_impl(type_def, child);
-                     }
-                     break;
-               }
-            }
-            return def;
-         }
-         CustomOrderDefinition outright_impl(CustomOrderDefinition def) {
-            if (def == null) {
-               def = new CustomOrderDefinition { Limit = new LimitOrderDefinition() };
-            } else {
-               switch (def.CustomCase) {
-                  case CustomOrderDefinition.CustomOneofCase.Spreader:
-                  case CustomOrderDefinition.CustomOneofCase.None:
-                     def = new CustomOrderDefinition { Limit = new LimitOrderDefinition() };
-                     break;
-                  default:
-                     foreach (var cad in get_child_algo_data(def)) {
-                        var child = outright_impl(cad.get_child(def));
-                        if (child == null)
-                           return null;
-                        cad.assign_child(def, child);
-                     }
-                     break;
-               }
-            }
-            return def;
+            case Security.ContractOneofCase.Synthetic: return CustomOrderDefinition.CustomOneofCase.Spreader;
+            default: return CustomOrderDefinition.CustomOneofCase.Limit;
          }
       }
 
@@ -678,7 +441,7 @@ namespace mbts.spark.utilities {
             CustomOrderDefinition.CustomOneofCase.Iceberg,
             CustomOrderDefinition.CustomOneofCase.Lean,
             CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover, 
+            CustomOrderDefinition.CustomOneofCase.Cover,
             CustomOrderDefinition.CustomOneofCase.Top,
             CustomOrderDefinition.CustomOneofCase.Splitter,
             CustomOrderDefinition.CustomOneofCase.Scratch,
@@ -764,12 +527,6 @@ namespace mbts.spark.utilities {
 
          public static bool equals(CustomOrderDefinitionImmutable a, CustomOrderDefinitionImmutable b) => custom_order_defi_comparer.Equals(a, b);
 
-         public static bool structure_equals(CustomOrderDefinitionImmutable a, CustomOrderDefinitionImmutable b) {
-            return impl(a.CloneToMutable(), b.CloneToMutable());
-            bool impl(CustomOrderDefinition ma, CustomOrderDefinition mb) =>
-               ma.CustomCase == mb.CustomCase && get_child_algo_data(ma).All(cad => impl(cad.get_child(ma), cad.get_child(mb)));
-         }
-
          public static bool iceberg_equals(IcebergOrderDefinitionImmutable a, IcebergOrderDefinitionImmutable b) =>
             a.ShowQuantity.Equals(b.ShowQuantity) && a.HasChild ? b.HasChild && equals(a.Child, b.Child) : !b.HasChild;
 
@@ -1031,7 +788,6 @@ namespace mbts.spark.utilities {
             }
          };
 
-         public static OrderDefinitionImmutable OrderDefinitionImmutable { get; } = new OrderDefinitionImmutable(OrderDefinition);
          public static OrderRecordImmutable OrderRecordImmutable { get; } = new OrderRecordImmutable(OrderRecord);
 
          public static CustomOrderDefinition CustomOrderDefinition() => new CustomOrderDefinition { Limit = LimitOrderDefinition() };
@@ -1084,9 +840,6 @@ namespace mbts.spark.utilities {
          public static TriggerOrderDefinition TriggerOrderDefinition() => new TriggerOrderDefinition { Child = CustomOrderDefinition(), TriggerCondition = new TriggerCondition(), TriggerAction = new TriggerAction(), };
          public static DynamicOrderDefinition DynamicOrderDefinition() => new DynamicOrderDefinition();
          public static SpreaderOrderDefinition SpreaderOrderDefinition() => new SpreaderOrderDefinition { ExecutionStrategy = new ExecutionStrategy() };
-
-         public static SplitterOrderDefinition SplitterOrderDefinition() => new SplitterOrderDefinition();
       }
    }
-   
 }

src/client/spark/Orders/OrderPropertiesEditorVM.cs

@@ -82,13 +82,8 @@ namespace mbts.spark {
             CustomSort = AlgoCollectionHelper.algo_definition_vm_comparer.to_non_generic(),
             IsLiveSorting = true
          };
-         algo_definitions_view.Filter = d => {
-            if (d.key.mode == AlgoKey.Mode.Persisted && _algo_metadata.TryGetValue(d.key, out AlgoDefinitionMetadata meta) && meta.is_hidden)
-               return false;
-            var sec_fmt = OrderDataHelper.get_algo_format(security.ContractCase);
-            return (d.selected_format & sec_fmt) == sec_fmt;
-         };
-         _algo_definitions_view.LiveFilteringProperties.Add(nameof(AlgoDefinitionVM.selected_format));
+         algo_definitions_view.Filter = d =>
+            !(d.key.mode == AlgoKey.Mode.Persisted && _algo_metadata.TryGetValue(d.key, out AlgoDefinitionMetadata meta) && meta.is_hidden);
          _algo_definitions_view.LiveSortingProperties.Add(nameof(AlgoDefinitionVM.group));
          _algo_definitions_view.LiveSortingProperties.Add(nameof(AlgoDefinitionVM.name));
          algo_definitions_view.GroupDescriptions.Add(new PropertyGroupDescription(nameof(AlgoDefinitionVM.group)));
@@ -167,33 +162,27 @@ namespace mbts.spark {
                undo_definition_observer,
                redo_definition_observer,
                revert_definition_observer),
-            security_md_source.Select(smd => model.factory.default_algos(smd.security)).Table(s => s.key).Subscribe(factory_defs => {
-               using (_selected_algo_definition.begin_update())
-                  AlgoCollectionHelper.update_persisted_definitions(_algo_definitions, factory_defs, fd => {
-                     var vm = AlgoDefinitionVM.create(_model, fd, init_format: OrderDataHelper.get_algo_format(security.ContractCase), editing_permissions: AlgoEditingPermissions.AllowEditValues);
-                     vm.selected_format = OrderDataHelper.get_algo_format(security.ContractCase);
-                     return vm;
-                  });
-            }),
-            model.algo_model.definitions.Subscribe(_definitions_buffer),
-            _definitions_buffer.OnDispatcher().Subscribe(u => {
-               AlgoCollectionHelper.update_persisted_definitions(_algo_definitions, u, def_data => {
-                  var def_vm = AlgoDefinitionVM.create(model, def_data, OrderDataHelper.get_algo_format(security.ContractCase), AlgoEditingPermissions.AllowEditValues);
-                  def_vm.selected_format = OrderDataHelper.get_algo_format(security.ContractCase);
-                  def_vm.favorite_index = _get_favorite_algo_index(def_vm.key);
-                  return def_vm;
-               });
+            security_source.SelectManyInSeries(model.subscribe_algo_meta_tuples)
+            .OnDispatcher()
+            .Subscribe(table => {
+               using (_selected_algo_definition.begin_update()) {
+                  foreach (var (algo, meta) in table.removed)
+                     _algo_definitions.Remove(algo.key);
+                  foreach (var (algo, meta) in table.changed) {
+                     var vm = _algo_definitions[algo.key];
+                     vm.load(algo);
+                     vm.favorite_index = _get_favorite_algo_index(vm.key);
+                  }
+                  foreach (var (algo, meta) in table.added) {
+                     var vm = AlgoDefinitionVM.create(_model, algo, AlgoEditingPermissions.AllowEditValues);
+                     vm.favorite_index = _get_favorite_algo_index(vm.key);
+                     _algo_definitions.Add(vm);
+                  }
+               }
             }),
             model.algo_model.metadata_service.subscribe_metadata().OnDispatcher().Subscribe(_on_template_metadata),
             ObservableAutoWrapper.connect_item_hooks(algo_definitions, vm =>
-               Disposable.Create(
-                  vm.connect_data(nullable_security_md_source),
-                  security_source.Subscribe(s => {
-                     var fmt = OrderDataHelper.get_algo_format(s.ContractCase);
-                     foreach (var d in algo_definitions)
-                        d.selected_format = fmt;
-                  }),
-                  vm),
+               Disposable.Create(vm.connect_data(nullable_security_md_source), vm),
                true)) {
             () => Disposable.dispose(ref _algo_param_rows_hook)
          };
@@ -325,13 +314,11 @@ namespace mbts.spark {
       public bool open_algo_popup_on_select { get => _open_algo_popup_on_select; set => Set(ref _open_algo_popup_on_select, value); }
       // commands
       public ICommand save_definition_cmd { get; }
-      public ICommand save_template_cmd { get; }
       public ICommand revert_definition_cmd { get; }
       public ICommand set_default_algo_params_cmd { get; }
       public ICommand revert_algo_params_cmd { get; }
       public ICommand undo_definition_cmd { get; }
       public ICommand redo_definition_cmd { get; }
-      public ICommand remove_template_cmd { get; }
       public ICommand toggle_is_favorite_cmd { get; }
 
       public IObservable<AccountTuple?> observe_account(bool start_immediately = false) =>
@@ -396,4 +383,4 @@ namespace mbts.spark {
          algo_definitions_view.Refresh();
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/CompositeAlgoVM.cs

@@ -4,9 +4,7 @@ using mbts.ui;
 using System;
 using System.Collections.Specialized;
 using System.Reactive.Linq;
-using System.Threading.Tasks;
 using AlgoDef = bts.api.orderman.CustomOrderDefinition;
-using AlgoDefi = bts.api.orderman.CustomOrderDefinitionImmutable;
 using CompositeDef = bts.api.orderman.CompositeOrderDefinition;
 using CompositeDefi = bts.api.orderman.CompositeOrderDefinitionImmutable;
 
@@ -21,6 +19,7 @@ namespace mbts.spark {
             : base(id, display_name, editing_permissions) { }
 
          public override string abbreviation => _studio_meta?.abbreviation ?? base.abbreviation;
+         public override AlgoDisplayGroup group => AlgoDisplayGroup.Studio;
          public override OrderMetadata meta {
             get {
                var m = new OrderMetadata { KeyBinding = string.Empty, NoProps = false };
@@ -66,4 +65,4 @@ namespace mbts.spark {
          protected override void validate() => base.validate();
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/DefaultAlgoVM.cs

@@ -8,7 +8,6 @@ using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.ComponentModel;
-using System.Threading.Tasks;
 using System.Windows.Input;
 using AlgoDef = bts.api.orderman.CustomOrderDefinition;
 using AlgoDefi = bts.api.orderman.CustomOrderDefinitionImmutable;
@@ -19,7 +18,6 @@ namespace mbts.spark {
       static readonly AlgoDefi _empty = new AlgoDefi(new AlgoDef());
       readonly ObservableAggregateCollection<IAlgoParamVM> _algo_params = new ObservableAggregateCollection<IAlgoParamVM>();
       readonly VersionContainer<AlgoDefi> _definition;
-      AlgoFormats _format;
       AlgoEditingPermissions _editing_permissions;
       SecurityMD? _security_md;
       OrdersModel _model;
@@ -84,10 +82,8 @@ namespace mbts.spark {
       void IObserver<SecurityMD?>.OnNext(SecurityMD? security_md) {
          var orig_group = group;
          _security_md = security_md;
-         if (security_md.HasValue)
-            _format = OrderDataHelper.get_algo_format(security_md.Value.security.ContractCase);
          var orig_def = definition;
-         var fmt_type = OrderDataHelper.get_default_algo(_format);
+         var fmt_type = OrderDataHelper.get_default_algo(security_md.HasValue ? security_md.Value.security.ContractCase : bts.api.Security.ContractOneofCase.None);
          switch (fmt_type) {
             case AlgoType.None: _clear_impl(); break;
             default:

src/client/spark/Orders/OrderTypeVMs/DynamicOrderHelper.cs

@@ -2,7 +2,6 @@ using bts.api;
 using bts.api.orderman;
 using bts.utils;
 using Google.Protobuf.Reflection;
-using mbts.utils;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
@@ -15,13 +14,15 @@ namespace mbts.spark {
       static readonly IReadOnlyDictionary<DynamicOrderParam.ValueOneofCase, (FieldDescriptor field_descriptor, Type field_type)> _descriptor_case_index =
          ProtoHelper.get_oneof_descriptor_index<DynamicOrderParam, DynamicOrderParam.ValueOneofCase>(nameof(DynamicOrderParam.ValueCase));
 
-      readonly RegisteredDynamicOrderTypes _dynamic_types;
+      readonly Dictionary<string, DynamicOrderDescriptor> _dyn_types = new Dictionary<string, DynamicOrderDescriptor>(StringComparer.CurrentCultureIgnoreCase);
 
       [ImportingConstructor]
-      DynamicOrderHelper(OrderService order_service) =>
-         _dynamic_types = order_service.get_registered_dynamic_order_types(new VoidMessage());
+      DynamicOrderHelper(OrderService order_service) {
+         foreach (var dt in order_service.get_registered_dynamic_order_types(new VoidMessage()).DynamicOrderDesc)
+            _dyn_types[dt.DynamicOrderType] = dt;
+      }
 
-      public IEnumerable<string> dynamic_types => _dynamic_types.DynamicOrderDesc.Select(d => d.DynamicOrderType);
+      public IEnumerable<string> dynamic_types => _dyn_types.Keys;
 
       public static string fmt_param(DynamicOrderParam dyn_param) {
          if (dyn_param == null)
@@ -50,7 +51,7 @@ namespace mbts.spark {
       }
 
       public void add_dynamic_order_defs(List<CustomOrderDefinitionImmutable> defs, Func<CustomOrderDefinition.CustomOneofCase, CustomOrderDefinition> factory) {
-         foreach (var d in _dynamic_types.DynamicOrderDesc) {
+         foreach (var d in _dyn_types.Values) {
             var dorder = factory(CustomOrderDefinition.CustomOneofCase.Dynamic);
             dorder.Dynamic.DynamicOrderType = d.DynamicOrderType;
             var ddef = dorder.Dynamic;
@@ -60,8 +61,20 @@ namespace mbts.spark {
          }
       }
 
+      public bool try_parse(string str, out string dynamic_type) {
+         if (_dyn_types.TryGetValue(str, out var desc)) {
+            dynamic_type = desc.DynamicOrderType;
+            return true;
+         }
+         dynamic_type = null;
+         return false;
+      }
+
       internal DynamicOrderDescriptor get_descriptor(string dynamic_type) =>
-         _dynamic_types.DynamicOrderDesc.FirstOrDefault(d => d.DynamicOrderType == dynamic_type) ?? new DynamicOrderDescriptor();
+         _dyn_types.TryGetValue(dynamic_type, out var desc) ? desc : new DynamicOrderDescriptor();
+
+      internal CustomOrderDefinitionImmutable create_custom_defi(string dynamic_type) =>
+         new CustomOrderDefinitionImmutable(new CustomOrderDefinition { Dynamic = create_def(dynamic_type) });
 
       internal DynamicOrderDefinition create_def(string dynamic_type) {
          var desc = get_descriptor(dynamic_type);

src/client/spark/Orders/OrderTypeVMs/IOrderTypeVM.cs

@@ -122,12 +122,6 @@ namespace mbts.spark {
       public static Color safe_get_badge_color(this IAlgoVM vm) =>
          vm == null ? default : vm.meta.Color.rgb_color();
 
-      public static IDisposable load_algo_data_recursive(this IAlgoSelector selector, IReadOnlyDictionary<AlgoKey, AlgoData> algo_data, AlgoKey? default_algo) {
-         selector.load_algo_data(algo_data, default_algo);
-         return ObservableAutoWrapper.connect_item_hooks(selector.algos, vm =>
-            ObservableAutoWrapper.connect_item_hooks(vm.child_algo_selectors, s => load_algo_data_recursive(s, algo_data, default_algo)));
-      }
-
       public static IEnumerable<IAlgoParamVM> enumerate_all_params(this IAlgoVM vm) =>
          vm == null ? Enumerable.Empty<IAlgoParamVM>() : vm.algo_params.Concat(vm.child_algo_selectors.SelectMany(s => enumerate_all_params(s.selected_algo)));
 

src/client/spark/Orders/OrderTypeVMs/IcebergOrderVM.cs

@@ -1,16 +1,82 @@
 using bts.api.orderman;
+using mbts.ui;
+using System;
+using System.Reactive;
+using System.Reactive.Linq;
 
 namespace mbts.spark {
    static partial class OrderTypeVMFactory {
       class IcebergOrderVM : OrderTypeVMBase<IcebergOrderDefinition, IcebergOrderDefinitionImmutable> {
          readonly Property<double> _show_quantity;
+         readonly DisplayQuantityMultiplier<double> _display_qty_translator = DisplayQuantityMultiplier.create_double();
+         new IDisposable _dispose;
 
          public IcebergOrderVM(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions)
             : base(key, display_name, editing_permissions) {
-            _show_quantity = add_property(nameof(show_quantity), IcebergOrderDefinition.ShowQuantityFieldNumber, double.NaN, double.NaN);
+            _show_quantity = add_property(
+               "show_quantity",
+               IcebergOrderDefinition.ShowQuantityFieldNumber,
+               double.NaN,
+               double.NaN,
+               to_display_value: v => _qty_conversion.to_display(v),
+               from_display_value: v => _qty_conversion.from_display(v),
+               refresh_display_value:
+               Observable.Create<Unit>(o => {
+                  _request_refresh_display += refresh;
+                  return Disposable.Create(() => _request_refresh_display -= refresh);
+                  void refresh() =>
+                    o.OnNext(Unit.Default);
+               }));
+            _show_quantity.Changed += on_show_quantity_changed;
+            _dispose = Disposable.Create(
+               _display_qty_translator.connect_owner(this, nameof(display_show_quantity)),
+               _display_qty_translator.subscribe_actual_value().Subscribe(v => _show_quantity.Value = v),
+               Disposable.Create(() => _show_quantity.Changed -= on_show_quantity_changed));
+
+            void on_show_quantity_changed(double oldval, double newval) =>
+               _display_qty_translator.actual_value = newval;
          }
 
-         public double show_quantity { get => _show_quantity.Value; set => _show_quantity.set(value); }
+         event Action _request_refresh_display;
+
+         public double display_show_quantity {
+            get => _display_qty_translator.display_value;
+            set => _display_qty_translator.display_value = value;
+         }
+
+         QuantityConversionData _qty_conversion {
+            get => _display_qty_translator.conversion_data;
+            set {
+               if (_display_qty_translator.conversion_data != value) {
+                  _display_qty_translator.conversion_data = value;
+                  _request_refresh_display?.Invoke();
+               }
+            }
+         }
+
+         protected override void on_model_internal() =>
+            _refresh_display();
+
+         protected override void on_security_internal() =>
+            _refresh_display();
+
+         void _refresh_display() {
+            if (model == null) {
+               _qty_conversion = default;
+            } else {
+               var divisor = model.security_auxiliary_model.get_display_quantity_divisor(security);
+               if (divisor == 1 || divisor == 0) {
+                  _qty_conversion = default;
+               } else {
+                  _qty_conversion = new QuantityConversionData(divisor, QuantityConversionData.OpType.DivideActual);
+               }
+            }
+         }
+
+         protected override void Dispose(bool disposing) {
+            Disposable.dispose(ref _dispose);
+            base.Dispose(disposing);
+         }
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs

@@ -18,6 +18,7 @@ using System.Collections.Specialized;
 using System.ComponentModel;
 using System.Dynamic;
 using System.Linq;
+using System.Reactive;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
 using System.Runtime.CompilerServices;
@@ -61,7 +62,9 @@ namespace mbts.spark {
       public static IAlgoVM composite(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions) => new CompositeAlgoVM(key, display_name, editing_permissions);
 
       public static IAlgoVM create(AlgoData def_data, AlgoEditingPermissions editing_permissions = default_editing_permissions, bool is_expanded = false) {
-         var vm = create(def_data.key, def_data.name, def_data.value.CustomCase, editing_permissions, is_expanded);
+         var vm = def_data.key.is_persisted ?
+            new CompositeAlgoVM(def_data.key, def_data.name, editing_permissions) :
+            create(def_data.key, def_data.name, def_data.value.CustomCase, editing_permissions, is_expanded);
          vm.load_from(def_data.value);
          return vm;
       }
@@ -79,7 +82,7 @@ namespace mbts.spark {
             case AlgoType.Join: vm = join(key, name, editing_permissions); break;
             case AlgoType.Top: vm = top(key, name, editing_permissions); break;
             case AlgoType.Cover: vm = cover(key, name, editing_permissions); break;
-            case AlgoType.Oco : return oco(key, name, editing_permissions);
+            case AlgoType.Oco: return oco(key, name, editing_permissions);
             case AlgoType.Pricer: return pricer(key, name, editing_permissions);
             case AlgoType.Offset: vm = offset(key, name, editing_permissions); break;
             case AlgoType.Trigger: vm = trigger(key, name, editing_permissions); break;
@@ -87,13 +90,10 @@ namespace mbts.spark {
             case AlgoType.Iceberg: vm = iceberg(key, name, editing_permissions); break;
             case AlgoType.Spreader: vm = spreader(key, name, editing_permissions); break;
             case AlgoType.Dynamic: vm = dynamic(key, name, name, editing_permissions); break;
-            case AlgoType.Splitter: vm = splitter(key,name,editing_permissions); break;
+            case AlgoType.Splitter: vm = splitter(key, name, editing_permissions); break;
             case AlgoType.Composite: vm = composite(key, name, editing_permissions); break;
             case AlgoType.None: vm = new DefaultAlgoVM(); break;
             default: vm = generic(key, name, algo_type, editing_permissions); break;
-            //default:
-            //   Log.Ui.debug_fatal($"Failed to create {typeof(IAlgoVM)}: unknown algo type '{algo_type}.'");
-            //   return new DefaultAlgoVM();
          }
          vm.is_expanded = is_expanded && vm.can_expand;
          return vm;
@@ -269,12 +269,7 @@ namespace mbts.spark {
 
             void selector_load_impl(AlgoSelectorData sd) {
                var inner_def = sd.get_order(value);
-               if (inner_def.HasValue) {
-                  if (security.HasValue) {
-                     inner_def = OrderDataHelper.fit_to_type(inner_def.Value, security.Value.ContractCase);
-                     if (!inner_def.HasValue)
-                        return;
-                  }
+               if (inner_def.HasValue && inner_def.Value.CustomCase != AlgoType.None) {
                   var ival = inner_def.Value;
                   var vm = sd.selector.algos.GetOrAdd(
                      AlgoKey.from_custom_def(ival),
@@ -317,18 +312,12 @@ namespace mbts.spark {
                }
                AlgoKey key;
                if (security.HasValue) {
-                  init = OrderDataHelper.fit_to_type(init.Value, security.Value.ContractCase);
-                  if (!init.HasValue)
-                     return;
                   key = AlgoKey.from_custom_def(init.Value);
                   child_list.Add(init.Value);
                   for (int i = 1; i < algos.Count; i++) {
                      var child = sd.get_from_impl(algos[i]);
                      if (!child.HasValue || key != AlgoKey.from_custom_def(child.Value))
                         return;
-                     child = OrderDataHelper.fit_to_type(child.Value, security.Value.ContractCase);
-                     if (!child.HasValue)
-                        return;
                      child_list.Add(child.Value);
                   }
                } else {
@@ -372,7 +361,6 @@ namespace mbts.spark {
                   }),
                   c.assign_child);
                }
-               _populate_selectors();
             }
             // local methods
             IDisposable reset() {
@@ -415,9 +403,19 @@ namespace mbts.spark {
          #endregion
 
          #region Auto-Properties (non-child-algo params)
-         protected Property<T> add_property<T>(string prop_name, int target_field_number, T init_value = default, T non_uniform_value = default, bool validate_on_changed = false, AlgoParamPermissions permissions = AlgoParamPermissions.ReadWrite) {
+         protected Property<T> add_property<T>(
+            string prop_name,
+            int target_field_number,
+            T init_value = default,
+            T non_uniform_value = default,
+            bool validate_on_changed = false,
+            AlgoParamPermissions permissions = AlgoParamPermissions.ReadWrite,
+            Func<T, T> to_display_value = null,
+            Func<T, T> from_display_value = null,
+            IObservable<Unit> refresh_display_value = null) {
+            //
             var prop = backing(prop_name, init_value);
-            _on_property_added(prop_name, target_field_number, prop, prop.set, non_uniform_value, validate_on_changed, permissions);
+            _on_property_added(prop_name, target_field_number, prop, prop.set, non_uniform_value, validate_on_changed, permissions, to_display_value, from_display_value, refresh_display_value);
             return prop;
          }
 
@@ -592,17 +590,25 @@ namespace mbts.spark {
             }
          }
 
-         void _on_property_added<T>(string prop_name, int target_field_number, IProperty<T> prop, Action<T> assign_to_prop, T non_uniform_value, bool validate_on_changed, AlgoParamPermissions permissions) {
+         void _on_property_added<T>(string prop_name, int target_field_number, IProperty<T> prop, Action<T> assign_to_prop, T non_uniform_value, bool validate_on_changed, AlgoParamPermissions permissions, Func<T, T> to_display_value = null, Func<T, T> from_display_value = null, IObservable<Unit> refresh_display_value = null) {
             var field_desc = new TMutable().Descriptor.FindFieldByNumber(target_field_number);
-            var param = new AlgoParamVM<T>(prop_name, DisplayNameHelper.snake_to_title_case(prop_name), prop.Value) { permissions = permissions };
+            var param = new AlgoParamVM<T>(prop_name, DisplayNameHelper.snake_to_title_case(prop_name), prop.Value, EqualityComparer<T>.Default, null, to_display_value, from_display_value) {
+               permissions = permissions
+            };
             _algo_params.Add(param);
+            var prop_hook = _link_props(param, prop, assign_to_prop, non_uniform_value, validate_on_changed);
+            if (refresh_display_value != null) {
+               prop_hook = Disposable.Create(
+                  refresh_display_value.Subscribe(_ => param.refresh_display_value()),
+                  prop_hook);
+            }
             _auto_props.Add(new AutoProp(prop_name, mval => {
                if (mval != null && param.has_value)
                   field_desc.Accessor.SetValue(mval, prop.Value);
             },
             mval => assign_to_prop_and_commit(mval == null ? default : ConvertUtils.try_convert<T>(field_desc.Accessor.GetValue(mval))),
             mvals => assign_to_prop_and_commit(mvals.Select(v => ConvertUtils.try_convert<T>(field_desc.Accessor.GetValue(v))).Distinct().SafeSingleOrDefault(non_uniform_value)),
-            _link_props(param, prop, assign_to_prop, non_uniform_value, validate_on_changed)));
+            prop_hook));
             void assign_to_prop_and_commit(T value) {
                assign_to_prop(value);
                param.mark_committed();
@@ -814,8 +820,6 @@ namespace mbts.spark {
             return Disposable.Create(() => {
                if (security != old_sec) {
                   on_security_internal();
-                  if (_editing_permissions.HasFlag(AlgoEditingPermissions.AllowEditStructure))
-                     _populate_selectors();
                   _security_changed?.Invoke(security);
                }
                if (security_md != old_sec_md)
@@ -823,21 +827,6 @@ namespace mbts.spark {
             });
          }
 
-         IDisposable _selector_data_hooks;
-         void _populate_selectors() {
-            if (model != null && security.HasValue) {
-               var algo_data = model.factory.default_algos(security.Value);
-               var default_algo = AlgoKey.from_known_type(OrderDataHelper.get_default_algo(OrderDataHelper.get_algo_format(security.Value.ContractCase)));
-               foreach (var sd in _child_algo_selectors)
-                  sd.load_algo_data_recursive(algo_data, default_algo);
-               Disposable.dispose(ref _selector_data_hooks);
-               _selector_data_hooks = ObservableAutoWrapper.connect_item_hooks(child_algo_selectors, s => {
-                  s.load_algo_data_recursive(algo_data, default_algo);
-                  return Disposable.empty;
-               });
-            }
-         }
-
          class AutoProp : IDisposable {
             public readonly string name;
             public readonly Action<TMutable> assign, load;
@@ -924,6 +913,7 @@ namespace mbts.spark {
          AlgoParamPermissions _permissions = AlgoParamPermissions.ReadWrite;
          IReadOnlyDictionary<AlgoKey, AlgoData> _pending_data;
          AlgoKey? _pending_default;
+         IDisposable _loaded_data_hook;
 
          public AlgoSelector(string id, string display_name) {
             this.id = id;
@@ -1020,8 +1010,11 @@ namespace mbts.spark {
 
          void _sync_formats() {
             if (initialized && _pending_data != null) {
-               var fmt_algos = _pending_data;
-               algos.SyncWithDictionary(fmt_algos, d => create(d, is_expanded: true));
+               Disposable.dispose(ref _loaded_data_hook);
+               var data = _pending_data;
+               algos.SyncWithDictionary(data, d => create(d, is_expanded: true));
+               _loaded_data_hook = ObservableAutoWrapper.connect_item_hooks(algos, vm =>
+                  ObservableAutoWrapper.connect_item_actions(vm.child_algo_selectors, s => s.load_algo_data(data, null)));
                if (_algo == null || !algos.ContainsKey(_algo.id)) {
                   if (_pending_default.HasValue && algos.TryGetValue(_pending_default.Value, out IAlgoVM match))
                      selected_algo = match;

src/client/spark/Orders/OrderTypeVMs/SpreaderOrderVM.cs

@@ -138,7 +138,7 @@ namespace mbts.spark {
                new DialogSkirtItem("Save Strategy", commit);
 
             void _ensure_init() {
-               if (_vm == null && _model != null && _template.HasValue && _template.Value.HasSecurity) {
+               if (_vm == null && _model != null && _template.HasValue && _template.Value.HasSecurity && _template.Value.HasExecutionStrategy) {
                   Log.isnull(_save_strategy_item);
                   Log.isnull(_vm_strategy_sub);
                   var tmp = _template.Value;

src/client/spark/Orders/OrdersModel.cs

@@ -32,6 +32,7 @@ namespace mbts.spark {
       public readonly ISecurityAuxiliaryDataModel security_auxiliary_model;
 
       readonly IBus _bus;
+      readonly IAlgoProvider _algo_provider, _attachable_algo_provider;
       readonly OrderFactory _factory;
       readonly OrderService _service;
       readonly PositionService _position_service;
@@ -57,6 +58,8 @@ namespace mbts.spark {
          IAlgoStudioModel algo_model,
          ISecurityAuxiliaryDataModel security_auxiliary_model,
          MarketDataAuxiliaryModel md_auxiliary_model,
+         IAlgoProvider algo_provider,
+         IAttachableAlgoProvider attachable_algo_provider,
          SecurityStore sstore,
          ErrorReporter error) {
          //
@@ -67,6 +70,8 @@ namespace mbts.spark {
          _mbo_service = mbo_service;
          _spreader = spreader;
          _error = error;
+         _algo_provider = algo_provider;
+         _attachable_algo_provider = attachable_algo_provider;
          _dynamic_helper = dynamic_helper;
          this.admin_model = admin_model;
          this.algo_model = algo_model;
@@ -314,6 +319,9 @@ namespace mbts.spark {
       public MBOService mbo_service => _mbo_service;
       public IAlgoStudioModel algo_model { get; }
 
+      public AlgoTypeSetImmutable get_permitted_algos(SecurityImmutable security) =>
+         _algo_provider.get_permitted_algos(security);
+
       public IObservable<TableUpdate<OrderSecurity, uint>> on_security_working_orders_with_piq(uint security) =>
          _security_working_orders_with_piq_demuxer.subscribe(security);
 
@@ -322,6 +330,15 @@ namespace mbts.spark {
 
       public IObservable<TableUpdate<(SecurityImmutable?, PositionUnitImmutable), int>> on_security_positions(uint security) =>
          _security_positions_demuxer.subscribe(security);
+
+      public IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos(bool attachable_only = false) =>
+         (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_algos();
+
+      public IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_algos(SecurityImmutable security, bool attachable_only = false) =>
+         (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_algos(security);
+
+      public IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_complete_algos(bool attachable_only = false, bool allow_undefined_child = false) =>
+         (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_complete_algos(allow_undefined_child);
       #endregion
 
       #region Input
@@ -634,14 +651,10 @@ namespace mbts.spark {
          }).Select(r => r.last_update);
 
       public static IObservable<TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>> subscribe_all_algos(this OrdersModel orders_model) =>
-         orders_model.studio_algos_with_meta.MergeTable(Observable.Return(
-            new TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>(
-               ImmutableDictionary<AlgoKey, (AlgoData algo, AlgoDefinitionMetadata? meta)>.Empty.AddRange(orders_model.factory.default_algos(OrderDataHelper.all_order_types).Select(kv => new KeyValuePair<AlgoKey, (AlgoData algo, AlgoDefinitionMetadata? meta)>(kv.Key, (kv.Value, null)))),
-               Array.Empty<(AlgoData, AlgoDefinitionMetadata?)>(),
-               Array.Empty<Change<(AlgoData, AlgoDefinitionMetadata?)>>(),
-               Array.Empty<(AlgoData, AlgoDefinitionMetadata?)>(),
-               d => d.algo.key
-            ).as_snapshot()));
+         _subscribe_algo_meta_tuples(orders_model, orders_model.subscribe_algos());
+
+      public static IObservable<TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>> subscribe_algo_meta_tuples(this OrdersModel orders_model, SecurityImmutable security) =>
+         _subscribe_algo_meta_tuples(orders_model, orders_model.subscribe_algos(security));
 
       public static async Task<OrderRecordImmutable?> order_from_key(this OrdersModel orders_model, uint order_key) {
          (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3));
@@ -684,6 +697,15 @@ namespace mbts.spark {
          return false;
       }
 
+      static IObservable<TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>> _subscribe_algo_meta_tuples(OrdersModel orders_model, IObservable<TableUpdate<AlgoData, AlgoKey>> algos) =>
+         algos.LeftJoin(
+            orders_model.algo_model.metadata_service.subscribe_metadata().Table(AlgoDefinitionMetadata.key_fn),
+            AlgoData.key_fn,
+            AlgoDefinitionMetadata.key_fn,
+            (algo, key, meta) => (algo, (AlgoDefinitionMetadata?)meta),
+            (algo, key) => (algo, (AlgoDefinitionMetadata?)null),
+            tup => tup.algo.key);
+
       class RootOrderData {
          public readonly SetMap<uint, uint> parent_to_children = new SetMap<uint, uint>();
          public TableUpdateTransform<OrderRecordImmutable, uint> last_update { get; set; } = new TableUpdateTransform<OrderRecordImmutable, uint>();

src/client/spark/view/Controls/Order/AlgoContentTemplateSelector.xaml

@@ -947,7 +947,7 @@
             <TextBlock HorizontalAlignment="Left" VerticalAlignment="Center">Show Quantity</TextBlock>
             <c:HelpButton Margin="{StaticResource Margin.LP1}" ToolTip="Maximum size to show on exchange, this is the BTS provided iceberg"/>
           </StackPanel>
-          <c:TickUpDownDouble x:Name="PART_ShowQuantity_TickUpDownDouble" Grid.Column="2" HorizontalAlignment="Stretch" Value="{Binding show_quantity}"/>
+          <c:TickUpDownDouble x:Name="PART_ShowQuantity_TickUpDownDouble" Grid.Column="2" HorizontalAlignment="Stretch" Value="{Binding display_show_quantity}"/>
           <uag:AlgoPickerNodeView
             x:Name="PART_Child_AlgoPickerNodeView"
             Grid.Row="2"

src/client/spark/view/Controls/Order/AlgoContentTemplateSelector.xaml.cs

@@ -15,8 +15,7 @@ namespace mbts.ui.spark {
          AlgoDropDownStyleKeyConverter = ValueConverter.create(obj => {
             if (obj is AlgoDefinitionVM) return DefinitionDropDownStyleKey;
             return null;
-         }),
-         DataContextConverter = ValueConverter.create(obj => obj is AlgoVMWrapper w ? w.value.algo_root : obj);
+         });
 
       public static IMultiValueConverter TriggerTimeoutStateConverter =
          ValueConverter.create<bool, AlgoEditingPermissions, string>((is_timeout_enabled, editing_permissions) =>

src/client/spark/view/Spark/ContractPicker/ContractPickerModel.cs

@@ -43,24 +43,24 @@ namespace mbts.spark {
                   foreach (var selected_item in all_securities.Where(s => selected_securities.Contains(s.Key))) {
                      // verify clause isn't already in the list
                      if (!security_criteria_clauses.Any(x => x.pretty_name == selected_item.Value.security.ToString())) {
-                        security_criteria_clauses.Insert(0, new SecurityCriteriaClause() {
-                           pretty_name = selected_item.Value.security.ToString(),
-                           exchanges = selected_exchanges.ToArray(),
-                           security_types = selected_security_types.ToArray(),
-                           symbols = selected_symbols.ToArray(),
-                           security_keys = new List<uint>() { selected_item.Key }
-                        });
+                        security_criteria_clauses.Insert(0, new SecurityCriteriaClause(
+                           pretty_name: selected_item.Value.security.ToString(),
+                           exchanges: selected_exchanges.ToArray(),
+                           security_types: selected_security_types.ToArray(),
+                           symbols: selected_symbols.ToArray(),
+                           security_keys: new[] { selected_item.Key }
+                        ));
                      }
                   }
                } else {
                   if (!security_criteria_clauses.Any(x => x.pretty_name == criteria_input_name)) {
-                     security_criteria_clauses.Insert(0, new SecurityCriteriaClause() {
-                        pretty_name = criteria_input_name,
-                        exchanges = selected_exchanges.ToArray(),
-                        security_types = selected_security_types.ToArray(),
-                        symbols = selected_symbols.ToArray(),
-                        security_keys = selected_securities
-                     });
+                     security_criteria_clauses.Insert(0, new SecurityCriteriaClause(
+                        pretty_name: criteria_input_name,
+                        exchanges: selected_exchanges.ToArray(),
+                        security_types: selected_security_types.ToArray(),
+                        symbols: selected_symbols.ToArray(),
+                        security_keys: selected_securities.ToArray()
+                     ));
                   }
                }
                criteria_input_name = string.Empty;
@@ -97,12 +97,12 @@ namespace mbts.spark {
          add_symbol_command = new DelegateCommand<string>(symbol => {
             _compose_criteria_clause_name(true);
             if (!security_criteria_clauses.Any(x => x.pretty_name == _criteria_input_name_from_double_click)) {
-               security_criteria_clauses.Insert(0, new SecurityCriteriaClause() {
-                  pretty_name = _criteria_input_name_from_double_click,
-                  exchanges = selected_exchanges.ToArray(),
-                  security_types = selected_security_types.ToArray(),
-                  symbols = new string[] { symbol },
-               });
+               security_criteria_clauses.Insert(0, new SecurityCriteriaClause(
+                  pretty_name: _criteria_input_name_from_double_click,
+                  exchanges: selected_exchanges.ToArray(),
+                  security_types: selected_security_types.ToArray(),
+                  symbols: new string[] { symbol }
+               ));
             }
             ok_button_command.TryRaiseCanExecuteChanged();
             remove_criteria_command.TryRaiseCanExecuteChanged();
@@ -110,11 +110,11 @@ namespace mbts.spark {
          add_security_type_command = new DelegateCommand<SecurityType>(sec_type => {
             _compose_criteria_clause_name(true, true);
             if (!security_criteria_clauses.Any(x => x.pretty_name == _criteria_input_name_from_double_click)) {
-               security_criteria_clauses.Insert(0, new SecurityCriteriaClause() {
-                  pretty_name = _criteria_input_name_from_double_click,
-                  exchanges = selected_exchanges.ToArray(),
-                  security_types = new SecurityType[] { sec_type },
-               });
+               security_criteria_clauses.Insert(0, new SecurityCriteriaClause(
+                  pretty_name: _criteria_input_name_from_double_click,
+                  exchanges: selected_exchanges.ToArray(),
+                  security_types: new SecurityType[] { sec_type }
+               ));
             }
             ok_button_command.TryRaiseCanExecuteChanged();
             remove_criteria_command.TryRaiseCanExecuteChanged();
@@ -122,10 +122,10 @@ namespace mbts.spark {
          add_exchange_command = new DelegateCommand<Venue>(exchange => {
             _compose_criteria_clause_name(true, true, true);
             if (!security_criteria_clauses.Any(x => x.pretty_name == _criteria_input_name_from_double_click)) {
-               security_criteria_clauses.Insert(0, new SecurityCriteriaClause() {
-                  pretty_name = _criteria_input_name_from_double_click,
-                  exchanges = new Venue[] { exchange },
-               });
+               security_criteria_clauses.Insert(0, new SecurityCriteriaClause(
+                  pretty_name: _criteria_input_name_from_double_click,
+                  exchanges: new Venue[] { exchange }
+               ));
             }
             ok_button_command.TryRaiseCanExecuteChanged();
             remove_criteria_command.TryRaiseCanExecuteChanged();
@@ -343,12 +343,16 @@ namespace mbts.spark {
          }
       }
 
+      // TODO: this should return a nullable (if no specific security is selected, the result should be null)
       public SecurityMD result {
          get {
-            SecurityCriteriaClause scc = new SecurityCriteriaClause();
-            scc.security_keys = selected_securities;
-            SecurityCriteria crit = new SecurityCriteria(new SecurityCriteriaClause[] { scc });
-            return all_securities.Where(s => security_in_criteria(crit, s.Value.security)).FirstOrDefault().Value;
+            if (selected_securities.Count == 0)
+               throw new InvalidOperationException("No securities selected.");
+            foreach (var k in selected_securities) {
+               if (all_securities.TryGetValue(k, out var smd))
+                  return smd;
+            }
+            throw new InvalidOperationException("None of the selected security keys are present in the security map.");
          }
       }
 
@@ -427,7 +431,7 @@ namespace mbts.spark {
             bool check_exchanges = !(x.exchanges == null || x.exchanges.Length == 0);
             bool check_security_types = !(x.security_types == null || x.security_types.Length == 0);
             bool check_symbols = !(x.symbols == null || x.symbols.Length == 0);
-            bool check_security_keys = !(x.security_keys == null || x.security_keys.Count == 0);
+            bool check_security_keys = !(x.security_keys == null || x.security_keys.Length == 0);
             bool check_something_in_clause = check_exchanges || check_security_types || check_symbols || check_security_keys;
 
             if (check_something_in_clause) {

src/client/spark/view/Spark/ContractPicker/SecurityCriteria.cs

@@ -8,8 +8,7 @@ using System.Reactive.Linq;
 
 namespace mbts.spark {
    [ProtoContract]
-   public class SecurityCriteria : IEquatable<SecurityCriteria>
-   {
+   class SecurityCriteria : IEquatable<SecurityCriteria> {
       public SecurityCriteria()
       {
          clauses = new SecurityCriteriaClause[] { };
@@ -29,9 +28,7 @@ namespace mbts.spark {
 
       internal static SecurityCriteria from_keys(IEnumerable<uint> keys) =>
          new SecurityCriteria(new[] {
-            new SecurityCriteriaClause {
-               security_keys = keys.ToList()
-            }
+            new SecurityCriteriaClause (security_keys: keys.ToArray())
          });
 
       internal static SecurityCriteria from_securities(ISet<SecurityImmutable> securities) =>
@@ -49,27 +46,38 @@ namespace mbts.spark {
    }
 
    [ProtoContract]
-   public class SecurityCriteriaClause : IEquatable<SecurityCriteriaClause>
-   {
-      public SecurityCriteriaClause() { }
-      internal SecurityCriteriaClause(IEnumerable<SecurityImmutable> securities) {
-         security_keys = securities.Select(s => s.Id).ToList();
+   readonly struct SecurityCriteriaClause : IEquatable<SecurityCriteriaClause> {
+      public SecurityCriteriaClause(
+         Venue[] exchanges = null,
+         SecurityType[] security_types = null,
+         string[] symbols = null,
+         uint[] security_keys = null,
+         string pretty_name = null) {
+         //
+         this.exchanges = exchanges;
+         this.security_types = security_types;
+         this.symbols = symbols;
+         this.security_keys = security_keys;
+         this.pretty_name = pretty_name;
       }
 
+      public SecurityCriteriaClause(IEnumerable<SecurityImmutable> securities)
+         : this(security_keys: securities.Select(s => s.Id).ToArray()) { }
+
       [ProtoMember(1)]
-      public string pretty_name;
+      public readonly string pretty_name;
 
       [ProtoMember(2)]
-      public Venue[] exchanges;
+      public readonly Venue[] exchanges;
 
       [ProtoMember(3)]
-      public SecurityType[] security_types;
+      public readonly SecurityType[] security_types;
 
       [ProtoMember(4)]
-      public string[] symbols;
+      public readonly string[] symbols;
 
       [ProtoMember(5)]
-      public List<uint> security_keys;
+      public readonly uint[] security_keys;
 
       public bool is_defined =>
          !string.IsNullOrEmpty(pretty_name) ||
@@ -84,7 +92,6 @@ namespace mbts.spark {
       }
 
       public bool Equals(SecurityCriteriaClause other) =>
-         other != null &&
          EqualityComparer.safe_sequence_equals(exchanges, other.exchanges) &&
          EqualityComparer.safe_sequence_equals(security_types, other.security_types) &&
          EqualityComparer.safe_sequence_equals(symbols, other.symbols) &&

src/client/spark/view/Spark/ContractPicker/SecurityCriteriaExtensions.cs

@@ -90,7 +90,6 @@ namespace mbts.spark {
       /// <param name="clause">A SecurityCriteriaClause.</param>
       /// <returns>true if the clause is non-null and specifies one security-key, and nothing else; otherwise false.</returns>
       public static bool is_single_security(this SecurityCriteriaClause clause) =>
-         clause != null &&
          clause.exchanges.is_null_or_empty() &&
          clause.symbols.is_null_or_empty() &&
          clause.security_types.is_null_or_empty() &&
@@ -98,6 +97,45 @@ namespace mbts.spark {
          !clause.security_keys.Distinct().Skip(1).Any();
 
       public static bool includes(this SecurityCriteria security_criteria, SecurityImmutable security) =>
-         security_criteria != null && ContractPickerModel.security_in_criteria(security_criteria, security);
+         security_criteria.is_defined() && ContractPickerModel.security_in_criteria(security_criteria, security);
+
+      public static bool includes(this SecurityCriteriaClause clause, SecurityImmutable security) =>
+         _to_predicates(clause).AnyAndAll(p => p(security));
+
+      public static bool is_subset_of(this SecurityCriteriaClause clause, SecurityCriteriaClause other) =>
+         // exchanges
+         (other.exchanges.is_null_or_empty() ||
+         (!clause.exchanges.is_null_or_empty() && clause.exchanges.All(other.exchanges.Contains))) &&
+         // symbols
+         (other.symbols.is_null_or_empty() ||
+         (!clause.symbols.is_null_or_empty() && clause.symbols.All(other.symbols.Contains))) &&
+         // security_types
+         (other.security_types.is_null_or_empty() ||
+         (!clause.security_types.is_null_or_empty() && clause.security_types.All(other.security_types.Contains))) &&
+         // security_keys
+         (other.security_keys.is_null_or_empty() ||
+         (!clause.security_keys.is_null_or_empty() && clause.security_keys.All(other.security_keys.Contains)));
+
+      public static IEnumerable<string> fmt_diagnostic_msgs(this SecurityCriteriaClause clause) {
+         if (clause.exchanges != null && clause.exchanges.Length > 0)
+            yield return "Exchanges: " + string.Join(", ", clause.exchanges);
+         if (clause.symbols != null && clause.symbols.Length > 0)
+            yield return "Symbols: " + string.Join(", ", clause.symbols);
+         if (clause.security_types != null && clause.security_types.Length > 0)
+            yield return "Security Types: " + string.Join(", ", clause.security_types);
+         if (clause.security_keys != null && clause.security_keys.Length > 0)
+            yield return "Security Keys: " + string.Join(", ", clause.security_keys);
+      }
+
+      static IEnumerable<Func<SecurityImmutable, bool>> _to_predicates(SecurityCriteriaClause clause) {
+         if (clause.exchanges != null && clause.exchanges.Length > 0)
+            yield return sec => clause.exchanges.Contains(sec.Exchange);
+         if (clause.symbols != null && clause.symbols.Length > 0)
+            yield return sec => clause.symbols.Contains(sec.Symbol);
+         if (clause.security_types != null && clause.security_types.Length > 0)
+            yield return sec => clause.security_types.Contains(sec.security_type());
+         if (clause.security_keys != null && clause.security_keys.Length > 0)
+            yield return sec => clause.security_keys.Contains(sec.Id);
+      }
    }
 }

src/client/spark/view/Utility/IDisplayTranslator.cs

@@ -40,25 +40,53 @@ namespace mbts.ui {
       public static bool operator !=(QuantityConversionData x, QuantityConversionData y) => !x.Equals(y);
    }
 
-   class DisplayQuantityMultiplier : IDisplayTranslator<uint> {
-      uint _actual_value;
+   static class DisplayQuantityMultiplier {
+      public static DisplayQuantityMultiplier<uint> create_uint() =>
+         new DisplayQuantityMultiplier<uint>(
+            DisplayTranslatorExtensions.to_display,
+            DisplayTranslatorExtensions.from_display);
+
+      public static DisplayQuantityMultiplier<double> create_double() =>
+         new DisplayQuantityMultiplier<double>(
+            DisplayTranslatorExtensions.to_display,
+            DisplayTranslatorExtensions.from_display);
+   }
+
+   class DisplayQuantityMultiplier<T> : IDisplayTranslator<T> {
+      readonly Func<QuantityConversionData, T, T> _to_display, _from_display;
+      readonly IEqualityComparer<T> _comparer;
+      T _actual_value;
       QuantityConversionData _conversion_data;
 
+      public DisplayQuantityMultiplier(
+         Func<QuantityConversionData, T, T> to_display,
+         Func<QuantityConversionData, T, T> from_display,
+         IEqualityComparer<T> comparer = null,
+         T actual_value = default,
+         QuantityConversionData conversion_data = default) {
+         //
+         _to_display = to_display ?? throw new ArgumentNullException(nameof(to_display));
+         _from_display = from_display ?? throw new ArgumentNullException(nameof(from_display));
+         _comparer = comparer ?? EqualityComparer<T>.Default;
+         _actual_value = actual_value;
+         _conversion_data = conversion_data;
+      }
+
       public event Action value_changed;
 
-      public uint actual_value {
+      public T actual_value {
          get => _actual_value;
          set {
-            if (_actual_value != value) {
+            if (!_comparer.Equals(_actual_value, value)) {
                _actual_value = value;
                value_changed?.Invoke();
             }
          }
       }
 
-      public uint display_value {
-         get => _conversion_data.to_display(_actual_value);
-         set => actual_value = _conversion_data.from_display(value);
+      public T display_value {
+         get => _to_display(_conversion_data, _actual_value);
+         set => actual_value = _from_display(_conversion_data, value);
       }
 
       public QuantityConversionData conversion_data {
@@ -168,5 +196,13 @@ namespace mbts.ui {
             default: return display_value;
          }
       }
+
+      public static double from_display(this QuantityConversionData quantity_conversion_data, double display_value) {
+         switch (quantity_conversion_data.op) {
+            case QuantityConversionData.OpType.DivideActual: return display_value * quantity_conversion_data.value;
+            case QuantityConversionData.OpType.MultiplyActual: return display_value.safe_divide(quantity_conversion_data.value);
+            default: return display_value;
+         }
+      }
    }
 }

src/client/spark_api/Co2/MDConsumerHandle.cs

@@ -259,6 +259,7 @@ namespace mbts.co2 {
          }
       }
 
+      // If Size is wrong remember to update from 
       [StructLayout(LayoutKind.Sequential, Size = 704)]
       internal struct ColdEntry {
          public int exchange_status;

src/client/spark_api/security/ExchangeSpecificUitlities.cs

@@ -0,0 +1,128 @@
+using bts.api;
+using bts.api.orderman;
+using bts.api.position;
+using bts.utils;
+using mbts;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+
+namespace bts.api {
+
+static class ExchangeSpecificUitlities{
+   private static string _default_hub = "Henry";
+   static class ICE{
+
+      #region Contract Type
+      public static bool is_spread(SecurityImmutable sec){
+         return sec.HasSpread && sec.Ice.StripName.Contains("/");
+      }
+
+      public static bool is_strip(SecurityImmutable sec){
+         return sec.HasSpread && !is_spread(sec);
+      }
+
+      public static bool is_ice_lots(SecurityImmutable sec){
+         return sec.Ice.MarketDescription.Contains("ICE Lots");
+      }
+
+      public static bool is_tas(SecurityImmutable sec){
+         return sec.Ice.MarketDescription.Contains("TAS");
+      }
+
+      #endregion
+
+      #region CPP
+      public static double get_future_cpp (SecurityImmutable sec){
+         if(sec.Symbol == "HHD")
+            // only futures in HHD are the dailies, everything
+            // else is a strip or spread
+            return 1;
+         if(is_ice_lots(sec))
+            return 2500;
+         else{
+            var days_in_month = DateTime.DaysInMonth(Convert.ToInt32(sec.Future.ExpirationYear),
+                                                   Convert.ToInt32(sec.Future.ExpirationMonth));
+            return days_in_month; // because the size is multiplied by 2500 already
+         }
+      }
+
+      public static double get_spread_cpp(SecurityImmutable sec){
+         return 0;
+      }
+
+      public static double get_strip_cpp(PNLUnitImmutable pnl,
+                                       SecurityImmutable sec){
+         // add up all the legs
+         double rv = 0;
+         foreach (var l in sec.Spread.Legs)
+            if(l.Security.HasFuture)
+               rv += get_cpp(pnl,l.Security);
+         return rv;
+      }
+
+      public static double get_cpp (PNLUnitImmutable pnl,
+                                    SecurityImmutable sec){
+         if(sec.HasFuture)
+            return get_future_cpp(sec);
+         else if(is_spread(sec))
+            return get_spread_cpp(sec);
+         else if(is_strip(sec))
+            return get_strip_cpp(pnl,sec);
+         return Double.NaN;
+      }
+      #endregion
+
+      #region Symbol      
+      public static string get_pnl_symbol (SecurityImmutable sec){
+         // for pnl purposes, for a symbol, the division is :
+         // Futures and Strips
+         // ICE Lots
+         // Spreads
+         var rv = String.Format("{0}{1}{2}",
+                              sec.Symbol,
+                              is_ice_lots(sec)? " ICE LOTS" : String.Empty,
+                              is_spread(sec)? " Spread" : String.Empty);
+         return rv;
+      }
+      #endregion
+
+      #region Formatting
+      public static string format_security (SecurityImmutable? security){
+         if(security.HasValue && security.Value.HasIce){
+            bool ice_lots = is_ice_lots(security.Value),
+               tas = is_tas(security.Value);
+            return String.Format("{0}{1}{2}{3}{4}",
+                                 security?.Symbol,
+                                 ice_lots? " ICE Lots" : String.Empty,
+                                 tas ? " TAS" : String.Empty,
+                                 String.Compare(security.Value.Ice.HubAlias, _default_hub,true)==0 ? String.Empty :
+                                             " " + security.Value.Ice.HubAlias,
+                                 " " + security?.Ice.StripName);
+         }
+         return String.Empty;
+      }
+      #endregion
+   }
+
+   public static double get_cpp (PNLUnitImmutable pnl,
+                                 SecurityImmutable sec){
+      if(sec.HasIce)
+         return ICE.get_cpp(pnl,sec);
+      return pnl.CashValue;      
+   }
+
+   public static string get_pnl_symbol (SecurityImmutable sec){
+      if(sec.HasIce)
+         return ICE.get_pnl_symbol(sec);
+      return sec.Symbol;
+   }
+
+   public static string format_security(SecurityImmutable? security){
+      if(security?.HasIce == true)
+         return ICE.format_security(security);
+      return security?.ToString();
+   }
+
+}}

src/client/spark_api/security/SecurityExtensions.cs

@@ -1,10 +1,15 @@
-using System;
-using bts.api;
-
-namespace bts.api
-{
-   public static class SecurityExtensions
-   {
+namespace bts.api {
+   public static class SecurityExtensions {
       public static SecurityType security_type(this Security s) { return ((SecurityImmutable)s).security_type(); }
+
+      public static SecurityType security_type(this Security.ContractOneofCase contract_case) {
+         switch (contract_case) {
+            case Security.ContractOneofCase.Future: return SecurityType.Future;
+            case Security.ContractOneofCase.Option: return SecurityType.Option;
+            case Security.ContractOneofCase.Spread: return SecurityType.Spread;
+            case Security.ContractOneofCase.Synthetic: return SecurityType.Synthetic;
+            default: return SecurityType.Unset;
+         }
+      }
    }
 }

src/client/spark_api/security/SecurityFormatter.cs

@@ -41,7 +41,7 @@ namespace bts.api {
          switch (security?.ContractCase) {
             case ContractOneofCase.Future:
                if(security?.HasIce == true)
-                  return _format_ICE(security);
+                  return bts.api.ExchangeSpecificUitlities.format_security(security);
                return format_future(security?.Symbol, security?.security_desc(), security.Value.Future);
             case ContractOneofCase.Spread:
                if (security?.HasGlbx == true) {
@@ -50,7 +50,7 @@ namespace bts.api {
                   return security?.Glbx.SecurityDesc;
                }
                else if(security?.HasIce == true)
-                  return _format_ICE(security);
+                  return bts.api.ExchangeSpecificUitlities.format_security(security);
                return string.Join("", _format_future_legs(SecurityHelper.get_flat_future_legs(security?.Spread.Legs)));
             case ContractOneofCase.Synthetic: return security?.Synthetic.Name;
             default: return "Unknown-security";
@@ -84,28 +84,5 @@ namespace bts.api {
             default: return $"{fl.quantity:+0;-0}{sym_exp_name}";
          }
       }
-
-      static string _format_ICE (SecurityImmutable? security){
-         string symbol = security?.Symbol,
-                strip = security?.Ice.StripName,
-                md_desc = security?.Ice.MarketDescription,
-                type_append = "";
-  
-         if(md_desc.Contains("25k ICE Lots")){
-            if(md_desc.Contains("TAS"))
-               type_append = "TAS";
-         }
-         else if(md_desc.Contains("ICE Lots")){
-            if(md_desc.Contains("TAS"))
-               type_append = "ICE Lots TAS";
-            else
-               type_append = "ICE Lots";
-         }
-         else if(md_desc.Contains("TAS"))
-            type_append = "TAS";
-         type_append = string.IsNullOrEmpty(type_append) ? 
-                                                " " : $" {type_append} ";
-         return $"{symbol}{type_append}{strip}";
-      }
    }
 }

src/client/spark_api/security/SecurityHelper.cs

@@ -83,26 +83,37 @@ namespace bts.api {
          }
 
          public static int Compare(SecurityImmutable x, SecurityImmutable y) {
-            if (x.ContractCase == ContractOneofCase.Future) {
-               return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
-            } else if (y.ContractCase == ContractOneofCase.Future) {
-               return 1;
+            int c = non_deterministic_impl();
+            if (c == 0) {
+               c = x.ToString().CompareTo(y.ToString());
+               if (c == 0)
+                  c = x.Id.CompareTo(y.Id);
             }
-            var xlegs = get_flat_future_legs(x).ToList();
-            var ylegs = get_flat_future_legs(y).ToList();
-            int max = Math.Min(xlegs.Count, ylegs.Count);
-            for (int i = 0; i < max; i++) {
-               int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future);
-               if (rv != 0)
-                  return rv;
-               rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
-               if (rv != 0)
-                  return rv;
-               rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
-               if (rv != 0)
-                  return rv;
+            return c;
+            // this has been observed to be non-determinisitc with certain ICE securities
+            // (it is determenistic with other exchanges..as far as we know).
+            int non_deterministic_impl() {
+               if (x.ContractCase == ContractOneofCase.Future) {
+                  return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
+               } else if (y.ContractCase == ContractOneofCase.Future) {
+                  return 1;
+               }
+               var xlegs = get_flat_future_legs(x).ToList();
+               var ylegs = get_flat_future_legs(y).ToList();
+               int max = Math.Min(xlegs.Count, ylegs.Count);
+               for (int i = 0; i < max; i++) {
+                  int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future);
+                  if (rv != 0)
+                     return rv;
+                  rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
+                  if (rv != 0)
+                     return rv;
+                  rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
+                  if (rv != 0)
+                     return rv;
+               }
+               return xlegs.Count.CompareTo(ylegs.Count);
             }
-            return xlegs.Count.CompareTo(ylegs.Count);
          }
 
          public static int Compare(string x_sym, FutureContract x, string y_sym, FutureContract y) {
@@ -116,4 +127,4 @@ namespace bts.api {
          }
       }
    }
-}
+}

src/client/spark_api/utils/Log.cs

@@ -36,6 +36,7 @@ namespace bts {
       Plugin,
       Sdk,
       Http,
+      iLink3
       // Please keep this list in sync with logging.h/logging.cpp
    }
    

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -7,6 +7,42 @@ using System.Reactive.Linq;
 
 namespace System.Collections.Generic {
    public static class EnumerableExtensions {
+      /// <summary>
+      /// Determines whether all elements of a sequence are true.
+      /// </summary>
+      /// <param name="source">An <see cref="IEnumerable{bool}"/> that contains the elements to which to apply the predicate.</param>
+      /// <returns>True if every element of the source sequence is true, or if the sequence is empty; otherwise false.</returns>
+      /// <exception cref="ArgumentNullException"><paramref name="source"/> is null</exception>
+      public static bool All(this IEnumerable<bool> source) {
+         if (source is null)
+            throw new ArgumentNullException(nameof(source));
+         foreach (var v in source) {
+            if (!v)
+               return false;
+         }
+         return true;
+      }
+
+      /// <summary>
+      /// Similar to <see cref="Enumerable.All{TSource}(IEnumerable{TSource}, Func{TSource, bool})"/>, but returns false if the <paramref name="source"/> is empty.
+      /// </summary>
+      /// <typeparam name="T">The type of elements in the source.</typeparam>
+      /// <param name="source">The sequence of elements to which the predicate will be applied.</param>
+      /// <param name="predicate">The predicate applied to each element in the specified sequence.</param>
+      /// <returns>True if the source sequence is not empty, and every element within it passes the test in the specified predicate; otherwise, false.</returns>
+      public static bool AnyAndAll<T>(this IEnumerable<T> source, Func<T, bool> predicate) {
+         using (var en = source.GetEnumerator()) {
+            if (en.MoveNext()) {
+               do {
+                  if (!predicate(en.Current))
+                     return false;
+               } while (en.MoveNext());
+               return true;
+            }
+            return false;
+         }
+      }
+
       #region Ordered
       public static IOrderedEnumerable<T> Ordered<T>(this IEnumerable<T> enumerable) =>
          enumerable.OrderBy(t => t);
@@ -165,6 +201,9 @@ namespace System.Collections.Generic {
          }
       }
 
+      public static IList<T> ToListLazy<T>(this IEnumerable<T> source) =>
+         new LazyList<T>(source);
+
       public static bool TryGetFirst<T>(this IEnumerable<T> source, out T result) {
          if (source == null) throw new ArgumentNullException(nameof(source));
          using (var en = source.GetEnumerator()) {
@@ -590,6 +629,9 @@ namespace System.Collections.Generic {
       public static bool is_null_or_empty<T>(this IEnumerable<T> source) =>
          source == null || !source.Any();
 
+      public static bool is_null_or_empty<T>(this T[] array) =>
+         array == null || array.Length == 0;
+
       public static IEnumerable<T> empty_if_null<T>(this IEnumerable<T> source) =>
          source ?? Enumerable.Empty<T>();
 
@@ -634,5 +676,44 @@ namespace System.Collections.Generic {
    
          public int? Count => _collection?.Count ?? _ro_collection?.Count;
       }
+
+      class LazyList<T> : IList<T> {
+         IEnumerable<T> _source;
+         IList<T> _list_backing;
+
+         public LazyList(IEnumerable<T> source) =>
+            _source = source;
+
+         public T this[int index] { get => _list[index]; set => _list[index] = value; }
+
+         public int Count => _list.Count;
+
+         public bool IsReadOnly => _list.IsReadOnly;
+
+         IList<T> _list {
+            get {
+               if (_list_backing == null) {
+                  if (_source == null) {
+                     _list_backing = new List<T>(0);
+                  } else {
+                     _list_backing = new List<T>(_source);
+                     _source = null;
+                  }
+               }
+               return _list_backing;
+            }
+         }
+
+         public void Add(T item) => _list.Add(item);
+         public void Clear() => _list.Clear();
+         public bool Contains(T item) => _list.Contains(item);
+         public void CopyTo(T[] array, int arrayIndex) => _list.CopyTo(array, arrayIndex);
+         public IEnumerator<T> GetEnumerator() => _list.GetEnumerator();
+         public int IndexOf(T item) => _list.IndexOf(item);
+         public void Insert(int index, T item) => _list.Insert(index, item);
+         public bool Remove(T item) => _list.Remove(item);
+         public void RemoveAt(int index) => _list.RemoveAt(index);
+         IEnumerator IEnumerable.GetEnumerator() => _list.GetEnumerator();
+      }
    }
 }

src/client/spark_core/utils/UpdateExtensions.cs

@@ -40,6 +40,17 @@ namespace bts.utils {
          });
       }
 
+      public static void FilterIncremental<T, TKey>(this TableUpdate<T, TKey> table, Func<T, bool> predicate, out IEnumerable<T> added, out IEnumerable<Change<T>> changes, out IEnumerable<T> removed) where TKey : IEquatable<TKey> {
+         // added
+         added = table.added.Where(predicate).Concat(
+            table.changes.Where(ch => predicate(ch.New) && !predicate(ch.Old)).Select(ch => ch.New));
+         // changes
+         changes = table.changes.Where(ch => predicate(ch.New) && predicate(ch.Old));
+         // removed
+         removed = table.removed.Where(predicate).Concat(
+            table.changes.Where(ch => !predicate(ch.New) && predicate(ch.Old)).Select(ch => ch.Old));
+      }
+
       public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
          TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
 
@@ -81,7 +92,8 @@ namespace bts.utils {
             void update(TableUpdate<T, TKey> input) {
                state = input.state;
                var builder = output.to_builder();
-               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
+               // execute all parent_to_children changes first, so descendent-propagation will be efficient
+               // (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
                foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
                   var k = input.key_extractor(v);
                   parent_to_children.remove(parent_key(v), v);

src/client/spark_test/Security/SecurityTests.cs

@@ -1,4 +1,5 @@
 using bts.api;
+using bts.collections;
 using bts.resources;
 using bts.test;
 using bts.utils.proto;
@@ -10,7 +11,6 @@ namespace bts.mcore {
    class SecurityTests {
       [Test]
       public void test_security_equal() {
-
          SecurityImmutable a = new SecurityImmutable(new Security() { Id = 42 });
          SecurityImmutable b = new SecurityImmutable(new Security() { Id = 42 });
          SecurityImmutable c = new SecurityImmutable(new Security() { Id = 4 });
@@ -29,20 +29,31 @@ namespace bts.mcore {
 
       [Test]
       public void test_security_compare() {
-         const string securities_resource = "KrandallOrderBookCrash.security_proto";
-         // verify that the sample size doesn't shrink for some reason.
-         // if it grows a bit that's fine; the generic comparison test
-         // asserts against collections that are too large.
-         const int min_sample_size = 91;
+         foreach (var file_path in SecurityFiles.all())
+            _test_security_compare(_read_securities(file_path));
+      }
 
-         var securities = StreamHelper.ReadProtos<Security>(
-            ResourceHelper.LoadResource(securities_resource))
-            .Select(s => new SecurityImmutable(s))
-            .ToList();
-         Assert.IsTrue(securities.Count >= min_sample_size);
+      void _test_security_compare(IEnumerable<SecurityImmutable> securities) {
+         const int max_count = 2048;
+         var sec_list = securities as IList<SecurityImmutable> ?? securities.ToList();
+         if (sec_list.Count > max_count)
+            sec_list = sec_list.WrapSubRange(0, max_count);
+         ComparisonTests.transitive_comparison(sec_list, SecurityHelper.Comparer.InstanceImmutable);
+         ComparisonTests.identity_comparison(sec_list, SecurityHelper.Comparer.InstanceImmutable);
+      }
 
-         ComparisonTests.transitive_comparison(securities, SecurityHelper.Comparer.InstanceImmutable);
-         ComparisonTests.identity_comparison(securities, SecurityHelper.Comparer.InstanceImmutable);
+      IEnumerable<SecurityImmutable> _read_securities(string file_path) =>
+         StreamHelper.ReadProtos<Security>(ResourceHelper.LoadResource(file_path)).Select(s => new SecurityImmutable(s));
+
+      static class SecurityFiles {
+         public const string
+            KrandallOrderBookCrash = "KrandallOrderBookCrash.security_proto",
+            NonDeterministicICEFutures = "NonDeterministicICEFutures.security_proto";
+
+         public static IEnumerable<string> all() {
+            yield return KrandallOrderBookCrash;
+            yield return NonDeterministicICEFutures;
+         }
       }
    }
 }

src/client/spark_test/Utils/ComparisonTests.cs

@@ -4,14 +4,14 @@ using System.Linq;
 
 namespace bts.mcore {
    static class ComparisonTests {
-      public static void transitive_comparison<T>(IEnumerable<T> items, IComparer<T> comparer) {
-         // Verify that the sample size isn't too big.
-         // This test compares each item to every other item
-         // (technically twice; lhs/rhs is flipped the second time),
-         // so performance is O(n^2).
-         const int max_sample_size = 4092;
+      /// <summary>
+      /// Compares every permutation of pairs in the specified items.
+      /// Be wary of use with large collections, as performance is O(n^2).
+      /// </summary>
+      public static void transitive_comparison<T>(IEnumerable<T> items, IComparer<T> comparer, int min_item_count = 2, int max_item_count = 5000) {
          var sorted_items = items.ToList();
-         Assert.That(sorted_items.Count <= max_sample_size);
+         Assert.That(sorted_items.Count >= min_item_count);
+         Assert.That(sorted_items.Count <= max_item_count);
          sorted_items.Sort(comparer);
          for (int i = 0; i < sorted_items.Count - 1; i++) {
             for (int j = i + 1; j < sorted_items.Count; j++) {

src/client/spark_test/spark_test.csproj

@@ -16,5 +16,6 @@
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="Resources\KrandallOrderBookCrash.security_proto" />
+    <EmbeddedResource Include="Resources\NonDeterministicICEFutures.security_proto" />
   </ItemGroup>
 </Project>

src/data_files/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
+set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)

src/data_files/files/audit_ilink3.py

@@ -0,0 +1,405 @@
+import json
+import csv
+import datetime, time
+import argparse, os
+
+
+def valid_file(f):
+    if not os.path.exists(f):
+        # Argparse uses the ArgumentTypeError to give a rejection message like:
+        # error: argument input: x does not exist
+        raise argparse.ArgumentTypeError("{0} does not exist".format(f))
+    return f
+
+def valid_date(s):
+    try:
+        return datetime.datetime.strptime(s, "%Y%m%d")
+    except ValueError:
+        raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
+
+def print_headers():
+    # From https://www.cmegroup.com/confluence/display/EPICSANDBOX/iLink+3+-+Minimum+Acceptable+Audit+Trail+Elements+-+Data+Definitions+and+Validation+Rules
+    line_to_write = "Sending Timestamps,Message Direction,Operator ID,Session ID,Manual Order Indicator,Message Type,Instrument,Order Flow ID,\
+CME Globex Order ID,Client Order ID,Order Request ID,Buy/Sell Indicator,Quantity,Limit Price,Stop Price,Order Type,Order Qualifier,IFM Flag,\
+Display Quantity,Minimum Quantity,Country of Origin,Fill Price,Fill Quantity,Cumulative Quantity,Remaining Quantity,Aggressor Flag,Source of Cancellation,\
+Reject Reason,CME Globex Message ID,Cross ID,Quote Request ID,Message Quote ID,Quote Set ID,Quote Entry ID,Bid Price,Bid Size,Offer Price,Offer Size,\
+Processed Entries,Party Details List Request ID,ListUpdateAction,Self-Matching Prevention ID,Self-Match Prevention Instr,Customer Type Indicator,\
+AvgPxGroupID,Average Px Ind,CmtaGiveUpCD,Origin,Clearing Trade Price Type,Customer Handling Instr,Executing Firm ID,Account Number,Take Up Firm,Take up Account"
+
+    with open (args.output_file, 'w') as the_file:
+                    the_file.write(line_to_write)
+                    the_file.write("\n")
+
+def set_blanks():
+    return ""
+
+def get_value(json_key):
+    prices = ["Price", "LastPx", "StopPx"]
+    if json_key in json_data:
+        if json_key in prices:
+            x = json_data[json_key]
+            return x['mantissa']/100000000000
+        else:
+            return json_data[json_key]
+    else:
+        return ""
+
+
+# Column 1  - Sending Timestamps
+def SendingTimeEpoch(epochtime):   #1
+    #TIME = time.strftime('%Y%m%d-%H:%M:%S.{}'.format(epochtime/1000000000), time.gmtime(epochtime/1000000000))
+    TIME = datetime.datetime.fromtimestamp(int(epochtime) / 1e9).strftime('%Y%m%d-%H:%M:%S.%f')
+    return TIME
+
+# Column 2 - Message Direction
+def direction(whichway):
+    if whichway == '<':    
+        DIRECTION = "FROM CME"
+    else:                       
+        DIRECTION = "TO CME"     
+    return DIRECTION
+
+# Column 3 - Operator ID
+def SenderID(sender):
+    if "SenderID" in json_data:
+        SENDER = json_data["SenderID"] 
+        return SENDER
+    else:
+        return SESSIONID
+
+
+# Column 4 - Session ID
+
+# Column 5 - Manual Order Indicator
+def ManualOrderIndicator(manual):
+    if "ManualOrderIndicator" in json_data:
+        if manual == "Manual":
+            return "1"
+        else:
+            return "0"
+
+# Column 6 - Message Type
+def MessageType(msg_type,orderstatus):
+    if msg_type == "NewOrderSingle514":
+       return "D" 
+    elif msg_type == "ExecutionReportNew522":
+        return "8--0"
+    if "ExecutionReportTrade" in msg_type:
+        if orderstatus == "PartiallyFilled":
+            return "8--1"
+        else:
+            return "8--2"
+    elif msg_type == "ExecutionReportCancel534":
+        return "8--4"
+    elif msg_type == "ExecutionReportModify531":
+        return "8--5"
+    elif msg_type == "ExecutionReportCancel534":
+        return "8--H"
+    elif msg_type == "ExecutionReportReject523":
+        return "8--8"
+    elif msg_type == "OrderCancelReplaceReject536":
+        return "9--2"
+    elif msg_type == "OrderCancelReject535":
+        return "9--1"
+    elif msg_type == "PartyDetailsDefinitionRequest518":
+        return "CX"
+    elif msg_type == "PartyDetailsDefinitionRequestAck519":
+        return "CY"
+    elif msg_type == "OrderCancelRequest516":
+        return "F"
+    elif msg_type == "OrderCancelReplaceRequest515":
+        return "G"
+    
+    
+
+# Column 12 - Buy-Sell Indicator/Side
+def Side(indicator):
+    if "Side" in json_data:
+        if indicator == "Buy":
+            return "1"
+        else:
+            return "2"
+
+# Column 16 - Order Type/OrdType
+def OrdType(ordertype):
+    if "OrdType" in json_data:
+        if ordertype == "MarketWithProtection" or ordertype == "MarketwithProtection":
+            return "1"
+        elif ordertype == "Limit":
+            return "2"
+        elif ordertype == "StopWithProtection" or ordertype == "StopwithProtection":
+            return "3"
+        elif ordertype == "StopLimit":
+            return "4"
+        elif ordertype == "MarketLimit":
+            return "K"
+        else:
+            return ""
+
+# Column 17 - Order Qualifier/TimeInForce
+def TimeInForce(orderqual):
+    if "TimeInForce" in json_data:
+        if orderqual == "Day":
+            return "0"
+        elif orderqual == "GTC":
+            return "1"
+        elif orderqual == "FAK":
+            return "3"
+        elif orderqual == "GTD":
+            return "6"
+        else:
+            return ""
+
+# Column 18 IFM Flag/OFMOverride
+def OFMOverride(imfflag):
+    if "OFMOverride" in json_data:
+        if imfflag == "Enabled":
+            return "1"
+        else:
+            return "0"
+
+# Column 19 Display Quantity/DisplayQty
+def DisplayQty(qty):
+    if "DisplayQty" in json_data:
+        if str(qty) == "0":
+            return ""
+        elif str(qty) == "4294967295":
+            return ""
+        else:
+            return qty
+
+
+# Column 26 - Aggressor Flag/AggressorIndicator
+def AggressorIndicator(aggressor):
+    if "AggressorIndicator" in json_data:
+        if aggressor == "True":
+            return "1"
+        else:
+            return "0"
+
+# Column 27 - Source of Cancellation/ExecRestatementReason
+def ExecRestatementReason(sourcecancel):
+    if "ExecRestatementReason" in json_data:
+        if sourcecancel == "Exchange":
+            return "8"
+        elif sourcecancel == "CancelledNotBest":
+            return "9"
+        elif sourcecancel == "CancelOnDisconnect":
+            return "100"
+        elif sourcecancel == "CancelRestingDueToSelfMatchPrevention":
+            return "103"
+        elif sourcecancel == "CancelFromCMEGlobexCreditControlsViolation":
+            return "104"
+        elif sourcecancel == "CancelFromCMEOne":
+            return "105"
+        elif sourcecancel == "CancelFromRiskManagementAPI":
+            return "106"
+        elif sourcecancel == "CancelAggressingDueToSelfMatchPrevention":
+            return "107"
+        elif sourcecancel == "CancelDueToVolatilityQuoteOptionLessThanMinOrderSize":
+            return "108"
+        else:
+            return ""
+
+# Column 43 - Self-Match Prevention Instr/SelfMatchPreventionInstruction
+def SelfMatchPreventionInstruction(instr):
+    if "SelfMatchPreventionInstruction" in json_data:
+        if instr == "CancelOldest":
+            return "O"
+        elif instr == "CancelNewest":
+            return "N"
+        else:
+            return ""
+
+# Column 44 - Customer Type Indicator/CustOrderCapacity
+def CustOrderCapacity(custord):
+    if "CustOrderCapacity" in json_data:
+        if custord == "NULL_VALUE":
+            return ""
+        elif custord == "Membertradingfortheirownaccount":
+            return "1"
+        elif custord == "ClearingFirmtradingforitsproprietaryaccount":
+            return "2"
+        elif custord == "Membertradingforanothermember":
+            return "3"
+        else:
+            return "4" 
+
+# Column 46 - Average Px Ind/AvgPxInd
+def AvgPxInd(avgpxindicator):
+    if "AvgPxInd" in json_data:
+        if avgpxindicator == "NULL_VALUE":
+            return ""
+        else:
+            return avgpxindicator
+
+# Column 48 - Origin/ClearingAccountType
+def ClearingAccountType(accounttype):
+    if "ClearingAccountType" in json_data:
+        if accounttype == "Customer":
+            return "0"
+        elif accounttype == "Firm":
+            return "1"
+        else:
+            return ""
+
+# Column 49 - Clearing Trade Price Type/ClearingTradePriceType
+def ClearingTradePriceType(clrtrdprice):
+    if "ClearingTradePriceType" in json_data:
+        if clrtrdprice == "NULL_VALUE":
+            return "0"
+        else:
+            return "3"
+
+# Column 50 - Customer Handling Instr/CustOrderHandlingInst
+def CustOrderHandlingInst(custorderinst):
+    if "CustOrderHandlingInst" in json_data:
+        if custorderinst == "ClientElectronic":
+            return "C"
+        elif custorderinst == "TEST":
+            return "D"
+        else:
+            return ""
+
+# Column 51 and 52 - Getting the Firm and Account Number
+def partyid(partydetailid,column):
+    for x in partydetailid:
+        role = x['PartyDetailRole']
+        id = x['PartyDetailID']
+        if column == "51" and role == "ExecutingFirm":
+            return id
+        if column == "52" and role == "CustomerAccount":
+            return id
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("json_log", help="the json log[.gz] you wish to process", type=valid_file)
+parser.add_argument("output_file", help="the destination csv file you wish to write")
+parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
+                    default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
+args = parser.parse_args()
+
+print_headers()
+
+list_types = ['NewOrderSingle514', 'OrderCancelReplaceRequest515', 'OrderCancelRequest516', 'PartyDetailsDefinitionRequest518', 'PartyDetailsDefinitionRequestAck519', \
+        'ExecutionReportNew522', 'ExecutionReportReject523', 'ExecutionReportTradeOutright525', 'ExecutionReportTradeSpread526', 'ExecutionReportTradeSpreadLeg527', \
+        'ExecutionReportModify531', 'ExecutionReportCancel534', 'OrderCancelReject535', 'OrderCancelReplaceReject536', ]
+
+# Note that the timedeltas specificed in trade_day_start and trade_day_end
+# are based on Chicago times; as such, this script should run on a machine
+# set to a Chicago timezone. A future impromovement would be to determine
+# the current machine's timezone and determine the appropriate offset
+# versus Chicago time, but for now we assume this will run on a Chicago
+# tz machine.
+input_as_datetime = args.date
+trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=16, minutes=30)
+trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
+trade_day_end = input_as_datetime + datetime.timedelta(hours=16)
+trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
+
+file_opener = None
+if args.json_log.endswith(".gz"):
+    import gzip
+    file_opener = gzip.open
+else:
+    file_opener = open
+
+
+#inputFile = open(fileInput) #open json file
+#outputFile = open(fileOutput, 'w') #load csv file
+# Get Session
+with file_opener(args.json_log) as json_file:
+    for line in json_file:
+        data = json.loads(line)
+        json_data = data['msg_json']
+        if "Session" in json_data:
+            SESSIONID = json_data["Session"]
+
+#inputFile = open(fileInput) #open json file
+#outputFile = open(fileOutput, 'w') #load csv file
+with file_opener(args.json_log) as json_file:
+    for line in json_file:
+        data = json.loads(line)
+        line_timestamp = time.strptime(data['time'], "%Y%m%d-%H:%M:%S.%f")
+        if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
+            json_data = data['msg_json']
+            NAME = json_data["Name"]
+            if NAME in list_types:
+                EXECUTING_FIRM_ID = ""
+                ACCOUNT_NUMBER = ""
+                DIRECTION = direction(data['direction'])
+                # Set Blanks
+                blank_json = {'QuoteReqID': '', 'QuoteID': '', 'QuoteSetID': '', 'QuoteEntryID': '', 'BidPx': '', 'BidSize': '', 'OfferPx': '', \
+                'OfferSize': '', 'NoProcessedEntries': '', 'ListUpdateAction': '', 'MinQty': '', 'AvgPxGroupID': '', \
+                'CmtaGiveUpCD': '', 'TAKE_UP_FIRM': '', 'TAKE_UP_ACCOUNT': '',}
+
+                # No Data Processing
+                straight_json = {'SecurityID': '', 'ClOrdID': '', 'OrderID': '', 'OrderRequestID': '', 'OrderQty': '', \
+                'Location': '', 'LastQty': '', 'CumQty': '', 'LeavesQty': '', 'Text': '', 'SideTradeID': '', 'CrossID': '', 'SelfMatchPreventionID': '', \
+                'Price': '', 'StopPx': '', 'LastPx': '', 'PartyDetailsListReqID': '', 'OrdStatus': ''}
+                        
+                # Processing Data
+                process_json = {'SendingTimeEpoch': '', 'ManualOrderIndicator': '', 'SenderID': '', 'Side': '', 'OrdType': '', 'TimeInForce': '', \
+                'OFMOverride': '', 'AggressorIndicator': '', 'ExecRestatementReason': '', 'SelfMatchPreventionInstruction': '', 'CustOrderCapacity': '', \
+                'AvgPxInd': '', 'ClearingAccountType': '', 'ClearingTradePriceType': '', 'CustOrderHandlingInst': '', 'DisplayQty': ''}
+                
+                for d in straight_json:
+                    test = get_value(d)
+                    if test != "None" or test != "":
+                        if "," in str(test):
+                            test = test.replace(",",";")
+                        if "4294967295" in str(test):
+                            test = ""
+                        if "us;" in str(test):
+                            test = test.upper()
+                        if d == "SelfMatchPreventionID" and len(str(test)) > 8:
+                            test = ""
+                        straight_json[d] = test
+                    else:
+                        straight_json[d] = ''
+                        
+                for d in process_json:
+                    if d in json_data:
+                        function = locals()[d]
+                        test = function(json_data[d])
+                        if test != "None":
+                            process_json[d] = test
+                        else:
+                            process_json[d] = ''
+                    elif d == "SenderID":
+                        function = locals()[d]
+                        test = function(SenderID)
+                        if test != "None":
+                            process_json[d] = test
+
+                # Party Message Fields
+                # Column 51 - Executing Firm ID
+                if "NoPartyDetails" in json_data:
+                    EXECUTING_FIRM_ID = partyid(json_data['NoPartyDetails'], "51")
+                    # Column 52 - Party Account Number
+                    ACCOUNT_NUMBER = partyid(json_data['NoPartyDetails'], "52")
+                
+                line_to_write = process_json['SendingTimeEpoch'] + "," + DIRECTION + "," + process_json['SenderID'] + "," + SESSIONID + "," + process_json['ManualOrderIndicator'] + "," + \
+                MessageType(NAME,straight_json['OrdStatus']) + "," + str(straight_json['SecurityID']) + "," + str(straight_json['ClOrdID']) + "," + str(straight_json['OrderID']) + "," + str(straight_json['ClOrdID']) + "," + \
+                str(straight_json['OrderRequestID']) + "," + process_json['Side'] + "," + str(straight_json['OrderQty']) + "," + str(straight_json['Price']) + "," + \
+                str(straight_json['StopPx'])  + "," + process_json['OrdType'] + "," + process_json['TimeInForce'] + "," + process_json['OFMOverride'] + "," + str(process_json['DisplayQty']) + "," + \
+                blank_json['MinQty'] + "," + straight_json['Location'] + "," + str(straight_json['LastPx']) + "," + str(straight_json['LastQty']) + "," + str(straight_json['CumQty']) + "," + \
+                str(straight_json['LeavesQty']) + "," + process_json['AggressorIndicator'] + "," + process_json['ExecRestatementReason'] + "," + str(straight_json['Text']) + "," + \
+                str(straight_json['SideTradeID']) + "," + str(straight_json['CrossID']) + "," + blank_json['QuoteReqID'] + "," + blank_json['QuoteID'] + "," + blank_json['QuoteSetID'] + "," + \
+                blank_json['QuoteEntryID'] + "," + blank_json['BidPx'] + "," + blank_json['BidSize'] + "," + blank_json['OfferPx'] + "," + blank_json['OfferSize'] + "," + \
+                blank_json['NoProcessedEntries'] + "," + str(straight_json['PartyDetailsListReqID']) + "," + blank_json['ListUpdateAction'] + "," + str(straight_json['SelfMatchPreventionID']) + "," + \
+                process_json['SelfMatchPreventionInstruction'] + "," + process_json['CustOrderCapacity'] + "," + blank_json['AvgPxGroupID'] + "," + process_json['AvgPxInd'] + "," + \
+                blank_json['CmtaGiveUpCD'] + "," + str(process_json['ClearingAccountType']) + "," + process_json['ClearingTradePriceType'] + "," + process_json['CustOrderHandlingInst'] + "," + \
+                EXECUTING_FIRM_ID + "," + str(ACCOUNT_NUMBER) + "," + blank_json['TAKE_UP_FIRM'] + "," + blank_json['TAKE_UP_ACCOUNT']
+
+                
+                #print(line_to_write)
+
+                with open (args.output_file, 'a') as the_file:
+                    the_file.write(line_to_write)
+                    the_file.write("\n")
+        
+
+
+
+ 

src/data_files/files/audit_upload.sh

@@ -0,0 +1,73 @@
+#/bin/bash
+
+function help_and_exit()
+{
+  echo
+  echo Usage: $0 [option]... config_file
+  echo 
+  echo Can also be used in cron like the following
+  echo 40 16 * * 1-5 /usr/bin/docker exec spark_10000 bash /sparkbin/data_files/audit_upload.sh /sparkdata/data/audit_uploads/example.conf
+  echo example.conf included.  The location of the conf file should be relative to the docker container, not the host
+  echo
+  echo Options are as follows:
+  echo "     -d|--date YYYYMMDD              Example is 20180731"
+  echo "  Help: "
+  echo "     -h|--help                       Display this message."
+  echo
+  exit 1
+}
+
+DATE=$(date +%Y%m%d)
+ORIG_DIR=$(pwd)
+ERROR_LEVEL="0"
+
+while getopts "d:h" option; do
+    case $option in
+        d) DATE=$OPTARG ;;
+        h) help_and_exit;;
+        \?) echo "Invalid option: -$OPTARG" >&2;
+        exit 1;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+CONFIG_FILE=$1
+
+declare -A conf_vars; while read line; do conf_vars[${line%=*}]=${line#*=}; done <$CONFIG_FILE
+
+SESSION_ID=${conf_vars[session_id]}
+FCM=${conf_vars[fcm]}
+CLIENT=${conf_vars[client]}
+
+cd /sparkbin/data_files
+
+for SESSION in $(ls /sparkdata/data/ilink3/); do
+   if [[ $SESSION == *$SESSION_ID* ]]; then
+      echo "Session is $SESSION"
+      SESSION_DAY=$(date +%Y-%m-%d)
+      if [[ $SESSION == "cme_"* ]]; then 
+         SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
+         AUDIT_SCRIPT="audit_ilink3.py"
+      fi
+      echo $SESSION_DAY
+      if [[ ! -d  /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
+         mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
+      fi
+      if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
+         echo "Starting the creation of the audit log" 
+         python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
+      fi
+      if [[ ! -d /sparkdata/data/md_audit ]]; then
+         mkdir /sparkdata/data/md_audit
+      fi
+      if [[ -d /sparkdata/data/md_audit ]]; then
+         # Running the MD Audit csv generation
+         cd /sparkbin
+         ./sparkd adminreport md > /sparkdata/data/md_audit/"$FCM"_md_"$CLIENT"_"$DATE".csv
+         cd /sparkbin/data_files
+      fi
+      echo "Uploading to the (s)FTP Server"
+      python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM" 
+   fi 
+done

src/data_files/files/audit_uploader.py

@@ -0,0 +1,150 @@
+import getopt, sys, datetime
+import paramiko, ftplib
+
+#########################################################
+#                  Commandline Args                     #
+#########################################################
+
+def usage():
+    print "-------------------------------------------------------"
+    print "-Usage:                                               -"
+    print "-------------------------------------------------------"
+    print "-    --config_file= : Path to upload config file      -"
+    print "-                      -if no path is specified look  -"
+    print "-                       in the working directory.     -"    
+    print "-                                                     -"
+    print "-    --help         : Print this message              -"
+    print "-                                                     -"
+    print "-    --datestamp=   : Optional, defaults to today     -"
+    print "-                       -replaces *datestamp in the   -"
+    print "-                        the upload config file       -"
+    print "-                                                     -"
+    print "-    --fcm          : Optional, spark input           -"
+    print "-                                                     -"
+    print "-    --session      : Optional, spark jf session      -"
+    print "-------------------------------------------------------"
+
+fcm = ''
+session = ''
+
+try:
+   options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
+except getopt.GetoptError, err:
+    print str(err)
+    usage()
+    sys.exit(2)
+
+for option, value in options:
+    if option in ("-h","--h","-help","--help"):
+        usage()
+        sys.exit()
+    elif option in ("-config_file","--config_file"):
+        config_file = value
+    elif option in ("-datestamp","--datestamp"):
+        datestamp = value
+    elif option in ("-fcm","--fcm"):
+        fcm = value
+    elif option in ("-client","--client"):
+        client = value
+    elif option in ("-session","--session"):
+        session = value
+
+for argument in arguments:
+    if argument in ("h", "help"):
+        usage()
+        sys.exit()
+    else:
+        print "Unrecognized commandline argument. Type help for usage."
+        sys.exit()
+
+if len(options) <= 0:
+    print "Requires a config file to run."
+    usage()
+    sys.exit()
+
+
+#########################################################
+#              Read Config Function                     #
+#########################################################
+
+def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('%Y%m%d'), fcm=fcm, session=session):
+        '''
+        Parse our config file that gets passed in as a commandline parameter.
+        All config keys must be of the form key=value. White space is ignored.
+        All config keys must be seperated by a newline character.
+        Called during initialization.
+        '''
+        upload_config = {}
+        config_file = open(config_file, 'r')        
+        config_file = config_file.read()
+        for config in config_file.split("\n"):
+            if '=' in config:
+                cleaned_key = config.split('=')[0].strip().upper()
+                cleaned_value = config.split('=')[1].strip()
+                if '*datestamp*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*datestamp*',datestamp)
+                if '*fcm*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*fcm*',fcm)
+                if '*client*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*client*',client)
+                if '*session*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*session*',session)                    
+                upload_config[cleaned_key] = cleaned_value
+            else:
+                pass
+        return upload_config   
+
+#########################################################
+#             FTP File Transfer Function                #
+#########################################################
+def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
+    if verbose: print 'Uploading', file
+    local = open(file, 'rb')    
+    remote = ftplib.FTP(site)   
+    remote.login(user, password)         
+    remote.storbinary(put_path, local, 1024)
+    remote.quit()
+    local.close()
+    if verbose: print 'Upload done.'
+
+#########################################################
+#            SFTP File Transfer Function                #
+#########################################################
+def putfile_sftp(file, put_path, site, username, password, port=22, verbose=True):
+    transport = paramiko.Transport((site, port))
+    transport.connect(username = username, password = password)
+    sftp = paramiko.SFTPClient.from_transport(transport)
+    sftp.put(file, put_path)
+    sftp.close()
+    transport.close()
+
+
+#########################################################
+#                       Main                            #
+#########################################################
+#if datestamp != None:
+#    upload_config = read_config_file(config_file, datestamp)
+#else:
+#    upload_config = read_config_file(config_file)
+
+try:
+    datestamp
+except NameError:
+    upload_config = read_config_file(config_file)
+else:
+    upload_config = read_config_file(config_file, datestamp)
+
+
+
+print upload_config
+
+if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
+    print "Protocol = FTP"
+    putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
+    putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
+elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
+    print "Protocol = SFTP"
+    putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+    putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+
+

src/data_files/files/cert_secure_logon.txt

@@ -1,12 +1,19 @@
-Session ID           Access ID                      Secret Key                                         Creation Date        Expiration Date      Environment          
-
-M4D                  lLtjH74y1Kh5XUJTLyH2           f1K_tROQp2lTIplVp34WxKa7xDUgfFnIteolfik061o        2019-09-03           2020-09-03           CERT                 
-
-MC1                  VEMlkDuLz74OVHSuacQB           IuHyYT3b-9KUo3WsuXykt-fXsjzp7L-JOrdCSQZLNl0        2019-09-03           2020-09-03           CERT                 
-
-MC5                  77lzSUwPLlM9obaXh2mK           z6aA9qfASDKfZkYTVKFqJd-78i0HZ2Q9ec3CKCYIqHg        2019-09-03           2020-09-03           CERT                 
-
-MC7                  RphzI3rn71oGUEiYnX9e           y6OvvPPyoyj2S8YAnE1gFduXT8v29HIMjO5EngYxSNg        2019-09-03           2020-09-03           CERT                 
-
-MC8                  PYBIm2TI9gFPIxmHbuYl           6o-ZolGH2DdeQOlGtJ5Zqoh_kg7VdbxkmjeOHmigKwU        2019-09-03           2020-09-03           CERT                 
-
+Session ID           Access ID                      Secret Key                                         Creation Date        Expiration Date      Environment          
+
+M4D                  lLtjH74y1Kh5XUJTLyH2           f1K_tROQp2lTIplVp34WxKa7xDUgfFnIteolfik061o        2019-09-03           2020-09-03           CERT                 
+
+MC1                  VEMlkDuLz74OVHSuacQB           IuHyYT3b-9KUo3WsuXykt-fXsjzp7L-JOrdCSQZLNl0        2019-09-03           2020-09-03           CERT                 
+
+MC5                  77lzSUwPLlM9obaXh2mK           z6aA9qfASDKfZkYTVKFqJd-78i0HZ2Q9ec3CKCYIqHg        2019-09-03           2020-09-03           CERT                 
+
+MC7                  RphzI3rn71oGUEiYnX9e           y6OvvPPyoyj2S8YAnE1gFduXT8v29HIMjO5EngYxSNg        2019-09-03           2020-09-03           CERT                 
+
+MC8                  PYBIm2TI9gFPIxmHbuYl           6o-ZolGH2DdeQOlGtJ5Zqoh_kg7VdbxkmjeOHmigKwU        2019-09-03           2020-09-03           CERT                 
+
+NGG                  RYcigz3bSLIIBBF2A2Hq           qB1dm9VkTKeXcdQpFzla8B3_jwTgya_-Fk5iyEJvr2E        2020-06-15           2021-06-15           NEWRELEASE           
+
+NGF                  o4OTmmTLQ7Lv4g6bCm26           HbD3FneMyb9Fu-FkCDLscCczHqFcFE2PyWv978RXGyA        2019-12-20           2020-12-20           NEWRELEASE           
+
+UZE                  EeFdZ1FPepk6U4k3xBZA           2LjEtdLvBLLB1QuDJ4KiJW6M4dF6IHWd_UKmo7p3_Us        2020-06-24           2021-06-24           NEWRELEASE           
+
+VAE                  Y8iqatTewuoHX16gghw8           FNcCB3Ks1qleS56quYzH4V7zAu78YY2bCdLRi9tmU9E        2020-06-24           2021-06-24           NEWRELEASE           

src/data_files/files/example.conf

@@ -0,0 +1,16 @@
+fcm=FCM
+client=NAME
+session_id=ses
+put_host=ftpc.example.com
+port_put=2222
+username_put=user
+password_put=password
+protocol_put=sftp
+get_ilink3_file_path=/sparkdata/data/ilink3/*session*/audit_logs/
+put_ilink3_file_path=/
+get_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
+put_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
+get_md_file_path=/sparkdata/data/md_audit/
+put_md_file_path=/
+get_md_file_name=*fcm*_md_*client*_*datestamp*.csv
+put_md_file_name=*fcm*_md_*client*_*datestamp*.csv

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metricsLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/AlgoTypes.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cLine truncated

src/lib/CMakeDeps_win.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthSessionManager.cpp;utils/AuthToken.cLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/AlgoTypes.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthSessioLine truncated

src/lib/api/service_compiler/plugin.py

@@ -319,7 +319,7 @@ if __name__ == "__main__":
 
     if DEBUG: 
         for f in response.file:
-            print f.content
+            print (f.content)
     else:
         output = response.SerializeToString()
         sys.stdout.write(output)

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -580,7 +580,9 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(const bt
      rv.set_session(conn_ptr->name());
      rv.set_operator_id(cme_user_data->operator_id());
      rv.set_location(cme_user_data->location());
+
      rv.set_smp_id(cme_user_data->smp_id());
+
      if (cme_user_data->smp_instruction()== "O")
         rv.set_smp_instruction(api::orderman::SelfMatchPreventionInstruction::SELF_MATCH_PREVENTION_INSTRUCTION_CANCEL_RESTING);
      else if (cme_user_data->smp_instruction()== "N")

src/lib/api/services/OrderService.proto

@@ -386,7 +386,9 @@ message NativeOrderState {
   RoutingRecord routing_record = 5;
   NativeOrderType native_type = 6;
   string last_transact_time = 7;
+  uint64 last_transact_time_unix_ns = 9;
   double stop_px = 8;
+  uint64 exchange_order_id_u64 = 10;
 }
 
 // Custom order states
@@ -455,7 +457,8 @@ enum TimeInForce {
   TIME_IN_FORCE_UNDEFINED = 0;
   TIME_IN_FORCE_DAY = 48;
   TIME_IN_FORCE_GTC = 49;
-  TIME_IN_FORCE_IOC = 51;
+  TIME_IN_FORCE_IOC = 51; // FAK
+  TIME_IN_FORCE_DATE = 52;
 }
 
 enum ManualOrderIndicator {

src/lib/config/defaults/MarketData.conf

@@ -45,6 +45,12 @@ CMEOptions {
     }
 }
 
+ICEPublisherOptions {
+   value { 
+      network_interface: "eth1"
+   }
+}
+
 #to make this work in dev clone the operations repo
 #go here: c:\\code\\bts\\operations\\ice\\ICE_iMpactMulticastFeed_GettingStarted-1_1_36\\bin\\easyTunnel>
 #modify the easyTunnelProxy.properties file to replace 

src/lib/config/protos/AlgoTypes.proto

@@ -0,0 +1,22 @@
+syntax='proto2';
+package bts.config;
+
+message AlgoTypeContext {
+   repeated string exchanges = 1;
+   repeated string symbols = 2;
+   repeated string security_types = 3;
+}
+
+message AlgoTypeDef {
+   optional string name = 1;
+   optional bool is_attachable = 2 [default = false];
+}
+
+message AlgoTypeRule {
+   optional AlgoTypeContext context = 1;
+   repeated AlgoTypeDef algos = 2; 
+}
+
+message AlgoTypeRuleSet {
+   repeated AlgoTypeRule rules = 1; 
+}

src/lib/config/protos/MarketData.proto

@@ -103,6 +103,7 @@ message CMEOptions
       // Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done
       optional uint32 spin_time = 4 [default = 1];
       optional uint32 channels_per_thread = 5 [default = 3];
+      // rmem_max
       optional uint32 udp_socket_buffer_size = 11 [default = 268435456]; //256mb
    }
    message CMEChannel {
@@ -110,9 +111,13 @@ message CMEOptions
       optional string feed_engine = 3 [default = "default"];
 
 
+      // for re-orders
       optional uint32 max_gap_recovery_cache_size = 11 [default = 6];
+      // for re-orders
       optional uint32 lost_packet_wait_timeout_ms = 12 [default = 10];
+      
       optional OnixSLogMode onixs_log_mode = 13 [default = Important];
+      // If using both feeds prefer the B feed for snapshots
       optional bool prefer_backup_feeds = 15 [default = false];
 
       optional bool use_tcp_replay = 20 [default = true];

src/lib/config/protos/jetfix.proto

@@ -9,9 +9,9 @@ message JetFIX {
     enum JetFIXPlatform {
 	   GLBX = 1;
 	   CBOE = 6;
-      SIM = 2;
-      ICE = 3;
-      SIMICE = 4;
+           SIM = 2;
+           ICE = 3;
+           SIMICE = 4;
     }
     optional JetFIXPlatform platform = 15                       [default = GLBX];
     
@@ -87,6 +87,7 @@ message JetFIX {
 
     optional uint32 max_connection_attempts = 346               [default=10];
     optional bool auto_fix_seq_num = 347                        [default=false];
+    optional bool use_ilink3       = 348                        [default=false] ;
    }
 
   repeated Connection connection = 10;
@@ -116,4 +117,4 @@ message CMECannedSecureLogon {
 "MC8                  AbEEtBn63WIAuItPDz1c           n3ndjYb03Hx6wBO3me35HoONyDgO5TZKWk0tDSaX9KA        2018-04-05           2019-04-05           CERT                 \n"
 "T39                  UWggBgHWAe07aqW3ps2I           tbb3it0rUQSSTa0TC9QcRwkrHEMDLWhaJH4VQ0FqGB4        2018-04-05           2019-04-05           NEWRELEASE           \n"
 					  ];
-}
+}

src/lib/config/protos/toggles.proto

@@ -73,4 +73,5 @@ message DynamicAlgos{
    optional bool enable_tas = 3 [default = false];
    optional bool enable_tascounter = 4 [default = false];
    optional bool enable_limit_cover = 5 [default = false];
+   optional bool enable_onsignal = 6 [default = false];
 }

src/lib/domain_model/Security.h

@@ -212,31 +212,20 @@ inline bool gr_than(double p1, double p2, double tick_size) {
    return gr_than_eq(p1, p2, tick_size) &&
           !test_price_equality(p1, p2, tick_size);
 }
+inline bool test_price_equality(double price_1, double price_2) {
+   return std::fabs(price_2 - price_1) < market_epsilon();
+}
 
-// Works only if ICEExchangeData is part of the a spread security
-// inline bool is_ice_strip(const bts::api::ICEExchangeData& ice_data){
-//    return ice_data.is_strip();
-// }
+// ice specialization /////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
 
-inline bool is_ice_strip(const bts::api::ICEExchangeData& ice_data){
-   // TODO, put this in the security
+inline bool is_ice_spread(const bts::api::ICEExchangeData& ice_data){
    const auto& strip = ice_data.strip_name();
-   auto find = [&](const std::string& sub) -> bool {      
-      return strip.find(sub) != std::string::npos;
-   };   
-   if(!find("/"))
-      return find("-") || 
-             find("Q1") || find("Q2") || find("Q3") || find("Q4") ||
-             find("Cal");
-   else
-      return false;
+   return strip.find("/") != std::string::npos;
 }
 
 inline bool is_strip(const bts::api::Security& sec){
-   if(sec.has_spread())
-      if (sec.has_ice())
-         return is_ice_strip(sec.ice());
-   return false;
+   return sec.has_ice() && sec.has_spread() &&  !is_ice_spread(sec.ice());
 }
 
 inline bool is_spread_of_strips(const bts::api::Security& sec){

src/lib/jetfix/codec/FIXEncodingStream.h

@@ -52,7 +52,7 @@ class FIXEncodingStream {
       auto msg_offset = _buffer.size();
 
       _current_msg = FIXFragment(_buffer);
-      _begin_string_bm = _current_msg.add<fix::BeginString>("FIX.4.2");
+      _begin_string_bm = _current_msg.add<fix::BeginString>(std::string_view {"FIX.4.2"});
       _body_length_bm = _current_msg.add<fix::BodyLength>(0, 6);
 
       char mt_encoding[4];

src/lib/jetfix/codec/FieldDefinitions.h

@@ -85,6 +85,7 @@ typedef StringField<307> UnderlyingSecurityDesc;
 typedef StringField<336> TradingSessionID;
 typedef StringField<35> MsgType;
 typedef StringField<37> OrderID;
+typedef BigIntField<37> OrderIDu64;
 typedef StringField<41> OrigClOrdID;
 typedef StringField<440> ClearingAccount;
 typedef StringField<48> SecurityID;

src/lib/jetfix/codec/Fields.h

@@ -2,6 +2,8 @@
 
 #include <memory>
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <string_view>
+
 #include "lib/jetfix/codec/ReaderCursor.h"
 #include "lib/utils/meta_programming.h"
 #include "lib/utils/datetime.h"
@@ -203,6 +205,13 @@ public:
    static std::pair<uint32_t,uint32_t> write (char*buffer,std::string const &str)
    {return write (buffer,str.data (),str.size ());}
 
+
+#if __cplusplus >= 201703L
+   static uint32_t max_value_size (std::string_view const&str) {return str.size ();}
+   static std::pair<uint32_t,uint32_t> write (char*buffer,std::string_view const &str)
+   {return write (buffer,str.data (),str.size ());}
+#endif
+
    static std::pair<uint32_t,uint32_t> write (char*buffer,const char*str,uint32_t sz)
    {
       memcpy (buffer,str,sz);

src/lib/jetfix/connection/SocketConnection.cpp

@@ -23,9 +23,9 @@ SocketConnection::SocketConnection(bts::config::JetFIX::Connection const &cfg,
     : _fd(-1), _cfg(cfg), _log(cfg.log_fix_stream()), _in_log(), _out_log(),
       _perf_log(perf_log) {
    if (_log) {
-      _in_log.open(std::string("/tmp/fix_raw_read_log_") + session_name(cfg),
+      _in_log.open(std::string("/tmp/fix_raw_read_log_") + session_name(cfg).as_string,
                    std::ios_base::app);
-      _out_log.open(std::string("/tmp/fix_raw_write_log_") + session_name(cfg),
+      _out_log.open(std::string("/tmp/fix_raw_write_log_") + session_name(cfg).as_string,
                     std::ios_base::app);
    }
 }

src/lib/jetfix/gateway/Session.cpp

@@ -786,9 +786,10 @@ OutFIXPtr make_logon(uint32_t seq_num, uint32_t last_seq_num_processed,
    if(session_cfg.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_ICE || 
       session_cfg.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_SIMICE)
    {
-      buffer.add<ICEApplicationSystemName>("Katallage");
-      buffer.add<ICEApplicationSystemVersion>("1.0");
-      buffer.add<ICEApplicationSystemVendor>("Katallage");      
+      std::string sender_comp_id = session_cfg.sender_subid();
+      buffer.add<ICEApplicationSystemName>(sender_comp_id);
+      buffer.add<ICEApplicationSystemVersion>(std::string_view{"1.0"});
+      buffer.add<ICEApplicationSystemVendor>(sender_comp_id);      
       
       buffer.add<UserName>(session_cfg.user_name());      
       buffer.add<RawDataLength>(session_cfg.raw_data().size());
@@ -796,19 +797,18 @@ OutFIXPtr make_logon(uint32_t seq_num, uint32_t last_seq_num_processed,
       buffer.add<EncryptMethod>(0);
       buffer.add<ICEStrategyPreference>(1);
       buffer.add<ICETimeStampPreference>(1);
-   }
-   else
-   {
-      buffer.add<ApplicationSystemName>(application);
-      buffer.add<ApplicationSystemVersion>("1.0");
-      buffer.add<ApplicationSystemVendor>("BTS");
+   } else {
+      buffer.add<ApplicationSystemName>(std::string_view{application});
+      buffer.add<ApplicationSystemVersion>(std::string_view{"1.0"});
+      buffer.add<ApplicationSystemVendor>(std::string_view{"BTS"});
 
       if (session_cfg.has_trading_session_id()) {
          buffer.add<fix::NoTradingSessions>(1);
          buffer.add<fix::TradingSessionID>(session_cfg.trading_session_id());
       }
       if (session_cfg.has_udf_support_indicator())
-         buffer.add<fix::UDFSupportIndicator>(session_cfg.udf_support_indicator());
+         buffer.add<fix::UDFSupportIndicator>(
+             session_cfg.udf_support_indicator());
 
       if (session_cfg.use_cme_secure_logon()) {
          bts_assert(session_cfg.sender_compid().size() >= 3);
@@ -819,25 +819,24 @@ OutFIXPtr make_logon(uint32_t seq_num, uint32_t last_seq_num_processed,
          std::string last_seq_num_str(9, ' ');
          i32_toa(last_seq_num_processed, &last_seq_num_str[0], 9);
          auto ld = glbx::calculate_hmac(
-            session_id, seq_num_str, session_cfg.sender_compid(),
-            session_cfg.sender_subid(), time_str, session_cfg.target_subid(),
-            std::to_string(session_cfg.seconds_between_hbs()),
-            session_cfg.sender_locationid(), last_seq_num_str, application, "1.0",
-            "BTS");
-   #ifndef BTS_EDGE
-         LOG(Jetfix, Info, "Canonical string for secure logon: {}", ld.canonical);
-   #endif
+             session_id, seq_num_str, session_cfg.sender_compid(),
+             session_cfg.sender_subid(), time_str, session_cfg.target_subid(),
+             std::to_string(session_cfg.seconds_between_hbs()),
+             session_cfg.sender_locationid(), last_seq_num_str, application,
+             "1.0", "BTS");
+#ifndef BTS_EDGE
+         LOG(Jetfix, Info, "Canonical string for secure logon: {}",
+             ld.canonical);
+#endif
          buffer.add<EncodedTextLen>(ld.access_key.size());
          buffer.add<EncodedText>(ld.access_key);
-         buffer.add<EncryptedPasswordMethod>("CME-1-SHA-256");
+         buffer.add<EncryptedPasswordMethod>(std::string_view{"CME-1-SHA-256"});
          buffer.add<EncryptedPasswordLen>(ld.signature.size());
          buffer.add<EncryptedPassword>(ld.signature);
-      } 
-      else 
-      {
+      } else {
          buffer.add<RawDataLength>(session_cfg.raw_data().size());
          buffer.add<RawData>(session_cfg.raw_data());
-         if(session_cfg.user_name() != "")
+         if (session_cfg.user_name() != "")
             buffer.add<UserName>(session_cfg.user_name());
       }
    }

src/lib/jetfix/store/FIXFileStore.cpp

@@ -185,7 +185,7 @@ void load_seq_nums(std::string const &path, uint32_t *last_seq_num,
 FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
                            std::string base_path)
     : me(new PImpl(cfg)) {
-   auto sn = session_name(cfg);
+   auto sn = session_name(cfg).as_string;
    me->active = sn.size() > 0;
    if (me->active) {
 

src/lib/jetfix/store/session_name.cpp

@@ -75,49 +75,55 @@ bool in_session (datetime::PTime const &ref_date,
    return closest_earlier_end<closest_earlier_start;
 }
 
-std::string session_name_from_ptime (datetime::PTime t,
+SessionName session_name_from_ptime (datetime::PTime t,
                                      boost::posix_time::time_duration time_of_day_start,
                                      boost::posix_time::time_duration time_of_day_end)
 {
-   std::string result (10,' ');
+   SessionName result;
+   result.as_string = std::string(10,' ');
    auto start_of_session = t;
    
    if (time_of_day_start > time_of_day_end && 
        start_of_session.time_of_day () <= time_of_day_end)
       start_of_session -= boost::gregorian::date_duration (1);
-      
-   sprintf (&result[0], "%04d-%02d-%02d",
+
+   using namespace boost::posix_time;
+   ptime epoch(boost::gregorian::date(1970,1,1));
+   ptime session_start_date(start_of_session.date());
+   result.as_uint64_t = (session_start_date - epoch).total_milliseconds();
+
+   sprintf (&(result.as_string[0]), "%04d-%02d-%02d",
             (unsigned short) start_of_session.date ().year (),
             start_of_session.date ().month ().as_number (),
             start_of_session.date ().day ().as_number ());
    return result;
 }
 
-std::string session_name (datetime::PTime const &ref_date,
-                          boost::gregorian::greg_weekday weekday_start,
-                          boost::posix_time::time_duration time_of_day_start,
-                          boost::gregorian::greg_weekday weekday_end,
-                          boost::posix_time::time_duration time_of_day_end)
+SessionName session_name (datetime::PTime const &ref_date,
+                         boost::gregorian::greg_weekday weekday_start,
+                         boost::posix_time::time_duration time_of_day_start,
+                         boost::gregorian::greg_weekday weekday_end,
+                         boost::posix_time::time_duration time_of_day_end)
 {
    if (in_session (ref_date,weekday_start,time_of_day_start,weekday_end,time_of_day_end)) 
       return session_name_from_ptime (closest_before (ref_date,weekday_start,time_of_day_start),
                                       boost::posix_time::time_duration(0, 0, 0),
                                       boost::posix_time::time_duration(0, 0, 1));
    else
-      return std::string ();
+      return {};
 }
 
-std::string session_name (datetime::PTime const &ref_date,
+SessionName session_name (datetime::PTime const &ref_date,
                           boost::posix_time::time_duration time_of_day_start,
                           boost::posix_time::time_duration time_of_day_end)
 {
    if (in_session (ref_date,time_of_day_start,time_of_day_end))
       return session_name_from_ptime (ref_date, time_of_day_start, time_of_day_end);
    else
-      return "";
+      return {};
 }
 
-std::string session_name (config::JetFIX::Connection const &session_cfg)
+SessionName session_name (config::JetFIX::Connection const &session_cfg)
 {
    auto now = datetime::local_now ();
    
@@ -134,7 +140,6 @@ std::string session_name (config::JetFIX::Connection const &session_cfg)
                            parse_time (session_cfg.end_time ()));
 }
 
-
 }}
  
 

src/lib/jetfix/store/session_name.h

@@ -1,9 +1,9 @@
 #pragma once
 
+#include "lib/config/protos/jetfix.pb.h"
+#include "lib/utils/datetime.h"
 #include <memory>
 #include <string>
-#include "lib/utils/datetime.h"
-#include "lib/config/protos/jetfix.pb.h"
 
 namespace boost {
 namespace gregorian {
@@ -11,39 +11,45 @@ class greg_weekday;
 }
 namespace posix_time {
 class time_duration;
-}}
-
+}
+} // namespace boost
 
 namespace bts {
 namespace jetfix {
 
-boost::gregorian::greg_weekday parse_weekday (std::string weekday_str);
-bool parse_time (std::string const &time_str,boost::posix_time::time_duration *out);
+boost::gregorian::greg_weekday parse_weekday(std::string weekday_str);
+bool parse_time(std::string const &time_str,
+                boost::posix_time::time_duration *out);
 
-datetime::PTime closest_before (datetime::PTime const &ref_date,
-                                boost::gregorian::greg_weekday weekday,
-                                boost::posix_time::time_duration time_of_day);
+datetime::PTime closest_before(datetime::PTime const &ref_date,
+                               boost::gregorian::greg_weekday weekday,
+                               boost::posix_time::time_duration time_of_day);
 
-bool in_session (datetime::PTime const &ref_date,
-                 boost::gregorian::greg_weekday weekday_start,
-                 boost::posix_time::time_duration time_of_day_start,
-                 boost::gregorian::greg_weekday weekday_end,
-                 boost::posix_time::time_duration time_of_day_end);
+bool in_session(datetime::PTime const &ref_date,
+                boost::gregorian::greg_weekday weekday_start,
+                boost::posix_time::time_duration time_of_day_start,
+                boost::gregorian::greg_weekday weekday_end,
+                boost::posix_time::time_duration time_of_day_end);
 
-bool in_session (datetime::PTime const &ref_date,
-                 boost::posix_time::time_duration time_of_day_start,
-                 boost::posix_time::time_duration time_of_day_end);
+bool in_session(datetime::PTime const &ref_date,
+                boost::posix_time::time_duration time_of_day_start,
+                boost::posix_time::time_duration time_of_day_end);
 
-std::string session_name (datetime::PTime const &ref_date,
-                          boost::gregorian::greg_weekday weekday_start,
-                          boost::posix_time::time_duration time_of_day_start,
-                          boost::gregorian::greg_weekday weekday_end,
-                          boost::posix_time::time_duration time_of_day_end);
+struct SessionName {
+   std::string as_string;
+   uint64_t as_uint64_t;
+   bool empty() const {return as_string.empty();};
+};
 
-std::string session_name (datetime::PTime const &ref_date,
-                          boost::posix_time::time_duration time_of_day_start,
-                          boost::posix_time::time_duration time_of_day_end);
+SessionName session_name(datetime::PTime const &ref_date,
+                         boost::gregorian::greg_weekday weekday_start,
+                         boost::posix_time::time_duration time_of_day_start,
+                         boost::gregorian::greg_weekday weekday_end,
+                         boost::posix_time::time_duration time_of_day_end);
+SessionName session_name(datetime::PTime const &ref_date,
+                         boost::posix_time::time_duration time_of_day_start,
+                         boost::posix_time::time_duration time_of_day_end);
+SessionName session_name(config::JetFIX::Connection const &session_cfg);
 
-std::string session_name (config::JetFIX::Connection const &session_cfg);
-
-}}
+} // namespace jetfix
+} // namespace bts

src/lib/jetfix/tests/fix_fragment_tests.cpp

@@ -81,9 +81,9 @@ TEST_CASE ("JetFIX Field2Tests", "[jetfix]")
       std::string out (20, ' ');
       typedef StringField<123> Field;
 
-      REQUIRE(Field::max_value_size ("hello")==5);
+      REQUIRE(Field::max_value_size (std::string_view{"hello"})==5);
 
-      auto result = Field::write (&out[0],"hello");
+      auto result = Field::write (&out[0],std::string_view{"hello"});
       REQUIRE (result.first==5);
       REQUIRE (result.second=='h'+'e'+'l'+'l'+'o');
       REQUIRE (out.compare (0,5,"hello") == 0);
@@ -112,7 +112,7 @@ TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]")
       FIXFragment g (0);
       g.add<CharField<1>> ('a');
       g.add<FloatField<2>> (1.2,-1,6);
-      g.add<StringField<3>> ("hello");
+      g.add<StringField<3>> (std::string_view{"hello"});
       g.add<IntField<4>> (123,3);
       const char*t_s = "20150427-19:05:18.922";
       g.add<TimeField<5>> (datetime::parse_FIX_timestamp (t_s));
@@ -141,14 +141,14 @@ TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]")
       FIXFragment g;
       auto c_bm = g.add<CharField<1>> ('a');
       auto f_bm = g.add<FloatField<2>> (1.2,-1,6);
-      auto s_bm = g.add<StringField<3>> ("hello");
+      auto s_bm = g.add<StringField<3>> (std::string_view{"hello"});
       auto i_bm = g.add<IntField<4>> (123,3);
       auto t_bm = g.add<TimeField<5>> (datetime::parse_FIX_timestamp (t_s1));
       auto b_bm = g.add<BoolField<6>> (true);
 
       g.overwrite (c_bm,'b');
       g.overwrite (f_bm,1.3,-1,6);
-      g.overwrite (s_bm,"olleh");
+      g.overwrite (s_bm,std::string_view {"olleh"});
       g.overwrite (i_bm,321,3);
       g.overwrite (t_bm,(datetime::parse_FIX_timestamp (t_s2)));
       g.overwrite (b_bm,false);
@@ -172,11 +172,11 @@ TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]")
       FIXFragment g1;;
       g1.add<CharField<1>> ('a');
       g1.add<FloatField<2>> (1.2,-1,6);
-      g1.add<StringField<3>> ("hello");
+      g1.add<StringField<3>> (std::string_view{"hello"});
       auto chk1 = g1.checksum ();
       
       FIXFragment g2;
-      g2.add<StringField<4>> ("goodbye");
+      g2.add<StringField<4>> (std::string_view{"goodbye"});
       g2.add<IntField<5>> (321,3);
       auto chk2 = g2.checksum ();
 

src/lib/jetfix/tests/store_tests.cpp

@@ -6,279 +6,270 @@
 #include "lib/jetfix/common/OutFIXPtr.h"
 #include "lib/jetfix/store/FIXFileStore.h"
 #include "lib/jetfix/store/session_name.h"
+#include "lib/logging/logging.h"
 #include "lib/utils/TempDir.h"
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/datetime.h"
-#include "lib/logging/logging.h"
 
+#include <boost/date_time.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
 #include <boost/filesystem.hpp>
-#include <boost/date_time.hpp>
 
 namespace bts {
 namespace jetfix {
 
-TEST_CASE ("JetFIX SessionNameTests", "[jetfix]")
-{
-   SECTION("greg_weekday_works")
-   {
-      const char*days [] = {"sun","Monday","tuesday","Wedn","Thursday","Friday","sat"};
+TEST_CASE("JetFIX SessionNameTests", "[jetfix]") {
+   SECTION("greg_weekday_works") {
+      const char *days[] = {"sun",      "Monday", "tuesday", "Wedn",
+                            "Thursday", "Friday", "sat"};
 
-      int i =0;
+      int i = 0;
       for (auto d : days)
-         REQUIRE (parse_weekday (d)==boost::gregorian::greg_weekday (i++));
-   }
-   
-   SECTION("closest_before_works")
-   {
-      auto same_day_before = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                             boost::gregorian::Friday,
-                                             boost::posix_time::time_duration(16, 59, 59));
-      REQUIRE (datetime::FIX_date_time_string (same_day_before)=="20150717-16:59:59");
-
-      auto same_day_after = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                            boost::gregorian::Friday,
-                                            boost::posix_time::time_duration(17,0,1));
-      REQUIRE (datetime::FIX_date_time_string (same_day_after)=="20150710-17:00:01");
-
-      auto another_day_same_week = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                                   boost::gregorian::Thursday,
-                                                   boost::posix_time::time_duration(16, 59, 59));
-      REQUIRE (datetime::FIX_date_time_string (another_day_same_week)=="20150716-16:59:59");
-
-      auto another_day_previous_week = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                                       boost::gregorian::Saturday,
-                                                       boost::posix_time::time_duration(16, 59, 59));
-      REQUIRE (datetime::FIX_date_time_string (another_day_previous_week)=="20150711-16:59:59");
-
+         REQUIRE(parse_weekday(d) == boost::gregorian::greg_weekday(i++));
    }
 
-   SECTION("in_session_works")
-   {
-      REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                               boost::gregorian::Sunday,
-                               boost::posix_time::time_duration(16, 59, 59),
-                               
-                               boost::gregorian::Friday,
-                               boost::posix_time::time_duration(17, 0, 1)));
+   SECTION("closest_before_works") {
+      auto same_day_before =
+          closest_before(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Friday,
+                         boost::posix_time::time_duration(16, 59, 59));
+      REQUIRE(datetime::FIX_date_time_string(same_day_before) ==
+              "20150717-16:59:59");
 
-      REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::gregorian::Sunday,
-                                boost::posix_time::time_duration(16, 59, 59),
-                               
-                                boost::gregorian::Friday,
-                                boost::posix_time::time_duration(16, 59, 59)));
+      auto same_day_after = closest_before(
+          datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+          boost::gregorian::Friday, boost::posix_time::time_duration(17, 0, 1));
+      REQUIRE(datetime::FIX_date_time_string(same_day_after) ==
+              "20150710-17:00:01");
 
-      REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                               boost::posix_time::time_duration(16, 59, 59),
-                               boost::posix_time::time_duration(17, 59, 59)));
+      auto another_day_same_week =
+          closest_before(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Thursday,
+                         boost::posix_time::time_duration(16, 59, 59));
+      REQUIRE(datetime::FIX_date_time_string(another_day_same_week) ==
+              "20150716-16:59:59");
 
-      REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::posix_time::time_duration(0,0,0),
-                                boost::posix_time::time_duration(16, 0,0)));
-
-      REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                               boost::posix_time::time_duration(17, 50, 59),
-                               boost::posix_time::time_duration(17, 45, 59)));
-
-      REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,47,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)));
+      auto another_day_previous_week =
+          closest_before(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Saturday,
+                         boost::posix_time::time_duration(16, 59, 59));
+      REQUIRE(datetime::FIX_date_time_string(another_day_previous_week) ==
+              "20150711-16:59:59");
    }
 
-   SECTION("session_name_works")
-   {
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::gregorian::Sunday,
-                                boost::posix_time::time_duration(16, 59, 59),
-                               
-                                boost::gregorian::Friday,
-                                boost::posix_time::time_duration(17, 0, 1)) ==
-                  "2015-07-12");
+   SECTION("in_session_works") {
+      REQUIRE(in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Sunday,
+                         boost::posix_time::time_duration(16, 59, 59),
 
+                         boost::gregorian::Friday,
+                         boost::posix_time::time_duration(17, 0, 1)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::gregorian::Sunday,
-                                boost::posix_time::time_duration(16, 59, 59),
-                                
-                                boost::gregorian::Friday,
-                                boost::posix_time::time_duration(16, 59, 59)) == "");
+      REQUIRE(!in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                          boost::gregorian::Sunday,
+                          boost::posix_time::time_duration(16, 59, 59),
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::posix_time::time_duration(16, 59, 59),
-                                boost::posix_time::time_duration(17, 59, 59)) == 
-                  "2015-07-17");
+                          boost::gregorian::Friday,
+                          boost::posix_time::time_duration(16, 59, 59)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,18,0,0,0),
-                                boost::posix_time::time_duration(16, 59, 59),
-                                boost::posix_time::time_duration(17, 59, 59)) == 
-                  "");
+      REQUIRE(in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::posix_time::time_duration(16, 59, 59),
+                         boost::posix_time::time_duration(17, 59, 59)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)) == 
-                  "2015-07-16");
+      REQUIRE(!in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                          boost::posix_time::time_duration(0, 0, 0),
+                          boost::posix_time::time_duration(16, 0, 0)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,51,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)) == 
-                  "2015-07-17");
-
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,47,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)) == 
-                  "");
+      REQUIRE(in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::posix_time::time_duration(17, 50, 59),
+                         boost::posix_time::time_duration(17, 45, 59)));
 
+      REQUIRE(!in_session(datetime::make_ptime(2015, 7, 17, 17, 47, 0, 0),
+                          boost::posix_time::time_duration(17, 50, 59),
+                          boost::posix_time::time_duration(17, 45, 59)));
+   }
+
+   SECTION("session_name_works") {
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::gregorian::Sunday,
+                           boost::posix_time::time_duration(16, 59, 59),
+
+                           boost::gregorian::Friday,
+                           boost::posix_time::time_duration(17, 0, 1)).as_string ==
+              "2015-07-12");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::gregorian::Sunday,
+                           boost::posix_time::time_duration(16, 59, 59),
+
+                           boost::gregorian::Friday,
+                           boost::posix_time::time_duration(16, 59, 59)).as_string == "");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::posix_time::time_duration(16, 59, 59),
+                           boost::posix_time::time_duration(17, 59, 59)).as_string ==
+              "2015-07-17");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 18, 0, 0, 0),
+                           boost::posix_time::time_duration(16, 59, 59),
+                           boost::posix_time::time_duration(17, 59, 59)).as_string == "");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::posix_time::time_duration(17, 50, 59),
+                           boost::posix_time::time_duration(17, 45, 59)).as_string ==
+              "2015-07-16");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 51, 0, 0),
+                           boost::posix_time::time_duration(17, 50, 59),
+                           boost::posix_time::time_duration(17, 45, 59)).as_string ==
+              "2015-07-17");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 47, 0, 0),
+                           boost::posix_time::time_duration(17, 50, 59),
+                           boost::posix_time::time_duration(17, 45, 59)).as_string == "");
    }
-   
 }
 
-
-TEST_CASE ("JetFIX FileStoreTests", "[jetfix]")
-{
-   SECTION("empty_store_works")
-   {
+TEST_CASE("JetFIX FileStoreTests", "[jetfix]") {
+   SECTION("empty_store_works") {
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
       utils::TempDir td;
-      
-      FIXFileStore fs (c,td.name ());
-      REQUIRE (fs.session_active ());
-      REQUIRE (fs.replay ().size ()==0);
-      REQUIRE (fs.last_seq_num ()==0);
-      REQUIRE (fs.last_seq_num_processed ()==0);
+
+      FIXFileStore fs(c, td.name());
+      REQUIRE(fs.session_active());
+      REQUIRE(fs.replay().size() == 0);
+      REQUIRE(fs.last_seq_num() == 0);
+      REQUIRE(fs.last_seq_num_processed() == 0);
    }
 
-   SECTION("in_msg_is_stored")
-   {
-      auto td = std::make_shared<utils::TempDir> ();
+   SECTION("in_msg_is_stored") {
+      auto td = std::make_shared<utils::TempDir>();
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
-      FIXEncodingStream stream (c);
-      auto &buf = stream.new_msg (MessageTypes::Logon);
-      buf.add<fix::TargetLocationID> ("US,NC");
-      buf.add<fix::EncryptMethod> (0);
+      FIXEncodingStream stream(c);
+      auto &buf = stream.new_msg(MessageTypes::Logon);
+      buf.add<fix::TargetLocationID>(std::string_view{"US,NC"});
+      buf.add<fix::EncryptMethod>(0);
       buf.add<fix::HeartBtInt>(30);
-      buf.add<fix::ApplicationSystemName> ("O2");
-      buf.add<fix::ApplicationSystemVersion> ("1.0");
-      buf.add<fix::ApplicationSystemVendor> ("BTS");
-      stream.close ();
-      stream.stamp (2,3,datetime::FIX_timestamp (datetime::now ()));
-      auto fix = stream.to_str ();
-      
+      buf.add<fix::ApplicationSystemName>(std::string_view{"O2"});
+      buf.add<fix::ApplicationSystemVersion>(std::string_view{"1.0"});
+      buf.add<fix::ApplicationSystemVendor>(std::string_view{"BTS"});
+      stream.close();
+      stream.stamp(2, 3, datetime::FIX_timestamp(datetime::now()));
+      auto fix = stream.to_str();
+
       {
-         FIXFileStore fs (c,td->name ());
-         fs.store (InMsgBuffer (fix));
-         fs.set_last_msg_processed (3);
+         FIXFileStore fs(c, td->name());
+         fs.store(InMsgBuffer(fix));
+         fs.set_last_msg_processed(3);
       }
 
-      FIXFileStore fs (c,td->name ());
-      auto replay = fs.replay ();
-      REQUIRE (replay.size ()==1);
-      REQUIRE (replay[0].type ()==MessageTypes::Logon);
-      REQUIRE (replay[0].to_str ()==fix);
-      REQUIRE (fs.last_seq_num ()==0);
-      REQUIRE (fs.last_seq_num_processed ()==3);
+      FIXFileStore fs(c, td->name());
+      auto replay = fs.replay();
+      REQUIRE(replay.size() == 1);
+      REQUIRE(replay[0].type() == MessageTypes::Logon);
+      REQUIRE(replay[0].to_str() == fix);
+      REQUIRE(fs.last_seq_num() == 0);
+      REQUIRE(fs.last_seq_num_processed() == 3);
    }
 
-   SECTION("out_msg_is_stored")
-   {
-      auto td = std::make_shared<utils::TempDir> ();
+   SECTION("out_msg_is_stored") {
+      auto td = std::make_shared<utils::TempDir>();
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
       auto msg_vec = std::make_shared<OutFIX>(c);
-      auto &stream = msg_vec->fix.back ();
-      auto &buf = stream.new_msg (MessageTypes::Logon);
-      buf.add<fix::TargetLocationID> ("US,NC");
-      buf.add<fix::EncryptMethod> (0);
+      auto &stream = msg_vec->fix.back();
+      auto &buf = stream.new_msg(MessageTypes::Logon);
+      buf.add<fix::TargetLocationID>(std::string_view{"US,NC"});
+      buf.add<fix::EncryptMethod>(0);
       buf.add<fix::HeartBtInt>(30);
-      buf.add<fix::ApplicationSystemName> ("O2");
-      buf.add<fix::ApplicationSystemVersion> ("1.0");
-      buf.add<fix::ApplicationSystemVendor> ("BTS");
-      stream.close ();
-      stream.stamp (2,3,datetime::FIX_timestamp (datetime::now ()));
-      
+      buf.add<fix::ApplicationSystemName>(std::string_view{"O2"});
+      buf.add<fix::ApplicationSystemVersion>(std::string_view{"1.0"});
+      buf.add<fix::ApplicationSystemVendor>(std::string_view{"BTS"});
+      stream.close();
+      stream.stamp(2, 3, datetime::FIX_timestamp(datetime::now()));
+
       {
-         FIXFileStore fs (c,td->name ());
-         fs.store (msg_vec);
+         FIXFileStore fs(c, td->name());
+         fs.store(msg_vec);
       }
 
-      FIXFileStore fs (c,td->name ());
-      auto replay = fs.replay ();
-      REQUIRE (replay.size ()==0);
-      REQUIRE (fs.last_seq_num ()==3);
-      REQUIRE (fs.last_seq_num_processed ()==0);
+      FIXFileStore fs(c, td->name());
+      auto replay = fs.replay();
+      REQUIRE(replay.size() == 0);
+      REQUIRE(fs.last_seq_num() == 3);
+      REQUIRE(fs.last_seq_num_processed() == 0);
    }
 
-   SECTION("test_performance")
-   {
-      auto td = std::make_shared<utils::TempDir> ();
+   SECTION("test_performance") {
+      auto td = std::make_shared<utils::TempDir>();
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
-      auto start = utils::TimeStamp::now ();
+      auto start = utils::TimeStamp::now();
       {
-         FIXFileStore fs (c,td->name ());
-         for (int i = 0; i < 10000;i++) {
+         FIXFileStore fs(c, td->name());
+         for (int i = 0; i < 10000; i++) {
             auto msg_vec = std::make_shared<std::vector<FIXEncodingStream>>();
-            msg_vec->emplace_back (c);
-            auto &stream = msg_vec->back ();
-            auto &buf = stream.new_msg (MessageTypes::Logon);
-            buf.add<fix::TargetLocationID> ("US,NC");
-            buf.add<fix::EncryptMethod> (0);
+            msg_vec->emplace_back(c);
+            auto &stream = msg_vec->back();
+            auto &buf = stream.new_msg(MessageTypes::Logon);
+            buf.add<fix::TargetLocationID>(std::string_view{"US,NC"});
+            buf.add<fix::EncryptMethod>(0);
             buf.add<fix::HeartBtInt>(30);
-            buf.add<fix::ApplicationSystemName> ("O2");
-            buf.add<fix::ApplicationSystemVersion> ("1.0");
-            buf.add<fix::ApplicationSystemVendor> ("BTS");
-            stream.close ();
-            stream.stamp (i,3,datetime::FIX_timestamp (datetime::now () - boost::gregorian::days (1)));
-            fs.store (InMsgBuffer (stream.to_str ()));
+            buf.add<fix::ApplicationSystemName>(std::string_view{"O2"});
+            buf.add<fix::ApplicationSystemVersion>(std::string_view{"1.0"});
+            buf.add<fix::ApplicationSystemVendor>(std::string_view{"BTS"});
+            stream.close();
+            stream.stamp(i, 3,
+                         datetime::FIX_timestamp(datetime::now() -
+                                                 boost::gregorian::days(1)));
+            fs.store(InMsgBuffer(stream.to_str()));
          }
       }
-      auto stop = utils::TimeStamp::now ();
-      LOG(Testing,Info,"jetfix store gen: {} ms", (stop-start).milliseconds () );
-      
-      start = utils::TimeStamp::now ();
-      FIXFileStore fs (c,td->name ());
-      stop = utils::TimeStamp::now ();
-      LOG(Testing,Info,"jetfix store load: {} ms", (stop-start).milliseconds () );
+      auto stop = utils::TimeStamp::now();
+      LOG(Testing, Info, "jetfix store gen: {} ms",
+          (stop - start).milliseconds());
+
+      start = utils::TimeStamp::now();
+      FIXFileStore fs(c, td->name());
+      stop = utils::TimeStamp::now();
+      LOG(Testing, Info, "jetfix store load: {} ms",
+          (stop - start).milliseconds());
    }
-   
 }
 
-}}
-
-    
-  
-
+} // namespace jetfix
+} // namespace bts

src/lib/logging/logging.cpp

@@ -120,6 +120,7 @@ void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupC
                                 {log::Plugin,"Plugin"},
                                 {log::Sdk,"Sdk"},
                                 {log::Http,"Http"},
+                                {log::iLink3,"iLink3"},
    };
 
    real_cfg.sinks["console"] =

src/lib/logging/logging.h

@@ -46,6 +46,7 @@ constexpr log::Subsystem DataFlow   {Latency.ordinal+1     };
 constexpr log::Subsystem Plugin     {DataFlow.ordinal+1    };
 constexpr log::Subsystem Sdk        {Plugin.ordinal+1      };
 constexpr log::Subsystem Http       {Sdk.ordinal+1   };
+constexpr log::Subsystem iLink3     {Http.ordinal+1   };
 
 struct LoggerHandle {
    LoggerHandle(LoggerHandle const&) = delete;

src/lib/md/ExchangeSecurityKey.h

@@ -0,0 +1,5 @@
+#pragma once
+#include "lib/utils/ValueWrapper.h"
+namespace bts {
+using ExchangeSecurityKey = utils::ValueWrapper<struct ExchangeSecurityKeyTag,uint64_t>;
+}

src/lib/md/IntPrice.h

@@ -60,10 +60,13 @@ class IntPrice {
          return delta.mantissa() < 0;
       }
    }
-   bool operator <=(const IntPrice& other) {
+   bool operator <=(const IntPrice& other) const {
       return *this == other || *this < other;
    }
-   bool operator >=(const IntPrice& other) {
+   bool operator >(const IntPrice& other) const {
+      return other < *this;
+   }
+   bool operator >=(const IntPrice& other) const {
       return !(*this < other);
    }
    IntPrice operator - (const IntPrice& o) const {
@@ -72,9 +75,31 @@ class IntPrice {
    IntPrice operator + (const IntPrice& o) const {
       return op(o, [](int64_t m1, int64_t m2) { return m1 + m2; });
    }
+   IntPrice& operator +=(const IntPrice& o) {
+      auto temp = *this + o;
+      _mantissa = temp._mantissa;
+      _exponent = temp._exponent;
+      return *this;
+   }
+   IntPrice& operator -=(const IntPrice& o) {
+      auto temp = *this - o;
+      _mantissa = temp._mantissa;
+      _exponent = temp._exponent;
+      return *this;
+   }
+   IntPrice operator -() {
+      return IntPrice(-_mantissa, _exponent);
+   }
    IntPrice operator * (const IntPrice& other) const {
       return op(other, [](int64_t m1, int64_t m2) { return m1 * m2; });
    }
+   IntPrice operator * (const int& other) const {
+      return IntPrice(_mantissa * other, _exponent);
+   }
+   IntPrice& operator *=(const int& other) {
+      _mantissa *= other;
+      return *this;
+   }
    IntPrice operator / (double v) const {
       return { (int64_t)(_mantissa / v), exponent() };
    }
@@ -183,4 +208,4 @@ inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) {
    return os;
 }
 
-}}
+}}

src/lib/md/MBOServiceServer.cpp

@@ -1,71 +1,72 @@
 #ifndef _WIN32
-#include "MBOServiceServer.h" 
+#include "MBOServiceServer.h"
 
-#include "lib/api/services/OrderService.client.pb.h"
-#include "lib/utils/Dispatcher.h"
-#include "lib/utils/TimeStamp.h"
-#include "lib/domain_model/Security.h"
-#include "MDClient.h"
 #include "ExchangeOrderKey.h"
 #include "MBOProcessor.h"
+#include "MDClient.h"
+#include "lib/api/services/OrderService.client.pb.h"
+#include "lib/domain_model/Security.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/utils/Dispatcher.h"
+#include "lib/utils/TimeStamp.h"
 
 #include "lib/utils/LockBox.h"
 
 #include <tuple>
-#include <vector>
 #include <unordered_map>
-
+#include <vector>
 
 using namespace bts::api::orderman;
 
-namespace bts { 
-namespace md { 
+namespace bts {
+namespace md {
 
 //
-// Simple class to handle MD recovery and maintain an index of SecurityKey=>MDConsumerHandle
+// Simple class to handle MD recovery and maintain an index of
+// SecurityKey=>MDConsumerHandle
 
 class HandleIndex {
-   MDClient& _client;
+   MDClient &_client;
    int _sub;
    std::optional<MDConsumer> _consumer;
 
-public:
-   utils::LockBox<std::unordered_map<SecurityKey,MDConsumerHandle>> index;
+ public:
+   utils::LockBox<std::unordered_map<SecurityKey, MDConsumerHandle>> index;
 
-   HandleIndex(md::MDClient& client): 
-      _client(client),
-      _sub(0),
-      _consumer(std::nullopt),
-      index() {
-         _sub = _client.on_consumer().subscribe([=](const MDConsumer& c) { on_consumer(c); });
-         on_consumer(_client.current_consumer());
-      }
+   HandleIndex(md::MDClient &client)
+       : _client(client), _sub(0), _consumer(std::nullopt), index() {
+      _sub = _client.on_consumer().subscribe(
+          [=](const MDConsumer &c) { on_consumer(c); });
+      on_consumer(_client.current_consumer());
+   }
 
    ~HandleIndex() { _client.on_consumer().unsubscribe(_sub); }
 
-
-private:
-   void on_consumer(const MDConsumer& c) {
+ private:
+   void on_consumer(const MDConsumer &c) {
       MDConsumer temp(c);
       if (temp.all_heartbeats_ok()) {
          auto existing = _consumer ? _consumer->id() : 0;
-         LOG(md,Info, "Updateing MBOService MDConsumer. Old {} new {}", existing, temp.id());
+         LOG(md, Info, "Updateing MBOService MDConsumer. Old {} new {}",
+             existing, temp.id());
          // clear the index
-         index.lock([](auto& index) { index.clear(); });
+         index.lock([](auto &index) { index.clear(); });
          // Swap to new consumer
          if (_consumer) {
             _consumer->swap(temp);
+         } else {
+            _consumer = temp;
          }
-         else { _consumer = temp; }
          // Subscribe to on_security
          auto id = _consumer->id();
-         _consumer->on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) {
-            if (_consumer->id() == id) {
-               index.lock([handle](auto& index) {
-                  index[SecurityKey(handle.key())] = handle;
-               });
-            }
-         });
+         _consumer->on_security(
+             [this, id](const SecurityPtr &, MDConsumerHandle handle) {
+                if (_consumer->id() == id) {
+                   index.lock([handle](auto &index) {
+                      index[SecurityKey(handle.key())] = handle;
+                   });
+                }
+             });
       }
    }
 };
@@ -73,21 +74,20 @@ private:
 class ActiveOrder {
    SecurityKey _security;
    SparkOrderKey _id;
-   ExchangeOrderKey  _ex_id;
+   ExchangeOrderKey _ex_id;
    int32_t _position;
 
-public:
+ public:
    // empty constructor needed for unordered map operator[]
-   // ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); }
-   ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex): _security(sec), _id(s), _ex_id(ex), _position(-1)  {}
-   ActiveOrder(SecurityKey sec, SparkOrderKey s, const std::string& ex): _security(sec), _id(s), _ex_id(), _position(-1) {
-      _ex_id = ExchangeOrderKey(std::stoul(ex));
-   }
+   // ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) {
+   // bts_assert(false); }
+   ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex)
+       : _security(sec), _id(s), _ex_id(ex), _position(-1) {}
 
    SecurityKey security_id() const { return _security; }
    SparkOrderKey id() const { return _id; }
    ExchangeOrderKey exchange_id() const { return _ex_id; }
-   int32_t& position() { return _position; };
+   int32_t &position() { return _position; };
    int32_t position() const { return _position; };
    bool set_position(int32_t pos) {
       if (pos != _position) {
@@ -108,23 +108,14 @@ public:
 // MBOService
 
 struct MBOServiceImpl {
-   MBOServiceImpl(
-      api::ServiceContext& ctx,
-      utils::Dispatcher& d,
-      md::MDClient& c,
-      MBOServiceBase::PiqPublisher&& pub
-      ):
-      dispatcher(d),
-      client(c),
-      orders(ctx.locate<api::orderman::OrderService>()),
-      handle_index(c),
-      publish(pub),
-      timer(d, 200, [&]() {timer_tick();}),
-      orders_sub()
-   {}
+   MBOServiceImpl(api::ServiceContext &ctx, utils::Dispatcher &d,
+                  md::MDClient &c, MBOServiceBase::PiqPublisher &&pub)
+       : dispatcher(d), client(c),
+         orders(ctx.locate<api::orderman::OrderService>()), handle_index(c),
+         publish(pub), timer(d, 200, [&]() { timer_tick(); }), orders_sub() {}
 
-   utils::Dispatcher& dispatcher;
-   md::MDClient& client;
+   utils::Dispatcher &dispatcher;
+   md::MDClient &client;
    OrderService orders;
 
    HandleIndex handle_index;
@@ -138,23 +129,23 @@ struct MBOServiceImpl {
    void flush();
 };
 
-
-
 ///////////////////////
 
-std::optional<uint32_t> search_orders(const ExchangeOrderKey oid, const std::vector<SnapshotOrder>& orders) {
+std::optional<uint32_t>
+search_orders(const ExchangeOrderKey oid,
+              const std::vector<SnapshotOrder> &orders) {
    IntPrice price = {};
    size_t position = 0;
    for (size_t x = 0; x < orders.size(); ++x) {
-      const auto& oo = orders[x];
+      const auto &oo = orders[x];
       if (oo.id == oid) {
-         if (oo.price == price) return { position };
-         else return 0;
-      }
-      else if (price == oo.price) {
+         if (oo.price == price)
+            return {position};
+         else
+            return 0;
+      } else if (price == oo.price) {
          position += oo.size;
-      }
-      else {
+      } else {
          price = oo.price;
          position = oo.size;
       }
@@ -162,11 +153,11 @@ std::optional<uint32_t> search_orders(const ExchangeOrderKey oid, const std::vec
    return std::nullopt;
 }
 
-bool update_position(const HandleIndex& handles, ActiveOrder& order) {
-   const auto& index = handles.index;
-   auto handle = index.lock([id=order.security_id()](const auto& index) {
+bool update_position(const HandleIndex &handles, ActiveOrder &order) {
+   const auto &index = handles.index;
+   auto handle = index.lock([id = order.security_id()](const auto &index) {
       auto iter = index.find(id);
-      return iter != index.end() ? iter->second : MDConsumerHandle {};
+      return iter != index.end() ? iter->second : MDConsumerHandle{};
    });
 
    if (handle.is_initialized()) {
@@ -185,9 +176,9 @@ bool update_position(const HandleIndex& handles, ActiveOrder& order) {
 }
 
 void MBOServiceImpl::timer_tick() {
-   //Dispatcher thread
+   // Dispatcher thread
    std::vector<OrderPIQ> rows;
-   for (auto& pair : active) {
+   for (auto &pair : active) {
       if (update_position(handle_index, pair.second)) {
          rows.push_back(pair.second.to_piq());
       }
@@ -197,29 +188,30 @@ void MBOServiceImpl::timer_tick() {
    }
 }
 
-const NativeOrderState* native_order_state(const OrderRecord& record) {
-   switch(record.state().custom_case()) {
-      case OrderState::kMarket:
-         return &record.state().market().native_state();
-      case OrderState::kLimit:
-         return &record.state().limit().native_state();
-      case OrderState::kStop:
-         return &record.state().stop().native_state();
-      case OrderState::kStopLimit:
-         return &record.state().stop_limit().native_state();
-      case OrderState::kGlbxIceberg:
-         return &record.state().glbx_iceberg().native_state();
-      default:
+const NativeOrderState *native_order_state(const OrderRecord &record) {
+   switch (record.state().custom_case()) {
+   case OrderState::kMarket:
+      return &record.state().market().native_state();
+   case OrderState::kLimit:
+      return &record.state().limit().native_state();
+   case OrderState::kStop:
+      return &record.state().stop().native_state();
+   case OrderState::kStopLimit:
+      return &record.state().stop_limit().native_state();
+   case OrderState::kGlbxIceberg:
+      return &record.state().glbx_iceberg().native_state();
+   default:
       return nullptr;
    }
 }
 
-bool is_active_exchange_order(const OrderRecord& record) {
+bool is_active_exchange_order(const OrderRecord &record) {
    return record.state().status() == ORDER_STATUS_WORKING &&
           native_order_state(record) != nullptr;
 }
 
-void try_remove(MBOServiceImpl& me, std::set<uint32_t>& to_remove, const OrderRecord& record) {
+void try_remove(MBOServiceImpl &me, std::set<uint32_t> &to_remove,
+                const OrderRecord &record) {
    auto key = SparkOrderKey(record.key());
    auto iter = me.active.find(key);
    if (iter != me.active.end()) {
@@ -227,103 +219,113 @@ void try_remove(MBOServiceImpl& me, std::set<uint32_t>& to_remove, const OrderRe
       to_remove.insert(key.value());
    }
 }
-void try_add(MBOServiceImpl& me, std::vector<OrderPIQ>& to_add, const OrderRecord& record) {
+void try_add(MBOServiceImpl &me, std::vector<OrderPIQ> &to_add,
+             const OrderRecord &record) {
    if (is_active_exchange_order(record)) {
       auto native = native_order_state(record);
+
+      auto ex_key = orderman::get_exchange_order_id(*native);
+      auto u64_key =
+          std::visit(overloaded{[](orderman::ExchangeOrderID const &str) {
+                                   return std::stoul(str.value().data());
+                                },
+                                [](orderman::ExchangeOrderID64 oid) {
+                                   return oid.value();
+                                }},
+                     ex_key);
       bts_assert(native);
-      ActiveOrder order(
-         SecurityKey(record.def().security_key()), 
-         SparkOrderKey(record.key()), 
-         native->exchange_order_id());
+      ActiveOrder order(SecurityKey(record.def().security_key()),
+                        SparkOrderKey(record.key()), ExchangeOrderKey{u64_key});
       update_position(me.handle_index, order);
       to_add.push_back(order.to_piq());
       me.active.insert(std::make_pair(order.id(), order));
    }
 }
 
-MBOServiceServer::MBOServiceServer(
-      api::ServiceContext& ctx,
-      utils::Dispatcher& d,
-      md::MDClient& c
-      ) : 
-         MBOServiceBase(ctx),
-         _me(new MBOServiceImpl(ctx,d,c,make_piq_publisher()))
-      {
-         // subscribe to mdclient ✔
-         // make publishers ✔
-         // subscribe to orders => stash active orders dict
-         _me->orders_sub =
-         _me->orders.subscribe_orders().subscribe([=](const auto& update) {
-            // api thread
-            const auto& added = update.added();
-            const auto& changed = update.changed();
-            const auto& removed = update.removed();
-            _me->dispatcher.post([this,added,changed,removed]() {
-               // dispatcher thread
-               std::vector<OrderPIQ> to_add, to_change;
-               std::set<uint32_t> to_remove;
+MBOServiceServer::MBOServiceServer(api::ServiceContext &ctx,
+                                   utils::Dispatcher &d, md::MDClient &c)
+    : MBOServiceBase(ctx),
+      _me(new MBOServiceImpl(ctx, d, c, make_piq_publisher())) {
+   // subscribe to mdclient ✔
+   // make publishers ✔
+   // subscribe to orders => stash active orders dict
+   _me->orders_sub =
+       _me->orders.subscribe_orders().subscribe([=](const auto &update) {
+          // api thread
+          const auto &added = update.added();
+          const auto &changed = update.changed();
+          const auto &removed = update.removed();
+          _me->dispatcher.post([this, added, changed, removed]() {
+             // dispatcher thread
+             std::vector<OrderPIQ> to_add, to_change;
+             std::set<uint32_t> to_remove;
 
-               // Remove
-               for(const auto& r : removed) {
-                  // If it was active remove it and add to to_remove
-                  try_remove(*_me, to_remove, *r);
-               }
-               // Added
-               for(const auto& add : added) {
-                  // If active add look up the position and add to the state
-                  try_add(*_me, to_add, *add);
-               }
-               // Changed
-               for(const auto& ch : changed) {
-                  // new order is active and old is not look up piq and add to state
-                  bool is_active = is_active_exchange_order(*ch.current());
-                  bool was_active = is_active_exchange_order(*ch.previous());
-                  if (is_active && !was_active) {
-                     try_add(*_me, to_add, *ch.current());
-                  }
-                  else if (was_active && !is_active) {
-                     try_remove(*_me, to_remove, *ch.current());
-                  }
-                  // is_active && was_active => don't care
-                  // !is_active && !was_active => don't care
-               }
+             // Remove
+             for (const auto &r : removed) {
+                // If it was active remove it and add to to_remove
+                try_remove(*_me, to_remove, *r);
+             }
+             // Added
+             for (const auto &add : added) {
+                // If active add look up the position and add to the state
+                try_add(*_me, to_add, *add);
+             }
+             // Changed
+             for (const auto &ch : changed) {
+                // new order is active and old is not look up piq and add to
+                // state
+                bool is_active = is_active_exchange_order(*ch.current());
+                bool was_active = is_active_exchange_order(*ch.previous());
+                if (is_active && !was_active) {
+                   try_add(*_me, to_add, *ch.current());
+                } else if (was_active && !is_active) {
+                   try_remove(*_me, to_remove, *ch.current());
+                }
+                // is_active && was_active => don't care
+                // !is_active && !was_active => don't care
+             }
 
-               _me->publish(&to_add, &to_change, &to_remove);
-               if (!_me->timer.running()) _me->timer.start();
-            });
-         });
-      }
+             _me->publish(&to_add, &to_change, &to_remove);
+             if (!_me->timer.running())
+                _me->timer.start();
+          });
+       });
+}
 
-MBOServiceServer::~MBOServiceServer() { }
+MBOServiceServer::~MBOServiceServer() {}
 
-
-void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, const api::Response<PositionData>& reply) {
+void MBOServiceServer::get_orders_for_security(
+    const MBOSecurityQuery &request, const api::Response<PositionData> &reply) {
    PositionData response;
    response.set_security_id(request.security_id());
-   auto handle = _me->handle_index.index.lock([id=request.security_id()](const auto& index) {
-      auto iter = index.find(SecurityKey(id));
-      return iter != index.end() ? iter->second : MDConsumerHandle {};
-   });
+   auto handle = _me->handle_index.index.lock(
+       [id = request.security_id()](const auto &index) {
+          auto iter = index.find(SecurityKey(id));
+          return iter != index.end() ? iter->second : MDConsumerHandle{};
+       });
 
    std::unordered_map<ExchangeOrderKey, SparkOrderKey> reverse;
-   for (const auto& pair : _me->active) {
+   for (const auto &pair : _me->active) {
       reverse.insert(std::make_pair(pair.second.exchange_id(), pair.first));
    }
 
    size_t count = 0;
    auto tick = handle.get_minimum_tick();
-   if(handle.is_initialized()) {
+   if (handle.is_initialized()) {
       auto mbo = handle.get_mbo();
       if (mbo) {
-         for (const auto& o : mbo->bids) {
+         for (const auto &o : mbo->bids) {
             double price = o.price.price();
             bool isnan = os::is_nan(request.price_level());
-            if (isnan || securities::test_price_equality(request.price_level(), price, tick)) {
+            if (isnan || securities::test_price_equality(request.price_level(),
+                                                         price, tick)) {
                ++count;
-               if (request.limit() && count > request.limit()) break;
-               auto* entry = response.add_bid();
+               if (request.limit() && count > request.limit())
+                  break;
+               auto *entry = response.add_bid();
                entry->set_exchange_id(o.id.value());
-               if (isnan) entry->set_price(price);
+               if (isnan)
+                  entry->set_price(price);
                entry->set_size(o.size);
                auto iter = reverse.find(o.id);
                if (iter != reverse.end()) {
@@ -331,15 +333,18 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
                }
             }
          }
-         for (const auto& o : mbo->asks) {
+         for (const auto &o : mbo->asks) {
             double price = o.price.price();
             bool isnan = os::is_nan(request.price_level());
-            if (isnan || securities::test_price_equality(request.price_level(), price, tick)) {
+            if (isnan || securities::test_price_equality(request.price_level(),
+                                                         price, tick)) {
                ++count;
-               if (request.limit() && count > request.limit()) break;
-               auto* entry = response.add_ask();
+               if (request.limit() && count > request.limit())
+                  break;
+               auto *entry = response.add_ask();
                entry->set_exchange_id(o.id.value());
-               if (isnan) entry->set_price(price);
+               if (isnan)
+                  entry->set_price(price);
                entry->set_size(o.size);
                auto iter = reverse.find(o.id);
                if (iter != reverse.end()) {
@@ -349,9 +354,10 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
          }
       }
    }
-   
+
    reply(response);
 }
 
-}} //namespace 
+} // namespace md
+} // namespace bts
 #endif

src/lib/md/MDClient.cpp

@@ -92,7 +92,7 @@ MDConsumer& MDClient::current_consumer() const {
    return _self->_consumer;
 }
 SecurityOracle MDClient::oracle() const {
-   return SecurityOracle(*this);
+   return SecurityOracle(_self->_cache);
 }
 std::set<UserSymbol> MDClient::symbols() const {
    return _self->_symbols;

src/lib/md/MDConsumer.cpp

@@ -897,7 +897,7 @@ public:
       scan_securities(resolve_legs);
    }
 
-   SecurityPtr get_security(SecurityKey key) const {
+   const Security* get_security(SecurityKey key) const {
       return _cache.get_security(key);
    }
 
@@ -979,7 +979,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol,
 
 size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); }
 void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); }
-const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); }
+const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); }
 std::vector<api::Security const*> MDConsumer::get_securities() const {return _pimpl->get_securities();}
 
 MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const {

src/lib/md/SecurityCache.cpp

@@ -1,8 +1,10 @@
 #include "SecurityCache.h"
 #include "SecurityDB.h"
 #include "lib/utils/Dispatcher.h"
+#include "lib/utils/LockBox.h"
 
 #include <unordered_map>
+#include <google/btree/btree_map.h>
 #include <tbb/concurrent_queue.h>
 #include <mutex>
 
@@ -10,8 +12,8 @@ namespace bts {
 namespace md {
 
 struct SecurityCacheImpl {
-   std::mutex cache_mutex;
-   std::unordered_map<SecurityKey, SecurityPtr> cache;
+   utils::LockBox<btree::btree_map<SecurityKey, SecurityPtr>> cache;
+   utils::LockBox<btree::btree_map<ExchangeSecurityKey, SecurityPtr>> ex_cache;
    tbb::concurrent_queue<SecurityPtr> update_queue;
    void enqueue_update_db(const SecurityPtr&);
    void flush();
@@ -42,61 +44,83 @@ void SecurityCacheImpl::flush() {
 SecurityCache::SecurityCache() : _me(std::make_shared<SecurityCacheImpl>()) {}
 SecurityCache::~SecurityCache() {}
 
-SecurityPtr SecurityCache::try_get_security(SecurityKey key) const {
-   std::lock_guard<std::mutex> lock(_me->cache_mutex);
-   auto iter = _me->cache.find(key);
-   if (iter != _me->cache.end()) {
-      return iter->second;
-   }
-   return nullptr;
+const Security* SecurityCache::try_get_security(SecurityKey key) const {
+   return _me->cache.lock([key](btree::btree_map<SecurityKey, SecurityPtr>& t) {
+      auto iter = t.find(key);
+      if (iter != t.end()) {
+         return iter->second.get();
+      }
+      return (const api::Security*)nullptr;
+   });
+}
+const Security* SecurityCache::try_get_security(ExchangeSecurityKey key) const {
+   return _me->ex_cache.lock([key](btree::btree_map<ExchangeSecurityKey,SecurityPtr>& t) {
+      auto iter = t.find(key);
+      if (iter != t.end()) {
+         return iter->second.get();
+      }
+      return (const api::Security*)nullptr;
+   });
 }
 
-SecurityPtr SecurityCache::get_security(SecurityKey key) {
-   std::unique_lock<std::mutex> lock(_me->cache_mutex);
-   auto iter = _me->cache.find(key);
-   if (iter != _me->cache.end()) {
-      return iter->second;
-   }
-   else {
-      // unlock for db read
-      lock.unlock();
+
+const Security* SecurityCache::get_security(SecurityKey key) {
+   const Security* result = try_get_security(key);
+   if (result == nullptr) {
+      // outside lock
       auto security = db::find_security(key);
       if (security) {
          LOG(md,Info, "Loaded security from database: {} {}", security->symbol(), security->id());
          return add_db_security(security);
       }
-      return security;
+      return nullptr;
    }
-}
-
-std::vector<api::Security const*> SecurityCache::get_securities()
-{
-   std::vector<api::Security const*> result;
-   result.reserve(100000);
-
-   std::unique_lock<std::mutex> lock(_me->cache_mutex);
-   for (auto const&p : _me->cache)
-      result.push_back(p.second.get());
    return result;
 }
 
-SecurityPtr SecurityCache::add_db_security(const SecurityPtr& security) {
+std::vector<api::Security const*> SecurityCache::get_securities() {
+   std::vector<api::Security const*> result;
+   result.reserve(100000);
+
+   _me->cache.lock([&result](auto& cache) {
+      for (auto const&p : cache) {
+         result.push_back(p.second.get());
+      }
+   });
+   return result;
+}
+
+const Security* SecurityCache::add_db_security(const SecurityPtr& security) {
    bts_assert(security);
-   std::lock_guard<std::mutex> lock(_me->cache_mutex);
-   SecurityKey key(security->id());
-   auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security));
-   // If result was inserted result.first->second is equal to security
-   // If the value was already present it must be a recently inserted 'live' security and we should use it
-   return result.first->second;
+   if (security->has_glbx()) {
+      _me->ex_cache.lock([&security](auto& cache) {
+         ExchangeSecurityKey key(security->id());
+         cache.insert(std::make_pair(key, security));
+      });
+   }
+   return _me->cache.lock([&security](auto& cache) {
+      SecurityKey key(security->id());
+      auto result = cache.insert(std::make_pair(key, security));
+      // If result was inserted result.first->second is equal to security
+      // If the value was already present it must be a recently inserted 'live' security and we should use it
+      return result.first->second.get();
+   });
 }
 bool SecurityCache::add_live_security(const SecurityPtr& security) {
    bts_assert(security);
-   std::lock_guard<std::mutex> lock(_me->cache_mutex);
-   SecurityKey key(security->id());
-   // Insert but don't overwrite existing
-   auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security));
-   _me->enqueue_update_db(security);
-   return result.second;
+   if (security->has_glbx()) {
+      _me->ex_cache.lock([&security](auto& cache) {
+         ExchangeSecurityKey key(security->id());
+         cache.insert(std::make_pair(key, security));
+      });
+   }
+   return _me->cache.lock([&](auto& cache) {
+      SecurityKey key(security->id());
+      // Insert but don't overwrite existing
+      auto result = cache.insert(std::make_pair(SecurityKey(security->id()), security));
+      _me->enqueue_update_db(security);
+      return result.second;
+   });
 }
 
 void SecurityCache::flush_to_disk() { _me->flush(); }

src/lib/md/SecurityCache.h

@@ -3,6 +3,7 @@
 #include "lib/domain_model/Security.pb.h"
 #include "MDConsumerHandle.h"
 #include "lib/domain_model/SecurityKey.h"
+#include "lib/md/ExchangeSecurityKey.h"
 #include "SecurityPtr.h"
 #include <memory>
 
@@ -20,17 +21,18 @@ public:
    ~SecurityCache();
 
    //Load security from cache with fallback to database
-   SecurityPtr get_security(SecurityKey key); 
+   const Security* get_security(SecurityKey key); 
 
    // Load all securities from cache, no fallback to database
    std::vector<api::Security const*> get_securities();
    
    // Get the security if it's in the cache otherwise return nullptr. Does not query security database.
-   SecurityPtr try_get_security(SecurityKey key) const;
+   const Security* try_get_security(SecurityKey key) const;
+   const Security* try_get_security(ExchangeSecurityKey key) const;
 
    // returns true if the cache was updated
    bool add_live_security(const SecurityPtr& security);
-   SecurityPtr add_db_security(const SecurityPtr& security);
+   const Security* add_db_security(const SecurityPtr& security);
 
    void flush_to_disk();
 

src/lib/md/SecurityConstructor.h

@@ -1,56 +0,0 @@
-#pragma once
-
-#include "core/bts_core/utils/bts_no_copy.h"
-#include "Shm/FixedSecurity.h"
-#include "../bts_conn_exports.h"
-#include <memory>
-#include <functional>
-#include <atomic>
-
-namespace bts {
-namespace md {
-
-
-class BTS_EXPORT SecurityConstructor {
-   // An object "pool" optimized for the case where only one instance is ever used at a time
-   template <class T>
-   class SingleObjectPool<T> {
-   public:
-      using ptr_type = std::unique_ptr<T, std::function<void(T*)>>;
-      ptr_type acquire() {
-         T* pt = _inst ? _inst.release() : new T();
-         return ptr_type(pt, [this](T* ptr) {
-               return_inst(std::unique_ptr<T>(ptr));
-            });
-      }
-   private:
-      void return_inst(std::unique_ptr<T> t) {
-         if (!_inst) {
-            reset(t);
-            _inst = std::move(t);
-         }
-      }
-      unique_ptr<T> _inst;
-   };
-
-public:
-   SecurityConstructor() : _future(), _option(), _spread() {}
-
-   SingleObjectPool<FixedFutureSecurity>& future() { return _future; }
-   SingleObjectPool<FixedOptionSecurity>& option() { return _option; }
-   SingleObjectPool<FixedSpreadSecurityL>& spread() { return _spread; }
-   
-private:
-   template <typename T>
-      void reset(T& t) {
-      new (&t)T();
-   }
-
-   SingleObjectPool<FixedFutureSecurity> _future;
-   SingleObjectPool<FixedOptionSecurity> _option;
-   SingleObjectPool<FixedSpreadSecurityL> _spread;
-};
-
-
-}}//namespace
-

src/lib/md/SecurityOracle.cpp

@@ -4,14 +4,18 @@
 namespace bts {
 namespace md {
 
-SecurityOracle::SecurityOracle(const MDClient& client): _client(client) {}
+SecurityOracle::SecurityOracle(SecurityCache& cache): _cache(cache) {}
 SecurityOracle::~SecurityOracle() { }
 const api::Security* SecurityOracle::get_security(SecurityKey key) const {
-   return _client.current_consumer().get_security(key);
+   return _cache.get_security(key);
+}
+
+const api::Security* SecurityOracle::get_security(ExchangeSecurityKey key) const {
+   return _cache.try_get_security(key);
 }
 
 std::vector<api::Security const*> SecurityOracle::get_securities() const {
-   return _client.current_consumer().get_securities();
+   return _cache.get_securities();
 }
 
 }}//namespace

src/lib/md/SecurityOracle.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include "lib/domain_model/SecurityKey.h"
+#include "lib/md/ExchangeSecurityKey.h"
+#include "lib/md/SecurityCache.h"
 #include <memory>
 #include <vector>
 
@@ -9,11 +11,12 @@ namespace api {class Security;}
 namespace md {
 class MDClient;
 class SecurityOracle {
-   const MDClient& _client;
+   SecurityCache& _cache;
 public:
-   SecurityOracle(const MDClient&);
+   SecurityOracle(SecurityCache&);
    ~SecurityOracle();
    const api::Security* get_security(SecurityKey key) const;
+   const api::Security* get_security(ExchangeSecurityKey key) const;
    std::vector<api::Security const*> get_securities() const;
 };
 

src/lib/md/readme.md

@@ -0,0 +1,209 @@
+
+# Overview
+
+The core abstraction that everything else is built on is the MDProducer+MDProducerHandle / MDConsumer+MDConsumerHandle. 
+
+A new market data source constructs an MDProducer and adds securities to it.  When a security is added you get back an MDProducerHandle representing that security in shared memory.  The memory is fully allocated and books can be written immidiately but the security will not be visible to the MDConsumer until the security key is stored on disk (SecurityIDRecorder).  The MDProducerHandle assumes a **single writer** for most operations and *may break silently if this constraint is violated*.
+
+The MDConsumer is the dual of the MDProducer.  It provides an on_security() callback which is driven by the call to scan_securities().  No threads are created by MDConsumer/MDProducer instead they get cpu time from periodic calls to scan_securities (consumer), SecurityIDRecorder::do_work (producer).  When a security is discovered by scan_securities the callback is called with the Security and the MDConsumerHandle.  The MDConsumerHandle is thread safe and can be safely copied and read from concurrently.
+
+## Single Writer Principle
+- https://mechanical-sympathy.blogspot.com/2011/09/single-writer-principle.html
+
+# Core Classes
+
+'*' = New in spark
+* MBOProcessor*
+* MBOServiceServer*
+* MDClient*
+* MDConsumer
+* MDConsumerHandle
+* MDProducer
+* MDProducerHandle
+* PlatformLoader
+* PriceSize/IntPrice
+* SecurityCache*
+* SecurityOracle*
+* SecurityDB*
+* SecurityIDRecorder
+* SecurityMapper*
+* SettlementRecorder
+* SyntheticSecurityService
+* FixedSecurity.h / MDStructure.h
+* SharedList.h
+
+## MBOProcessor*
+
+This class is an optional dependency of MDProducer.  It takes the low level incremental updates to the MBO book and constructs the book on a different thread.  This is optimized for the UI use case so it tries to get the mbo messages out of the critical path as soon as possible. This uses regular heap memory so is not compatible with out of process operation.  It exposes a reader and writer interface.
+
+Each update goes into a queue and gets processed by MBOProcessorImpl::process_incremental() on a single background thread.  This thread drains the queue and constructs the books in place.  Then it copies snapshots of the books and publishes them as immutable snapshots which are consumed by the MBOServiceServer/UI.  If mbo load gets very high this function will stop publishing snapshots and only update books until the load reduces.
+
+## MBOServiceServer*
+
+Reads the snapshots from the MBOProcessor/MBOReader and serves them for the UI.
+
+## MDClient*
+
+New in spark.
+
+This class handles constructing the MDConsumer, calling scan_securities on it and checking for bad heartbeats and re-constructing the MDConsumer if there was a failure.  In edge those tasks were the responsibility of each consumer of the market data but it was mitigated because most things used streams rather than MDConsumer directly.  MDConsumer is reference counted internally so when the consuming code gets an on_consumer() callback with a new MDConsumer it should keep a copy of the old MDConsumer until it has finished any asyncronous processes involving the MDConsumerHandles from that first consumer.  Once an MDConsumer is destroyed the MDConsumerHandles it produced will cause an **Access Violation** on reading because the memory has been unmapped.
+
+Spark has always used MD in process so, while based on existing functionality, this recovery process hasn't been well tested.
+
+## MDConsumer
+
+An MDConsumer maps the shared memory sections and produces MDConsumerHandles.  These handles are only valid as long as the MDConsumer exists.  It would make some sense to store the MDConsumer reference in the MDConsumerHandle to avoid this but MDConsumerHandle is designed to be *very* small for fast iteration.  The MDConsumer also holds a reference to the ISyntheticSecurityService and makes synthetic securities 'look like' regular shared memory securities using polymorphism in the MDConsumerHandle.
+
+Internally MDConsumer is a collection of MDConsumerInst which each handle a single symbol+type+exchange and the corresponding shared memory blocks.
+
+* scan_securities - in spark this is always called by MDClient
+* get_handle - Low level way to get a handle. Does a linear walk though shared memory and can return a void handle (MDConsumerHandle::is_initialize() == false). 
+* get_managed_handle - returns a 'managed handle' this is a shared_ptr to an MDConsumerhandle. The only difference is that this is tracked by the MDConsumer and may transition from uninitialized to initialized.  Storing these in a vector and walking them is almost *50% slower* than doing the same thing with an unmanaged handle.  If you use these it's probably best to store the uninitialized handles in a nursery and check them periodically.  You can dereference a managed handle into a regular handle at any time.
+* get_channel - Gets a MDConsumerChannel representing common data for a symbol+type+exchange.  This is mostly for the trade ticker / rfq and also exposes various counters which get published to dev-stats.
+
+On Linux MDConsumer maps the shared memory as read-only pages.  On Windows it cannot do this because std::atomic uses a different instruction which is considered a write by the CPU so we map it read/write.
+
+If we have multiple platforms that can produce md for an exchange (i.e. onixs vs emumba) the MDConsumer consults the PlatformMap config key to determine which to read from.  The md replayer impersonates the producer it was recorded from.
+
+### MDConsumerChannel
+
+read_trades() and read_rfqs() read from a ring buffer stored in the header section of the shared memory. They read directly into a google::protobuf::RepeatedPtrField to avoid extra copies in the QueryServer but can just as easily read into a std::vector. They expose a 'cursor' representing the position in the ring buffer and this cursor is sent all the way to the client so it can request updates.
+
+## MDConsumerHandle
+
+An MDConsumerHandle 4 states and would proably be impemented as an std::variant<> if done today. If not for the high performance requirements it could also be a virtual class but I didn't want to pay for a vtable.  It is the size of 4 pointers plus the enum tag.
+
+* VOID - an uninitialized handle
+* BBO - Legacy from Edge.  This represents an option book where we only kept the top to save space.  I think this has rotted by now and wont work but I kept it as an example of how to add polymorphism to the shared memory types.
+* DEPTH - Used for all exchange securities in Spark.
+* SYNTHETIC - A Synthetic handle. These are bigger so they get stored as a pointer.
+
+Reading from a BBO/DEPTH handle is done with an optimistic copy.  If you inrease the size of the book data this copy can fail more often and get significantly worse perf. The result of the read can be changed, unchanged or busy (timeout).  In edge 'busy' happened 1-2 times per day across all reads and gets logged by QueryServer.
+
+## MDProducer
+
+Handle to writeable shared memory.  Each market data producer "platform" (onixs_cme, onixs_ice, emumba, etc) constructs one.
+
+## MDProducerHandle
+
+All updates to a single security go through here.  Assumes a single writer thread.
+
+## PlatformLoader
+
+In Edge this was the entry point to the run() for the market data producer process. It loaded the bts_md_*.so so each producer process only loaded the third party objects for a particular producer.  In spark this is all in process so this is defunct.  It looks like it still exists because it has some utility functions that probably belong in another file. 
+
+## PriceSize/IntPrice
+
+Market data uses integer prices to avoid double problems.
+
+## SecurityCache*
+
+Created/Owned by MDClient this owns the Securities.  
+
+It has methods to get a security from the live collection or load them from the SecurityDB on disk if someone asks for one that hasn't been seen yet.  
+* If a security is loaded from disk it 'freezes' it so when the live security arrives it will continue to serve the old security so we don't end up with 2 copies of a single security with different properties.  
+* This class is also responsible for sending the live securities to the SecurityDB where we *always* overwrite stored securities with what comes from the exchange.
+* Survives beyond the MDConsumer in case of crash/recovery of out of process md.
+
+## SecurityOracle*
+Interface on top of SecurityCache exposing the read get_ operations.
+
+## SecurityDB*
+* Just a collection of methods for save/load of securities to sqlite.
+* Stored in binary format so we can use the protobuf versioning
+
+## SecurityIDRecorder
+* Manages security to unique 32 bit id mapping across multiple processes
+* Provides a callback when ids are safely persisted.
+
+## SecurityMapper*
+
+* Happens *before* the security gets stored in shared memory.
+* Replaces the mapping.h from edge with a more organized mechanism.  
+* Owned by the MDProducer each security is passed though SecurityMapper::apply() before it is first stored in shared memory
+* SecurityMapper is made up of multiple ISecurityMapper instances so each mapping can be in it's own class so it's easier to read each mapping.  Currently all mappings are defined in SecurityMaper.cpp
+* The mappings each have an opportunity to mutate the FixedSecurity.
+
+## SettlementRecorder
+
+In Edge SettlementRecorder stored the settlements for a security so we could preserve yesterday/day before settlement.  In Spark this broke and nobody seemed to care so it was never fixed.
+
+## SyntheticSecurityService
+
+Each synthetic security is a api::Security and a chunk of metadata.  This service handles API requests to create/update synthetics and fires callbacks for MDConsumer which does the work of producing synethic books.
+
+## MDStructure.h
+
+### Hot vs Cold
+
+## FixedSecurity.h
+* variable sized structure in shared mem
+* For FixedSpreadSecurity we stack allocate a lot of legs then memcpy() into shared memory only the correct number of legs
+* When accessing FixedSpreadSecurity it is cast back to a large number of legs. Iterate legs based on leg_count() not the static leg count.
+
+## SharedList.h
+* Primitive list-like abstraction over shared memory
+* Allows variable sized entries so always accessed in O(N)
+* Data never moves so we point into it for O(1)
+
+# Market Data Relay / Recorder
+* QueryServer
+    - Stateless request/response
+    - Uses multiple threads for throughput
+    - Uses change timestamps per channel instead of per security
+    - Will over-send books that change around the time of the request
+
+* RelayClient
+    - Runs in the shell (or outside)
+    - Replicates books from the server (QueryServer) to local shared memory
+
+* RelayRecorder
+    - Records Protobufs from QueryServer to a file
+
+# Diagnostic tools
+* sparkd md dump [symbol] [type] [exchange] [hot?]
+    - **Primary diagnostic tool**
+    - ShmDumper.cpp
+    - Shows current contents of shared memory
+    - append 'hot' to the end to see hot data i.e. books
+    - If you get a "cannot open" check that the file exists in /run/shm/
+* dumpsecurities
+    - Inspect security database
+* sparkd md tail
+    - MDPeriodicDumper.cpp
+    - for validating new market data
+    - prints books at given sequence number interval
+* rectool
+    - print recording to text / convert back to futures.coconut
+        - This is most useful to checking if a particular symbol is in a replay file
+    - Occasionally useful for testing very specific market data transitions
+
+# Tuning / Dealing with Loss
+
+## Packet loss
+#### If a small number 1-5 is lost it's upstream
+- Use tcpdump to capture
+- Check the tcpdump for gaps
+- Ask Pete to contact hosting provider
+#### If large gap probably overflow
+- make sure socket buffers sizes, rmem_max, spin_time, data_wait are all set
+- check affinity is set and working
+- check ethtool
+
+
+#### Show ethernet ring buffer sizes
+ethtool -g eth0
+#### change ethernet ring buffer rx size
+sudo ethtool -G [device] rx 4096
+#### interface statistics
+ethtool -S eth0
+- particularly look at:
+   ethtool -S p1p1 | egrep "(bad)|(nodesc)"
+
+### If increasing the ring buffer doesn't work you can tweak interrupt coalescing using:
+- /proc/sys/net/core/dev_weight
+- This is the maximum number of packets that can be dequeued in a single interrupt
+- https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Performance_Tuning_Guide/s-network-common-queue-issues.html
+
+- **NOTE**: We've never used this and it's unlikely it will fix anything
+

src/lib/md/readme.txt

@@ -1,83 +0,0 @@
-
-
-********************
-classes
-********************
-
-btscmd md => PlatformLoader
-
-PlatformLoader:
-   * Loads the platform specific dll
-   * Calls ->start_marketdata() with a full vector<security_info>
-
-SharedList<T> / SharedListWriter
-   * Shared memory list object which spans across shared memory segments as necessary
-   * O(N) walk
-   * atomic add()
-   * Single adder / multiple readers
-   * no delete
-   * Custom header
-
-SecurityIDRecorder
-   * add_security(security, lambda callback) -> int     returns security id instantly
-   * security_persisted callback when security id is safely persisted
-
-SecurityIndex
-   * Platform Implementation helper
-   * Maps exhange_id to (ProducerHandle + Data)
-   * 
-
-CacheEntry / CacheEntryDepth  (CacheStructure.h)
-   * CacheEntry(symbol, depth, # legs) -- size determined in constructor
-   * Simple inheritance tree for optional depth
-   * Has FixedSecurity
-   * Fits in SharedList<T>
-   * CRITICAL fields are at the front cache aligned.  bbo/depth. Security goes at the end
-
-   * size, is_public(), is_persisted, is_resolved (spread)
-   * statistics like bbo/trade counters
-   * is_status_ok
-   * bbo, depth, trade, status, open, close
-
-FixedSecurity / FixsdSpreadSecurity
-   * Normalized version of our protobuf security
-   * Has methods to convert to pb Security
-   * Size known at construction w/ # legs - spread legs of type security_key
-   * Can use a union for option/future fields
-
-MDProducer(symbol, type)
-   * manages lifetime of SharedList instances
-   * Hands out MDProducerHandle - entry for one security
-
-MDProducerHandle
-   * pointers to shared memory
-   * update methods
-
-MDConsumer
-   * list of securities
-   * find_security_id(symbol, type, etc)
-   * get_cache_handle(security id#)
-   * Hands out MDCacheHandle
-      - if security is null add MDCacheHandle to a list to be updated
-        next time securities are polled
-
-MDCacheHandle
-   * If security is not present add this is "null"
-   * has pointers to relevant shared memory
-
-
-
-********************
-* Cache Structure
-********************
-
-2 Shared memory segment for each  (SYMBOL, TYPE)  - md_hot (bbo,depth,status), md_cold (others)
-
-When a new security is discovered
-   1. id is allocated to the security
-   2. New CacheEntry is allocated in appropriate arena (is_public=false)
-   3. md_cache entry count is incremented
-   4. When id is persisted is_public=true, increment version on cache arena
-
-
-