Fix System tests

RTM: SPARK-2363 - PIQ popup

Approved-by: Jake Norbut <jake.norbut@bluetradesys.com>

.vscode/launch.json

@@ -1,83 +1,14 @@
 {
     "version": "0.2.0",
     "configurations": [
-
-        {
-            "name": "(gdb) Launch",
-            "type": "cppdbg",
-            "request": "launch",
-            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd/sparkd",
-            "args": ["run"],
-            "stopAtEntry": false,
-            "cwd": "${workspaceRoot}/build/ninja_debug/src/server/sparkd",
-            "environment": [],
-            "externalConsole": true,
-            "MIMode": "gdb",
-            "setupCommands": [
-                {
-                    "description": "Enable pretty-printing for gdb",
-                    "text": "-enable-pretty-printing",
-                    "ignoreFailures": true
-                }
-            ]
-        },
-        {
-         "name": "Client Launch",
-         "type": "clr",
-         "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/spark.exe",
-         "args": [],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-        },
-      {
-         "name": "SDK Launch",
-         "type": "clr",
-         "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/spark_sdk.exe",         
-         "args": ["md","GLBX","Future","CL", "CLZ9", "--user=bts", "--pass="],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-      },        
-      {
-         "name": "Excel Launch",
-         "type": "clr",
-         "request": "launch",
-         "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
-         "args": ["${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-      },          
-      {
-         "name": "Client Tests",
-         "type": "clr",
-         "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
-         "args": ["test","test_filter"],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-      },           
-      {
-            "name": "Client Attach",
-            "type": "clr",
-            "request": "attach",
-            "processId":"${command:pickProcess}"
-        },
         {
             "name": "SparkVM Launch",
             "type": "cppdbg",
             "request": "launch",
-            
             "program": "/home/bts/spark/build/ninja_debug/server/sparkd",
-            "args": ["run"],
+            "args": [
+                "run"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server/",
             "environment": [],
@@ -102,24 +33,113 @@
                     "description": "LD LIBRARY",
                     "text": "set env LD_LIBRARY_PATH=.",
                     "ignoreFailures": false
-                }                
+                }
             ],
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             }
         },
+        {
+            "name": "(gdb) Launch",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd/sparkd",
+            "args": [
+                "run"
+            ],
+            "stopAtEntry": false,
+            "cwd": "${workspaceRoot}/build/ninja_debug/src/server/sparkd",
+            "environment": [],
+            "externalConsole": true,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                }
+            ]
+        },
+        {
+            "name": "Client Launch",
+            "type": "clr",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/Debug/bin/net471/spark.exe",
+            "args": [],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "SDK Launch",
+            "type": "clr",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/Debug/bin/net471/spark_sdk.exe",
+            "args": [
+                "md",
+                "GLBX",
+                "Future",
+                "CL",
+                "CLZ9",
+                "--user=bts",
+                "--pass="
+            ],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "Excel Launch",
+            "type": "clr",
+            "request": "launch",
+            "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
+            "args": [
+                "${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"
+            ],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "Client Tests",
+            "type": "clr",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+            "args": [
+                "test",
+                "test_filter"
+            ],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "Client Attach",
+            "type": "clr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+        },
         {
             "name": "Spreader Tests",
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test", "[spreader]"],
+            "args": [
+                "test",
+                "[spreader]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],
@@ -139,7 +159,9 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
@@ -151,7 +173,10 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test", "[admin_service]"],
+            "args": [
+                "test",
+                "[admin_service]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],
@@ -171,7 +196,9 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
@@ -183,7 +210,10 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test", "[limiter]"],
+            "args": [
+                "test",
+                "[limiter]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],
@@ -203,7 +233,9 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
@@ -215,7 +247,10 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/server/sparkd",
-            "args": ["test", "[mbo]"],
+            "args": [
+                "test",
+                "[mbo]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server/",
             "environment": [],
@@ -240,14 +275,16 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             }
         },
-         {
+        {
             "name": "SparkVM attach",
             "type": "cppdbg",
             "request": "attach",
@@ -255,16 +292,18 @@
             "processId": "${command:pickRemoteProcess}",
             "MIMode": "gdb",
             "pipeTransport": {
-               "pipeCwd": "${workspaceRoot}",
-               "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-               "pipeArgs": ["bts@sparkvm"],
-               "debuggerPath": "/usr/bin/gdb"
+                "pipeCwd": "${workspaceRoot}",
+                "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
+                "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
-               "/home/bts/spark/": "${workspaceRoot}"
+                "/home/bts/spark/": "${workspaceRoot}"
             }
-         },
-         {
+        },
+        {
             "name": "Python",
             "type": "python",
             "request": "launch",
@@ -274,12 +313,59 @@
             "cwd": "${workspaceRoot}",
             "env": {},
             "envFile": "${workspaceRoot}/.env",
-            "args": [ "DEBUG" ],
+            "args": [
+                "DEBUG"
+            ],
             "debugOptions": [
                 "WaitOnAbnormalExit",
                 "WaitOnNormalExit",
                 "RedirectOutput"
             ]
         },
+        {
+            "name": ".NET Core Remote Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickRemoteProcess}",
+            "pipeTransport": {
+                "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
+                "debuggerPath": "/home/bts/vsdbg/vsdbg",
+                "pipeCwd": "${workspaceRoot}",
+                "quoteArgs": true
+            },
+            "sourceFileMap": {
+                "/home/bts/spark/": "${workspaceRoot}"
+            },
+            "justMyCode": true
+        },
+        {
+            "name": ".NET Core Remote Launch",
+            "type": "coreclr",
+            "request": "launch",
+            "program": "/usr/bin/dotnet",
+            "args": [
+                "sparkcmd.dll",
+                "test",
+                "test_join_create_and_takeout_market"
+            ],
+            "stopAtEntry": false,
+            "cwd": "/home/bts/spark/build/ninja_debug/server",
+            "pipeTransport": {
+                "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
+                "debuggerPath": "/home/bts/vsdbg/vsdbg",
+                "pipeCwd": "${workspaceRoot}",
+                "quoteArgs": true
+            },
+            "sourceFileMap": {
+                "/home/bts/spark/": "${workspaceRoot}"
+            },
+            "justMyCode": false
+        },
     ]
 }

doc/user/content/release_notes/_index.md

@@ -4,6 +4,43 @@ weight = 3
 sort_by = "weight"
 insert_anchor_links = "right"
 +++
+## 2.0.2
+### User Visible Changes
+  - Middle click the ladder price column to show the orders for that price level 
+  - Middle click the bid or ask column in the ladder no longer sends an order
+
+### Internal changes
+  - Improved market by order performance under heavy load
+  - Fixed a concurrency bug in SIM
+
+## 2.0.0
+### User Visible Changes
+  - Support for attaching and detaching algos to existing orders
+  - Support for display and update of all parameters in running algos
+  - Streamlining of order badge decorations in ladder
+  - New order search/filtering algorithm for order book
+  - Simpler representation of complex algo chains in order book and mini order book
+  - Visual and usability improvements to mini order book
+  - Streamlining of algo parameter dialog in ladder
+  - New spreader queue holder behavior
+  - Visual improvements to the order book
+  - New server side API
+  - Improved admin UI performance
+  - Optimized PIQ rendering
+### Internal changes
+  - Exchange simulator no longer requires daily sequence number reset
+  - Improved MD resilience to corrupted shared memory
+  - Native orders can now buffer updates/cancels when disconnected or pending first acknowledgment
+  - Improved settings manager tool output
+  - Added diagnostics in the event MD memory corruption is detected
+  - Added systems tests for join,lean, iceberg and spreader orders
+  - Made exchange order recovery more resilient in the presence of database loss 
+  
+## 1.6.2
+### User Visible Changes
+### Exchange simulator
+  - Exchange simulator now now produce position in queue data
+
 ## 1.6.0
 ### User Visible Changes
 #### New Features

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -73,7 +73,7 @@ namespace mbts.spark {
          _has_working_orders = backing(nameof(has_working_orders), false);
          contract_picker_model = new ContractPickerModel(bus, false, null, new SecurityCriteria(new[] {
             new SecurityCriteriaClause() {
-               exchanges = EnumHelper.get_values_except(Venue.Bts).ToLArray()
+               exchanges = EnumHelper.get_values_except(Venue.Bts).ToArray()
             }
          }));
          contract_picker_model.cancel_action = _close_new_leg_popup;

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoData.cs

@@ -2,6 +2,7 @@
 using bts.utils;
 using mbts.spark.utilities;
 using System;
+using System.Windows.Media;
 
 namespace mbts.spark {
    struct AlgoData : IEquatable<AlgoData> {
@@ -39,20 +40,28 @@ namespace mbts.spark {
 
    struct AlgoDataVM : IEquatable<AlgoDataVM> {
       public readonly AlgoData data;
+      public readonly AlgoDefinitionMetadata? meta;
 
-      public AlgoDataVM(AlgoData data) =>
+      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
          this.data = data;
+         this.meta = meta;
+      }
 
+      public string name => data.name;
+      public Brush badge_fill => OrderUtils.badge_fill(meta, OrderDataHelper.get_metadata(data.value.CustomCase));
       public AlgoDisplayGroup group => data.key.display_group();
 
       public override string ToString() =>
          data.name;
 
-      public static AlgoDataVM from_data(AlgoData data) =>
-         new AlgoDataVM(data);
+      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
+         new AlgoDataVM(data, meta);
 
-      public bool Equals(AlgoDataVM other) => data == other.data;
+      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
+         new AlgoDataVM(tup.algo, tup.meta);
+
+      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
       public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
-      public override int GetHashCode() => data.GetHashCode();
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
    }
 }

src/client/spark/Modules/Trading/AlgoStudio/Utils/AlgoCollectionHelper.cs

@@ -47,15 +47,19 @@ namespace mbts.spark {
          }
       }
 
-      public static void update_persisted_definition_wrappers(ObservableKeyedCollection<IAlgoVM, AlgoKey> target, TableUpdate<AlgoData, AlgoKey> update, Func<AlgoData, AlgoDefinitionVM> factory) {
+      public static void update_persisted_definition_wrappers(IDictionary<AlgoKey, IAlgoVM> target, TableUpdate<AlgoData, AlgoKey> update, Func<AlgoData, AlgoDefinitionVM> factory) {
          foreach (var d in update.removed)
             target.Remove(d.key);
-         foreach (var d in update.changed)
-            target[d.key].load_from(d.value);
+         foreach (var d in update.changed) {
+            var vm = target[d.key];
+            if (vm is AlgoVMWrapper wrapper) {
+               wrapper.value.load(d);
+            } else vm.load_from(d.value);
+         }
          foreach (var d in update.added) {
             if (target.TryGetValue(d.key, out IAlgoVM def)) {
                def.load_from(d.value);
-            } else target.Add(new AlgoVMWrapper(factory(d)));
+            } else target.Add(d.key, new AlgoVMWrapper(factory(d)));
          }
       }
 
@@ -114,4 +118,4 @@ namespace mbts.spark {
             DisplayNameHelper.compare_alpha_numeric(a.display_name, b.display_name);
       }
    }
-}
+}

src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoDefinitionVM.cs

@@ -202,6 +202,7 @@ namespace mbts.spark {
       public string name { get => _name.Value; set => _name.set(value); }
       public string abbreviation { get => _abbreviation.Value; set => _abbreviation.set(value); }
       public ObservableCollection<IAlgoVM> algo_roots => _algo_roots;
+      public IDictionary<AlgoKey, IAlgoVM> algo_roots_index => _algo_roots;
       public IReadOnlyObservableCollection<IAlgoVM> algos_view { get; }
       public IAlgoVM algo_root { get => _algo_root.Value; set => _algo_root.set(value); }
       public IAlgoVM visible_algo_root => _visible_algo_root.Value;

src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoParamRowVM.cs

@@ -18,12 +18,16 @@ namespace mbts.spark {
          this.algo = algo;
          this.parent = parent;
          _template_key = template_key;
+         param.permissions_changed += on_permissions_changed;
+         _dispose = Disposable.Create(() => param.permissions_changed -= on_permissions_changed);
+         void on_permissions_changed(AlgoParamPermissions oldval, AlgoParamPermissions newval) => RaisePropertyChanged(nameof(permissions));
       }
 
       public event Action invalidated;
 
       public IAlgoVM algo { get; set; }
       public IAlgoParamVM param { get; }
+      public AlgoParamPermissions permissions => param.permissions;
       public AlgoParamRowVM parent { get; }
       public int level => parent == null ? 0 : parent.level + 1;
       public string path_disambiguation {

src/client/spark/Modules/Trading/AlgoStudio/VM/AlgoVMWrapper.cs

@@ -29,13 +29,12 @@ namespace mbts.spark {
             nameof(editing_permissions),
             nameof(definition),
             nameof(display_name)
-         },
-         _value_echo_props = new HashSet<string> {
-            nameof(AlgoDefinitionVM.name),
-            nameof(AlgoDefinitionVM.abbreviation),
-            nameof(AlgoDefinitionVM.badge_fill),
-            nameof(AlgoDefinitionVM.can_undo),
-            nameof(AlgoDefinitionVM.can_redo)
+         };
+      static readonly IReadOnlyDictionary<string, string>
+         _value_echo_props = new Dictionary<string, string> {
+            { nameof(AlgoDefinitionVM.name), nameof(display_name) },
+            { nameof(AlgoDefinitionVM.abbreviation), nameof(abbreviation) },
+            { nameof(AlgoDefinitionVM.badge_fill), nameof(meta) }
          };
       readonly ObservableAggregateCollection<IAlgoSelector> _child_algo_selectors = new ObservableAggregateCollection<IAlgoSelector>();
       readonly ObservableAggregateCollection<IAlgoParamVM> _algo_params = new ObservableAggregateCollection<IAlgoParamVM>();
@@ -143,8 +142,8 @@ namespace mbts.spark {
                RaisePropertyChanged(e.PropertyName);
          }
          void on_value_prop_changed(object sender, PropertyChangedEventArgs e) {
-            if (_value_echo_props.Contains(e.PropertyName))
-               RaisePropertyChanged(e.PropertyName);
+            if (_value_echo_props.TryGetValue(e.PropertyName, out string wrapper_prop))
+               RaisePropertyChanged(wrapper_prop);
          }
       }
 

src/client/spark/Modules/Trading/AlgoStudio/View/AlgoEditorGridView/AlgoEditorGridView.xaml.cs

@@ -52,6 +52,8 @@ namespace mbts.ui.spark {
          // AlgoParamRows
          AlgoParamRowsViewSource = new CollectionViewSource<ListCollectionView<AlgoParamRowVM>>();
          BindingOperations.SetBinding(AlgoParamRowsViewSource, CollectionViewSource.SourceProperty, new Binding(nameof(AlgoParamRows)) { Source = this });
+         AlgoParamRowsViewSource.IsLiveFilteringRequested = true;
+         AlgoParamRowsViewSource.LiveFilteringProperties.Add(nameof(AlgoParamRowVM.permissions));
          AlgoParamRowsViewSource.Filter += param_rows_filter;
          DataGridEditHelper.set_hooks(this);
          void param_rows_filter(object sender, FilterEventArgs e) =>

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -233,7 +233,7 @@ namespace mbts.spark {
          public OrderType order_type => _order.order_type();
          public string display_name => OrderUtils.display_name(_order, _fill_tuple.algo);
          public OrderMetadata order_metadata => _order_metadata ?? (_order_metadata = _model.factory.get_metadata(_order.Def.CustomDef.CustomCase));
-         public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(_order, _fill_tuple.algo_meta, order_metadata));
+         public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(_fill_tuple.algo_meta, order_metadata));
          public string account { get; private set; }
          public string user => _order.user();
          public Venue exchange => security.Exchange;

src/client/spark/Modules/Trading/Ladder/LadderModel.cs

@@ -2,6 +2,7 @@
 using bts.api;
 using bts.api.core;
 using bts.api.orderman;
+using bts.md;
 using mbts.co2;
 using System;
 using System.Threading.Tasks;
@@ -10,12 +11,14 @@ namespace mbts.spark {
    class LadderModel {
       readonly OrdersModel _order_gateway;
 
-      public LadderModel(SecurityMD source, OrdersModel ordergw, Func<OrderSecurity, bool> order_filter = null) {
+      public LadderModel(SecurityMD source, OrdersModel ordergw, MBOService mbo, Func<OrderSecurity, bool> order_filter = null) {
          security_md = source;
          _order_gateway = ordergw;
          orders = ordergw.on_security_working_orders_with_piq(source.security.Id);
-         if (order_filter != null)
+         this.mbo_service = mbo;
+         if (order_filter != null) {
             orders = orders.Filter(order_filter, o => o.order.Key);
+		 }
       }
 
       public enum CancelType { All, Bids, Asks }
@@ -26,6 +29,8 @@ namespace mbts.spark {
       public OrdersModel order_gateway => _order_gateway;
       public IObservable<TableUpdate<OrderSecurity, uint>> orders { get; }
 
+      public MBOService mbo_service { get; }
+
       public Task cancel_orders(CancelType ct) {
          switch (ct) {
             case CancelType.All: return _order_gateway.cancel_security_orders(security.Id);

src/client/spark/Modules/Trading/Ladder/LadderWindowFactory.cs

@@ -1,4 +1,5 @@
 using bts.api;
+using bts.md;
 using mbts;
 using mbts.config;
 using mbts.persist;
@@ -21,17 +22,26 @@ namespace bts.shell {
       readonly LadderConfig _ladder_conf;
       readonly IBus _bus;
       readonly IWindowManager _wm;
+      readonly MBOService _mbo;
       readonly UserSettingsHub _settings_hub;
       readonly OrdersModel _orders_model;
       IReadOnlyDictionary<uint, SecurityMD> _securities = new Dictionary<uint, SecurityMD>();
 
-      public LadderWindowFactory(IShellModule owner, IBus bus, IWindowManager wm, UserSettingsHub settings_hub, OrdersModel orders_model)
+      public LadderWindowFactory(
+         IShellModule owner, 
+         IBus bus, 
+         IWindowManager wm, 
+         UserSettingsHub settings_hub, 
+         OrdersModel orders_model,
+         MBOService mbo)
+
          : base(owner, "SparkLadder") {
          _bus = bus;
          _wm = wm;
          _settings_hub = settings_hub;
          _orders_model = orders_model;
          _ladder_conf = new LadderConfig(Configuration.get<bts.config.LadderKnobs>(), Configuration.get<bts.config.MultiThreadedWindows>().ladder);
+         _mbo = mbo;
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s),
             bus.subscribe(SparkTags.ShowLadderTag).OnDispatcher().Subscribe(s => show_windows(new[] { s })),
@@ -54,7 +64,7 @@ namespace bts.shell {
       IShellWindow create_ladder_window(SecurityMD md, LadderSettings state_settings) {
          var id = Interlocked.Increment(ref _ladder_count) % Environment.ProcessorCount;
          var view = new Lazy<UIElement>(() => {
-            var model = new LadderModel(md, _orders_model);
+            var model = new LadderModel(md, _orders_model, _mbo);
             var settings_manager = new WindowSettingsManager<LadderSettings>(
                _settings_hub, state_settings == null || state_settings.use_defaults ? null : state_settings,
                us => sanitize_ladder_settings(us.ladder),

src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs

@@ -44,6 +44,9 @@ namespace mbts.spark {
       double _real_price = double.NaN, _phantom_price = double.NaN;
       uint filled, quantity, num_root_orders;
       int working_quantity;
+      uint? _min_piq = UInt32.MaxValue;
+
+      public event Action piq_changed;
 
       LadderModel model => _owner.model;
       Dictionary<uint, (OrderRecordImmutable order, uint? p, AlgoData? algo, AlgoDefinitionMetadata? meta)> _orders = new Dictionary<uint, (OrderRecordImmutable order, uint? p, AlgoData? algo, AlgoDefinitionMetadata? meta)>();
@@ -166,12 +169,32 @@ namespace mbts.spark {
          RaisePropertyChanged(nameof(text_lines));
       }
 
+      /// <summary>
+      /// Add Order performs an add the first time it's called via the constructor. When it's called, directly
+      /// it really serves as an update/replace. Also note that since the min_piq is a separate piece of state representing
+      /// the lowest piq of the orders the badge represents, we must reevaluate if that has changed on said update.
+      /// </summary>
       public void add_order((OrderRecordImmutable order, uint? p, AlgoData? algo, AlgoDefinitionMetadata? meta) new_op) {
          _orders[new_op.order.Key] = new_op;
+         // reevaluate min piq if this order's piq is less
+         if (new_op.p.HasValue) {
+            // if this lovm only represents 1 order and the piq is now different, regardless of what it is that's our piq
+            if (_orders.Count == 1 && new_op.p.Value != min_piq) {
+               min_piq = new_op.p.Value;
+            } else if (new_op.p.Value < min_piq) {
+               min_piq = new_op.p.Value;
+            }
+         }
       }
 
       public void remove_order(uint order_key) {
          _orders.Remove(order_key);
+         // on a remove, potentially the order w/ the lowest piq has been removed, so
+         // if the min() is now > than the current min_piq, update the min_piq
+         var prospective_piq = _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min();
+         if (prospective_piq > min_piq) {
+            min_piq = prospective_piq;
+         }
       }
 
       //public void update(OrderRecordImmutable neworder) {
@@ -183,7 +206,16 @@ namespace mbts.spark {
 
       public uint security_key => _owner.security.Id;
       public double price { get => double.IsNaN(_phantom_price) ? _real_price : _phantom_price; }
-      public uint? min_piq => _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min();
+      public uint? min_piq {
+         get {
+            return _min_piq;
+         }
+         set {
+            if (Set(ref _min_piq, value)) {
+               piq_changed?.Invoke();
+            }
+         }
+      }
       public double real_price => _real_price;
       public Side side {
          get {

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -116,7 +116,7 @@ namespace mbts.spark {
          //_window_height = backing(nameof(window_height), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 600);
          //_window_width = backing(nameof(window_width), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 420);
          position_click_command = new DelegateCommand(() => size_calculator.value = DoubleHelper.safe_convert_to_uint(Math.Abs(position)));
-         send_order_command = new DelegateCommand<(double price, Side? side, bool size_from_alternate)>(send_order);
+         send_order_command = new DelegateCommand<(double price, Side? side, MouseButton button)>(send_order);
          cancel_command = new AsyncCommand<LadderModel.CancelType>(_model.cancel_orders, true);
          show_book_command = new DelegateCommand(_show_book);
          show_ticket_command = new DelegateCommand(_show_ticket);
@@ -232,7 +232,7 @@ namespace mbts.spark {
       // we return only entries who have piqs and whose piqs are not uint.Max
       // uint.Max happens when the server is restarted but the client is still up
       // maybe long term we don't show this data while the exchange connection is down
-      public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value));  
+      public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value));
       public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic;
 
       Property<string> _status;
@@ -348,13 +348,15 @@ namespace mbts.spark {
          }
       }
 
-      public async void send_order((double price, Side? side, bool size_from_alternate) args) {
+      public async void send_order((double price, Side? side, MouseButton buton) args) {
+         if (args.buton == MouseButton.Middle) return;
+         var size_from_alternate = args.buton == MouseButton.Right;
          if (order_properties_vm == null)
             return;
          var s = _settings_manager.current_settings();
-         if (DoubleHelper.IsNaNOrInfinity(args.price) || !args.side.HasValue || HasErrors || order_properties_vm.HasErrors || (args.size_from_alternate && s.alternate_order_size == 0))
+         if (DoubleHelper.IsNaNOrInfinity(args.price) || !args.side.HasValue || HasErrors || order_properties_vm.HasErrors || (size_from_alternate && s.alternate_order_size == 0))
             return;
-         else if (!args.size_from_alternate && (!size_calculator.value.HasValue || size_calculator.value.Value == 0)) {
+         else if (!size_from_alternate && (!size_calculator.value.HasValue || size_calculator.value.Value == 0)) {
             size_calculator.flash_error();
          } else {
             var order = order_properties_vm.create(args.side.Value);
@@ -371,7 +373,7 @@ namespace mbts.spark {
                }
             }
                
-            uint size = args.size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
+            uint size = size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
             await _model.order_gateway.send_order(order, args.price, size);
             if (auto_clear)
                size_calculator.value = 0;

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -1,8 +1,10 @@
 using bts;
 using bts.api;
 using bts.api.md.auxiliary;
+using bts.md;
 using bts.collections;
 using bts.utils;
+using bts.config;
 using mbts.co2;
 using mbts.ui;
 using mbts.ui.controls;
@@ -17,11 +19,13 @@ using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading.Tasks;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Input;
 using System.Windows.Media;
 using System.Windows.Media.Animation;
+using mbts.config;
 
 namespace mbts.spark {
    partial class LadderGridHost : Control {
@@ -56,7 +60,7 @@ namespace mbts.spark {
       LadderTactileAdorner _tactile_adorner;
       ulong _prices_updated;
       NativePriceSize[] _bids = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH], _asks = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH];
-      bool _price_dirty = false;
+      bool _price_dirty = false, _piq_dirty = false;
       double[] _price_list = null;
       IComparer<double> _price_comparer;
       double _last_origin_price;
@@ -257,11 +261,15 @@ namespace mbts.spark {
             case NotifyCollectionChangedAction.Add:
                foreach (LadderOrderVM item in e.NewItems) {
                   _ordershost.add_order(item);
+                  item.piq_changed += _set_piq_dirty;
+                  _set_piq_dirty();
                }
                break;
             case NotifyCollectionChangedAction.Remove:
                foreach (LadderOrderVM item in e.OldItems) {
                   _ordershost.remove_order(item);
+                  item.piq_changed -= _set_piq_dirty;
+                  _set_piq_dirty();
                }
                break;
             case NotifyCollectionChangedAction.Reset:
@@ -272,6 +280,9 @@ namespace mbts.spark {
                throw new NotImplementedException();
          }
       }
+
+      void _set_piq_dirty() { _piq_dirty = true; }
+
       #endregion
 
       Column[] columns { get; set; } = new Column[0];
@@ -313,25 +324,56 @@ namespace mbts.spark {
          Focus();
          var point = e.GetPosition(this);
          var column = _column_for_point(point);
-         if (column.column_type == ColumnType.Price && e.ChangedButton == MouseButton.Middle)
-            _scroll_to_center();
+         if (column.column_type == ColumnType.Price && e.ChangedButton == MouseButton.Middle) {
+            show_mbo_popup(point).ignore();
+         }
          else if (_settings != null) {
             var price = price_for_point(point);
             Side? bs = null;
             if (column.column_type == ColumnType.Bid) bs = Side.Buy;
             if (column.column_type == ColumnType.Ask) bs = Side.Sell;
-            var pbs = (price, bs, e.ChangedButton == MouseButton.Right);
+            var pbs = (price, bs, e.ChangedButton);
             if (Command != null && Command.CanExecute(pbs)) {
                if (bs.HasValue) {
                   using (_tactile_adorner.show_busy(_row_for_point(point), bs.Value == Side.Buy ? ColumnType.Bid : ColumnType.Ask))
                      await Command.ExecuteAsync(pbs);
-               } else
+               }
+               else {
                   Command.Execute(pbs);
+               }
             }
          }
          base.OnMouseDown(e);
       }
 
+      private async Task show_mbo_popup(Point point) {
+         var query = MBOPopupModel.default_query.with(
+            security_id: _vm.model.security.Id,
+            price_level: price_for_point(point)
+         );
+         var model = new MBOPopupModel(_vm.model.mbo_service, _vm.model.security, query);
+         // Load the first set of orders so we know if there are any before showing the popup
+         var result = await model.load_orders();
+         if (result.success && result.value.orders.Count > 0) {
+            var view = new MBOPopupView(model, result.value);
+            var popup = new AutoClosePopup {
+               PlacementTarget = this,
+               PlacementRectangle = new Rect(point, point),
+               IsOpen = true,
+               Child = new Border {
+                  BorderBrush = Brushes.Black,
+                  BorderThickness = new Thickness(0.5),
+                  Child = new ScrollViewer {
+                     VerticalScrollBarVisibility = ScrollBarVisibility.Hidden,
+                     HorizontalScrollBarVisibility = ScrollBarVisibility.Disabled,
+                     Content = view,
+                  }
+               }
+            };
+            view.RequestClose += (o, _) => popup.IsOpen = false;
+         }
+      }
+
       protected override void OnMouseDoubleClick(MouseButtonEventArgs e) {
          var column = _column_for_point(e.GetPosition(this));
          if (column.column_type == ColumnType.Price)
@@ -485,6 +527,7 @@ namespace mbts.spark {
             _prices_updated = 0;
             _price_comparer = Comparer<double>.Create((x, y) => security.ContractSpec.tick_equals(x, y) ? 0 : y.CompareTo(x));
             _price_dirty = true;
+            _piq_dirty = true;
             _vm.on_tick += _tick;
             _settings_sub = _vm.settings.OnDispatcher().Subscribe(_settings_updated);
             _volume_sub = _vm.model.order_gateway.md_auxiliary_model.security_volume_at_price(security.Id).Subscribe(t => _volume_container.write_value(t.state.Values));
@@ -580,8 +623,10 @@ namespace mbts.spark {
                if (depth_changed)
                   _refresh_sizes();
             }
-            // TODO - detect changes in PIQ to selectively redraw
-            _refresh_piq();
+            if (_piq_dirty) {
+               _refresh_piq();
+               _piq_dirty = false;
+            }
             _refresh_volume_at_price();
             _last_origin_price = _mkt_break_lines.origin;
             _price_dirty = false;
@@ -622,8 +667,7 @@ namespace mbts.spark {
          if (!_settings.piq_column.show)
             return;
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         var piq_list = _vm.price_vs_piq;
-         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, piq_list.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column);
+         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, _vm.price_vs_piq, _settings.row_height, security.ContractSpec, _settings.piq_column);
       }
 
       void _refresh_sizes() {

src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs

@@ -72,8 +72,7 @@ namespace mbts.spark {
                   panel.Children.Remove(item);
                }
             }
-         } //else
-            //Log.Ui.warn($"Failed to remove order ({order.id}): order not in {nameof(LadderOrdersHost)}'s {nameof(_orders)}.");
+         }
       }
 
       public void clear_orders() {
@@ -192,11 +191,8 @@ namespace mbts.spark {
                Margin = Badge.DefaultMargin,
                Orientation = Orientation.Horizontal,
                MaxWidth = _col_width
-               //MaxHeight = _row_height,
-               //ClipToBounds = true
             };
 
-            //var view_box = new Viewbox { Child = panel, Width = _col_width, Height = _row_height };
             _position_panel(panel, price);
             _stacks[price] = new PriceCellComponents(panel);
             Children.Add(panel);

src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs

@@ -288,22 +288,22 @@ namespace mbts.spark {
       }
 
       // this is basically a copy of redner_sizes up above
-      internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, (double price, uint piq)[] piqs, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) {
+      internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, IEnumerable<(double price, uint piq)> pvp, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) {
          using (var dc = dv.RenderOpen()) {
             using (_draw_with_clip(dc, settings, row_height, prices.Length)) {
-               if (!prices.within_range(origin_index) || piqs.is_null_or_empty())
+               if (!prices.within_range(origin_index) || pvp.is_null_or_empty())
                   return;
                var foreground = settings.foreground.rgb_brush();
                int curr_index = origin_index;
                double curr_price = prices[curr_index];
-               for (int i = 0; i < piqs.Length; i++) {
-                  double price = piqs[i].price;
+               foreach (var entry in pvp) {
+                  double price = entry.price;
                   // prices are in descending order, so get diff from curr_price - price
                   int tick_diff = spec.tickdiff_unchecked(curr_price, price);
                   curr_index += tick_diff;
                   curr_price = price;
                   if (curr_index >= 0 && curr_index < prices.Length) {
-                     var gr = format_text(piqs[i].piq.ToString(), curr_index, row_height, settings, glyph_context);
+                     var gr = format_text(entry.piq.ToString(), curr_index, row_height, settings, glyph_context);
                      dc.DrawGlyphRun(foreground, gr);
                   } else break;
                }

src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs

@@ -566,7 +566,7 @@ namespace mbts.spark {
 
          public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
             CellRender.draw_text(dc, _get_glyph_context(row), font_size, _get_text(row), position, row_height,
-               font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding);
+               RenderUtils.get_foreground(row, font_brush), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding);
       }
 
       #region Explicit Column Defs
@@ -598,12 +598,15 @@ namespace mbts.spark {
       }
 
       public class OrderBookOrderTypeColumn : OrderTypeColumn<Columns>, ICVOrderBookColumn {
+         readonly BindingBase _foreground_binding;
          ListSortDirection? _cached_sort_direction;
+
          public OrderBookOrderTypeColumn(Columns key, string badge_fill_binding_path, string display_name_binding_path, IComparer<OrderBookRowVM> row_comparer, IDataGridColumnPersistentFilter<OrderSecurityAlgoMeta, AlgoID[]> filter, string header = DefaultColumnHeaders.OrderType)
             : base(key, new Binding(badge_fill_binding_path), new Binding(display_name_binding_path), filter, header) {
             id = key.ToString();
             this.filter = filter;
             font_weight_binding = RenderUtils.get_font_weight_binding();
+            _foreground_binding = RenderUtils.get_foreground_binding(this);
             _cached_sort_direction = SortDirection;
             this.comparer = row_comparer;
          }
@@ -615,6 +618,12 @@ namespace mbts.spark {
          IDataGridColumnFilter<OrderSecurityAlgoMeta> ICVOrderBookColumn.filter => filter;
          ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; }
 
+         public override FrameworkElement view_factory() {
+            var view = base.view_factory();
+            view.SetBinding(OrderTypeItem.ForegroundProperty, _foreground_binding);
+            return view;
+         }
+
          protected override void OnSortDirectionChanged(ListSortDirection? new_value) {
             _cached_sort_direction = new_value;
             base.OnSortDirectionChanged(new_value);
@@ -646,7 +655,7 @@ namespace mbts.spark {
       }
 
       public class SecurityColumn : FrameworkElementColumnBase<Columns>, ICVOrderBookColumn, IFilterableColumn {
-         readonly BindingBase _font_weight_binding = RenderUtils.get_font_weight_binding();
+         readonly BindingBase _font_weight_binding = RenderUtils.get_font_weight_binding(), _foreground_binding;
          readonly IValueConverter _expander_visiblilty_converter = new ui.converters.BooleanToVisibilityConverter { VisibilityForFalse = Visibility.Hidden };
          readonly IValueConverter _margin_converter = new AnonymousValueConverter<int, Thickness>(level => new Thickness(10 * level, 0, 0, 0));
 
@@ -655,6 +664,7 @@ namespace mbts.spark {
          public SecurityColumn(IDataGridColumnPersistentFilter<OrderSecurityAlgoMeta, SecurityCriteria> filter) : base(Columns.Security) {
             id = key.ToString();
             this.filter = filter;
+            _foreground_binding = RenderUtils.get_foreground_binding(this);
             Header = DefaultColumnHeaders.Security;
             Width = DefaultColumnWidths.Security;
             _cached_sort_direction = SortDirection;
@@ -681,7 +691,7 @@ namespace mbts.spark {
             security_view.SetBinding(TextBlock.FontSizeProperty, new Binding(nameof(font_size)) { Source = this });
             security_view.SetBinding(TextBlock.TextProperty, nameof(OrderBookRowVM.security_text));
             security_view.SetBinding(TextBlock.HorizontalAlignmentProperty, new Binding(nameof(horizontal_alignment)) { Source = this });
-            security_view.SetBinding(TextBlock.ForegroundProperty, new Binding(nameof(font_brush)) { Source = this });
+            security_view.SetBinding(TextBlock.ForegroundProperty, _foreground_binding);
             security_view.SetBinding(TextBlock.FontFamilyProperty, new Binding(nameof(font_family)) { Source = this });
             security_view.SetBinding(TextBlock.FontWeightProperty, _font_weight_binding);
             //
@@ -1062,6 +1072,10 @@ namespace mbts.spark {
             popup.SetBinding(Popup.DataContextProperty, new Binding("PlacementTarget.DataContext") {
                RelativeSource = new RelativeSource(RelativeSourceMode.Self)
             });
+            popup.Closed += (s, e) => {
+               if (popup.try_find_ancestor_of_type(out MiniOrderBookView mini_ob) && mini_ob.DataContext is OrderBookVM ob_vm)
+                  ob_vm.is_attaching_orders = false;
+            };
 
             var the_whole_thing = new Grid();
             the_whole_thing.Children.Add(update_button);
@@ -1149,19 +1163,24 @@ namespace mbts.spark {
       static class RenderUtils {
          public static FontWeight
             root_font_weight = FontWeights.Normal,
-            child_font_weight = FontWeights.Light;
+            child_font_weight = FontWeights.Normal;
+         static readonly IMultiValueConverter _foreground_converter =
+            ValueConverter.create<bool, Brush, Brush>(_get_foreground);
 
          public static double font_size = FontSizes.Small;
 
          public static readonly IValueConverter
             _sub_type_text_converter = new ui.spark.converters.OrderSubTypeDisplayTextConverter(),
             _side_brush_converter = new AnonymousValueConverter<Side, Brush>(s => s == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask),
-            _font_weight_converter = new AnonymousValueConverter<bool, FontWeight>(is_child => is_child ? FontWeights.Light : FontWeights.Normal);
+            _font_weight_converter = new AnonymousValueConverter<bool, FontWeight>(is_child => is_child ? child_font_weight : root_font_weight);
+
          public static readonly IMultiValueConverter
             _price_text_converter = new ui.spark.converters.TickDisplayConverter(),
             _background_converter = new AnonymousMultiValueConverter<bool, OrderStatus, Brush>(
                (selected, status) => selected || status != OrderStatus.Working ? Brushes.Transparent : BTSColors.Orders.working_order_background_brush);
 
+         static readonly bool _animate_child_order_rows = mbts.config.Configuration.get<bts.config.OrderBookOptions>().animate_child_order_rows;
+
          public static BindingBase _get_background_binding() {
             var mb = new MultiBinding { Converter = _background_converter };
             mb.Bindings.Add(new Binding(nameof(ListBoxItem.IsSelected)) { RelativeSource = new RelativeSource(RelativeSourceMode.FindAncestor, typeof(ListBoxItem), 1) });
@@ -1169,9 +1188,29 @@ namespace mbts.spark {
             return mb;
          }
 
-         public static Brush get_side_brush(OrderBookRowVM r) => r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask;
-         public static BindingBase get_font_weight_binding() => new Binding(nameof(OrderBookRowVM.is_child)) { Converter = _font_weight_converter };
-      }
+         public static BindingBase get_foreground_binding(ColumnBase col) {
+            var col_font_brush_binding = new Binding(nameof(ColumnBase.font_brush)) { Source = col };
+            if (!_animate_child_order_rows)
+               return col_font_brush_binding;
+            var mb = new MultiBinding { Converter = _foreground_converter };
+            mb.Bindings.Add(new Binding(nameof(OrderBookRowVM.is_child)));
+            mb.Bindings.Add(col_font_brush_binding);
+            return mb;
+         }
 
+         public static Brush get_side_brush(OrderBookRowVM r) => get_foreground(r, r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask);
+         public static BindingBase get_font_weight_binding() => new Binding(nameof(OrderBookRowVM.is_child)) { Converter = _font_weight_converter };
+
+         public static Brush get_foreground(OrderBookRowVM row, Brush brush) =>
+            _animate_child_order_rows ? _get_foreground(row.is_child, brush) : brush;
+
+         static Brush _get_foreground(bool is_child, Brush brush) {
+            if (is_child) {
+               brush = brush.Clone();
+               brush.Opacity = 0.75;
+            }
+            return brush;
+         }
+      }
    }
 }

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -58,7 +58,7 @@ namespace mbts.spark {
       public OrderSecurityAlgoMeta order_tuple { get; private set; }
       public OrderRecordImmutable order => order_tuple.order;
       public OrderMetadata order_metadata => _owner.get_metadata(this);
-      public Brush badge_fill => _badge_fill ?? (_badge_fill = order_tuple.badge_fill(order_metadata));
+      public Brush badge_fill => _badge_fill ?? (_badge_fill = OrderUtils.badge_fill(order_tuple.meta, order_metadata));
       public uint order_key => order.Key;
       public uint parent_order_key => order.ParentKey;
       public uint security_key => order.security_key();
@@ -201,7 +201,7 @@ namespace mbts.spark {
 
       // private props
       Color _root_algo_badge_color {
-         get => _root_algo_badge_color_backing ?? (_root_algo_badge_color_backing = order_tuple.root_badge_color(root_algo_order_metadata)).Value;
+         get => _root_algo_badge_color_backing ?? (_root_algo_badge_color_backing = OrderUtils.badge_color(order_tuple.root_meta, root_algo_order_metadata)).Value;
          set {
             if (_root_algo_badge_color_backing != value) {
                _root_algo_badge_color_backing = value;
@@ -282,7 +282,7 @@ namespace mbts.spark {
          // root badge fill
          if (_root_algo_badge_color_backing.HasValue) {
             // only need to invalidate it if it had a value (it is lazily instantiated)
-            _root_algo_badge_color = order_tuple.root_badge_color(root_algo_order_metadata);
+            _root_algo_badge_color = OrderUtils.badge_color(order_tuple.root_meta, root_algo_order_metadata);
             RaisePropertyChanged(nameof(root_algo_badge_fill));
          }
          // root display name

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -3,6 +3,7 @@ using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
+using bts.md;
 using bts.shell;
 using bts.shell.orderbook;
 using bts.spark;
@@ -64,7 +65,14 @@ namespace mbts.spark {
          _detachable_selected_orders = new HashSet<uint>();
       FilterData _cached_filter_data;
 
-      public OrderBookVM(IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model, IWindowSettingsManager<OrderBookGridSettings> settings_manager, bool rows_are_initially_expanded = false) {
+      public OrderBookVM(
+         IObservable<string> owner_title, 
+         IBus bus, 
+         INotificationService notification_service, 
+         OrdersModel model, 
+         IWindowSettingsManager<OrderBookGridSettings> settings_manager, 
+         bool rows_are_initially_expanded = false) {
+
          _bus = bus;
          _model = model;
          _selected_params_mon = new SelectedOrderParamsMon(model.dynamic_helper);
@@ -248,10 +256,7 @@ namespace mbts.spark {
                var order_tups = selected_rows.Select(r => r.order_tuple).ToList();
                var mod_algo_vm = attach_vm.algo_vm.algo_root;
                await Task.WhenAll(order_tups.Select(t => {
-                  var algo = t.order.Def.CustomDef.CloneToMutable();
-                  if (algo.CustomCase == CustomOrderDefinition.CustomOneofCase.Composite) {
-                     algo = algo.Composite.Definition;
-                  }
+                  var algo = t.order.Def.CustomDef.actual().CloneToMutable();
                   mod_algo_vm.assign_to(algo);
                   if (algo_template_key != 0) {
                      algo = new CustomOrderDefinition {
@@ -278,15 +283,14 @@ namespace mbts.spark {
             Disposable.replace(ref attach_vm_hooks,
                Disposable.Create(
                   connect_common_selected_params(attach_vm),
-                  _selected_params_mon.subscribe_algos().Subscribe(algos => {
+                  attach_vm.algo_vm.observe_algo_root().Subscribe(algo_root => {
                      bool show_price, show_size;
-                     using (var en = algos.Select(a => a.CustomCase).Distinct().GetEnumerator()) {
-                        if (show_price = show_size = en.MoveNext()) {
-                           do {
-                              if (show_price) show_price = AttachConf.show_price(en.Current);
-                              if (show_size) show_size = AttachConf.show_size(en.Current);
-                           } while ((show_price || show_size) && en.MoveNext());
-                        }
+                     if (algo_root == null) {
+                        show_price = false;
+                        show_size = false;
+                     } else {
+                        show_price = AttachConf.show_price(algo_root.order_type);
+                        show_size = AttachConf.show_size(algo_root.order_type);
                      }
                      attach_vm.price_permissions = bool_to_perm(show_price);
                      attach_vm.size_permissions = bool_to_perm(show_size);
@@ -295,7 +299,7 @@ namespace mbts.spark {
                   }),
                   // custom templates (created by user)
                   _model.algo_model.definitions.OnDispatcher().Subscribe(u =>
-                     AlgoCollectionHelper.update_persisted_definition_wrappers(attach_vm.algo_vm.algo_roots as ObservableKeyedCollection<IAlgoVM, AlgoKey>, u, mk_factory(true)))));
+                     AlgoCollectionHelper.update_persisted_definition_wrappers(attach_vm.algo_vm.algo_roots_index, u, mk_factory(true)))));
             // vm factory
             Func<AlgoData, AlgoDefinitionVM> mk_factory(bool connect) =>
                fd => {
@@ -878,7 +882,10 @@ namespace mbts.spark {
                return;
             Disposable.replace(ref _ladder_vm,
                _security_md.HasValue ?
-                  LadderVM.create_readonly(_owner._bus, new LadderModel(security_md.Value, _owner._model, new Func<OrderSecurity, bool>(osec => _selected_order_tree.Contains(osec.order.Key)))) :
+                  LadderVM.create_readonly(
+                     _owner._bus, 
+                     new LadderModel(security_md.Value, _owner._model, _owner._model.mbo_service,
+                        new Func<OrderSecurity, bool>(osec => _selected_order_tree.Contains(osec.order.Key)))) :
                   null);
             RaisePropertyChanged(nameof(ladder_vm));
          }

src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml

@@ -4,13 +4,12 @@
   x:Name="_grid"
   xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
   xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:order="clr-namespace:bts.api.orderman;assembly=spark_api"
   xmlns:sp="clr-namespace:mbts.spark"
-  xmlns:u="clr-namespace:mbts.ui"
   xmlns:c="clr-namespace:mbts.ui.controls"
   xmlns:usp="clr-namespace:mbts.ui.spark"
   xmlns:uspcv="clr-namespace:mbts.ui.spark.converters"
   xmlns:sys="clr-namespace:System;assembly=mscorlib"
+  InactiveSelectionOpacity="0.3"
 >
   <c:DIYGrid.Resources>
     <ResourceDictionary>
@@ -109,63 +108,4 @@
     <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
     <KeyBinding Key="Return" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
   </c:DIYGrid.InputBindings>
-  <c:DIYGrid.RowStyle>
-    <Style TargetType="ListBoxItem" BasedOn="{StaticResource DIYGridRowStyle}">
-      <Style.Triggers>
-        <!-- Completed order background -->
-        <MultiDataTrigger>
-          <MultiDataTrigger.Conditions>
-            <Condition Binding="{Binding IsSelected, RelativeSource={RelativeSource Self}}" Value="False"/>
-            <Condition Binding="{Binding status}" Value="{u:Enum Completed, Type=order:OrderStatus}"/>
-          </MultiDataTrigger.Conditions>
-          <MultiDataTrigger.Setters>
-            <Setter Property="Background" Value="{x:Static Member=completed_order_background_brush, MemberType={u:Nested u:BTSColors.Orders}}"/>
-          </MultiDataTrigger.Setters>
-        </MultiDataTrigger>
-        <!-- Error, RejectedBySpark, Disconnected order background -->
-        <MultiDataTrigger>
-          <MultiDataTrigger.Conditions>
-            <Condition Binding="{Binding IsSelected, RelativeSource={RelativeSource Self}}" Value="False"/>
-            <Condition Binding="{Binding status}" Value="{u:Enum Error, Type=order:OrderStatus}"/>
-          </MultiDataTrigger.Conditions>
-          <MultiDataTrigger.Setters>
-            <Setter Property="Background" Value="{x:Static Member=error_order_background_brush, MemberType={u:Nested u:BTSColors.Orders}}"/>
-          </MultiDataTrigger.Setters>
-        </MultiDataTrigger>
-        <MultiDataTrigger>
-          <MultiDataTrigger.Conditions>
-            <Condition Binding="{Binding IsSelected, RelativeSource={RelativeSource Self}}" Value="False"/>
-            <Condition Binding="{Binding status}" Value="{u:Enum RejectedBySpark, Type=order:OrderStatus}"/>
-          </MultiDataTrigger.Conditions>
-          <MultiDataTrigger.Setters>
-            <Setter Property="Background" Value="{x:Static Member=error_order_background_brush, MemberType={u:Nested u:BTSColors.Orders}}"/>
-          </MultiDataTrigger.Setters>
-        </MultiDataTrigger>
-        <MultiDataTrigger>
-          <MultiDataTrigger.Conditions>
-            <Condition Binding="{Binding IsSelected, RelativeSource={RelativeSource Self}}" Value="False"/>
-            <Condition Binding="{Binding status}" Value="{u:Enum Disconnected, Type=order:OrderStatus}"/>
-          </MultiDataTrigger.Conditions>
-          <MultiDataTrigger.Setters>
-            <Setter Property="Background" Value="{x:Static Member=error_order_background_brush, MemberType={u:Nested u:BTSColors.Orders}}"/>
-          </MultiDataTrigger.Setters>
-        </MultiDataTrigger>
-        <!-- Cancel order background -->
-        <MultiDataTrigger>
-          <MultiDataTrigger.Conditions>
-            <Condition Binding="{Binding IsSelected, RelativeSource={RelativeSource Self}}" Value="False"/>
-            <Condition Binding="{Binding status}" Value="{u:Enum Canceled, Type=order:OrderStatus}"/>
-          </MultiDataTrigger.Conditions>
-          <MultiDataTrigger.Setters>
-            <Setter Property="Background" Value="{x:Static Member=canceled_order_background_brush, MemberType={u:Nested u:BTSColors.Orders}}"/>
-          </MultiDataTrigger.Setters>
-        </MultiDataTrigger>        
-      </Style.Triggers>
-    </Style>
-  </c:DIYGrid.RowStyle>
-  <c:DIYGrid.RowTemplate>
-    <DataTemplate>
-      <sp:OrderBookRowView/>
-    </DataTemplate>
-  </c:DIYGrid.RowTemplate>
 </c:DIYGrid>

src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml.cs

@@ -1,4 +1,11 @@
+using bts.api.orderman;
+using mbts.ui;
 using mbts.ui.controls;
+using mbts.ui.converters;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Data;
+using System.Windows.Media;
 
 namespace mbts.spark {
    internal partial class OrderBookGridView : DIYGrid {
@@ -45,8 +52,49 @@ namespace mbts.spark {
          RootAlgoType = 1 << 29
       }
 
-      public OrderBookGridView() =>
+      public OrderBookGridView() {
          InitializeComponent();
+         var row_template_factory = new FrameworkElementFactory(typeof(OrderBookRowView));
+         row_template_factory.SetValue(OrderBookRowView.BackgroundProperty, RowBackgroundHelper.row_background_binding);
+         RowTemplate = new DataTemplate { VisualTree = row_template_factory };
+      }
+
+      static class RowBackgroundHelper {
+         public static readonly BindingBase row_background_binding;
+
+         static RowBackgroundHelper() {
+            var bg_binding = new MultiBinding();
+            bg_binding.Bindings.Add(new Binding(nameof(OrderBookRowVM.status)));
+            bg_binding.Bindings.Add(new Binding(nameof(ListBoxItem.IsSelected)) { RelativeSource = new RelativeSource(RelativeSourceMode.FindAncestor, typeof(ListBoxItem), 1) });
+            double overlay_opacity = config.Configuration.get<bts.config.OrderBookOptions>().status_color_opacity_when_selected;
+            if (overlay_opacity > 0.1 && overlay_opacity < 0.9) {
+               bg_binding.Converter = ValueConverter.create<OrderStatus, bool, Brush>((status, is_selected) => {
+                  var color = _status_color(status);
+                  return color.HasValue ? new SolidColorBrush(color.Value) { Opacity = is_selected ? overlay_opacity : 1.0 } : null;
+               });
+            } else {
+               bg_binding.Converter = ValueConverter.create<OrderStatus, bool, Brush>((status, is_selected) => {
+                  if (is_selected)
+                     return null;
+                  var color = _status_color(status);
+                  return color.HasValue ? new SolidColorBrush(color.Value) : null;
+               });
+            }
+            row_background_binding = bg_binding;
+         }
+
+         static Color? _status_color(OrderStatus status) {
+            switch (status) {
+               case OrderStatus.Canceled: return BTSColors.Orders.canceled_order_background;
+               case OrderStatus.Completed: return BTSColors.Orders.completed_order_background;
+               case OrderStatus.Disconnected:
+               case OrderStatus.RejectedBySpark:
+               case OrderStatus.Error:
+                  return BTSColors.Orders.error_order_background;
+               default: return null;
+            }
+         }
+      }
    }
 
    class OrderBookRowView : DIYGrid.HighPerformanceRow<OrderBookRowVM, OrderBookGridView.ColumnKeys> {

src/client/spark/Modules/Trading/TradingShellModule.cs

@@ -6,6 +6,7 @@ using mbts.ui;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using System.Windows.Media;
 using static mbts.ui.shell.ProtoBufSerializationHelper;
@@ -20,9 +21,17 @@ namespace bts.shell {
       AlgoStudioWindowFactory _aswf;
 
       [ImportingConstructor]
-      public TradingShellModule(IBus bus, IWindowManager wm, INotificationService notification_service, INotificationServiceDataCache notification_service_data_cache,
-         UserSettingsHub settings_hub, OrdersModel orders_model, PAndLModel pandl_model, ErrorReporter error_reporter) {
-         _lwf = new LadderWindowFactory(this, bus, wm, settings_hub, orders_model);
+      public TradingShellModule(
+            IBus bus, 
+            IWindowManager wm, 
+            INotificationService notification_service, 
+            INotificationServiceDataCache notification_service_data_cache,
+            UserSettingsHub settings_hub, 
+            OrdersModel orders_model, 
+            PAndLModel pandl_model,
+            md.MBOService mbo,
+            ErrorReporter error_reporter) {
+         _lwf = new LadderWindowFactory(this, bus, wm, settings_hub, orders_model, mbo);
          _obwf = new OrderBookWindowFactory(this, bus, notification_service, notification_service_data_cache, orders_model, settings_hub);
          _fwf = new FillsWindowFactory(this, bus, settings_hub, orders_model);
          _plwf = new PAndLWindowFactory(this, bus, settings_hub, pandl_model);
@@ -52,9 +61,8 @@ namespace bts.shell {
          yield return _aswf;
       }
 
-      public Task<IShellWindow> create(Stream load_stream) {
-         return dispatch_to_factory(load_stream, window_factories().ToLArray());
-      }
+      public Task<IShellWindow> create(Stream load_stream) =>
+         dispatch_to_factory(load_stream, window_factories().ToArray());
 
       public void Dispose() {
          foreach (var i in window_factories())

src/client/spark/Orders/OrderUtils.cs

@@ -240,22 +240,12 @@ namespace mbts.spark {
          }
          return false;
       }
-      public static Brush badge_fill(this OrderSecurityAlgoMeta order_sam, OrderMetadata order_meta) =>
-         badge_fill(order_sam.order, order_sam.meta, order_meta);
 
-      public static Brush badge_fill(OrderRecordImmutable order, AlgoDefinitionMetadata? algo_meta, OrderMetadata order_meta) {
-         if (algo_meta?.badge_color is Color c) {
-            return c.to_brush();
-         } else {
-            return order_meta.Color.rgb_brush();
-         }
-      }
+      public static Brush badge_fill(AlgoDefinitionMetadata? algo_meta, OrderMetadata order_meta) =>
+         new SolidColorBrush(badge_color(algo_meta, order_meta));
 
-      public static Color root_badge_color(this OrderSecurityAlgoMeta order_sam, OrderMetadata root_order_meta) =>
-         order_sam.root_meta?.badge_color is Color c ? c : root_order_meta.Color.rgb_color();
-
-      public static Brush root_badge_fill(this OrderSecurityAlgoMeta order_sam, OrderMetadata root_order_meta) =>
-         new SolidColorBrush(root_badge_color(order_sam, root_order_meta));
+      public static Color badge_color(AlgoDefinitionMetadata? algo_meta, OrderMetadata order_meta) =>
+         algo_meta?.badge_color ?? order_meta.Color.rgb_color();
 
       public static string order_collection_summary(double best_price, int best_price_volume, int total_volume, ContractSpecificationImmutable spec) {
          if (total_volume == 0)

src/client/spark/Orders/OrdersModel.cs

@@ -84,15 +84,16 @@ namespace mbts.spark {
          _security_orders_demuxer = new TableUpdateDemuxer<OrderRecordImmutable, uint, uint>(o => o.Key, o => o.security_key());
          _security_working_orders_with_piq_demuxer = new TableUpdateDemuxer<OrderSecurity, uint, uint>(o => o.order.Key, o => o.order.security_key());
          _security_positions_demuxer = new TableUpdateDemuxer<PositionUnitImmutable, int, uint>(o => o.RecordId, p => p.SecurityId);
-         var algo_meta_join = algo_model.definitions.LeftJoin<AlgoData, AlgoDefinitionMetadata, (AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>(
+         studio_algos_with_meta = algo_model.definitions.LeftJoin<AlgoData, AlgoDefinitionMetadata, (AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>(
             algo_model.metadata_service.subscribe_metadata().Table(m => m.id),
             d => d.key,
             m => m.id,
             (algo, k, meta) => (algo, meta),
             (algo, k) => (algo, null),
-            t => t.algo.key);
+            t => t.algo.key)
+            .Cache();
          var order_secs_algo_meta_join = on_orders.LeftJoin(
-            algo_meta_join,
+            studio_algos_with_meta,
             t => t.order.Key,
             am => am.algo.key,
             t => new AlgoKey(new bts.persist.algo.AlgoPersistedKey(t.order.algo_template_key())),
@@ -178,8 +179,10 @@ namespace mbts.spark {
       public IObservable<TableUpdate<FillTuple, string>> on_fill_tuples { get; }
       public IObservable<IReadOnlyDictionary<uint,OrderSecurity>> working_orders { get; }
       public IObservable<TableUpdate<PositionUnitImmutable, int>> on_positions { get; }
+      public IObservable<TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>> studio_algos_with_meta { get; }
 
       public AdminModel admin_model { get; }
+      public MBOService mbo_service => _mbo_service;
       public IAlgoStudioModel algo_model { get; }
 
       public IObservable<TableUpdate<OrderSecurity, uint>> on_security_working_orders_with_piq(uint security) =>
@@ -436,16 +439,18 @@ namespace mbts.spark {
             return rod;
          }).Select(r => r.last_update);
 
-      public static IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_all_algos(this OrdersModel orders_model) =>
-         orders_model.algo_model.definitions.MergeTable(Observable.Return(
-            new TableUpdate<AlgoData, AlgoKey>(
-               ImmutableDictionary<AlgoKey, AlgoData>.Empty.AddRange(orders_model.factory.default_algos(OrderDataHelper.all_order_types)),
-               Array.Empty<AlgoData>(),
-               Array.Empty<Change<AlgoData>>(),
-               Array.Empty<AlgoData>(),
-               d => d.key
+      public static IObservable<TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>> subscribe_all_algos(this OrdersModel orders_model) =>
+         orders_model.studio_algos_with_meta.MergeTable(Observable.Return(
+            new TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>(
+               ImmutableDictionary<AlgoKey, (AlgoData algo, AlgoDefinitionMetadata? meta)>.Empty.AddRange(orders_model.factory.default_algos(OrderDataHelper.all_order_types).Select(kv => new KeyValuePair<AlgoKey, (AlgoData algo, AlgoDefinitionMetadata? meta)>(kv.Key, (kv.Value, null)))),
+               Array.Empty<(AlgoData,AlgoDefinitionMetadata?)>(),
+               Array.Empty<Change<(AlgoData,AlgoDefinitionMetadata?)>>(),
+               Array.Empty<(AlgoData,AlgoDefinitionMetadata?)>(),
+               d => d.algo.key
             ).as_snapshot()));
 
+      //public static IObservable<TableUpdate<AlgoData, AlgoKey>> subscribe_all_algos_with_meta(this OrdersModel orders_model) =>
+
       public static async Task<OrderRecordImmutable?> order_from_key(this OrdersModel orders_model, uint order_key) {
          (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3));
          if (success && tu.state.TryGetValue(order_key, out var rec)) {

src/client/spark/Spark.csproj

@@ -18,9 +18,9 @@
     <RootNamespace>bts</RootNamespace>
     <LangVersion>7.2</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <WarningsAsErrors /> 
-    <Version>1.0.0</Version>   
-  </PropertyGroup> 
+    <WarningsAsErrors />
+    <Version>1.0.0</Version>
+  </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="../spark_api/spark_api.csproj" />
     <ProjectReference Include="../spark_core/spark_core.csproj" />
@@ -121,6 +121,7 @@
       <Generator>MSBuild:Compile</Generator>
     </None>
   </ItemGroup>
+
   <Target Name="diagnostic" AfterTargets="PreBuildEvent">
   </Target>
 </Project>

src/client/spark/core/Collections/ObservantDictionary.cs

@@ -42,7 +42,7 @@ namespace mbts.collections
                foreach (TModel m in e.OldItems) _remove(m);
                break;
             case NotifyCollectionChangedAction.Reset:
-               var old_views = Values.ToLArray();
+               var old_views = Values.ToArray();
                Clear();
                foreach (TValue v in old_views) ValueDestroyed?.Invoke(v);
                foreach (TModel m in _models) _add(m);

src/client/spark/view/Controls/DIYGrid/DIYGrid.cs

@@ -242,6 +242,7 @@ namespace mbts.ui.controls {
       public static readonly SolidColorBrush DefaultSelectionHighlight = new SolidColorBrush(Colors.MediumSlateBlue) { Opacity = 0.4 };
       public static readonly Pen DefaultSelectionStroke = new Pen(Brushes.MediumSlateBlue, 1);
       readonly NotifyCollectionChangedEventHandler _sort_descriptions_collection_changed_cb;
+      double _inactive_selection_opacity = 0.2;
       DataContextHooks _data_context_hooks;
 
       public static readonly DependencyProperty SelectionHighlightProperty =
@@ -249,7 +250,7 @@ namespace mbts.ui.controls {
             nameof(SelectionHighlight),
             (o, v) => {
                o.Resources[ActiveSelectionHighlightKey] = v;
-               o.Resources[InactiveSelectionHighlightKey] = new SolidColorBrush(v.Color) { Opacity = 0.2 }.frozen();
+               o._reset_inactive_selection_brush();
                o.Resources[SelectionStrokeKey] = new Pen(o.SelectionHighlight.Color.to_solid_brush().frozen(), 1).frozen();
                foreach (var col in o.Columns.empty_if_null().OfType<ISelectionHighlightListener>())
                   col.on_selection_fill(v);
@@ -322,14 +323,24 @@ namespace mbts.ui.controls {
                }
             }
          }));
-         Resources[InactiveSelectionHighlightKey] = new SolidColorBrush(SelectionHighlight.Color) { Opacity = 0.2 };
          Resources[ActiveSelectionHighlightKey] = SelectionHighlight;
+         _reset_inactive_selection_brush();
          _data_context_hooks = new DataContextHooks(this, DataContext);
          DataContextChanged += OnDataContextChanged;
       }
 
       public event DIYGridSortingEventHandler Sorting;
 
+      public double InactiveSelectionOpacity {
+         get => _inactive_selection_opacity;
+         set {
+            if (value != _inactive_selection_opacity && value.is_within_range(0.1, 0.9)) {
+               _inactive_selection_opacity = value;
+               _reset_inactive_selection_brush();
+            }
+         }
+      }
+
       public ItemContainerGenerator RowContainerGenerator =>
          _row_presenter.ItemContainerGenerator;
 
@@ -376,6 +387,18 @@ namespace mbts.ui.controls {
       void OnDataContextChanged(object sender, DependencyPropertyChangedEventArgs e) =>
          Disposable.replace(ref _data_context_hooks, new DataContextHooks(this, e.NewValue));
 
+      void _reset_inactive_selection_brush() {
+         try {
+            if (TryFindResource(ActiveSelectionHighlightKey) is SolidColorBrush brush) {
+               var clone = brush.Clone();
+               if (_inactive_selection_opacity < clone.Opacity) {
+                  clone.Opacity = _inactive_selection_opacity;
+               }
+               Resources[InactiveSelectionHighlightKey] = clone;
+            }
+         } catch { }
+      }
+
       #region Column Handling
       private void invalidate_column_layout(object sender = null, EventArgs args = null) {
          var geometry = GetColumnGeometry().ToList();
@@ -591,7 +614,7 @@ namespace mbts.ui.controls {
             _row_presenter.ItemContainerStyle = _row_style;
       }
 
-      public Style RowStyle { get => GetValue(RowStyleProperty) as Style; set => SetValue(StyleProperty, value); }
+      public Style RowStyle { get => GetValue(RowStyleProperty) as Style; set => SetValue(RowStyleProperty, value); }
       #endregion
 
       public static readonly DependencyProperty CurrentRowProperty = DependencyProperty.Register(
@@ -915,4 +938,4 @@ namespace mbts.ui.controls {
             }
          });
    }
-}
+}

src/client/spark/view/Controls/DIYGrid/Row/HighPerformanceRow.cs

@@ -11,7 +11,22 @@ using System.Windows.Media;
 
 namespace mbts.ui.controls {
    partial class DIYGrid {
-      public class HighPerformanceRow<TRowVM, TColKey> : FrameworkElement where TRowVM : class, IRowVM {
+      public class HighPerformanceRowBase : FrameworkElement {
+         protected Brush cached_background;
+
+         public static readonly DependencyProperty BackgroundProperty =
+            WPFHelper.create_dp<Brush, HighPerformanceRowBase>(
+               nameof(Background),
+               FrameworkPropertyMetadataOptions.AffectsRender,
+               (o, v) => o.cached_background = v);
+
+         public Brush Background {
+            get => GetValue(BackgroundProperty) as Brush;
+            set => SetValue(BackgroundProperty, value);
+         }
+      }
+
+      public class HighPerformanceRow<TRowVM, TColKey> : HighPerformanceRowBase where TRowVM : class, IRowVM {
          double _row_height;
          ObservableCollection<ColumnBase> _columns;
          List<ColumnGeometry> _column_layout = new List<ColumnGeometry>();
@@ -85,6 +100,9 @@ namespace mbts.ui.controls {
             _element_cells.VisualAtIndex(index);
 
          protected override void OnRender(DrawingContext dc) {
+            if (cached_background != null) {
+               dc.DrawRectangle(cached_background, null, new Rect(0, 0, ActualWidth, ActualHeight));
+            }
             if (_row != null) {
                foreach (var cell in _column_layout) {
                   if (!cell.column.is_collapsed && cell.column is IRenderColumn<TRowVM> rc)

src/client/spark/view/Controls/DataGridEx/Filters/VM/DiscreteValuesFilterVM.cs

@@ -3,17 +3,14 @@ using bts.collections;
 using bts.utils;
 using mbts.collections;
 using mbts.ui.shell;
-using mbts.ui.utilities;
 using mbts.ui.utility;
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.ComponentModel;
 using System.Linq;
 using System.Reactive.Linq;
-using System.Windows.Data;
 
 namespace mbts.spark.filters {
    static class DiscreteValuesFilterVM {
@@ -87,7 +84,7 @@ namespace mbts.spark.filters {
          public IReadOnlyCollection<TValue> items { get; }
          public ObservableCollectionEx<TValue> selected_items { get; } = new ObservableCollectionEx<TValue>();
 
-         public override TValue[] get_state() => selected_items.ToLArray();
+         public override TValue[] get_state() => selected_items.ToArray();
          protected override void _clear() {
             _destroy_snapshot();
             selected_items.Clear();
@@ -147,10 +144,10 @@ namespace mbts.spark.filters {
          ) : this(get_row_val, get_state, load_state, prop_name, equality_comparer, comparer) {
             //
             _items_sub = items.OnDispatcher().Subscribe(update => {
-               foreach (var v in update.added)
-                  _items_set.Add(v);
                foreach (var v in update.removed.Except(_selected_items_set))
                   _items_set.Remove(v);
+               foreach (var v in update.added)
+                  _items_set.Add(v);
                _unsorted_items.sync_with(_items_set);
             });
          }
@@ -262,7 +259,7 @@ namespace mbts.spark.filters {
          readonly Func<TState, IEnumerable<TKey>> _load_state;
          readonly Func<IEnumerable<TKey>, TState> _get_state;
          readonly IEqualityComparer<TValue> _comparer;
-         readonly HashSet<TKey> _selected_keys = new HashSet<TKey>();
+         readonly HashSet<TKey> _selected_keys = new HashSet<TKey>(), _desired_keys = new HashSet<TKey>();
          ListCollectionView<TValue> _items_view;
 
          public TableUpdateItemsFilter(Func<TRow, TValue> factory, Func<TRow, TKey> row_key, Func<TValue, TKey> key, Func<TState, IEnumerable<TKey>> load_state, Func<IEnumerable<TKey>, TState> get_state)
@@ -290,6 +287,12 @@ namespace mbts.spark.filters {
                _items.Replace(_key(ch.Old), ch.New);
             foreach (var v in table.added)
                _items.Add(v);
+            var newly_selected_keys = _desired_keys.Where(_selection_model.try_select).ToList();
+            if (newly_selected_keys.Count > 0) {
+               _selected_keys.AddRange(newly_selected_keys);
+               _desired_keys.removeRange(newly_selected_keys);
+               _raise_filter_changed();
+            }
          }
 
          public override TState get_state() => _get_state(_selection_model.selected_items.Select(_key));
@@ -309,8 +312,10 @@ namespace mbts.spark.filters {
 
          protected override bool _filter(TValue value) => _selected_keys.Contains(_key(value));
          protected override void _load_state_internal(TState state) {
-            foreach (var k in _load_state(state).empty_if_null())
-               _selection_model.try_select(k);
+            foreach (var k in _load_state(state).empty_if_null()) {
+               if (!_selection_model.try_select(k))
+                  _desired_keys.Add(k);
+            }
          }
       }
       #endregion
@@ -337,4 +342,4 @@ namespace mbts.spark.filters {
          }
       }
    }
-}
+}

src/client/spark/view/Controls/DataGridEx/Filters/Views/FilterTemplates.xaml

@@ -163,6 +163,11 @@
         <c:ListBoxEx.GroupStyle>
           <GroupStyle ContainerStyle="{StaticResource ComboBox.DelineatedGroupContainerStyle}"/>
         </c:ListBoxEx.GroupStyle>
+        <c:ListBoxEx.ItemTemplate>
+          <DataTemplate>
+            <usp:OrderTypeItem DisplayName="{Binding name}" BadgeFill="{Binding badge_fill}" Margin="6,0,0,0"/>
+          </DataTemplate>
+        </c:ListBoxEx.ItemTemplate>
       </c:ListBoxEx>
     </usp:FilterContentControl>
   </DataTemplate>

src/client/spark/view/Controls/Order/AlgoTemplateToolBar.xaml

@@ -243,7 +243,7 @@
                   </MenuItem.Header>
                   <MenuItem.ToolTip>
                     <TextBlock>
-                      Make current paramaters the local defaults
+                      Make current paramaters the local defaults.
                       <LineBreak/>
                       <Italic>These values will be lost when the ladder closes.</Italic>
                     </TextBlock>

src/client/spark/view/Converters/EnumConverters/EnumCaseStringConverter.cs

@@ -1,28 +1,15 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using System.Windows.Data;
 
-namespace mbts.ui.converters
-{
-	public class EnumCaseStringConverter<T> : IValueConverter where T : struct
-	{
+namespace mbts.ui.converters {
+   class EnumCaseStringConverter<T> : IValueConverter where T : struct {
 		public object Convert(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) {
-			T t = default(T);
-			if (value is T)
-				t = (T)value;
-			string raw_str = t.ToString();
-			return raw_str.Substring(0, 1) + new String(raw_str.Skip(1).Select(Char.ToLower).ToLArray());
+			string raw_str = (value is T t ? t : default).ToString();
+			return raw_str.Substring(0, 1) + new string(raw_str.Skip(1).Select(char.ToLower).ToArray());
 		}
 
-		public object ConvertBack(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) {
-			T t = default(T);
-			string str = value as string;
-			if (str != null)
-				Enum.TryParse<T>(str, true, out t);
-			return t;
-		}
+      public object ConvertBack(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) =>
+         value is string str && Enum.TryParse(str, true, out T result) ? result : default;
 	}
-}
+}

src/client/spark/view/DragDrop/DragDropHelper.cs

@@ -106,12 +106,20 @@ namespace mbts.ui
                   var effect = DragDrop.DoDragDrop(_source, data, DragDropEffects.Move);
                }
                catch (Exception error) {  Log.Ui.warn("Dragdrop failed: " + error.ToString()); }
-               await show_document(_source, _source_window, true);
-               _source = null;
-               _source_window = null;
+               /// Reentrancy Risk:
+               ///   During the await period for <see cref="show_document(TabDocument, FloatingWindow, bool)"/>,
+               ///   the new window hooks up to the dragdrop events via <see cref="attach_dragdrop_events(ItemsControl)"/>.
+               ///   It must be assured that the new window does not get a callback into <see cref="container_PreviewMouseMove(ItemsControl, MouseEventArgs)"/>
+               ///   and attempt to create a second <see cref="DragDropDocumentPreview"/> for the same instance <see cref="_source"/>.
+               ///   For this reason, <see cref="_source"/> is set to null before <see cref="show_document(TabDocument, FloatingWindow, bool)"/> is called.
                _source_image = null;
                _target = null;
                _dddp = null;
+               var src = _source;
+               var src_win = _source_window;
+               _source = null;
+               _source_window = null;
+               await show_document(src, src_win, true);
             }
          }
       }

src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs

@@ -19,6 +19,21 @@ namespace bts.settings {
          var from_persist = new UserUISettings(user_list.Users.First(u => u.Login == from).Id, persist, null);
 
          var workspaces = from_persist.query<WorkspaceDefinition>(WorkspacesModel.WORKSPACE_NAMESPACE, ".*").Result;
+
+         ConsoleEx.WriteLine("Workspace Descriptions", ConsoleColor.Cyan);
+         foreach(var w in workspaces) {
+            var r = from ws in w.value.windows
+                    group ws by ws.owner into grouping
+                    select new { key = grouping.Key.Replace("bts.shell.", ""), window_count = grouping.Count() };
+            string desc = string.Empty;
+            foreach(var i in r) {
+               desc += $"{Environment.NewLine}\t{i.window_count} - {i.key}";
+            }
+            ConsoleEx.Write($"{w.key}", ConsoleColor.Yellow);
+            ConsoleEx.WriteLine($" - {w.value.windows.Count} windows - {w.value.named_windows.Count} named windows - {desc}");
+         }
+         ConsoleEx.WriteLine("");
+
          var selected_ws = ConsoleEx.prompt("Choose workspace:", workspaces.Select(w => w.key));
 
          var to = ConsoleEx.prompt("To User", user_names);

src/client/spark/view/Spark/Ladder/MBO/MBOPopupModel.cs

@@ -0,0 +1,133 @@
+using bts.api;
+using bts.api.core;
+using bts.md;
+using mbts.config;
+using mbts.ui.utility.text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   public struct MBOPopupData {
+      public readonly Side side;
+      public readonly IReadOnlyList<OrderPositionImmutable> orders;
+
+      public MBOPopupData(Side side, IReadOnlyList<OrderPositionImmutable> orders) {
+         this.side = side;
+         this.orders = orders;
+      }
+   }
+   public struct MBOPopupDrawParams {
+      public MBOPopupData data;
+      public double width => 300.0;
+      public double height;
+      public FormattedText header;
+
+      public static readonly Thickness margin = new Thickness(10, 5, 5, 5);
+
+      //columns
+      public double row_height => header.Height + 2;
+      public double bar_height => 0.6 * row_height;
+
+      public List<FormattedText> row_numbers;
+      public double numbers_width;
+      public List<FormattedText> row_labels;
+      public double scale;
+   }
+
+   public class MBOPopupModel {
+      const double font_size = 9.0;
+      readonly MBOService _mbo;
+      readonly SecurityImmutable _security;
+      readonly MBOSecurityQueryImmutable _query;
+      readonly Dictionary<string, FormattedText> _cache;
+      int _max_order_size = 1;
+      double _height = 0;
+      double _number_width = 0;
+
+
+      public MBOPopupModel(MBOService mbo, SecurityImmutable security, MBOSecurityQueryImmutable query) {
+         _mbo = mbo;
+         _security = security;
+         _query = query;
+
+         // prime the cache with numbers on the left hand side
+         _cache =
+            Enumerable.Range(1, 1+(int)default_query.Limit)
+            .AsParallel()
+            .Select(i => TextUtils.GetFormattedText(i.ToString(), font_size))
+            .ToDictionary(ft => ft.Text);
+      }
+
+      FormattedText format_text(string s, bool bold=false) {
+         // cache FormattedText for the life of the popup
+         if (_cache.TryGetValue(s, out FormattedText ft)) {
+            return ft;
+         }
+         else
+         ft = bold ? TextUtils.GetFormattedTextBold(s, font_size) : TextUtils.GetFormattedText(s, font_size);
+         _cache.TryAdd(s, ft);
+         return ft;
+      }
+
+      public async Task<Result<MBOPopupData>> load_orders() {
+         if (_mbo == null) return new Result<MBOPopupData>(ApiError.NoServer);
+         var data = await _mbo.get_orders_for_security_async(_query);
+         if (data.success) {
+            var side = data.value.side();
+            IEnumerable<OrderPosition> orders = new OrderPosition[0];
+            if (side == Side.Buy) orders = data.value.Bid;
+            if (side == Side.Sell) orders = data.value.Ask;
+            return new Result<MBOPopupData>(new MBOPopupData(side, orders.Select(o => (OrderPositionImmutable)o).ToList()));
+         }
+         else {
+            return new Result<MBOPopupData>(data.code);
+         }
+      }
+
+      string buy_sell_text(Side side) {
+         switch (side) {
+            case Side.Buy: return "BUY";
+            case Side.Sell: return "SELL";
+            default:
+            case Side.Undefined: return "";
+         }
+      }
+
+      public MBOPopupDrawParams compute_layout(MBOPopupData p) {
+         var pdp = new MBOPopupDrawParams { data = p };
+         pdp.header = format_text(
+            $"{buy_sell_text(p.side)} {_security.security_desc()} @ {_security.ContractSpec.format(_query.PriceLevel)}", true);
+
+         var margin = MBOPopupDrawParams.margin;
+         var text_margin = margin.Left + margin.Right;
+
+         pdp.row_numbers = p.orders.Select((o, i) => format_text((i+1).ToString())).ToList();
+         pdp.numbers_width = _number_width = Math.Max(_number_width, (int)pdp.row_numbers.Max(r => r.Width) + text_margin);
+         pdp.row_labels = p.orders.Select(o => format_text(o.Size.ToString())).ToList();
+         var max_label_width = pdp.row_labels.Max(l => l.Width) + text_margin;
+
+         var available_width = pdp.width - pdp.numbers_width - max_label_width;
+         _max_order_size = Math.Max(p.orders.Max(o => o.Size), _max_order_size);
+         pdp.scale = available_width / _max_order_size;
+
+         pdp.height = _height =
+            Math.Max(_height,
+            pdp.header.Height + margin.Top + margin.Bottom + (pdp.row_height * p.orders.Count) + margin.Bottom);
+
+         return pdp;
+      }
+
+      #region static
+      public static readonly MBOSecurityQueryImmutable default_query = (MBOSecurityQueryImmutable) 
+         new MBOSecurityQuery {
+            Limit = Configuration.get<bts.config.LadderKnobs>().max_mbo_orders
+         };
+      #endregion
+   }
+
+}

src/client/spark/view/Spark/Ladder/MBO/MBOPopupView.cs

@@ -0,0 +1,86 @@
+using System.Windows;
+using System.Windows.Media;
+using System.Collections.Generic;
+using bts.api.orderman;
+using System.Linq;
+using bts.md;
+using bts;
+using mbts.ui;
+using System.Windows.Input;
+using System.Threading.Tasks;
+using System;
+using mbts.ui.utility.text;
+using bts.api;
+using System.Windows.Threading;
+
+namespace mbts.spark {
+   internal class MBOPopupView : UIElement, IRequestViewClose {
+      readonly Brush _background = BTSColors.LadderSpecial.default_column_background;
+      readonly MBOPopupModel _model;
+      MBOPopupDrawParams _dp;
+
+      public MBOPopupView(MBOPopupModel model, MBOPopupData p) {
+         _model = model;
+         _dp = model.compute_layout(p);
+         IsVisibleChanged += (o, _) => poll_orders();
+      }
+
+      private async void poll_orders() {
+         while (IsVisible) {
+            var result = await _model.load_orders();
+            if (result.success && result.value.orders.Count > 0) {
+               Log.Ui.diag($"MBOpopup got {result.value.orders.Count} orders");
+               _dp = _model.compute_layout(result.value);
+               InvalidateMeasure();
+               InvalidateVisual();
+            }
+            else RequestClose?.Invoke(this, null);
+            await Task.Delay(250);
+         }
+      }
+
+      public event EventHandler RequestClose;
+
+      protected override Size MeasureCore(Size availableSize) {
+         return new Size(_dp.width, _dp.height);
+      }
+
+      static Thickness margin => MBOPopupDrawParams.margin;
+      protected override void OnRender(DrawingContext dc) {
+         base.OnRender(dc);
+         // draw background
+         dc.DrawRectangle(_background, null, new Rect(0, 0, _dp.width, _dp.height));
+         // draw header
+         dc.DrawText(_dp.header, new Point(margin.Left, margin.Top));
+         // draw grid
+         draw_grid(dc, _dp, _dp.header.Height + margin.Top + margin.Bottom);
+      }
+
+      static Brush brush_for(Side side, OrderPositionImmutable order) {
+         if (order.OrderId != 0) return Brushes.Black;
+         switch (side) {
+            case Side.Buy: return BTSColors.MarketData.bid_brush;
+            case Side.Sell: return BTSColors.MarketData.ask_brush;
+            default:
+               return Brushes.DarkGray;
+         }
+      }
+
+      static void draw_grid(DrawingContext dc, MBOPopupDrawParams dp, double offset) {
+         for (int x = 0; x < dp.data.orders.Count(); ++x) {
+            // draw numbers
+            var number = dp.row_numbers[x];
+            var left_offset = dp.numbers_width - number.Width - 5;
+            dc.DrawText(number, new Point(left_offset, offset));
+            // draw rectangles
+            var order = dp.data.orders[x];
+            var rect = new Rect(dp.numbers_width, offset + (dp.row_height-dp.bar_height)/2, order.Size * dp.scale, dp.bar_height);
+            dc.DrawRectangle(brush_for(dp.data.side, order), null,rect);
+            // draw sizes
+            var label = dp.row_labels[x];
+            dc.DrawText(label, new Point(rect.Right+margin.Right, offset));
+            offset += dp.row_height;
+         }
+      }
+   }
+}

src/client/spark/view/Spark/Ladder/MBO/PositionDataExtensions.cs

@@ -0,0 +1,19 @@
+using bts.api;
+using bts.md;
+
+namespace mbts.spark {
+   static class PositionDataExtensions {
+      public static Side side(this PositionData pd) {
+         var bidlen = pd.Bid.Count;
+         var asklen = pd.Ask.Count;
+         if (bidlen > 0 && asklen == 0) {
+            return Side.Buy;
+         }
+         else if (asklen > 0 && bidlen == 0) {
+            return Side.Sell;
+         }
+         return Side.Undefined;
+      }
+   }
+
+}

src/client/spark/view/Utility/AutoClosePopup.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Input;
+
+namespace mbts.ui.controls {
+   class AutoClosePopup : System.Windows.Controls.Primitives.Popup {
+      public AutoClosePopup() {
+         StaysOpen = false;
+         CommandBindings.Add(new CommandBinding(
+            ApplicationCommands.Close, 
+            (o,_) => { IsOpen = false; }
+         ));
+         InputBindings.Add(new KeyBinding(ApplicationCommands.Close, Key.Escape, ModifierKeys.None));
+      }
+
+      public new FrameworkElement Child {
+         get => base.Child as FrameworkElement;
+         set {
+            var old_child = Child;
+            base.Child = value;
+            hook_child_events(value);
+         }
+      }
+
+      private void hook_child_events(FrameworkElement fe) {
+         MouseDown += OnMouseDown;
+         fe.MouseDown += OnMouseDown;
+         fe.PreviewMouseDown += OnMouseDown;
+         if (fe is IRequestViewClose vc) {
+            vc.RequestClose += (o,_) => IsOpen = false;
+         }
+      }
+
+      private void OnMouseDown(object sender, MouseButtonEventArgs e) {
+         if (sender == this && e.MiddleButton == MouseButtonState.Pressed) IsOpen = false;
+         else {
+            if (!e.Handled) {
+               IsOpen = false;
+            }
+         }
+      }
+   }
+}

src/client/spark_api/Co2/Co2NativeMethods.cs

@@ -67,6 +67,10 @@ namespace mbts.co2
       [DllImport("libspark")]
       internal unsafe static extern void md_consumer_get_system_md_symbols(raw_usersymbol_callback cb);
 
+      
+      [DllImport("libspark")]
+      internal unsafe static extern void dump_co2_memory_to_disk(IntPtr consumer, string path);
+
 
       //
       // Relay control

src/client/spark_api/Co2/MDConsumer.cs

@@ -37,7 +37,10 @@ namespace mbts.co2
       unsafe void init() {
          _raw_sec_cb = (key, buf, len, handle) => {
             var security = Co2NativeMethods.deserialize<Security>(buf, len);
-            var next = new SecurityAndHandle() { security = security, handle = new MDConsumerHandle(handle) };
+            var next = new SecurityAndHandle() { 
+               security = security, 
+               handle = new MDConsumerHandle(_consumer, security, handle)
+            };
             _securities.Add(next);
             _on_security?.Invoke(new List<SecurityAndHandle>(1) { next });
          };
@@ -67,7 +70,7 @@ namespace mbts.co2
       public MDConsumerHandle try_get_handle(security_key key) {
          Co2NativeMethods.BlitPointerTuple raw;
          if (Co2NativeMethods.md_consumer_get_handle(_consumer, key, out raw)) {
-            return new MDConsumerHandle(raw);
+            return new MDConsumerHandle(this._consumer, null, raw);
          }
          return null;
       }
@@ -77,7 +80,10 @@ namespace mbts.co2
          List<SecurityAndHandle> result = new List<SecurityAndHandle>();
          Co2NativeMethods.raw_security_callback cb = (key, buf, len, handle) => {
             var security = Co2NativeMethods.deserialize<Security>(buf, len);
-            result.Add(new SecurityAndHandle() { security = new Security(security), handle = new MDConsumerHandle(handle) });
+            result.Add(
+               new SecurityAndHandle() {
+                  security = new Security(security), 
+                  handle = new MDConsumerHandle(_consumer, security, handle) });
          };
          Co2NativeMethods.md_consumer_walk_security_list(_consumer, cb);
          return result;

src/client/spark_api/Co2/MDConsumerHandle.cs

@@ -156,13 +156,22 @@ namespace mbts.co2 {
       //
       // Internal
 
-      internal MDConsumerHandle(Co2NativeMethods.BlitPointerTuple tuple) {
+      internal MDConsumerHandle(
+         IntPtr consumer,
+         Security security,
+         Co2NativeMethods.BlitPointerTuple tuple) {
          _header = (ColdHeader*)tuple.cold_header.ToPointer();
          _cold = (ColdEntry*)tuple.cold_data.ToPointer();
 
          if (has_depth) {
             _hot_depth = (HotDepthEntry*)tuple.hot.ToPointer();
-            Log.assert(_hot_depth->key == _cold->security_key);
+            if (_hot_depth->key != _cold->security_key) {
+               var temp_path = Path.Combine(Path.GetTempPath(), "btsshm");
+               Co2NativeMethods.dump_co2_memory_to_disk(consumer, temp_path);
+               Log.assert(
+                  _hot_depth->key == _cold->security_key, 
+                  $"Please attach the contents of '${temp_path}' to the bug report you are about to file. security:{security?.Id} cold:{_cold->security_key} hot:{_hot->key}");
+            }
          }
          else {
             _hot = (HotEntry*)tuple.hot.ToPointer();

src/client/spark_api/api/Channels/QuickBusClientChannel.cs

@@ -67,8 +67,8 @@ namespace bts.api.core {
             while (true) {
                var response = await check_stream_access(method_id, qualifier);
                if (response.success) {
-                  if (response.value.Authorized) {
-                     // success. Break loop and subscribe
+                  if (response.value.Authorized) {
+                     // success. Break loop and subscribe
                      break;
                   }
                   else {
@@ -94,7 +94,8 @@ namespace bts.api.core {
             // Authorized. Stream events
             unsubscribe_temp_sub();
             return SubscribeNative(method, qualifier, parser, converter, observer);
-         });
+         });
+         Log.QBus.diag($"Subscribing for sub: '{method}/{qualifier}'");
          return result;
       }
       private Task<Result<StreamAccessResponse>> check_stream_access(int method_id, string qualifier) {
@@ -145,6 +146,7 @@ namespace bts.api.core {
          };
          var subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero);
          return FinalizeableAnonymousDisposable.Create((disposing) => {
+            Log.QBus.diag($"Unsubscribing for sub: '{method}/{qualifier}'");
             ApiNativeMethods.api_client_unsubscribe(_client, subscription);
             subscription = IntPtr.Zero;
             GC.KeepAlive(cb);

src/client/spark_api/utils/MathUtils.cs

@@ -53,6 +53,10 @@ namespace bts.utils {
          for (int i = 1; i < num2; i++) if ((num1 * i) % num2 == 0) return i * num1;
          return num1 * num2;
       }
+
+      public static bool is_within_range(this double value, double min, double max) =>
+         value >= min && value <= max;
+
       public static int within_range(int min, int max, int value) {
          if (value < min) return min;
          if (value > max) return max;

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -30,11 +30,8 @@ namespace System.Collections.Generic {
       public static IEnumerable<T> OfTypeSafe<T>(this IEnumerable source) =>
          source == null ? Enumerable.Empty<T>() : source.OfType<T>();
 
-      public static T[] ToLArray<T>(this IEnumerable<T> enumerable) =>
-         enumerable.ToList().ToArray();
-
       public static TAccumulate AggregateWhile<T, TAccumulate>(this IEnumerable<T> source, TAccumulate seed, Func<TAccumulate, T, TAccumulate> aggregate, Func<TAccumulate, T, bool> predicate) =>
-       AggregateWhile(source, seed, aggregate, predicate, (curr, el, next) => curr);
+         AggregateWhile(source, seed, aggregate, predicate, (curr, el, next) => curr);
 
       public static TAccumulate AggregateWhile<T, TAccumulate>(this IEnumerable<T> source, TAccumulate seed, Func<TAccumulate, T, TAccumulate> aggregate, Func<TAccumulate, T, bool> predicate, TAccumulate early_break_rv) =>
          AggregateWhile(source, seed, aggregate, predicate, (curr, el, next) => early_break_rv);

src/client/spark_core/TestRunner/TestAttributes.cs

@@ -17,7 +17,7 @@ namespace bts.test {
    }
    [AttributeUsage(AttributeTargets.Method)]
    public class TestAttribute : Attribute {
-      public uint Timeout = 10;
+      public uint Timeout = 10; //timeout in seconds
    }
 
    // Use constructor for setup

src/client/spark_sdk/spark_sdk.csproj

@@ -61,7 +61,7 @@
      <None Include="$(OutputPath)\net471\spark_excel_example_2018.xlsm" pack="true" PackagePath=".\excel\spark_excel_example_2018.xlsm" />
      <None Include="$(OutputPath)\net471\spark_excel-addin64-packed.xll" pack="true" PackagePath=".\excel\spark_excel-addin64-packed.xll" />
      <None Include="$(OutputPath)\net471\spark_excel-addin64.dna" pack="true" PackagePath=".\excel\spark_excel-addin64.dna" />
-     <None Include="../../../doc/user/content/bts_spark_sdk/Client/**" pack="true" PackagePath=".\doc\" />
+     <None Include="doc/**" pack="true" PackagePath=".\doc\" />
   </ItemGroup>
 
   <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences" Condition= " '$(SparkRoot)' != '' ">

src/client/spark_test/System/AlgoDefinitionTests.cs

@@ -0,0 +1,132 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.api.test;
+using bts.test;
+using bts.api.orderman;
+using mbts;
+using mbts.tests;
+using bts.co2;
+using bts.api.orderman.algo;
+
+namespace bts.tests {
+   [TestGroup(TestType = TestType.System)]
+   public class AlgoDefinitionTests : IDisposable {
+      List<IDisposable> _resources;
+      ServiceContext _context;
+      //OrderService _order_service;
+      AdminService _admin_service;
+      AlgoDefinitionService _algo_definition_service;
+      IEnumerable<AlgoDefinitionRecordImmutable> _non_deleted_defs;
+      Random _rng = new Random();
+
+      public AlgoDefinitionTests() {
+         _context = new ServiceContext();
+         //_order_service = _context.locate<OrderService>();
+         _admin_service = _context.locate<AdminService>();
+         _algo_definition_service = _context.locate<AlgoDefinitionService>();
+
+         _resources = new List<IDisposable> {
+            _context,
+            _admin_service,
+            _algo_definition_service,
+            _algo_definition_service.subscribei_algo_definitions().Subscribe(defs => _non_deleted_defs = defs.all.Where( x => x.Metadata.IsActive))
+         };
+
+         var auth = _context.login("bts", "");
+         auth.Wait(3000);
+      }
+
+      [Test(Timeout = 25)]
+      public async Task test_crud_algo_definitions() {
+         var created_algo_records = new List<AlgoDefinitionRecord>();
+         var watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
+         var user_data = await AdminUtilities.get_user_data(_admin_service, "bts");
+         int num_created = 0;
+         if (user_data.HasValue) {
+            // create 100 algo defs
+            while (num_created < 100) {
+               var result = await _algo_definition_service.save_algo_definition_async(
+                  new AlgoDefinitionUpdateRequest {
+                     UserId = user_data.Value.Id,
+                     Key = 0,
+                     Algo = generate_random_algo_definition()
+                  }
+               );
+               Assert.IsTrue(result.value.Success, "There was a problem creating an algo defintion");
+               created_algo_records.Add(result.value.Record);
+               num_created++;
+            }
+
+            // update the 100 we just created algo defs
+            foreach(var r in created_algo_records) {
+               var def = r.Algo;
+               def.Name += " modified";
+               await _algo_definition_service.save_algo_definition_async(
+                  new AlgoDefinitionUpdateRequest {  
+                     UserId = user_data.Value.Id,
+                     Key = r.Key,
+                     Algo = def
+                  }
+               );
+            }
+
+            // delete all non-deleted algo defs         
+            Assert.IsTrue(_non_deleted_defs.Count() > 0, "There are no algo definitions to delete.");
+            foreach (var i in _non_deleted_defs.Select(x => x.Key).ToList()) {
+               var result = await _algo_definition_service.remove_algo_definition_async(new AlgoDefinitionRemoveRequest { Key = i });
+               Assert.IsTrue(result.value.Success, "There was a problem removing an algo definition");
+            }
+            Assert.IsTrue(_non_deleted_defs.Count() == 0, "There was a problem removing all algo definitions");
+
+            watch.Stop();
+            Console.WriteLine($"Execution Time: {watch.ElapsedMilliseconds} ms");
+         }
+      }
+
+      // not quite random yet
+      private AlgoDefinition generate_random_algo_definition() {
+         return new AlgoDefinition {
+            Definition = new CustomOrderDefinition {
+               Iceberg = new IcebergOrderDefinition {
+                  ShowQuantity = _rng.NextDouble() * 100,
+                  Child = new CustomOrderDefinition {
+                     Limit = new LimitOrderDefinition { }
+                  }
+               },
+            },
+            Name = $"Test Algo Def: {generate_name(_rng, _rng.Next(4, 15))}"
+         };
+      }
+
+      // https://stackoverflow.com/questions/14687658/random-name-generator-in-c-sharp
+      public static string generate_name(Random r, int len) {
+         string[] consonants = { "b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "l", "n", "p", "q", "r", "s", "sh", "zh", "t", "v", "w", "x" };
+         string[] vowels = { "a", "e", "i", "o", "u", "ae", "y" };
+         string name = "";
+         name += consonants[r.Next(consonants.Length)].ToUpper();
+         name += vowels[r.Next(vowels.Length)];
+         int b = 2; //b tells how many times a new letter has been added. It's 2 right now because the first two letters are already in the name.
+         while (b < len) {
+            name += consonants[r.Next(consonants.Length)];
+            b++;
+            name += vowels[r.Next(vowels.Length)];
+            b++;
+         }
+         return name;
+      }
+
+      public void Dispose() {
+         Disposable.dispose(_resources);
+      }
+
+   }
+
+}

src/client/spark_test/System/IcebergAlgoTests.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.test;
+using bts.api.orderman;
+using mbts;
+using bts.co2;
+using bts.tests.utils;
+
+namespace bts.tests {
+
+   [TestGroup(TestType = TestType.System)]
+   class IcebergAlgoTests : TestBase {
+      public IcebergAlgoTests() { }
+
+      [Test(Timeout = 25)]
+      public async Task test_ib_create_and_takeout_market() {
+         // find a security with no market
+         var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0);
+         Assert.IsTrue(s.HasValue, "Could not find a security with no maket");
+
+         // create a market in the instrument
+         List<Result<Response>> orders_response_list = new List<Result<Response>>();
+         var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user);
+         var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user);
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 50));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.05, 50));
+         List<OrderRecordImmutable> test_orders = new List<OrderRecordImmutable>();
+         foreach (var i in orders_response_list) {
+            var iret = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+            Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'");
+            test_orders.AddRange(iret);
+         }
+
+         // create an IB that will cross this market
+         var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, _test_account, _test_user, 3);
+         var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 55.05, 30);
+         var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 1,
+            rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         var ib_child_orders = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 10,
+            rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+
+         Assert.IsTrue(ib_filtered_order.Count() == 1, "This IB order should have been 'Completed'");
+         Assert.IsTrue(ib_child_orders.Count() == 10, "There should have been 10 completed child orders");
+
+         var filled_childen = (await OrderUtilities.wait_for_filtered_fills(
+            _order_service,
+            rows => rows.Where(x => ib_child_orders.Select(o => o.Key).Contains(x.OrderKey)).Count() == 10,
+            rows => rows.Where(x => ib_child_orders.Select(o => o.Key).Contains(x.OrderKey)))).result ?? Enumerable.Empty<FillImmutable>();
+                       
+         Assert.IsTrue(filled_childen.Count() == 10, "There should have been 10 fills");
+         Assert.IsTrue(filled_childen.All(x => 
+            TestUtils.test_price_equality(x.LastPrice, 55.05, s.Value.ContractSpec.ticksize(x.LastPrice)) && x.LastShares == 3), "All child fills should have been filled at price 55.50 with quantity 3");
+
+         // cleanup
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders");
+      }
+
+      [Test(Timeout = 25)]
+      public async Task test_ib_avg_fill() {
+         // find a security with no market
+         var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0);
+         Assert.IsTrue(s.HasValue, "Could not find a security with no maket");
+
+         // create a market in the instrument
+         List<Result<Response>> orders_response_list = new List<Result<Response>>();
+         var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user);
+         var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user);
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.02, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.01, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 56.00, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.50, 5));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, 55.01, 20));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.99, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.98, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.97, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.96, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.95, 1));
+         List<OrderRecordImmutable> test_orders = new List<OrderRecordImmutable>();
+         foreach (var i in orders_response_list) {
+            var iret = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+            Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'");
+            test_orders.AddRange(iret);
+         }
+
+         // create an IB that will cross this market
+         var ib_order_def = OrderDefinitionUtilities.create_iceberg_order_def(s.Value.Id, Side.Buy, _test_account, _test_user, 1);
+         var ib_order_result = await OrderUtilities.send_order(_order_service, ib_order_def, 56.02, 26);
+         var ib_filtered_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 1,
+            rows => rows.Where(x => x.Key == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         var ib_child_orders = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey && x.HasState && x.State.Status == OrderStatus.Completed).Count() == 10,
+            rows => rows.Where(x => x.ParentKey == ib_order_result.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+
+         Assert.IsTrue(ib_filtered_order.Count() == 1, "This IB order should have been 'Completed'");
+         Assert.IsTrue(ib_filtered_order.All(x =>
+            TestUtils.test_price_equality(x.State.AvgPrice, 55.1423, s.Value.ContractSpec.ticksize(x.State.AvgPrice))), "IB avg price should be 55.1423");
+         
+         // cleanup
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders");
+      }
+
+   }
+}

src/client/spark_test/System/JoinAlgoTests.cs

@@ -0,0 +1,194 @@
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.test;
+using bts.tests.utils;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts.tests {
+
+   /// <summary>
+   /// For buy orders:
+   ///    triggers when: curr_bid.price > initial_bid && current_bid.qty >= join_threshold.
+   /// i.e. When the join is first entered, it looks at the market to capture the initial bid and
+   /// then it waits until the current bid is above that level AND that the qty at that level is greater
+   /// that some threshold.
+   /// </summary>
+
+   [TestGroup(TestType = TestType.System)]
+   public class JoinAlgoTests : TestBase {
+      public JoinAlgoTests() { }
+
+      [Test(Timeout = 25)]
+      public async Task test_join_price() {
+         double initial_price = 54.99;
+         double ask_price = 55.01;
+
+
+         // find a security with no market
+         var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0);
+         Assert.IsTrue(s.HasValue, "Could not find a security with no maket");
+
+         // create a market in the instrument
+         List<Result<Response>> orders_response_list = new List<Result<Response>>();
+         var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user);
+         var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user);
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 10));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price, 1));
+         List<OrderRecordImmutable> test_orders = new List<OrderRecordImmutable>();
+         foreach (var i in orders_response_list) {
+            var iret = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+            Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'");
+            test_orders.AddRange(iret);
+         }
+
+         // create a join order
+         var join_order_def = OrderDefinitionUtilities.create_join_order_definition(s.Value.Id, Side.Buy, _test_account, _test_user, new spark.QuantityMeasure() { Absolute = 1 });
+         var join_order_response = await OrderUtilities.send_order(_order_service, join_order_def, initial_price, 1);
+         var join_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == join_order_response.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == join_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_order.Count() == 1, "The join order should have 'working'");
+         test_orders.Add(join_order.First());
+
+         // verify the join order's child is working at the specefied price
+         // note: the price doesn't actually matter in the join order
+         var join_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key && 
+               o.HasState && (o.State.Status == OrderStatus.Working &&
+               TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should have been 'working' at {initial_price}");
+
+         // now we move the best bid to become the best offer
+         var original_bid_order = test_orders.Where(o => o.Def.Side == Side.Buy).First();
+         var bid_order_response = await OrderUtilities.modify_order(_order_service, original_bid_order, ask_price);
+         var modified_bid_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey && o.HasState && 
+                  TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price))).Count() == 1,
+               rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(modified_bid_order.Count() == 1, "The working bid should have become the best offer");
+
+         // and we finally verify that the child order has joined the market at the appropriate price
+         join_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key &&
+               o.HasState && (o.State.Status == OrderStatus.Working &&
+               TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should now have been 'working' at {ask_price}");
+         test_orders.Add(join_child_order.First());
+
+         // cleanup
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders");
+
+      }
+
+      [Test(Timeout = 25)]
+      public async Task test_join_qty_threshold() {
+         double initial_price = 54.99;
+         double ask_price = 55.01;
+         uint ask_size = 1;
+         uint qty_threshold = 15;
+
+
+         // find a security with no market
+         var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0);
+         Assert.IsTrue(s.HasValue, "Could not find a security with no maket");
+
+         // create a market in the instrument
+         List<Result<Response>> orders_response_list = new List<Result<Response>>();
+         var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user);
+         var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user);
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 55.00, 10));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price, ask_size));
+         List<OrderRecordImmutable> test_orders = new List<OrderRecordImmutable>();
+         foreach (var i in orders_response_list) {
+            var iret = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+            Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'");
+            test_orders.AddRange(iret);
+         }
+
+         // create a join order
+         var join_order_def = OrderDefinitionUtilities.create_join_order_definition(s.Value.Id, Side.Buy, _test_account, _test_user, new spark.QuantityMeasure() { Absolute = qty_threshold });
+         var join_order_response = await OrderUtilities.send_order(_order_service, join_order_def, initial_price, 1);
+         var join_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == join_order_response.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == join_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_order.Count() == 1, "The join order should have 'working'");
+         test_orders.Add(join_order.First());
+
+         // verify the join order's child is working at the specefied price
+         // note: the price doesn't actually matter in the join order
+         var join_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key &&
+               o.HasState && (o.State.Status == OrderStatus.Working &&
+               TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should have been 'working' at {initial_price}");
+
+         // now we move the best bid to become the best offer
+         var top_bid_order = test_orders.Where(o => o.Def.Side == Side.Buy).First();
+         var bid_order_response = await OrderUtilities.modify_order(_order_service, top_bid_order, ask_price);
+         var modified_bid_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey && o.HasState &&
+                  TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price))).Count() == 1,
+               rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(modified_bid_order.Count() == 1, "The working bid should have become the best offer");
+
+         // verify the join order's child is *still* working at the initial price
+         // b/c the quantity threshold has not yet been met
+         join_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key &&
+               o.HasState && (o.State.Status == OrderStatus.Working &&
+               TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should still have been 'working' at {initial_price}");
+
+         // now we modify our top bid's volume to meet the join's threshold
+         // note that we must add the intial ask_size to the qty_threshold b/c we have partially filled our bid order
+         // when we moved the bid up.
+         var target_qty = qty_threshold + ask_size;
+         bid_order_response = await OrderUtilities.modify_order(_order_service, modified_bid_order.First(), target_qty);
+         modified_bid_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey && o.HasState && o.State.Quantity == target_qty).Count() == 1,
+               rows => rows.Where(o => o.Key == bid_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(modified_bid_order.Count() == 1, $"The working bid should have its quantity modified to {target_qty}");
+
+         // now that the qty threshold has been met, verify that the child order has joined the market at the appropriate price
+         join_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key &&
+               o.HasState && (o.State.Status == OrderStatus.Working &&
+               TestUtils.test_price_equality(o.State.Price, ask_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == join_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(join_child_order.Count() == 1, $"The join child order should now have been 'working' at {ask_price}");
+         test_orders.Add(join_child_order.First());
+
+         // cleanup
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders");
+      }
+
+
+   }
+
+}

src/client/spark_test/System/LeanAlgoTests.cs

@@ -0,0 +1,144 @@
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.test;
+using bts.tests.utils;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts.tests {
+
+   [TestGroup(TestType = TestType.System)]
+   public class LeanAlgoTests : TestBase {
+
+      public LeanAlgoTests() { }
+
+      [Test(Timeout = 25)]
+      public async Task test_lean_price() {
+         double initial_price = 54.98;
+         double ask_price_1 = 55.00;
+         double ask_price_2 = 55.01;
+         uint ask_size = 10;
+         uint lean_order_size = 30;
+         uint cross_threshold = ask_size - 1;
+
+         // find a security with no market
+         var s = await MarketDataUtilities.get_security(_md_relay, "CL", SecurityType.Future, update => update.Direct.Asks.Count == 0 && update.Direct.Bids.Count == 0);
+         Assert.IsTrue(s.HasValue, "Could not find a security with no maket");
+
+         // create a market in the instrument
+         List<Result<Response>> orders_response_list = new List<Result<Response>>();
+         var bid_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Buy, _test_account, _test_user);
+         var ask_order_def = OrderDefinitionUtilities.create_limit_order_def(s.Value.Id, Side.Sell, _test_account, _test_user);
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, bid_order_def, 54.99, 1));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_1, ask_size));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_2, ask_size));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_2 + 0.01, ask_size));
+         orders_response_list.Add(await OrderUtilities.send_order(_order_service, ask_order_def, ask_price_2 + 0.02, ask_size));
+         List<OrderRecordImmutable> test_orders = new List<OrderRecordImmutable>();
+         foreach (var i in orders_response_list) {
+            var iret = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == i.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == i.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+            Assert.IsTrue(iret.Count() == 1, "This order should have been 'working'");
+            test_orders.AddRange(iret);
+         }
+
+         // create a lean order, 2 tick chase (default)
+         var lean_order_def = OrderDefinitionUtilities.create_lean_order_definition(s.Value.Id, Side.Buy, _test_account, _test_user, new spark.QuantityMeasure() { Absolute = cross_threshold });
+         var lean_order_response = await OrderUtilities.send_order(_order_service, lean_order_def, initial_price, lean_order_size);
+         var lean_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == lean_order_response.value.OrderKey && o.HasState && o.State.Status == OrderStatus.Working).Count() == 1,
+               rows => rows.Where(o => o.Key == lean_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(lean_order.Count() == 1, "The lean order should have 'working'");
+         test_orders.Add(lean_order.First());
+
+         // verify the lean order's child is working at the specified price
+         // note: the price doesn't actually matter in the lean order
+         var lean_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == lean_order.First().Key &&
+               o.HasState && (o.State.Status == OrderStatus.Working &&
+               TestUtils.test_price_equality(o.State.Price, initial_price, s.Value.ContractSpec.ticksize(o.State.Price)))).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == lean_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(lean_child_order.Count() == 1, $"The lean child order should have been 'working' at {initial_price}");
+
+         // now we modify the opposite inside market qty to breach the lean threshold
+         var target_ask_order = test_orders.Where(o => o.Def.Side == Side.Sell &&
+            TestUtils.test_price_equality(o.State.Price, ask_price_1, s.Value.ContractSpec.ticksize(o.State.Price))).First();
+         var ask_order_response = await OrderUtilities.modify_order(_order_service, target_ask_order, cross_threshold);
+         var modified_ask_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey && o.HasState &&
+                  o.State.Quantity == cross_threshold).Count() == 1,
+               rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(modified_ask_order.Count() == 1, "The opposite inside market should have been adjusted to the Lean's cross threshold");
+
+         // verify that the child order has leaned into the market at the appropriate price and been parially filled appropriately
+         // this is the initial trigger payup
+         lean_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == lean_order.First().Key &&
+               o.HasState && (
+                  o.State.Status == OrderStatus.Working &&
+                  TestUtils.test_price_equality(o.State.Price, ask_price_1, s.Value.ContractSpec.ticksize(o.State.Price)) && 
+                  o.State.FilledQuantity == 9)
+               ).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == lean_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(lean_child_order.Count() == 1, $"Initial trigger: The lean child order should now have been 'working' at {ask_price_1}");
+         test_orders.Add(lean_child_order.First());
+
+         // now we modify the opposite inside market qty again to start checking that the case is working
+         target_ask_order = test_orders.Where(o => o.Def.Side == Side.Sell &&
+         TestUtils.test_price_equality(o.State.Price, ask_price_2, s.Value.ContractSpec.ticksize(o.State.Price))).First();
+         ask_order_response = await OrderUtilities.modify_order(_order_service, target_ask_order, cross_threshold);
+         modified_ask_order = (await OrderUtilities.wait_for_filtered_orders(
+               _order_service,
+               rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey && o.HasState &&
+                  o.State.Quantity == cross_threshold).Count() == 1,
+               rows => rows.Where(o => o.Key == ask_order_response.value.OrderKey))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(modified_ask_order.Count() == 1, "The opposite inside market should have been adjusted to the Lean's cross threshold");
+
+         // verify that the child order has chased and that it's filled quantity has increased
+         lean_child_order = (await OrderUtilities.wait_for_filtered_orders(
+            _order_service,
+            rows => rows.Where(o => o.ParentKey == lean_order.First().Key &&
+               o.HasState && (
+                  o.State.Status == OrderStatus.Working &&
+                  TestUtils.test_price_equality(o.State.Price, ask_price_2, s.Value.ContractSpec.ticksize(o.State.Price)) &&
+                  o.State.FilledQuantity == 9)
+               ).Count() == 1,
+            rows => rows.Where(o => o.ParentKey == lean_order.First().Key))).result ?? Enumerable.Empty<OrderRecordImmutable>();
+         Assert.IsTrue(lean_child_order.Count() == 1, $"First Chase: The lean child order should now have been 'working' at {ask_price_1}");
+         test_orders.Add(lean_child_order.First());
+
+         // TODO:
+         // verify the parent order also reflects the increase filled quantity from the child order 
+
+         // TODO:
+         // instigate another chase
+
+         // TODO:
+         // verify parent order again reflects
+
+         // TODO:
+         // instigate another chanse
+
+         // TODO:
+         // verify we are *not* chasing any longer, verify appropriate order statuses
+
+
+
+         // cleanup
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, test_orders), "Unable to cancel test orders");
+
+      }
+
+   }
+}

src/client/spark_test/System/LimitsTests.cs

@@ -13,6 +13,7 @@ using bts.api.orderman;
 using mbts;
 using mbts.tests;
 using bts.co2;
+using bts.tests.utils;
 
 namespace bts.tests
 {
@@ -23,7 +24,10 @@ namespace bts.tests
       AdminService _admin_service;
       IDisposable _order_subscription;
       Update<OrderRecordImmutable, uint> _orders;
-      public LimitsTests (){
+      public LimitsTests (){}
+
+      private async Task setup_limits_tests() {
+         await TestUtils.wait_for_system_ready();
          _context = new ServiceContext();
 
          _order_service = _context.locate<OrderService>();
@@ -43,6 +47,7 @@ namespace bts.tests
          Disposable.dispose(ref _order_service);
          Disposable.dispose(ref _admin_service);
          Disposable.dispose(ref _context);
+         Disposable.dispose(ref _order_subscription);
       }
 
       async Task<IEnumerable<SecurityImmutable>> all_futures(string symbol) {
@@ -78,28 +83,30 @@ namespace bts.tests
          return securities;
       }
 
-      [Test(Timeout = 30)]
+   [Test(Timeout = 300000)]
       public async Task test_position_limits() {
+         await setup_limits_tests();
+
          string test_symbol = "CL";
 
          var test_user = ( name: "bts", id: 0u );
          var test_account = ( name: "ACC A", id: 0u );
 
-         BTSAccountImmutable? account_data = await AdminUtilities.get_account_data(_admin_service, test_account.name);
+         var account_task = AdminUtilities.get_account_data(_admin_service, test_account.name);
+         BTSAccountImmutable? account_data = await TestUtils.await_with_timeout<BTSAccountImmutable?>(account_task, TimeSpan.FromMinutes(0.5));
+
          Assert.IsTrue(account_data.HasValue);
          test_account.id = account_data.Value.Id;
-
          var user_data = await AdminUtilities.get_user_data(_admin_service, test_user.name);
          Assert.IsTrue(user_data.HasValue);
          test_user.id = user_data.Value.Id;
-
          var clip_set_success = await AdminUtilities.set_clip_limit(_admin_service, 10000, account_data, test_symbol);
          Assert.IsTrue(clip_set_success, "Failed to set clip limit ");
 
          var all_securities = await all_futures(test_symbol);
          List<SecurityImmutable> security_list = all_securities.ToList();
 
-         Assert.IsTrue(all_securities.Count() >= 2);
+         Assert.IsTrue(all_securities.Count() >= 2, "At least two securities are needed");
          SecurityImmutable first_sec = security_list[0];
          SecurityImmutable second_sec = security_list[1];
 
@@ -109,10 +116,11 @@ namespace bts.tests
          var second_order_def = create_order_def(second_sec.Id, Side.Buy, test_account, test_user);
 
 
-         Result<Response> no_limit_order_result = await send_order(first_order_def, 5, 1);
+         Result<Response> no_limit_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 1);
 
          IEnumerable<OrderRecordImmutable> no_limit_filtered_orders = await get_filtered_orders(x => x.Key == no_limit_order_result.value.OrderKey);
 
+         Assert.IsNotNull(no_limit_filtered_orders, "No orders retrieved");
          Assert.IsTrue(no_limit_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "'Absent Limit' Order is not 'Rejected by Spark'");
 
          bool position_set_success = await AdminUtilities.set_position_limit(_admin_service, test_account.name, test_symbol, 60, 120);
@@ -121,11 +129,11 @@ namespace bts.tests
          List<Result<Response>> orders_response_list = new List<Result<Response>>();
 
          for(int i = 0; i <3; i++){
-            orders_response_list.Add(await send_order(first_order_def, 5, 10));
+            orders_response_list.Add(await OrderUtilities.send_order(_order_service, first_order_def, 5, 10));
          }
 
          for(int i = 0; i <3; i++){
-            orders_response_list.Add(await send_order(second_order_def, 5, 10));
+            orders_response_list.Add(await OrderUtilities.send_order(_order_service, second_order_def, 5, 10));
          }
 
          foreach(var i in orders_response_list){
@@ -133,24 +141,24 @@ namespace bts.tests
             Assert.IsTrue(iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'");
          }
 
-         var fail_order_result = await send_order(first_order_def, 5, 1);
+         var fail_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 1);
          var fail_order_filtered_orders = await get_filtered_orders(x => x.Key == fail_order_result.value.OrderKey);
          Assert.IsTrue(fail_order_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'");
 
          bool position_increase_success = await AdminUtilities.set_position_limit(_admin_service, test_account.name, test_symbol, 70, 120);
          Assert.IsTrue(position_increase_success);
 
-         var increased_order_result = await send_order(first_order_def, 5, 10);
+         var increased_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 10);
          var increased_order_filtered_orders = await get_filtered_orders(x => x.Key == increased_order_result.value.OrderKey);
          Assert.IsTrue(increased_order_filtered_orders.First().State.Status == OrderStatus.Working, "This order should have been 'Working'");
 
-         var second_fail_order_result = await send_order(first_order_def, 5, 10);
+         var second_fail_order_result = await OrderUtilities.send_order(_order_service, first_order_def, 5, 10);
          var second_fail_filtered_orders = await get_filtered_orders(x => x.Key == second_fail_order_result.value.OrderKey);
          Assert.IsTrue(second_fail_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'");
 
          /* Test Short Position Limit */
 
-         Assert.IsTrue(await cancel_orders(), "Unable to cancel orders");
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, _orders.all), "Unable to cancel orders");
 
          await AdminUtilities.remove_position_limit(_admin_service, test_account.name, test_symbol);
 
@@ -162,7 +170,7 @@ namespace bts.tests
 
          uint short_no_limit_order_key;
 
-         Result<Response> short_no_limit_order_result = await send_order(first_short_order_def, 50000, 7);
+         Result<Response> short_no_limit_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 7);
 
          short_no_limit_order_key = short_no_limit_order_result.value.OrderKey;
          IEnumerable<OrderRecordImmutable> short_no_limit_filtered_orders = await get_filtered_orders(x => x.Key == short_no_limit_order_key);
@@ -175,11 +183,11 @@ namespace bts.tests
          List<Result<Response>> short_orders_response_list = new List<Result<Response>>();
 
          for(int i = 0; i <3; i++){
-            short_orders_response_list.Add(await send_order(first_short_order_def, 50000, 10));
+            short_orders_response_list.Add(await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 10));
          }
 
          for(int i = 0; i <3; i++){
-            short_orders_response_list.Add(await send_order(second_short_order_def, 50000, 10));
+            short_orders_response_list.Add(await OrderUtilities.send_order(_order_service, second_short_order_def, 50000, 10));
          }
 
          foreach(var i in short_orders_response_list){
@@ -187,27 +195,29 @@ namespace bts.tests
             Assert.IsTrue(iret.First().State.Status == OrderStatus.Working, "This order should have been 'working'");
          }
 
-         var short_fail_order_result = await send_order(first_short_order_def, 50000, 1);
+         var short_fail_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 1);
          var short_fail_order_filtered_orders = await get_filtered_orders(x => x.Key == short_fail_order_result.value.OrderKey);
          Assert.IsTrue(short_fail_order_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'");
 
          bool short_position_increase_success = await AdminUtilities.set_position_limit(_admin_service, test_account.name, test_symbol, 120, 70);
          Assert.IsTrue(short_position_increase_success);
 
-         var short_increased_order_result = await send_order(first_short_order_def, 50000, 10);
+         var short_increased_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 10);
          var short_increased_order_filtered_orders = await get_filtered_orders(x => x.Key == short_increased_order_result.value.OrderKey);
          Assert.IsTrue(short_increased_order_filtered_orders.First().State.Status == OrderStatus.Working, "This order should have been 'Working'");
 
-         var short_second_fail_order_result = await send_order(first_short_order_def, 50000, 10);
+         var short_second_fail_order_result = await OrderUtilities.send_order(_order_service, first_short_order_def, 50000, 10);
          var short_second_fail_filtered_orders = await get_filtered_orders(x => x.Key == short_second_fail_order_result.value.OrderKey);
          Assert.IsTrue(short_second_fail_filtered_orders.First().State.Status == OrderStatus.RejectedBySpark, "This order should have been 'Rejected by Spark'");
 
-         Assert.IsTrue(await cancel_orders(), "Unable to cancel orders");
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, _orders.all), "Unable to cancel orders");
       }
 
 
-      [Test(Timeout = 10)]
+      [Test(Timeout = 300000)]
       public async Task test_clip_limits() {
+         await setup_limits_tests();
+
          //Get and set limit values from the admin service
 
          uint test_limit_value = 10u;
@@ -241,6 +251,8 @@ namespace bts.tests
             Quantity = test_limit_value
          });
 
+         Assert.IsTrue(no_limit_order_result.value.OrderKey > 0, "failed to enter order");
+
          var set_success = await AdminUtilities.set_clip_limit(_admin_service, test_limit_value, account_data, test_symbol);
          Assert.IsTrue(set_success, "Failed to set limit");
 
@@ -249,12 +261,15 @@ namespace bts.tests
             Price = 5,
             Quantity = test_limit_value
          });
+         Assert.IsTrue(under_limit_order_result.value.OrderKey > 0, "failed to enter order");
 
          Result<Response> over_limit_order_result = await _order_service.create_order_async(new Create() {
             NewOrder = order_def,
             Price = 5,
             Quantity = test_limit_value + 1
          });
+         Assert.IsTrue(over_limit_order_result.value.OrderKey > 0, "failed to enter order");
+
 
          under_limit_order_key = under_limit_order_result.value.OrderKey;
          over_limit_order_key = over_limit_order_result.value.OrderKey;
@@ -286,6 +301,7 @@ namespace bts.tests
          }
 
          Assert.IsTrue(success, "Failed to restore Limit");
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, _orders.all), "Unable to cancel orders");
       }
 
 
@@ -305,40 +321,12 @@ namespace bts.tests
          }
          await Task.Delay(100);
       }
-      while(it < 100 &&(_orders == null || current_count < count));
+      while(it < 100 && (_orders == null || current_count < count));
 
       return null;
 
    }
 
-
-   public async Task<Result<Response>> send_order(OrderDefinition order_def, double price, uint size){
-       return await _order_service.create_order_async(new Create() {
-            NewOrder = order_def,
-            Price = price,
-            Quantity = size
-         });
-   }
-
-   public async Task<bool> cancel_orders(uint key = 0){
-      if (key > 0) {
-         Result<Response> resp = await _order_service.cancel_order_async(new Cancel() { OrderKey = key });
-         return resp.success;
-      }
-      else {
-         bool suc = true;
-         var working_orders = await get_filtered_orders(x => x.State.Status == OrderStatus.Working);
-         if(working_orders != null){
-            foreach (var o in working_orders){
-               Result<Response> resp = await _order_service.cancel_order_async(new Cancel() { OrderKey = o.Key });
-               suc &= resp.success;
-            }
-            return suc;
-         }
-         return true;// there are no working orders... so true enough
-      }
-   }
-
    public OrderDefinition create_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user){
       return new OrderDefinition {
          SecurityKey = security_id,
@@ -358,218 +346,4 @@ namespace bts.tests
 
    }
 
-   public class AdminUtilities {
-
-      public static async Task<BTSAccountImmutable?> get_account_data (AdminService admin_service, string account_name)
-      {
-         Update<AdminDataImmutable, int> update = await admin_service.subscribei_admin_data().FirstAsync();
-
-         foreach (var acc in update.all.First().Accounts) {
-            if(acc.Name == account_name) {
-               return acc;
-            }
-         }
-         return null;
-      }
-
-      public static async Task<BTSUserImmutable?> get_user_data (AdminService admin_service, string user_name)
-      {
-         Update<AdminDataImmutable, int> update = await admin_service.subscribei_admin_data().FirstAsync();
-
-         foreach (var usr in update.all.First().Users) {
-            if(usr.Name == user_name) {
-               return usr;
-            }
-         }
-         return null;
-      }
-
-      public static AccountSecurityClipLimitUnitImmutable? get_clip_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         IEnumerable<AccountSecurityClipLimitUnitImmutable> clip_limit_rows = account_data.Value.SecurityClipLimits.Limit.Where(
-               r => r.Security.SecuritySymbol == symbol
-               && r.Security.Exchange == venue
-               && r.Security.SecurityType == security_type);
-         return clip_limit_rows.Any() ? clip_limit_rows.First() : (AccountSecurityClipLimitUnitImmutable?) null;
-      }
-
-      public static uint? get_clip_limit(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         var row = get_clip_limit_row(account_data,symbol,security_type,venue);
-         return row.HasValue ? row.Value.ClipSize : (uint?) null;
-      }
-
-      public static async Task<uint?> get_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         var account_data = await get_account_data(admin_service, account_name);
-         return get_clip_limit(account_data, symbol, security_type, venue);
-      }
-
-      public static async Task<bool> set_clip_limit(AdminService admin_service, uint clip_value, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-            var account_data = await get_account_data(admin_service, account_name);
-            return await set_clip_limit(admin_service, clip_value,account_data, symbol,security_type, venue);
-      }
-
-      public static async Task<bool> set_clip_limit(AdminService admin_service, uint clip_value, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-
-            AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue);
-            AccountSecurityClipLimitUnitImmutable new_clip_limit_row;
-
-            List<AccountSecurityClipLimitUnitImmutable> clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList();
-
-            if(origin_clip_limit_row.HasValue) {
-               clip_limits_list.Remove(origin_clip_limit_row.Value);
-            }
-
-            new_clip_limit_row = (AccountSecurityClipLimitUnitImmutable) new AccountSecurityClipLimitUnit {
-                           Security = new SecurityScope(){ SecuritySymbol = symbol, Exchange = venue, SecurityType = security_type },
-                           ClipSize = clip_value
-                        };
-
-            clip_limits_list.Add(new_clip_limit_row);
-
-            AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits();
-            temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable()));
-
-            var new_account_data = account_data.Value.with(security_clip_limits: temp_scl);
-
-            var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
-               CompleteData = new_account_data.CloneToMutable()
-            });
-
-            return update_ret.success;
-      }
-
-      public static async Task<bool> clear_all_limits(AdminService admin_service, string account_name) {
-            var account_data = await get_account_data(admin_service, account_name);
-            return await clear_all_limits(admin_service, account_data);
-      }
-
-      public static async Task<bool> clear_all_limits(AdminService admin_service, BTSAccountImmutable? account_data) {
-            var new_account_data = account_data.Value.with(security_clip_limits: new AccountSecurityClipLimits(), product_position_limits: new List<AccountProductPositionLimitUnit>());
-
-            var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
-               CompleteData = new_account_data.CloneToMutable()
-            });
-
-            return update_ret.success;
-      }
-
-
-      public static async Task<bool> remove_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-            var account_data = await get_account_data(admin_service, account_name);
-            return await remove_clip_limit(admin_service, account_data, symbol, security_type, venue);
-      }
-
-      public static async Task<bool> remove_clip_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-
-            AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue);
-            List<AccountSecurityClipLimitUnitImmutable> clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList();
-
-            if(origin_clip_limit_row.HasValue) {
-               clip_limits_list.Remove(origin_clip_limit_row.Value);
-               AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits();
-               temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable()));
-               var new_account_data = account_data.Value.with(security_clip_limits: temp_scl);
-
-               var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
-                  CompleteData = new_account_data.CloneToMutable()
-               });
-
-               return update_ret.success;
-            }
-            else
-            {
-               return false;
-            }
-      }
-
-
-      public static async Task<AccountProductPositionLimitUnitImmutable?> get_position_limit_row(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         var account_data = await get_account_data(admin_service, account_name);
-         return get_position_limit_row(account_data, symbol, security_type, venue);
-      }
-
-      public static AccountProductPositionLimitUnitImmutable? get_position_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         IEnumerable<AccountProductPositionLimitUnitImmutable> limit_rows = account_data.Value.ProductPositionLimits.Where(
-               r => r.Product.SecuritySymbol == symbol
-               && r.Product.Exchange == venue
-               && r.Product.SecurityType == security_type);
-         return limit_rows.Any() ? limit_rows.First() : (AccountProductPositionLimitUnitImmutable?) null;
-
-      }
-
-      public static async Task<bool> remove_position_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         var account_data = await get_account_data(admin_service, account_name);
-         return await remove_position_limit(admin_service, account_data, symbol, security_type, venue);
-      }
-
-      public static async Task<bool> remove_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-
-            AccountProductPositionLimitUnitImmutable? origin_limit_row = get_position_limit_row(account_data,symbol,security_type,venue);
-            List<AccountProductPositionLimitUnitImmutable> limits_list = account_data.Value.ProductPositionLimits.ToList();
-
-            if(origin_limit_row.HasValue) {
-               limits_list.Remove(origin_limit_row.Value);
-               List<AccountProductPositionLimitUnit> temp_plu = new List<AccountProductPositionLimitUnit>();
-               temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable()));
-               var new_account_data = account_data.Value.with(product_position_limits: temp_plu);
-
-               var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
-                  CompleteData = new_account_data.CloneToMutable()
-               });
-
-               return update_ret.success;
-            }
-            else
-            {
-               return false;
-            }
-      }
-
-      public static async Task<bool> set_position_limit(AdminService admin_service, string account_name, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-         var account_data = await get_account_data(admin_service, account_name);
-         return await set_position_limit(admin_service, account_data, symbol, long_limit, short_limit, security_type, venue);
-      }
-
-      public static async Task<bool> set_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
-
-            AccountProductPositionLimitUnitImmutable? limit_row = get_position_limit_row(account_data,symbol,security_type,venue);
-            List<AccountProductPositionLimitUnitImmutable> limits_list = account_data.Value.ProductPositionLimits.ToList();
-
-            if(limit_row.HasValue) {
-               // update the row
-               limits_list.Remove(limit_row.Value);
-               limit_row = limit_row.Value.with(long_position_limit: long_limit, short_position_limit: short_limit);
-            }
-            else {
-               // create a new row
-               AccountProductPositionLimitUnit acplu = new AccountProductPositionLimitUnit();
-               acplu.LongPositionLimit = long_limit;
-               acplu.ShortPositionLimit = short_limit;
-               acplu.Product = new ProductScope {
-                  SecuritySymbol = symbol,
-                  SecurityType = security_type,
-                  Exchange = venue
-               };
-
-               limit_row = acplu;
-            }
-
-            limits_list.Add(limit_row.Value);
-
-            List<AccountProductPositionLimitUnit> temp_plu = new List<AccountProductPositionLimitUnit>();
-            temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable()));
-
-            var new_account_data = account_data.Value.with(product_position_limits: temp_plu);
-
-            var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
-               CompleteData = new_account_data.CloneToMutable()
-            });
-
-            return update_ret.success;
-      }
-
-
-   }
-
-
-
 }

src/client/spark_test/System/SpreaderTests.cs

@@ -0,0 +1,154 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.api.test;
+using bts.test;
+using bts.api.orderman;
+using mbts;
+using mbts.tests;
+using bts.co2;
+using bts.tests.utils;
+using bts.api.synthetic;
+using bts.spark;
+
+namespace bts.tests
+{
+   [TestGroup(TestType = TestType.System)]
+   public class SpreaderTests : IDisposable {
+      ServiceContext _context;
+      OrderService _order_service;
+      AdminService _admin_service;
+      MarketDataRelay _md_relay;
+      SyntheticSecurityService _synthetic_service;
+      IDisposable _order_subscription;
+      Update<OrderRecordImmutable, uint> _orders;
+      public SpreaderTests (){}
+
+      public async Task setup_spreader_tests () {
+         await TestUtils.wait_for_system_ready();
+         _context = new ServiceContext();
+
+         _order_service = _context.locate<OrderService>();
+         _admin_service = _context.locate<AdminService>();
+         _md_relay = _context.locate<MarketDataRelay>();
+         _synthetic_service = _context.locate<SyntheticSecurityService>();
+
+         var auth = _context.login("bts","");
+         auth.Wait(3000);
+
+         _order_subscription = _order_service.subscribei_orders().Subscribe(ords =>
+         {
+            _orders = ords;
+         });
+      }
+
+      public void Dispose() {
+         Disposable.dispose(ref _order_service);
+         Disposable.dispose(ref _admin_service);
+         Disposable.dispose(ref _context);
+         Disposable.dispose(ref _order_subscription);
+      }
+
+      [Test(Timeout = 300000)]
+      public async Task spreader_simple_sanity() {
+         await setup_spreader_tests();
+
+         string test_symbol = "SI";// don't use ES, the prices will be rounded to the same number and the result will be off
+         var test_user = (name: "bts", id: 1u);
+         var test_account = (name: "ACC A", id: 1u);
+
+         (SecurityImmutable s1, SecurityImmutable s2) test_securities = await SpreaderUtilities.get_securities_for_common_setup_one(_md_relay, test_symbol);
+
+         var setup_product = await SpreaderUtilities.common_setup_one(_order_service, _synthetic_service, test_securities, test_account, test_user);
+
+         var order1_def = OrderDefinitionUtilities.create_spreader_order_def(
+                                                                           setup_product.synthetic.security,
+                                                                           Side.Buy,
+                                                                           setup_product.synthetic.strategy,
+                                                                           test_account,
+                                                                           test_user
+                                                                           );
+
+         var spreader_order1_response = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1);
+         Assert.IsTrue(spreader_order1_response.success, "Failed to send first spreader order");
+
+         var (spreader_order_is_working, working_spreader_order1) = await verify_order_status(spreader_order1_response.value.OrderKey, OrderStatus.Working);
+         Assert.IsTrue(spreader_order_is_working, "Spreader order is not Working");
+
+         var (quoting_order_is_working, working_quoting_order) = await verify_child_order_status(spreader_order1_response.value.OrderKey, OrderStatus.Working);
+         Assert.IsTrue(quoting_order_is_working, "Order is not Working");
+         Assert.IsTrue(working_quoting_order.Value.State.Price == 18.6, "Quoting order has the wrong price");
+
+         Assert.IsTrue(await OrderUtilities.cancel_orders(_order_service, new List<OrderRecordImmutable> {working_spreader_order1.Value}), "Failed to cancel order");
+
+         var (quoting_order_is_canceled, canceled_quoting_order) = await verify_child_order_status(spreader_order1_response.value.OrderKey, OrderStatus.Canceled);
+         Assert.IsTrue(quoting_order_is_canceled, "Quoting order may not have been canceled");
+
+         // Spreader Order 2
+         var spreader_order2_response = await OrderUtilities.send_order(_order_service, order1_def, -0.30, 1);
+         Assert.IsTrue(spreader_order2_response.success, "Failed to send spreader order 2");
+
+         var (spreader_order2_is_working, working_spreader_order2) = await verify_order_status(spreader_order2_response.value.OrderKey, OrderStatus.Working);
+
+         Assert.IsTrue(spreader_order2_is_working, "Spreader order 2 is not 'Working'");
+
+         var spreader_order2_change_response = await OrderUtilities.modify_order(_order_service, working_spreader_order2.Value, -0.6);
+         Assert.IsTrue(spreader_order2_change_response.success, "failed to modify spreader order");
+
+         Assert.IsTrue(await verify_order_price(spreader_order2_response.value.OrderKey, -0.6), "Failed to change price of spreader order");
+
+         // verify quoting order price
+         Assert.IsTrue(await verify_child_order_price(spreader_order2_response.value.OrderKey, 18.9), "Quoting order incorrect or not found");
+
+         //fill spreader order
+         var spreader_order2_fill_change_response = await OrderUtilities.modify_order(_order_service, working_spreader_order2.Value, -0.2);
+         Assert.IsTrue(spreader_order2_fill_change_response.success, "failed to modify spreader order");
+
+         Assert.IsTrue(await verify_order_price(spreader_order2_response.value.OrderKey, -0.2), "Failed to change price of spreader order");
+
+         // verify quoting order price
+         Assert.IsTrue(await verify_child_order_price(spreader_order2_response.value.OrderKey, 18.5), "Quoting order incorrect or not found");
+
+         // verify Hedging order price
+         Assert.IsTrue(await verify_child_order_price(spreader_order2_response.value.OrderKey, 18.3), "Hedging order incorrect or not found");
+
+      }
+
+      private async Task<(Boolean, OrderRecordImmutable?)> verify_child_order_status(uint parent_order_key, OrderStatus expected_status) {
+         var order = await get_order(ordi => ordi.ParentKey == parent_order_key && ordi.State.Status == expected_status);
+         return (order.HasValue && order.Value.State.Status == expected_status, order);
+      }
+      private async Task<(Boolean, OrderRecordImmutable?)> verify_order_status(uint order_key, OrderStatus expected_status) {
+         var order = await get_order(ordi => ordi.Key == order_key);
+         return (order.HasValue && order.Value.State.Status == expected_status, order);
+      }
+
+      private async Task<Boolean> verify_order_price(uint order_key, double expected_price, double precision = 10) {
+         var order = await get_order(ordi => ordi.Key == order_key && Math.Truncate(ordi.State.Price * precision) == precision * expected_price);
+         return order.HasValue && (Math.Truncate(order.Value.State.Price * precision) == precision * expected_price);
+      }
+
+      private async Task<Boolean> verify_child_order_price(uint parent_order_key, double expected_price, double precision = 10.0)
+      {
+         var order = await get_order(ordi => ordi.ParentKey == parent_order_key && Math.Truncate(ordi.State.Price * precision) == precision * expected_price);
+         return order.HasValue && (Math.Truncate(order.Value.State.Price * precision) == precision * expected_price);
+      }
+
+      private async Task<OrderRecordImmutable?> get_order(Func<OrderRecordImmutable, bool> predicate)
+      {
+         var orders = await OrderUtilities.wait_for_filtered_orders(_order_service,
+                                                                        o  => o.Where(predicate).Count() > 0,
+                                                                        ord => ord.Where(predicate),
+                                                                        TimeSpan.FromSeconds(30));
+         return orders.result.TryFirst();
+      }
+
+
+   }
+}

src/client/spark_test/System/TestBase.cs

@@ -0,0 +1,59 @@
+using bts.api.admin;
+using bts.api.core;
+using bts.api.orderman;
+using bts.co2;
+using bts.test;
+using mbts;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace bts.tests {
+   public abstract class TestBase : IDisposable {
+
+      protected (string name, uint id) _test_account = ("ACC A", 0u);
+      protected (string name, uint id) _test_user = ("bts", 0u);
+      protected ServiceContext _context;
+      protected AdminService _admin_service;
+      protected OrderService _order_service;
+      protected MarketDataRelay _md_relay;
+      protected List<IDisposable> _resources;
+
+      public TestBase() {
+         _context = new ServiceContext();
+         _admin_service = _context.locate<AdminService>();
+         _order_service = _context.locate<OrderService>();
+         _md_relay = _context.locate<MarketDataRelay>();
+
+         _resources = new List<IDisposable> {
+            _context,
+            _admin_service,
+            _order_service,
+            _md_relay
+         };
+
+         var auth = _context.login("bts", "");
+         auth.Wait(3000);
+
+         // get the ACC A account
+         var account_data = AdminUtilities.get_account_data(_admin_service, _test_account.name);
+         account_data.Wait(3000);
+         Assert.IsTrue(account_data.Result.HasValue);
+         _test_account.id = account_data.Result.Value.Id;
+
+         // get the bts user
+         var user_data = AdminUtilities.get_user_data(_admin_service, _test_user.name);
+         user_data.Wait(3000);
+         Assert.IsTrue(user_data.Result.HasValue);
+         _test_user.id = user_data.Result.Value.Id;
+
+      }
+
+      public void Dispose() {
+         // seg faults on linux during this dispose, not sure why yet
+         if (System.Environment.OSVersion.Platform == PlatformID.Win32NT) {
+            Disposable.dispose(_resources);
+         }
+      }
+   }
+}

src/client/spark_test/Utils/AdminUtilities.cs

@@ -0,0 +1,230 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.api.test;
+using bts.test;
+using bts.api.orderman;
+using mbts;
+using mbts.tests;
+using bts.co2;
+
+namespace bts.tests
+{
+   public class AdminUtilities {
+
+      public static async Task<BTSAccountImmutable?> get_account_data (AdminService admin_service, string account_name)
+      {
+         Update<AdminDataImmutable, int> update = await admin_service.subscribei_admin_data().FirstAsync();
+
+         foreach (var acc in update.all.First().Accounts) {
+            if(acc.Name == account_name) {
+               return acc;
+            }
+         }
+         return null;
+      }
+
+      public static async Task<BTSUserImmutable?> get_user_data (AdminService admin_service, string user_name)
+      {
+         Update<AdminDataImmutable, int> update = await admin_service.subscribei_admin_data().FirstAsync();
+
+         foreach (var usr in update.all.First().Users) {
+            if(usr.Name == user_name) {
+               return usr;
+            }
+         }
+         return null;
+      }
+
+      public static AccountSecurityClipLimitUnitImmutable? get_clip_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         IEnumerable<AccountSecurityClipLimitUnitImmutable> clip_limit_rows = account_data.Value.SecurityClipLimits.Limit.Where(
+               r => r.Security.SecuritySymbol == symbol
+               && r.Security.Exchange == venue
+               && r.Security.SecurityType == security_type);
+         return clip_limit_rows.Any() ? clip_limit_rows.First() : (AccountSecurityClipLimitUnitImmutable?) null;
+      }
+
+      public static uint? get_clip_limit(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         var row = get_clip_limit_row(account_data,symbol,security_type,venue);
+         return row.HasValue ? row.Value.ClipSize : (uint?) null;
+      }
+
+      public static async Task<uint?> get_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         var account_data = await get_account_data(admin_service, account_name);
+         return get_clip_limit(account_data, symbol, security_type, venue);
+      }
+
+      public static async Task<bool> set_clip_limit(AdminService admin_service, uint clip_value, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+            var account_data = await get_account_data(admin_service, account_name);
+            return await set_clip_limit(admin_service, clip_value,account_data, symbol,security_type, venue);
+      }
+
+      public static async Task<bool> set_clip_limit(AdminService admin_service, uint clip_value, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+
+            AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue);
+            AccountSecurityClipLimitUnitImmutable new_clip_limit_row;
+
+            List<AccountSecurityClipLimitUnitImmutable> clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList();
+
+            if(origin_clip_limit_row.HasValue) {
+               clip_limits_list.Remove(origin_clip_limit_row.Value);
+            }
+
+            new_clip_limit_row = (AccountSecurityClipLimitUnitImmutable) new AccountSecurityClipLimitUnit {
+                           Security = new SecurityScope(){ SecuritySymbol = symbol, Exchange = venue, SecurityType = security_type },
+                           ClipSize = clip_value
+                        };
+
+            clip_limits_list.Add(new_clip_limit_row);
+
+            AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits();
+            temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable()));
+
+            var new_account_data = account_data.Value.with(security_clip_limits: temp_scl);
+
+            var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
+               CompleteData = new_account_data.CloneToMutable()
+            });
+
+            return update_ret.success;
+      }
+
+      public static async Task<bool> clear_all_limits(AdminService admin_service, string account_name) {
+            var account_data = await get_account_data(admin_service, account_name);
+            return await clear_all_limits(admin_service, account_data);
+      }
+
+      public static async Task<bool> clear_all_limits(AdminService admin_service, BTSAccountImmutable? account_data) {
+            var new_account_data = account_data.Value.with(security_clip_limits: new AccountSecurityClipLimits(), product_position_limits: new List<AccountProductPositionLimitUnit>());
+
+            var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
+               CompleteData = new_account_data.CloneToMutable()
+            });
+
+            return update_ret.success;
+      }
+
+
+      public static async Task<bool> remove_clip_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+            var account_data = await get_account_data(admin_service, account_name);
+            return await remove_clip_limit(admin_service, account_data, symbol, security_type, venue);
+      }
+
+      public static async Task<bool> remove_clip_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+
+            AccountSecurityClipLimitUnitImmutable? origin_clip_limit_row = get_clip_limit_row(account_data,symbol,security_type,venue);
+            List<AccountSecurityClipLimitUnitImmutable> clip_limits_list = account_data.Value.SecurityClipLimits.Limit.ToList();
+
+            if(origin_clip_limit_row.HasValue) {
+               clip_limits_list.Remove(origin_clip_limit_row.Value);
+               AccountSecurityClipLimits temp_scl = new AccountSecurityClipLimits();
+               temp_scl.Limit.AddRange(clip_limits_list.Select(x => x.CloneToMutable()));
+               var new_account_data = account_data.Value.with(security_clip_limits: temp_scl);
+
+               var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
+                  CompleteData = new_account_data.CloneToMutable()
+               });
+
+               return update_ret.success;
+            }
+            else
+            {
+               return false;
+            }
+      }
+
+
+      public static async Task<AccountProductPositionLimitUnitImmutable?> get_position_limit_row(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         var account_data = await get_account_data(admin_service, account_name);
+         return get_position_limit_row(account_data, symbol, security_type, venue);
+      }
+
+      public static AccountProductPositionLimitUnitImmutable? get_position_limit_row(BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         IEnumerable<AccountProductPositionLimitUnitImmutable> limit_rows = account_data.Value.ProductPositionLimits.Where(
+               r => r.Product.SecuritySymbol == symbol
+               && r.Product.Exchange == venue
+               && r.Product.SecurityType == security_type);
+         return limit_rows.Any() ? limit_rows.First() : (AccountProductPositionLimitUnitImmutable?) null;
+
+      }
+
+      public static async Task<bool> remove_position_limit(AdminService admin_service, string account_name, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         var account_data = await get_account_data(admin_service, account_name);
+         return await remove_position_limit(admin_service, account_data, symbol, security_type, venue);
+      }
+
+      public static async Task<bool> remove_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+
+            AccountProductPositionLimitUnitImmutable? origin_limit_row = get_position_limit_row(account_data,symbol,security_type,venue);
+            List<AccountProductPositionLimitUnitImmutable> limits_list = account_data.Value.ProductPositionLimits.ToList();
+
+            if(origin_limit_row.HasValue) {
+               limits_list.Remove(origin_limit_row.Value);
+               List<AccountProductPositionLimitUnit> temp_plu = new List<AccountProductPositionLimitUnit>();
+               temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable()));
+               var new_account_data = account_data.Value.with(product_position_limits: temp_plu);
+
+               var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
+                  CompleteData = new_account_data.CloneToMutable()
+               });
+
+               return update_ret.success;
+            }
+            else
+            {
+               return false;
+            }
+      }
+
+      public static async Task<bool> set_position_limit(AdminService admin_service, string account_name, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+         var account_data = await get_account_data(admin_service, account_name);
+         return await set_position_limit(admin_service, account_data, symbol, long_limit, short_limit, security_type, venue);
+      }
+
+      public static async Task<bool> set_position_limit(AdminService admin_service, BTSAccountImmutable? account_data, string symbol, uint long_limit, uint short_limit, SecurityType security_type = SecurityType.Future, Venue venue = Venue.Sim) {
+
+            AccountProductPositionLimitUnitImmutable? limit_row = get_position_limit_row(account_data,symbol,security_type,venue);
+            List<AccountProductPositionLimitUnitImmutable> limits_list = account_data.Value.ProductPositionLimits.ToList();
+
+            if(limit_row.HasValue) {
+               // update the row
+               limits_list.Remove(limit_row.Value);
+               limit_row = limit_row.Value.with(long_position_limit: long_limit, short_position_limit: short_limit);
+            }
+            else {
+               // create a new row
+               AccountProductPositionLimitUnit acplu = new AccountProductPositionLimitUnit();
+               acplu.LongPositionLimit = long_limit;
+               acplu.ShortPositionLimit = short_limit;
+               acplu.Product = new ProductScope {
+                  SecuritySymbol = symbol,
+                  SecurityType = security_type,
+                  Exchange = venue
+               };
+
+               limit_row = acplu;
+            }
+
+            limits_list.Add(limit_row.Value);
+
+            List<AccountProductPositionLimitUnit> temp_plu = new List<AccountProductPositionLimitUnit>();
+            temp_plu.AddRange(limits_list.Select(x => x.CloneToMutable()));
+
+            var new_account_data = account_data.Value.with(product_position_limits: temp_plu);
+
+            var update_ret = await admin_service.update_account_async(new AccountUpdateAPIArgument {
+               CompleteData = new_account_data.CloneToMutable()
+            });
+
+            return update_ret.success;
+      }
+
+
+   }
+}

src/client/spark_test/Utils/MarketDataUtilities.cs

@@ -0,0 +1,74 @@
+using bts.api;
+using bts.co2;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts.tests {
+   class MarketDataUtilities {
+
+      const string _MD_HOST = "system_test";
+
+      public static async Task<SecurityImmutable?> get_security(MarketDataRelay relay, string symbol, SecurityType security_type, Func<ChannelUpdate, bool> match, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) {
+         var request = new MDRequest() {
+            Host = _MD_HOST,
+            Channels = {
+               new SecurityChannel() {
+                  Time = 0,
+                  Flags = (uint)flags,
+                  Type = security_type,
+                  Exchange = v,
+                  Symbol = symbol,
+                  TradeCursor = 0,
+                  RfqCursor = 0
+               }
+            }
+         };
+         var result = await relay.requestmd_async(request);
+
+         if (result.success) {
+            foreach (var ch in result.value.Channels) {
+               foreach (var update in ch.Updates) {
+                  if (match(update)) {
+                     return new SecurityImmutable(update.Security);
+                  }
+               }
+            }
+         }
+         return null;
+      }
+
+      public static async Task<IEnumerable<FutureContract>> get_expirations(MarketDataRelay relay, string symbol, SecurityType security_type, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) {
+          var request = new MDRequest() {
+            Host = _MD_HOST,
+            Channels = {
+               new SecurityChannel() {
+                  Time = 0,
+                  Flags = (uint)flags,
+                  Type = security_type,
+                  Exchange = v,
+                  Symbol = symbol,
+                  TradeCursor = 0,
+                  RfqCursor = 0
+               }
+            }
+         };
+
+         var result = await relay.requestmd_async(request);
+
+         var futures = new List<FutureContract>();
+
+         if (result.success) {
+            foreach (var ch in result.value.Channels) {
+               foreach (var update in ch.Updates) {
+                  futures.Add(update.Security.Future);
+               }
+            }
+         }
+
+         return (IEnumerable<FutureContract>) futures;
+      }
+
+   }
+}

src/client/spark_test/Utils/OrderUtilities.cs

@@ -0,0 +1,200 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.api.test;
+using bts.test;
+using bts.api.orderman;
+using mbts;
+using mbts.tests;
+using bts.co2;
+using bts.tests.utils;
+using bts.spark;
+
+namespace bts.tests
+{
+   public class OrderUtilities {
+      public static async Task<Result<Response>> send_order(OrderService order_service, OrderDefinition order_def, double price, uint size){
+         return await order_service.create_order_async(new Create() {
+            NewOrder = order_def,
+            Price = price,
+            Quantity = size
+         });
+      }
+      public static async Task<Result<Response>> modify_order(OrderService order_service, OrderRecordImmutable o, uint size) {
+         return await modify_order(order_service, o.Key, o.State.Price, size);
+      }
+      public static async Task<Result<Response>> modify_order(OrderService order_service, OrderRecordImmutable o, double price) {
+         return await modify_order(order_service, o.Key, price, o.State.Quantity);
+      }
+      public static async Task<Result<Response>> modify_order(OrderService order_service, OrderRecordImmutable o, double price, uint size) {
+         return await modify_order(order_service, o.Key, price, size);
+      }
+      private static async Task<Result<Response>> modify_order(OrderService order_service, uint order_key, double price, uint size) {
+         return await order_service.update_order_async(new Update() {
+            OrderKey = order_key,
+            Price = price,
+            Quantity = size
+         });
+      }
+
+      public static async Task<bool> cancel_orders(OrderService order_service, IEnumerable<OrderRecordImmutable> orders, uint key = 0){
+         if (key > 0) {
+            Result<Response> resp = await order_service.cancel_order_async(new Cancel() { OrderKey = key });
+            return resp.success;
+         }
+         else {
+            bool suc = true;
+            var working_orders = orders.Where(x => x.State.Status == OrderStatus.Working);
+            if(working_orders != null){
+               foreach (var o in working_orders){
+                  Result<Response> resp = await order_service.cancel_order_async(new Cancel() { OrderKey = o.Key });
+                  suc &= resp.success;
+               }
+               return suc;
+            }
+            return true;// there are no working orders... so true enough
+         }
+      }
+
+      public async static Task<utils.TestUtils.TimeoutResult<IEnumerable<OrderRecordImmutable>>>
+         wait_for_filtered_orders(OrderService order_service, Func<IEnumerable<OrderRecordImmutable>, bool> match, Func<IEnumerable<OrderRecordImmutable>, IEnumerable<OrderRecordImmutable>> selector, TimeSpan? timeout = null) {
+         if (timeout == null) {
+            timeout = TimeSpan.FromSeconds(3);
+         }
+         var tcs = new TaskCompletionSource<IEnumerable<OrderRecordImmutable>>();
+         var sub = order_service.subscribei_orders().Subscribe(update => {
+            if (match(update.all)) {
+               tcs.TrySetResult(selector(update.all));
+            }
+         });
+         var tr = await tcs.Task.timeout(timeout.Value);
+         sub.Dispose();
+         return tr;
+      }
+
+      public async static Task<utils.TestUtils.TimeoutResult<IEnumerable<FillImmutable>>>
+         wait_for_filtered_fills(OrderService order_service, Func<IEnumerable<FillImmutable>, bool> match, Func<IEnumerable<FillImmutable>, IEnumerable<FillImmutable>> selector, TimeSpan? timeout = null) {
+         if (timeout == null) {
+            timeout = TimeSpan.FromSeconds(3);
+         }
+         var tcs = new TaskCompletionSource<IEnumerable<FillImmutable>>();
+         var sub = order_service.subscribei_fills().Subscribe(update => {
+            if (match(update.all)) {
+               tcs.TrySetResult(selector(update.all));
+            }
+         });
+         var tr = await tcs.Task.timeout(timeout.Value);
+         sub.Dispose();
+         return tr;
+      }
+
+      public async static Task<IEnumerable<OrderRecordImmutable>> create_book(OrderService order_service, (OrderDefinition order_def, double price, uint size)[] order_defs) {
+
+         List<Result<Response>> results = new List<Result<Response>>();
+
+         foreach(var ord in order_defs) {
+            Assert.IsTrue(ord.order_def != null, "Order definition can not be null");
+            results.Add(await send_order(order_service, ord.order_def, ord.price, ord.size));
+         }
+
+         Func<IEnumerable<OrderRecordImmutable>, IEnumerable<OrderRecordImmutable>> selector = existing_orders => {
+            return existing_orders.Where(z => results.Exists(y => z.Key == y.value.OrderKey));
+         };
+
+         Func<IEnumerable<OrderRecordImmutable>, bool> match = existing_orders => {
+            return selector(existing_orders).Count() > 0;
+         };
+
+         await Task.Delay(5000);// I'm not sure how to get around this
+
+         utils.TestUtils.TimeoutResult<IEnumerable<OrderRecordImmutable>> filtered_orders = await wait_for_filtered_orders(order_service, match, selector, TimeSpan.FromSeconds(30));
+
+         Assert.IsNotNull(filtered_orders.result, "Book is incomplete");
+         Assert.IsTrue(filtered_orders.result.Count() == order_defs.Length, "Book is incomplete");
+
+         return filtered_orders.result;
+      }
+
+   }
+
+   /// <summary>
+   ///  We should replace with something like the OrderDataHeler from the Spark project.
+   /// </summary>
+   public static class OrderDefinitionUtilities {
+
+      private static OrderDefinition create_base_order(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) {
+         return new OrderDefinition {
+            SecurityKey = security_id,
+            Side = side,
+            TimeInForce = TimeInForce.Day,
+            CtiCode = CTICode.CtiCode1,
+            CustomerOrFirm = CustomerOrFirm.Firm,
+            ManualOrderIndicator = ManualOrderIndicator.Yes,
+            Account = account.name,
+            AccountId = account.id,
+            User = user.name,
+            UserId = user.id
+         };
+      }
+      public static OrderDefinition create_limit_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user) {
+         var def = create_base_order(security_id, side, account, user);
+         def.CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition { } };
+         return def;
+      }
+
+      public static OrderDefinition create_spreader_order_def(Security security, Side side, ExecutionStrategy strategy, (String name, uint id) account, (String name, uint id) user) {
+         var def = create_base_order(security.Id, side, account, user);
+         def.CustomDef = new CustomOrderDefinition { Spreader = new SpreaderOrderDefinition {Security = security, ExecutionStrategy = strategy} };
+         return def;
+      }
+
+      public static OrderDefinition create_iceberg_order_def(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, double show_quantity) {
+         var def = create_base_order(security_id, side, account, user);
+         def.CustomDef = new CustomOrderDefinition {
+            Iceberg = new IcebergOrderDefinition {
+               ShowQuantity = show_quantity,
+               Child = new CustomOrderDefinition {
+                  Limit = new LimitOrderDefinition { }
+               }
+            }
+         };
+         return def;
+      }
+
+      public static OrderDefinition create_join_order_definition(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, spark.QuantityMeasure join_threshold) {
+         var def = create_base_order(security_id, side, account, user);
+         def.CustomDef = new CustomOrderDefinition {
+            Join = new JoinOrderDefinition {
+               JoinThreshold = join_threshold,
+               Child = new CustomOrderDefinition {
+                  Limit = new LimitOrderDefinition { }
+               }
+            }
+         };
+         return def;
+      }
+
+      public static OrderDefinition create_lean_order_definition(uint security_id, Side side, (String name, uint id) account, (String name, uint id) user, spark.QuantityMeasure lean_threshold, uint max_chase_ticks = 2, LeanOrderDefinition.Types.OnToward ontoward_behavor = LeanOrderDefinition.Types.OnToward.NoChange) {
+         var def = create_base_order(security_id, side, account, user);
+         def.CustomDef = new CustomOrderDefinition {
+            Lean = new LeanOrderDefinition {
+               CrossThreshold = lean_threshold,
+               MaxChaseTicks = max_chase_ticks,
+               TradeTowardBehavior = ontoward_behavor,
+               Child = new CustomOrderDefinition {
+                  Limit = new LimitOrderDefinition { }
+               }
+            }
+         };
+         return def;
+      }
+
+   }
+
+}

src/client/spark_test/Utils/SpreaderUtilities.cs

@@ -0,0 +1,180 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.api.test;
+using bts.test;
+using bts.api.orderman;
+using mbts;
+using mbts.tests;
+using bts.co2;
+using bts.tests.utils;
+using bts.api.synthetic;
+using bts.spark;
+
+namespace bts.tests
+{
+   public class SpreaderUtilities {
+
+      public static async Task<(SecurityImmutable s1, SecurityImmutable s2)> get_securities_for_common_setup_one(MarketDataRelay relay, string symbol) {
+         var exps  = (await MarketDataUtilities.get_expirations(relay, symbol, SecurityType.Future)).ToList();
+
+         Assert.IsNotNull(exps);
+         Assert.IsTrue(exps.Count() > 2, "Need at least two expirations");
+
+         var exp1 = exps[0];
+         var exp2 = exps[1];
+
+         Func<ChannelUpdate, bool> sec1_match = ch =>
+            ch.Security.Future.ExpirationYear == exp1.ExpirationYear &&
+            ch.Security.Future.ExpirationMonth == exp1.ExpirationMonth &&
+            ch.Security.Future.ExpirationDay == exp1.ExpirationDay;
+
+         Func<ChannelUpdate, bool> sec2_match = ch =>
+            ch.Security.Future.ExpirationYear == exp2.ExpirationYear &&
+            ch.Security.Future.ExpirationMonth == exp2.ExpirationMonth &&
+            ch.Security.Future.ExpirationDay == exp2.ExpirationDay;
+
+         var sec_task1 = MarketDataUtilities.get_security(relay, symbol, SecurityType.Future, sec1_match);
+         var sec_task2 = MarketDataUtilities.get_security(relay, symbol, SecurityType.Future, sec2_match);
+
+         var sec1 = await sec_task1;
+         var sec2 = await sec_task2;
+
+         Assert.IsFalse(sec1 == null, "security is null");
+         Assert.IsFalse(sec2 == null, "security is null");
+
+         return (s1: sec1.Value, s2: sec2.Value);
+      }
+      public static async Task<(IEnumerable<OrderRecordImmutable> book, (Security security, ExecutionStrategy strategy) synthetic)> common_setup_one(OrderService order_service, SyntheticSecurityService synthetic_service, (SecurityImmutable? s1, SecurityImmutable? s2) securities, (String name, uint id) account, (String name, uint id) user) {
+         var order_defs = new (OrderDefinition order_def, double price, uint size)[10];
+
+         for (int i = 0; i < 4; i ++) {
+            double p = (0.01 * i) + 18.230;
+            order_defs[i] = (
+               order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s1.Value.Id, Side.Buy, account, user),
+               price: p,
+               size: 1
+            );
+         }
+
+         for (int i = 4; i < 8; i ++) {
+            double p = (0.01 * (i - 4)) + 18.300;
+            order_defs[i] = (
+               order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s1.Value.Id, Side.Sell, account, user),
+               price: p,
+               size: 1
+            );
+         }
+
+         order_defs[8] = (
+               order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s2.Value.Id, Side.Buy, account, user),
+               price: 18.500,
+               size: 5
+            );
+
+         order_defs[9] = (
+               order_def: OrderDefinitionUtilities.create_limit_order_def(securities.s2.Value.Id, Side.Sell, account, user),
+               price: 18.53,
+               size: 5
+            );
+
+         var book = await OrderUtilities.create_book(order_service, order_defs);
+
+         List<(ContractLeg contract, LegSpreaderParams parameters)> legs = new List<(ContractLeg, LegSpreaderParams)>();
+
+         legs.Add(create_leg(securities.s1.Value.CloneToMutable(),1,1,false,1));
+         legs.Add(create_leg(securities.s2.Value.CloneToMutable(),-1,-1,true,1));
+
+         (Security security, ExecutionStrategy strategy) synthetic = await create_synthetic(synthetic_service, legs);
+
+         return(book, synthetic);
+
+      }
+
+      public static async Task<(Security, ExecutionStrategy)> create_synthetic(SyntheticSecurityService service, IEnumerable<(ContractLeg contract, LegSpreaderParams parameters)> legs) {
+         var new_synth = new Security(){};
+         new_synth.Synthetic = new SyntheticContract();
+         new_synth.Synthetic.Legs.Add(legs.Select(l => l.contract));
+         new_synth.Symbol = deduce_symbol(new_synth);
+         new_synth.ContractSpec = new ContractSpecification();
+         new_synth.ContractSpec.LowTick = 0.01;
+         new_synth.ContractSpec.HighTick = 0.01;
+         new_synth.ContractSpec.PriceBreak = double.MaxValue;
+         new_synth.ContractSpec.MainFraction = 1u;
+         new_synth.ContractSpec.PriceDisplayFormat = 3u;
+         new_synth.Exchange = Venue.Bts;
+         new_synth.Synthetic.Name = "TEST_SYNTH" + DateTime.Now.ToString();
+
+         var result = await service.create_synthetic_async(new_synth);
+         Assert.IsTrue(result.success, "Unable to create synthetic");
+         Assert.IsTrue(result.value.Error == CreateResult.Types.Error.Ok, "Create synthetic was not OK");
+
+         uint? id = result.value.Key;
+         Assert.IsTrue(id.HasValue && id.Value != 0, "Failed to get synthetic security key");
+
+         new_synth.Id = result.value.Key;
+
+         ExecutionStrategy strategy = new ExecutionStrategy();
+         foreach(var leg in legs){
+            strategy.LegParamsMap.Add(leg.contract.Security.Id, leg.parameters);
+         }
+         bool success = await save_strategy(service, new_synth.Id, new ExecutionStrategyImmutable(strategy));
+         Assert.IsTrue(success, "Failed to save synthetic strategy");
+
+         return (new_synth, strategy);
+
+      }
+
+      public static (ContractLeg contract, LegSpreaderParams parameters) create_leg(
+            Security security,
+            double quantity,
+            double price_multiplier,
+            bool is_quoting,
+            double lean_qty_factor,
+            LegSpreaderParams.Types.MarketDataType market_data_type = 0
+            )
+      {
+         var contract = new ContractLeg();
+         contract.Security = security;
+         contract.PriceMultiplier = price_multiplier;
+         contract.Side = quantity > 0 ? Side.Buy : Side.Sell;
+         contract.Quantity = Math.Abs(quantity);
+
+         var parameters = new LegSpreaderParams();
+         parameters.IsActiveQuoting = is_quoting;
+         parameters.MarketDataType = market_data_type; // 0 == Merged, 1 == Direct
+         parameters.EnableHedging = !is_quoting;
+         parameters.PostHedgeParams = new PostHedgeParams();
+         parameters.PreHedgeParams = new PreHedgeParams();
+
+         parameters.QuoteParams = new QuoteParams();
+         parameters.QuoteParams.LeanQuantityRatio = lean_qty_factor;
+         parameters.QuoteParams.QuoteMuteZone = null; // == Throttle unchecked
+
+         return (contract: contract, parameters: parameters);
+      }
+
+      public static string deduce_symbol(SecurityImmutable? security) {
+         if (!string.IsNullOrWhiteSpace(security?.Symbol)) return security?.Symbol;
+         if (security?.Synthetic.Legs.Count() > 0) return security?.Synthetic.Legs.First().Security.Symbol;
+         return string.Empty;
+      }
+
+      public static async Task<uint?> create_synthetic(SyntheticSecurityService service, SecurityImmutable? synthetic) {
+         var result = await service.create_synthetic_async(synthetic.Value);
+         Assert.IsTrue(result.success, "failed to create synthetic");
+         return result.value.Key;
+      }
+
+      public static async Task<bool> save_strategy(SyntheticSecurityService service, uint synthetic_id, ExecutionStrategyImmutable strategy) {
+         var result = await service.update_security_metadata_async(new Metadata { SecurityKey = synthetic_id, Strategy = strategy.CloneToMutable() });
+         return result.success;
+      }
+   }
+}

src/client/spark_test/Utils/TestUtils.cs

@@ -1,10 +1,17 @@
-using bts.collections;
+using bts.api;
+using bts.api.admin;
+using bts.api.core;
+using bts.api.orderman;
+using bts.metrics;
+using bts.collections;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Reactive.Linq;
+using System.Threading.Tasks;
 
 namespace bts.tests.utils {
-   static class TestUtils {
+   public static class TestUtils {
       public static List<T> randomize_values<T>(IReadOnlyList<T> input) {
          var remaining_indices = Enumerable.Range(0, input.Count).ToList();
          var rand = new Random();
@@ -38,5 +45,167 @@ namespace bts.tests.utils {
          }
          return reshuffled;
       }
+
+      public static bool test_price_equality(double price_1, double price_2, double tick_size) {
+         return Math.Abs(price_2 - price_1) < 0.01 * tick_size;
+      }
+
+      // this is a copy from TickUtils -- ask what's legal for the spark_test project...
+      public static double ticksize(this ContractSpecificationImmutable spec, double price) =>
+         Math.Abs(price) > spec.PriceBreak ? spec.HighTick : spec.LowTick;
+
+      // this was copied from GlobalExtensions
+      public struct TimeoutResult<T> {
+         public TimeoutResult(T result, TimeSpan ts) {
+            wait_duration = ts;
+            this.result = result;
+         }
+         public readonly TimeSpan wait_duration;
+         public readonly T result;
+      }
+
+      // this was copied from GlobalExtensions
+      public static async Task<TimeoutResult<T>> timeout<T>(this Task<T> task, TimeSpan timeout) {
+         System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
+         sw.Start();
+         var timeout_task = Task.Delay(timeout);
+         var completed = await Task.WhenAny(task, timeout_task);
+         return (completed == timeout_task) ?
+            new TimeoutResult<T>(default(T), timeout) :
+            new TimeoutResult<T>(task.Result, sw.Elapsed);
+      }
+
+      public static async Task<T> await_with_timeout<T>(Task<T> task, TimeSpan timeout){
+         var to = timeout<T>(task, timeout);
+         var tr = await to;
+         return tr.result;
+      }
+
+      public static async Task wait_for_system_ready(TimeSpan? timeout = null) {
+         if(timeout == null) {
+            timeout = TimeSpan.FromSeconds(120000);
+         }
+
+         System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
+         sw.Start();
+         // bool shell_latency_service_ready = await wait_for_shell_latency_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         // Console.WriteLine("Shell Latency Service Ready: " + shell_latency_service_ready);
+         // bts.test.Assert.IsTrue(shell_latency_service_ready, "System is not ready");
+
+         bool md_ready = await wait_for_md_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         Console.WriteLine("MD Service Ready: " + md_ready);
+         bts.test.Assert.IsTrue(md_ready, "System is not ready");
+
+         bool orders_ready = await wait_for_orders_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         Console.WriteLine("Order Service Ready: " + orders_ready);
+         bts.test.Assert.IsTrue(orders_ready, "System is not ready");
+
+         // bool admin_ready = await wait_for_admin_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         // Console.WriteLine("Admin Service Ready: " + admin_ready);
+         // bts.test.Assert.IsTrue(admin_ready, "System is not ready");
+
+
+         Console.WriteLine("System Ready after: " + sw.Elapsed.TotalSeconds + " seconds");
+
+         bts.test.Assert.IsTrue(orders_ready, "System is not ready");
+      }
+
+      public static async Task<bool> wait_for_shell_latency_service_ready(TimeSpan timeout) {
+         System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
+
+         Update<TickImmutable, int> service_value = null;
+         while(service_value == null && watch.Elapsed < timeout) {
+            var context = new ServiceContext();
+            var shell_latency_service = context.locate<ShellLatencyService>();
+
+            var clock_sub = shell_latency_service.subscribei_clock().Subscribe(update => service_value = update);
+
+            Console.WriteLine("Waiting for shell latency service...");
+            await Task.Delay(15000);
+
+            clock_sub.Dispose();
+            context.Dispose();
+         }
+         return service_value != null;
+      }
+
+      public static async Task<bool> wait_for_orders_service_ready(TimeSpan timeout) {
+         System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
+
+         var odef = OrderDefinitionUtilities.create_limit_order_def(1, Side.Buy, ("bts", 1), ("ACC A", 1));
+
+         Update<OrderRecordImmutable, uint> service_value = null;
+         while(service_value == null && watch.Elapsed < timeout) {
+            var context = new ServiceContext();
+            var order_service = context.locate<OrderService>();
+
+            Result<Response> oret = await OrderUtilities.send_order(order_service, odef, 0,1);
+
+            var sub = order_service.subscribei_orders().Subscribe(async update => {
+                  if(service_value == null) {
+                     await OrderUtilities.cancel_orders(order_service, null, oret.value.OrderKey);
+                     service_value = update;
+                  }
+               });
+
+            if(service_value == null) {
+               Console.WriteLine("Waiting for order service.");
+               await Task.Delay(10000);
+            }
+
+            sub.Dispose();
+            context.Dispose();
+         }
+         return service_value != null;
+      }
+
+      public static async Task<bool> wait_for_admin_service_ready(TimeSpan timeout) {
+         System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
+
+         Update<AdminDataImmutable, int> service_value = null;
+         while(service_value == null && watch.Elapsed < timeout) {
+            var context = new ServiceContext();
+            var admin_service = context.locate<AdminService>();
+
+            var sub = admin_service.subscribei_admin_data().Subscribe(update => service_value = update);
+
+            Console.WriteLine("Waiting for admin service...");
+            await Task.Delay(15000);
+
+            sub.Dispose();
+            context.Dispose();
+            //mbts.config.Configuration.initialize_for_test();
+
+         }
+         return service_value != null;
+      }
+
+
+      public static async Task<bool> wait_for_md_service_ready(TimeSpan timeout) {
+         var md_request = new co2.MDRequest() {
+            Host = "system_test",
+            Channels = {
+               new co2.SecurityChannel() {
+                  Time = 0,
+                  Flags = 0,
+                  Type = SecurityType.Future,
+                  Exchange = Venue.Sim,
+                  Symbol = "*",
+                  TradeCursor = 0,
+                  RfqCursor = 0
+               }
+            }
+         };
+         var context = new ServiceContext();
+         var md_relay = context.locate<co2.MarketDataRelay>();
+         Result<co2.MDReply>? md_result = await await_with_timeout(
+                                                               md_relay.requestmd_async(md_request),
+                                                               timeout);
+
+         return md_result != null;
+      }
    }
 }

src/client/spark_test/spark_test.csproj

@@ -11,4 +11,8 @@
     <ProjectReference Include="..\spark_core\spark_core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="../spark_test/system_test_scripts/*.sh" CopyToOutputDirectory="PreserveNewest"></Content>
+  </ItemGroup>
+
 </Project>

src/client/spark_test/system_test_scripts/delete_dbs.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ -d ~/sparkdata/ ]
+then
+   sparkdata_dir=~/sparkdata
+else
+   sparkdata_dir=/sparkdata
+fi
+
+if [ -d "$sparkdata_dir/data" ]
+then
+   [ -d $sparkdata_dir/data/databases/orders ] && rm -rf $sparkdata_dir/data/databases/orders
+   [ -d $sparkdata_dir/data/databases/fills ] && rm -rf $sparkdata_dir/data/databases/fills
+   [ -d $sparkdata_dir/data/databases/ex_sim_orders ] && rm -rf $sparkdata_dir/data/databases/ex_sim_orders
+
+   [ -d $sparkdata_dir/data/jetfix ] && rm -rf $sparkdata_dir/data/jetfix
+   [ -d $sparkdata_dir/data/exchange_sim ] && rm -rf $sparkdata_dir/data/exchange_sim
+
+fi

src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [ -d ~/sparkdata/ ]
+then
+   sparkdata_dir=~/sparkdata
+else
+   sparkdata_dir=/sparkdata
+fi
+
+./delete_dbs.sh
+
+config=$sparkdata_dir/conf/spark.conf
+bak=$sparkdata_dir/conf/spark.conf.bak
+
+[ ! -d "$sparkdata_dir/conf" ] && mkdir $sparkdata_dir/conf/
+[ -f "$conf" ] && mv $conf $bak
+
+cat << EOF > $sparkdata_dir/conf/spark.conf
+   ExchangeSim {
+      value {
+         merge_with_md:false
+      }
+   }
+   Logging {
+      value {
+            console { default_level: "Error" }
+            syslog { default_level: "Error" }
+            file { default_level: "Error" }
+      }
+}
+EOF
+
+
+nohup ./sparkd run &
+pid=$!
+echo "sparkd pid: " $pid
+
+dotnet sparkcmd.dll test spreader_simple_sanity
+exit_status=$?
+
+pkill -P $$
+
+[ -f "$bak" ] && mv $bak $conf
+[ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
+exit $exit_status

src/client/spark_test/system_test_scripts/test_clip_limits.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [ -d ~/sparkdata/ ]
+then
+   sparkdata_dir=~/sparkdata
+else
+   sparkdata_dir=/sparkdata
+fi
+
+./delete_dbs.sh
+
+config=$sparkdata_dir/conf/spark.conf
+bak=$sparkdata_dir/conf/spark.conf.bak
+
+[ ! -d "$sparkdata_dir/conf" ] && mkdir $sparkdata_dir/conf/
+[ -f "$conf" ] && mv $conf $bak
+
+cat << EOF > $sparkdata_dir/conf/spark.conf
+   ExchangeSim {
+      value {
+         merge_with_md:true
+      }
+   }
+   Logging {
+      value {
+            console { default_level: "Error" }
+            syslog { default_level: "Error" }
+            file { default_level: "Error" }
+      }
+}
+EOF
+
+nohup ./sparkd run &
+pid=$!
+echo "sparkd pid: " $pid
+
+dotnet sparkcmd.dll test test_clip_limits
+exit_status=$?
+
+pkill -P $$
+[ -f "$bak" ] && mv $bak $conf
+
+[ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
+exit $exit_status

src/client/spark_test/system_test_scripts/test_join_algo.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+# Stop script on NZEC
+#set -e # the will allow sparkd to keep running if a test fails, preventing Jenkins from continuing
+# Stop script if unbound variable found (use ${var:-} if intentional)
+set -u
+
+if [ -d ~/sparkdata/ ]
+then
+   sparkdata_dir=~/sparkdata
+else
+   sparkdata_dir=/sparkdata
+fi
+
+# Use in the the functions: eval $invocation
+invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"'
+
+# standard output may be used as a return value in the functions
+# we need a way to write text on the screen in the functions so that
+# it won't interfere with the return value.
+# Exposing stream 3 as a pipe to standard output of the script itself
+exec 3>&1
+
+# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors.
+# See if stdout is a terminal
+if [ -t 1 ] && command -v tput > /dev/null; then
+    # see if it supports colors
+    ncolors=$(tput colors)
+    if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then
+        bold="$(tput bold       || echo)"
+        normal="$(tput sgr0     || echo)"
+        black="$(tput setaf 0   || echo)"
+        red="$(tput setaf 1     || echo)"
+        green="$(tput setaf 2   || echo)"
+        yellow="$(tput setaf 3  || echo)"
+        blue="$(tput setaf 4    || echo)"
+        magenta="$(tput setaf 5 || echo)"
+        cyan="$(tput setaf 6    || echo)"
+        white="$(tput setaf 7   || echo)"
+    fi
+fi
+
+say_warning() {
+    printf "%b\n" "${yellow:-}$test_script_name: Warning: $1${normal:-}"
+}
+
+say_err() {
+    printf "%b\n" "${red:-}$test_script_name: Error: $1${normal:-}" >&2
+}
+
+say() {
+    # using stream 3 (defined in the beginning) to not interfere with stdout of functions
+    # which may be used as return value
+    printf "%b\n" "${cyan:-}$test_script_name:${normal:-} $1" >&3
+}
+
+say_verbose() {
+    if [ "$verbose" = true ]; then
+        say "$1"
+    fi
+}
+
+is_merge_with_md_on() {
+    eval $invocation
+
+    if [ $(./sparkd configcheck ExchangeSim | grep merge_with_md: | cut -d " " -f 2) == "false" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+set_merge_with_md_off() {
+    eval $invocation
+
+    if [ -f $config_file_path ]; then
+        say_verbose "config file exists ... removing"
+        rm $config_file_path
+    fi
+
+    echo "$merge_with_md_key" >> $config_file_path && test_config_file_created=true
+
+    return 0
+}
+
+# args
+# return_var - $1
+# class_name - $2
+get_tests_in_class() {
+    eval $invocation
+
+    local return_var=$1
+    local class_name=$2
+    local arr=($(dotnet sparkcmd.dll test --type System --list | perl -ne '/^( *)/; $i = length $1; $j && $i >= $j and print, next; $j = 0; /'"$class_name"'/ and $j = $i + 1, print' | tail -n +2))
+    eval $return_var='("${arr[@]}")'
+
+    return 0
+}
+
+# Start sparkd
+nohup ./sparkd run &
+pid=$!
+echo "sparkd pid: " $pid
+echo "sleeping for 120 seconds..."
+sleep 120
+
+
+# args
+# class_name - $1
+run_bts_test() {
+    eval $invocation
+
+    class_name=$1
+
+    local test_names_tuple=()
+    get_tests_in_class test_names_tuple $class_name
+    for test_name in ${test_names_tuple[@]}; do
+        say_verbose "running test with name: $test_name ...";
+        dotnet sparkcmd.dll test $test_name
+        exit_status=$?
+    done
+
+    return 0
+}
+
+test_script_name=$(basename "$0")
+verbose=true
+config_file_path="$sparkdata_dir/conf/test_join_algo_tests.conf"
+test_config_file_created=false
+merge_with_md_key=$(/bin/cat <<'END_HEREDOC'
+ExchangeSim {
+  value {
+    merge_with_md: false
+  }
+}
+END_HEREDOC
+)
+
+if is_merge_with_md_on; then
+    set_merge_with_md_off
+    if is_merge_with_md_on; then
+        say_err "Could not turn off merge_with_md"
+        say_err "Please investigate manually"
+        exit 1
+    fi
+else
+    say_verbose "merge with md is already off"
+fi
+
+run_bts_test "JoinAlgoTests"
+
+if [ "$test_config_file_created" = true ]; then
+    rm $config_file_path
+fi
+
+# Kill sparkd
+pkill -P $$
+
+[ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
+exit $exit_status

src/client/spark_test/system_test_scripts/test_position_limits.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+if [ -d ~/sparkdata/ ]
+then
+   sparkdata_dir=~/sparkdata
+else
+   sparkdata_dir=/sparkdata
+fi
+
+./delete_dbs.sh
+
+config=$sparkdata_dir/conf/spark.conf
+bak=$sparkdata_dir/conf/spark.conf.bak
+
+[ ! -d "$sparkdata_dir/conf" ] && mkdir $sparkdata_dir/conf/
+[ -f "$conf" ] && mv $conf $bak
+
+cat << EOF > $sparkdata_dir/conf/spark.conf
+   ExchangeSim {
+      value {
+         merge_with_md:true
+      }
+   }
+   Logging {
+      value {
+            console { default_level: "Error" }
+            syslog { default_level: "Error" }
+            file { default_level: "Error" }
+      }
+}
+EOF
+
+
+nohup ./sparkd run &
+pid=$!
+echo "sparkd pid: " $pid
+
+dotnet sparkcmd.dll test test_position_limits
+exit_status=$?
+
+pkill -P $$
+
+[ -f "$bak" ] && mv $bak $conf
+
+[ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
+exit $exit_status

src/client/sparkcmd/Cmd/MBOCommand.cs

@@ -23,26 +23,14 @@ namespace bts {
          return 0;
       }
       
-      static void dump_mbo(UInt32 key, api.Side side) {
+      static void dump_mbo(uint key, api.Side side) {
          using (var ctx = new ServiceContext()) {
             var mbo = ctx.locate<MBOService>();
             var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key });
 
             Console.WriteLine($"Order data for {data.SecurityId}");
-            IEnumerable<OrderPosition> orders = data.Bid;
-            switch(side) {
-               case api.Side.Undefined:
-                  orders = data.Ask.Concat(data.Bid);
-               break;
-               case api.Side.Buy:
-                  orders = data.Bid;
-               break;
-               case api.Side.Sell:
-                  orders = data.Ask;
-               break;
-            }
             var price = 0.0;
-            foreach(var e in orders) {
+            foreach(var e in data.Bid.Concat(data.Ask)) {
                if (e.Price != price) {
                   price = e.Price;
                   Console.WriteLine("================");

src/client/sparkcmd/Cmd/ServiceTail.cs

@@ -45,6 +45,9 @@ namespace bts {
          if (o.json) {
             _format = PrintFormat.Json;
          }
+         if (o.time) {
+            _timestamp = true;
+         }
 
          tail(split[0], split[1]);
          return 0;

src/lib/api/services/MBOService.proto

@@ -13,16 +13,18 @@ message OrderPIQ {
 message OrderPosition {
   uint64 exchange_id = 1;
   uint32 order_id = 2;
-  float price = 10;
+  double price = 10;
   int32 size = 11;
 }
 message PositionData {
   uint32 security_id = 1;
   repeated OrderPosition bid = 10;
-  repeated OrderPosition ask = 20;
+  repeated OrderPosition ask = 11;
 }
 message MBOSecurityQuery {
   uint32 security_id = 1;
+  double price_level = 2;
+  uint32 limit = 3;
 }
 
 service MBOService {

src/lib/config/protos/ExchangeSimCfg.proto

@@ -9,5 +9,6 @@ message ExchangeSim {
         string replay_file = 12 [default = "futures.coconut"];
         bool local_relay = 13;
     }
+    optional uint32 orders_per_level = 20 [default = 10];
 }
 

src/lib/config/protos/UI.proto

@@ -69,6 +69,12 @@ message LadderKnobs {
   optional uint32 security_timeout_secs = 5 [default = 60];
   optional bool open_algo_popup_on_select = 6 [default = false];
   optional bool use_algo_param_grid = 7 [default = true];
+  optional uint32 max_mbo_orders = 8 [default = 200];
+}
+
+message OrderBookOptions {
+   optional bool animate_child_order_rows = 1 [default = false];
+   optional double status_color_opacity_when_selected = 2 [default = 0.35];
 }
 
 message ShellPingOptions {

src/lib/md/MBOProcessor.cpp

@@ -1,15 +1,21 @@
 #include "MBOProcessor.h"
-#include  "lib/utils/Dispatcher.h"
 #include "lib/utils/ConcurrentPool.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/toggles.pb.h"
+#include "lib/utils/thread_manager.h"
 
 #include <map>
 #include <functional>
 #include <vector>
 #include <sstream>
+#include <thread>
+#include <atomic>
 
+#ifndef _WIN32
+#include <google/btree/btree_set.h>
+#endif
 #include <tbb/concurrent_hash_map.h>
+#include <tbb/concurrent_queue.h>
 
 
 namespace bts {
@@ -57,34 +63,126 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
 }
 
 typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
+struct Incremental {
+   MBOWorkingState* state;
+   inc::Update update;
+};
 struct MBOProcessorImpl {
    utils::ConcurrentPool<MBOSnapshot> pool;
+   tbb::concurrent_queue<Incremental> incremental_input;
    MapType snapshots;
-   utils::Dispatcher& dispatcher;
-   utils::DispatcherTimer timer;
-   MBOProcessorImpl(utils::Dispatcher& d) :
+   bool running;
+   utils::TimeStamp last_pool_print;
+   std::thread thread;
+
+   MBOProcessorImpl() :
       pool(2048,512),
       snapshots(),
-      dispatcher(d),
-      timer(d, 600*1000, [&]() { print_pool_stats(pool); })
-      {
-         if (config::get<config::toggles::MBOStats>().enabled()) {
-            timer.start();
-         }
+      running(true),
+      last_pool_print(utils::TimeStamp::invalid()),
+      thread(threading::make_thread("MBO", [=]() { process_incremental(); }))
+   {
+      if (config::get<config::toggles::MBOStats>().enabled()) {
+         last_pool_print = utils::TimeStamp::now();
       }
+
+   }
+
+   ~MBOProcessorImpl() {
+      running = false;
+      thread.join();
+   }
+
+   void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
+      bts_assert(*snapshot != nullptr);
+      MapType::accessor accessor;
+      snapshots.insert(accessor, key);
+      std::swap(accessor->second, *snapshot);
+      if (*snapshot != nullptr) {
+         // Note: relase() must happen inside accessor lifetime to ensure correctness
+         pool.release(snapshot);
+      }
+   }
+
+   void publish_incremental(MBOWorkingState& state) {
+      // determine if we need to publish
+      auto key = SecurityKey(state.handle.key());
+      if (key.value() != invalid_security_key) {
+         auto snap = pool.allocate();
+         snap->bids.reserve(state.bid.size());
+         snap->asks.reserve(state.ask.size());
+         copy_to_snapshot(state, snap);
+         publish_snapshot(key, &snap);
+      }
+   }
+
+   void process_incremental() {
+      #ifndef _WIN32
+      // drain queue and update state
+      // insert changed state pointers into a set
+      // after draining publish the set in small batches
+      // re-check input queue regularly 
+
+      btree::btree_set<MBOWorkingState*> dirty;
+      std::vector<MBOWorkingState*> clean;
+
+      while(running) {
+         Incremental temp;
+
+         // Drain entire input queue quickly
+         while(incremental_input.try_pop(temp)) {
+            bts_assert_d(temp.state != nullptr);
+            if (temp.state != nullptr) {
+               apply_incremental(*temp.state, temp.update);
+               dirty.insert(temp.state);
+            }
+         }
+
+         // publish a small batch of snapshots (then check queue again)
+         for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
+            publish_incremental(**iter);
+            clean.push_back(*iter);
+            if (clean.size() >= 128) break;
+         }
+
+         // Remove 'clean' from set
+         for (const auto& iter : clean) {
+            dirty.erase(iter);
+         }
+         clean.clear();
+
+         // If the queue is still empty sleep briefly
+         if (incremental_input.empty()) { 
+            if (last_pool_print != TimeStamp::invalid()) {
+               auto now = TimeStamp::now();
+               if ((now - last_pool_print).seconds() > 600) {
+                  last_pool_print =  now;
+                  print_pool_stats(pool);
+               }
+            }
+            std::this_thread::sleep_for(
+               std::chrono::milliseconds(1)
+            );
+         }
+
+      }
+      #endif
+   }
+
 };
 
 //
 // Processor
 
-MBOProcessor::MBOProcessor(utils::Dispatcher* d) :
-   _me(std::make_shared<MBOProcessorImpl>(d == nullptr ? utils::Dispatcher::background() : *d)),
-   _producer(_me),
-   _consumer(_me)
-{}
+MBOProcessor::MBOProcessor() :
+   _me(std::make_shared<MBOProcessorImpl>()),
+   _writer(_me),
+   _reader(_me)
+{
+}
 
-MBOWriter& MBOProcessor::writer() { return _producer;}
-MBOReader& MBOProcessor::reader() { return _consumer; }
+MBOWriter& MBOProcessor::writer() { return _writer;}
+MBOReader& MBOProcessor::reader() { return _reader; }
 
 //
 // Writer
@@ -98,18 +196,11 @@ void MBOWriter::release_snapshot(MBOSnapshot** s) {
    _me->pool.release(s);
 }
 void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
-   bts_assert(*snapshot != nullptr);
-   MapType::accessor accessor;
-   _me->snapshots.insert(accessor, key);
-   std::swap(accessor->second, *snapshot);
-   if (*snapshot != nullptr) {
-      // Note: relase() must happen inside accessor lifetime to ensure correctness
-      _me->pool.release(snapshot);
-   }
+   _me->publish_snapshot(key,snapshot);
 }
 
+#if 0
 bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) {
-   // bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS");
    #define REQUIRE(a,b,y) \
       if ((a) != (b)) { \
          LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \
@@ -137,25 +228,11 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot*
    #undef REQUIRE
    return true;
 }
+#endif
 
 void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) {
    LOG(md,Diag, "sec:{} : {}", state.handle.key(), update);
-
-   _me->dispatcher.post([this,&state,update]() {
-      // fast part
-      apply_incremental(state,update);
-      // slow part
-      publish_incremental(state);
-   });
-}
-void MBOWriter::publish_incremental(MBOWorkingState& state) {
-   // determine if we need to publish
-   auto key = SecurityKey(state.handle.key());
-   if (key.value() != invalid_security_key) {
-      auto snap = allocate_snapshot(state.bid.size(), state.ask.size());
-      copy_to_snapshot(state, snap);
-      publish_snapshot(key, &snap);
-   }
+   _me->incremental_input.push({&state, update});
 }
 
 void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) {
@@ -267,4 +344,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) {
    return os;
 }
 
-}
+}

src/lib/md/MBOProcessor.h

@@ -2,7 +2,6 @@
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/domain_model/Security.h"
 #include "lib/utils/TimeStamp.h"
-#include "lib/utils/Dispatcher.h"
 #include "IntPrice.h"
 #include "ExchangeOrderKey.h"
 #include "MDProducerHandle.h"
@@ -152,8 +151,6 @@ class MBOWriter {
 
    // Enqueue the update to the mbo work queue.
    void enqueue_incremental(MBOWorkingState&, const inc::Update&);
-   // Update working state and publish directly.
-   void publish_incremental(MBOWorkingState&);
 };
 
 class MBOReader {
@@ -169,11 +166,11 @@ class MBOReader {
 
 class MBOProcessor {
    std::shared_ptr<MBOProcessorImpl> _me;
-   MBOWriter _producer;
-   MBOReader _consumer;
+   MBOWriter _writer;
+   MBOReader _reader;
 
 public:
-   MBOProcessor(utils::Dispatcher* _=nullptr);
+   MBOProcessor();
    MBOWriter& writer();
    MBOReader& reader();
 };
@@ -184,3 +181,4 @@ public:
 namespace std {
 std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&);
 }
+

src/lib/md/MBOServiceServer.cpp

@@ -4,6 +4,7 @@
 #include "lib/api/services/OrderService.client.pb.h"
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/TimeStamp.h"
+#include "lib/domain_model/Security.h"
 #include "MDClient.h"
 #include "ExchangeOrderKey.h"
 #include "MBOProcessor.h"
@@ -307,27 +308,41 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
       reverse.insert(std::make_pair(pair.second.exchange_id(), pair.first));
    }
 
+   size_t count = 0;
+   auto tick = handle.get_minimum_tick();
    if(handle.is_initialized()) {
       auto mbo = handle.get_mbo();
       if (mbo) {
          for (const auto& o : mbo->bids) {
-            auto* entry = response.add_bid();
-            entry->set_exchange_id(o.id.value());
-            entry->set_price(o.price.price());
-            entry->set_size(o.size);
-            auto iter = reverse.find(o.id);
-            if (iter != reverse.end()) {
-               entry->set_order_id(iter->second.value());
+            double price = o.price.price();
+            bool isnan = os::is_nan(request.price_level());
+            if (isnan || securities::test_price_equality(request.price_level(), price, tick)) {
+               ++count;
+               if (request.limit() && count > request.limit()) break;
+               auto* entry = response.add_bid();
+               entry->set_exchange_id(o.id.value());
+               if (isnan) entry->set_price(price);
+               entry->set_size(o.size);
+               auto iter = reverse.find(o.id);
+               if (iter != reverse.end()) {
+                  entry->set_order_id(iter->second.value());
+               }
             }
          }
          for (const auto& o : mbo->asks) {
-            auto* entry = response.add_ask();
-            entry->set_exchange_id(o.id.value());
-            entry->set_price(o.price.price());
-            entry->set_size(o.size);
-            auto iter = reverse.find(o.id);
-            if (iter != reverse.end()) {
-               entry->set_order_id(iter->second.value());
+            double price = o.price.price();
+            bool isnan = os::is_nan(request.price_level());
+            if (isnan || securities::test_price_equality(request.price_level(), price, tick)) {
+               ++count;
+               if (request.limit() && count > request.limit()) break;
+               auto* entry = response.add_ask();
+               entry->set_exchange_id(o.id.value());
+               if (isnan) entry->set_price(price);
+               entry->set_size(o.size);
+               auto iter = reverse.find(o.id);
+               if (iter != reverse.end()) {
+                  entry->set_order_id(iter->second.value());
+               }
             }
          }
       }

src/lib/md/MDConsumer.cpp

@@ -269,6 +269,16 @@ public:
       if (!_cold.is_initialized()) return true;
       return is_heart_beat_ok(cold_header());
    }
+
+   void dump_shm_to(const char* directory) const {
+      _cold.dump_to(directory);
+      if (_hot) {
+         _hot->dump_to(directory);
+      }
+      if (_hot_depth) {
+         _hot_depth->dump_to(directory);
+      }
+   }
   
 private:
 
@@ -910,6 +920,15 @@ public:
       return result;
    }
 
+   void dump_shm_to(const char* p) const {
+      // To be safe we should lock _work_mutex -but-
+      // we need this function to diagnose problems happening in the on_security callback
+      // which holds the _work_mutex.  Since anyone calling this function is going down
+      // anyway we as well live dangerously.
+      for (auto& inst : _instances) {
+         inst->dump_shm_to(p);
+      }
+   }
 };
 
 
@@ -957,6 +976,8 @@ MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityTyp
 MDConsumerChannel MDConsumer::get_channel(const UserSymbol& u) const {
    return _pimpl->get_channel(u.symbol(), u.type(), u.exchange());
 }
+void MDConsumer::dump_shm_to(const char* p) const { _pimpl->dump_shm_to(p); }
+
 
 MDConsumerChannel::MDConsumerChannel(const shared_ptr<MDConsumerInst>& inst) : _pimpl(inst) {}
 int64_t MDConsumerChannel::read_trades(google::protobuf::RepeatedPtrField<co2::TradeMessage>& trades, int64_t cursor) { return _pimpl->read_trades(trades, cursor); }

src/lib/md/MDConsumer.h

@@ -67,6 +67,8 @@ public:
    }
    uintptr_t id() const { return reinterpret_cast<uintptr_t>(_pimpl.get()); }
 
+   void dump_shm_to(const char* directory) const;
+
 private:
    //BTS_NO_COPY(MDConsumer); COPY OK
    std::shared_ptr<MDConsumerImpl> _pimpl;

src/lib/md/MDConsumerInterop.cpp

@@ -177,5 +177,11 @@ CAPI void post_on_scan_security_thread(just_action a) {
    md::scan_securities_thread().post(a);
 }
 
+CAPI void dump_co2_memory_to_disk(MDConsumer* consumer, const char* path) {
+   if (consumer) {
+      consumer->dump_shm_to(path);
+   }
+}
+
 }} //namespace
 #endif

src/lib/md/MDProducer.cpp

@@ -169,6 +169,7 @@ MDProducerChannel MDProducer::get_channel(const char* symbol, SecurityType type,
 }
 
 void MDProducer::heartbeat() {
+   std::lock_guard<std::mutex> lock(*_mutex);
    for (auto& impl : _producers) {
       impl->heartbeat();
    }

src/lib/md/relay/RelayClient.cpp

@@ -143,7 +143,9 @@ public:
                   LOG(md,Diag, "Added {} trades", ch.trade_log_size());
                }
                for (const auto& trade : ch.trade_log()) {
-                  channel.add_trade(trade);
+                  if (trade.key()) {
+                     channel.add_trade(trade);
+                  }
                }
                for (const auto& rfq : ch.rfq_log()) {
                   channel.add_rfq(rfq);

src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp

@@ -10,6 +10,18 @@ namespace orderman {
 
 using namespace api::orderman;
 
+enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE };
+const char *CoverAlgo::status_name(Status st) {
+   switch (st) {
+   case Status::WORKING:
+      return "WORKING";
+   case Status::WAITING_FOR_HEDGES:
+      return "WAITING_FOR_HEDGES";
+   case Status::DONE:
+      return "Done";
+   }
+   return nullptr;
+}
 struct CoverAlgo::PImpl {
    IAlgoServices *algo;
    OrderKey child_key = null_order_key();
@@ -17,6 +29,8 @@ struct CoverAlgo::PImpl {
    ChildTemplate *child_template = nullptr;
    ChildTemplate *hedge_template = nullptr;
    Quantity hedged_fills = Quantity(0);
+   Status status = Status::WORKING;
+   std::vector<OrderKey> hedges;
 };
 
 CoverAlgo::CoverAlgo() : me(new PImpl()) {}
@@ -77,64 +91,96 @@ void CoverAlgo::do_launch_attached(IAlgoServices &algo, OrderKey k,
 }
 
 void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
-   bts_assert(k == me->child_key); // We are not subscribed to hedge changes
+   if (k == me->child_key) {
+      if (st.total_fills.value() > me->hedged_fills.value()) {
+         auto hedge_qty =
+             Quantity(st.total_fills.value() - me->hedged_fills.value());
+         auto hedge_price = securities::improve_price(
+             algo().definition().side(), st.last_price,
+             me->price_improvement_delta);
 
-   if (st.total_fills.value() > me->hedged_fills.value()) {
-      auto hedge_qty =
-          Quantity(st.total_fills.value() - me->hedged_fills.value());
-      auto hedge_price =
-          securities::improve_price(algo().definition().side(), st.last_price,
-                                    me->price_improvement_delta);
+         auto hedge =
+             algo().launch_child(*me->hedge_template, hedge_price, hedge_qty);
+         me->hedges.push_back(hedge);
 
-      algo().launch_child(*me->hedge_template, hedge_price, hedge_qty,
-                          false); // do not subscribe to hedge changes
-
-      me->hedged_fills = Quantity(hedge_qty.value() + me->hedged_fills.value());
-      LOG(OrderMan, Diag,
-          "CoverAlgo {}, launched new hedge in response to child fill. Hedge: "
-          "{} {}@{}",
-          algo().order_key().value(),
-          securities::side_name(algo().definition().side()), hedge_qty.value(),
-          hedge_price.value());
-   }
-   algo().update_fill_state(st.total_fills, st.last_price);
-   switch (st.status) {
-   case ChildStatus::WORKING:
-      return;
-   case ChildStatus::COMPLETED:
-      algo().detach_all_children();
-      algo().complete();
-      return;
-   case ChildStatus::CANCELED:
-      algo().terminate_error("Unexpected child cancel");
-      return;
+         me->hedged_fills =
+             Quantity(hedge_qty.value() + me->hedged_fills.value());
+         LOG(OrderMan, Diag,
+             "CoverAlgo {}, launched new hedge in response to child fill. "
+             "Hedge: "
+             "{} {}@{}",
+             algo().order_key().value(),
+             securities::side_name(algo().definition().side()),
+             hedge_qty.value(), hedge_price.value());
+      }
+      algo().update_fill_state(st.total_fills, st.last_price);
+      switch (st.status) {
+      case ChildStatus::WORKING:
+         break;
+      case ChildStatus::COMPLETED:
+         LOG(OrderMan, Info,
+             "CoverAlgo {}. Child is completed, waiting for hedges ",
+             algo().order_key().value());
+         me->status = Status::WAITING_FOR_HEDGES;
+         break;
+      case ChildStatus::CANCELED:
+         LOG(OrderMan, Info, "CoverAlgo {}. Child was canceled, terminating",
+             algo().order_key().value());
+         me->status = Status::DONE;
+         algo().terminate_error("Unexpected child cancel");
+         break;
+      }
+   } else {
+      switch (st.status) {
+      case ChildStatus::WORKING:
+         break;
+      case ChildStatus::COMPLETED:
+         if (auto it = std::find(me->hedges.begin(), me->hedges.end(), k);
+             it != me->hedges.end())
+            me->hedges.erase(it);
+         break;
+      case ChildStatus::CANCELED:
+         LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating",
+             algo().order_key().value());
+         me->status = Status::DONE;
+         algo().terminate_error("Unexpected child cancel");
+         break;
+      }
    }
+   check_terminate();
 }
 
 void CoverAlgo::do_update(api::orderman::Update const &update) {
-   LOG(OrderMan, Diag, "CoverAlgo order {} receiving update: {}",
-       algo().order_key().value(), utils::pb::to_string(update));
+   if (me->status == Status::WORKING) {
+      LOG(OrderMan, Diag, "CoverAlgo order {} receiving update: {}",
+          algo().order_key().value(), utils::pb::to_string(update));
 
-   if (update.has_custom() && !update.custom().has_cover()) {
-      algo().terminate_error("Invalid update");
-      return;
-   }
-
-   Update child_update;
-   child_update.set_price(update.price());
-   child_update.set_quantity(update.quantity());
-
-   // process custom part of the udpate if any
-   if (update.has_custom()) {
-      auto &cover_update = update.custom().cover();
-      set_price_improvement_delta(algo(), cover_update);
-      if (cover_update.has_child()) {
-         *child_update.mutable_custom() = cover_update.child();
+      if (update.has_custom() && !update.custom().has_cover()) {
+         algo().terminate_error("Invalid update");
+         return;
       }
-      if (cover_update.has_hedge())
-         init_hedge_template(algo(), cover_update.hedge());
+
+      Update child_update;
+      child_update.set_price(update.price());
+      child_update.set_quantity(update.quantity());
+
+      // process custom part of the udpate if any
+      if (update.has_custom()) {
+         auto &cover_update = update.custom().cover();
+         set_price_improvement_delta(algo(), cover_update);
+         if (cover_update.has_child()) {
+            *child_update.mutable_custom() = cover_update.child();
+         }
+         if (cover_update.has_hedge())
+            init_hedge_template(algo(), cover_update.hedge());
+      }
+      algo().update_child(me->child_key, child_update);
+   } else {
+      LOG(OrderMan, Info,
+          "CoverAlgo {}. Received update after child has completed. Ignoring "
+          "it.",
+          algo().order_key().value());
    }
-   algo().update_child(me->child_key, child_update);
 }
 
 void orderman::CoverAlgo::init_hedge_template(
@@ -155,5 +201,18 @@ void CoverAlgo::set_price_improvement_delta(
        securities::tick_size(serv.security()).value() * cover.hedge_ticks());
 }
 
+void CoverAlgo::check_terminate() {
+   LOG(OrderMan, Diag,
+       "CoverAlgo {} check_terminate. State: {} Hedge count: {}",
+       algo().order_key(), log::CharPtrWrapper{status_name(me->status)},
+       me->hedges.size());
+
+   if (me->status == Status::WAITING_FOR_HEDGES && me->hedges.empty()) {
+      LOG(OrderMan, Info, "CoverAlgo {}. Last hedge completed, terminating",
+          algo().order_key().value());
+      algo().complete();
+   }
+}
+
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h

@@ -21,6 +21,7 @@ public:
 
 private:
    struct PImpl;
+   enum class Status; 
    std::unique_ptr<PImpl> me;
    IAlgoServices &algo() const;
    void
@@ -28,6 +29,8 @@ private:
                                api::orderman::CoverOrderDefinition const &);
    void init_hedge_template(IBeforeLaunchAlgoServices &,
                             api::orderman::CustomOrderDefinition const &);
+   void check_terminate();
+   const char *status_name(Status st);
 };
 
 } // namespace orderman

src/lib/orderman/orders/algo/algo_orders/JoinAlgo.cpp

@@ -3,6 +3,7 @@
 
 #include "lib/orderman/order_factory/IOrderFactory.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
+#include "lib/os/os.h"
 #include "lib/utils/PBUtils.h"
 
 namespace bts {
@@ -20,6 +21,7 @@ struct JoinAlgo::PImpl {
    ChildTemplate *child_template = nullptr;
    Book book_at_inception;
    Book last_book;
+   Price joined_price=Price(os::nan());
 };
 
 JoinAlgo::JoinAlgo() : me(new PImpl()) {}
@@ -80,7 +82,8 @@ void JoinAlgo::do_update(api::orderman::Update const &update) {
    }
 
    Update child_update;
-   child_update.set_price(update.price());
+   Price price_to_pass = get_price_to_pass(Price(update.price()));
+   child_update.set_price(price_to_pass.value());
    child_update.set_quantity(update.quantity());
    if (update.has_custom()) {
       update_join_threshold();
@@ -146,6 +149,7 @@ void JoinAlgo::try_to_join() {
 
       // Time to join
       Update child_update;
+      me->joined_price = Price(my_side_psz.price());
       child_update.set_price(my_side_psz.price());
       child_update.set_quantity(algo().quantity().value());
       algo().update_child(me->child_key, child_update);
@@ -161,6 +165,20 @@ void JoinAlgo::try_to_join() {
    }
 }
 
+Price JoinAlgo::get_price_to_pass (Price price) const{
+   Price rv;
+   switch (me->status) {
+   case Status::WAITING_FOR_MD:      
+   case Status::WAITING_TO_JOIN:
+      rv = price;
+      break;
+   case Status::JOINED:
+      rv = me->joined_price;
+      break;
+   }
+   return rv;
+}
+
 void JoinAlgo::update_join_threshold() {
    me->min_quantity_to_join = SpreaderUtils::_quantity_threshold(
       algo().definition().custom_def().join().join_threshold(), algo().quantity());

src/lib/orderman/orders/algo/algo_orders/JoinAlgo.h

@@ -31,6 +31,7 @@ private:
    void update_join_threshold();
    bool has_running_child() const;
    void try_to_join();
+   Price get_price_to_pass (Price price) const;
 };
 
 }}

src/lib/orderman/orders/algo/algo_orders/LeanAlgo.cpp

@@ -28,6 +28,7 @@ struct LeanAlgo::PImpl {
    Book curr_book;
    Price out_price = Price(os::nan());
    Quantity out_qty = Quantity(0);
+   Price cross_price = Price(os::nan());
 };
 
 LeanAlgo::LeanAlgo() : me(new PImpl()) {}
@@ -79,7 +80,8 @@ void LeanAlgo::do_update(const api::orderman::Update &update) {
       return;
    }
    Update child_update;
-   child_update.set_price(update.price());
+   Price price_to_pass = _get_price_to_pass(Price(update.price()));
+   child_update.set_price(price_to_pass.value());
    child_update.set_quantity(update.quantity());
    if (update.has_custom()) {
       auto &lean_update = update.custom().lean();
@@ -195,6 +197,13 @@ LeanAlgo::MarketChange LeanAlgo::_market_change() const{
    else
       return MarketChange::SIZE_CHANGE;
 }
+Price LeanAlgo::_get_price_to_pass (Price price) const{
+   // if it has crossed once, then the
+   // child order is always passed that price
+   if(!os::is_nan(me->cross_price.value()))
+      return me->cross_price;
+   return price;
+}
 // actions ////////////////////////////////////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
 void LeanAlgo::_analyze_and_act() {
@@ -256,12 +265,11 @@ void LeanAlgo::_on_toward_me() {
 // it returns true if it successfully crossed.
 bool LeanAlgo::_try_cross(Price price) {
    if(_within_chase_band(price)){
-      Quantity level_qty, cumm_qty;
-      SpreaderUtils::find_qty_at_price_level(me->curr_book,
+      Quantity cumm_qty;
+      SpreaderUtils::find_qty_upto_price_level(me->curr_book,
                                              _opp_side(),
                                              price,
                                              me->tick_size,
-                                             level_qty,
                                              cumm_qty);
       Quantity trigger_quantity = _cross_threshold();
       if(cumm_qty.value()<trigger_quantity.value()){         
@@ -298,8 +306,9 @@ void LeanAlgo::_udpate_out_price_qty(Price price, Quantity qty){
 }
 bool LeanAlgo::_modify_price_price_qty(Price price, Quantity qty){
    if(me->out_price.value()!=price.value() ||
-      me->out_qty.value()!=qty.value()){
-      _udpate_out_price_qty(price,qty);   
+      me->out_qty.value()!=qty.value()){         
+      _udpate_out_price_qty(price,qty);
+      me->cross_price = price;
       Update child_update;
       child_update.set_price(price.value()),
       child_update.set_quantity(qty.value());

src/lib/orderman/orders/algo/algo_orders/LeanAlgo.h

@@ -34,7 +34,8 @@ class LeanAlgo : public IAlgoOrder {
    api::Side _opp_side() const;
    bool _within_chase_band(Price price) const;
    Price _chase_limit() const;
-   MarketChange _market_change() const;   
+   MarketChange _market_change() const;
+   Price _get_price_to_pass (Price price) const;
    void _analyze_and_act();
    bool _try_cross(Price);
    void _go_furthest_possible();

src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h

@@ -239,19 +239,18 @@ public:
    // cum qty upto that price.
    // returns false if the target_price is not found.
    // this is used to find the qty and cumm_qty   
-   static bool find_qty_at_price_level (const Book& book,
+   static bool find_qty_upto_price_level (const Book& book,
                                         api::Side book_side,
                                         Price target_price,
                                         double,
-                                        Quantity& price_qty,
                                         Quantity& cumm_qty){
-      price_qty = Quantity(0); cumm_qty = Quantity(0);
+      cumm_qty = Quantity(0);
       const size_t n_levels = book.n_levels();
       const bts::orderman::BookSide& book_depth = book.side(book_side);      
       for (size_t level = 0; level < n_levels; level++){
          const md::PriceSize& level_price_qty = book_depth[level];         
          double level_price = level_price_qty.price();         
-         price_qty = Quantity(level_price_qty.size());
+         auto price_qty = Quantity(level_price_qty.size());
          if(securities::is_better(book_side,target_price,Price(level_price)) ||
             price_qty.value()==0)
             break;
@@ -407,8 +406,8 @@ public:
                                                              Price& out_price){
       out_price = current_price;
       api::Side book_side = invert_side(hedge_side);
-      Quantity price_level_qty, cumm_qty;
-      bool find_qty_at_price = find_qty_at_price_level (book, book_side, current_price, tick_size, price_level_qty, cumm_qty);
+      Quantity cumm_qty;
+      bool find_qty_at_price = find_qty_upto_price_level (book, book_side, current_price, tick_size, cumm_qty);
       if (find_qty_at_price){
          Quantity qty_threshold = _quantity_threshold(params.quantity_threshold(), current_qty);
          if (cumm_qty.value() > qty_threshold.value()){
@@ -432,8 +431,8 @@ public:
                                                            Price& out_price){
       out_price = current_price;
       api::Side book_side = invert_side(hedge_side);      
-      Quantity price_level_qty, cumm_qty;
-      bool find_qty_at_price = find_qty_at_price_level (book, book_side, current_price, tick_size, price_level_qty, cumm_qty);
+      Quantity cumm_qty;
+      bool find_qty_at_price = find_qty_upto_price_level (book, book_side, current_price, tick_size, cumm_qty);
       if (find_qty_at_price){
          Quantity qty_threshold = _quantity_threshold(params.quantity_threshold(), current_qty);
          if (cumm_qty.value() < qty_threshold.value()){
@@ -636,18 +635,16 @@ public:
                                        const Book& book,
                                        Price& price, 
                                        Quantity& quantity){
-      Quantity out_qty(0), out_cum_qty(0);
+      Quantity out_cum_qty(0);
       if(leg.trading_side()==api::Side::SIDE_BUY)
-         find_qty_at_price_level(book,api::Side::SIDE_SELL,
+         find_qty_upto_price_level(book,api::Side::SIDE_SELL,
                                  price,
                                  leg.tick_size(),
-                                 out_qty,
                                  out_cum_qty);
       else if(leg.trading_side()==api::Side::SIDE_SELL)
-         find_qty_at_price_level(book,api::Side::SIDE_BUY,
+         find_qty_upto_price_level(book,api::Side::SIDE_BUY,
                                  price,
                                  leg.tick_size(),
-                                 out_qty,
                                  out_cum_qty);
       quantity = Quantity(std::min(quantity.value(),out_cum_qty.value()));
    }

src/lib/orderman/orders/spreader_orders/tests/spreader_tests.cpp

@@ -2340,34 +2340,43 @@ TEST_CASE ("spreader_WAITING_FOR_CANCEL_ACK", "[spreader]")
 }
 TEST_CASE ("find_target_price_in_book", "[spreader]"){
    SECTION("0.05"){
-      Quantity out_level_qty, out_cum_qty;     
+      Quantity out_cum_qty;     
       bts::orderman::Book book;      
       uint16_t sizes[] = { 11, 13, 17, 19, 23 };
       double prices[] = { 22, 21.15, 20.10, 20.05, 20.00 };
       fill_book (book, prices, sizes, bts::api::Side::SIDE_BUY);
       // middle
-      bool res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_BUY, Price(20.10), 0.05, out_level_qty, out_cum_qty);
+      bool res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(20.10), 0.05, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==17);
       REQUIRE(out_cum_qty.value()==41);
       // first      
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_BUY, Price(22.00), 0.05, out_level_qty, out_cum_qty);
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(22.00), 0.05, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==11);
       REQUIRE(out_cum_qty.value()==11);
       // last
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_BUY, Price(20.00), 0.05, out_level_qty, out_cum_qty);
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(20.00), 0.05, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==23);
       REQUIRE(out_cum_qty.value()==83);
-      // not found
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_BUY, Price(21.00), 0.05, out_level_qty, out_cum_qty);
-      REQUIRE(!res);
+      // missing
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(21.00), 0.05, out_cum_qty);
+      REQUIRE(res);
+      REQUIRE(out_cum_qty.value()==24);
+      // missing
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(20.075), 0.05, out_cum_qty);
+      REQUIRE(res);
+      REQUIRE(out_cum_qty.value()==41);
       // rounding error
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_BUY, Price(20.04999999), 0.05, out_level_qty, out_cum_qty);
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(20.04999999), 0.05, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==19);
       REQUIRE(out_cum_qty.value()==60);
+      // before
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(22.05), 0.05, out_cum_qty);
+      REQUIRE(!res);
+      REQUIRE(out_cum_qty.value()==0);
+      // after
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_BUY, Price(19), 0.05, out_cum_qty);
+      REQUIRE(res);
+      REQUIRE(out_cum_qty.value()==83);
    }
    SECTION("1/32"){
       Quantity out_level_qty, out_cum_qty;     
@@ -2376,34 +2385,33 @@ TEST_CASE ("find_target_price_in_book", "[spreader]"){
       double prices[] = { 100.21875, 100.28125, 101, 101.53125, 101.71875 };
       fill_book (book, prices, sizes, bts::api::Side::SIDE_SELL);
       // first
-      bool res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_SELL, Price(100.21875), 0.03125, out_level_qty, out_cum_qty);
+      bool res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(100.21875), 0.03125, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==12);
       REQUIRE(out_cum_qty.value()==12);
       // middle
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_SELL, Price(101.53125), 0.03125, out_level_qty, out_cum_qty);
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(101.53125), 0.03125, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==24);
       REQUIRE(out_cum_qty.value()==132);
       // end
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_SELL, Price(101.71875), 0.03125, out_level_qty, out_cum_qty);
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(101.71875), 0.03125, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==33);
       REQUIRE(out_cum_qty.value()==165);
       // absent
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_SELL, Price(101.5625), 0.03125, out_level_qty, out_cum_qty);
-      REQUIRE(!res);
-      REQUIRE(out_level_qty.value()==0);
-      REQUIRE(out_cum_qty.value()==0);
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_BUY, Price(101.71875), 0.03125, out_level_qty, out_cum_qty);
-      REQUIRE(!res);
-      REQUIRE(out_level_qty.value()==0);
-      REQUIRE(out_cum_qty.value()==0);
-      // rounding
-      res = SpreaderUtils::find_qty_at_price_level(book, bts::api::Side::SIDE_SELL, Price(101.531249999999), 0.03125, out_level_qty, out_cum_qty);
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(101.5625), 0.03125, out_cum_qty);
       REQUIRE(res);
-      REQUIRE(out_level_qty.value()==24);
       REQUIRE(out_cum_qty.value()==132);
+      // rounding
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(101.531249999999), 0.03125, out_cum_qty);
+      REQUIRE(res);
+      REQUIRE(out_cum_qty.value()==132);
+      // before
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(100.21875-0.03125), 0.03125, out_cum_qty);
+      REQUIRE(!res);
+      REQUIRE(out_cum_qty.value()==0);
+      // after
+      res = SpreaderUtils::find_qty_upto_price_level(book, bts::api::Side::SIDE_SELL, Price(101.71875+0.03125), 0.03125, out_cum_qty);
+      REQUIRE(res);
+      REQUIRE(out_cum_qty.value()==165);
    }
 }
 TEST_CASE ("prehedge_payless", "[spreader]"){

src/lib/orderman/server/OrderServiceServer.cpp

@@ -361,7 +361,7 @@ void OrderServiceServer::on_order_op_result(
    bts_assert(it!=me->in_flight_order_ops.end());
    auto &responder = it->second;
    api::orderman::Response r;
-   r.set_order_key(0);
+   r.set_order_key(op_result.ok.value());
    r.set_error_code(op_result.result.error);
    r.set_msg(op_result.result.error_msg);
    responder(r);

src/lib/utils/SharedList.h

@@ -15,8 +15,10 @@
 #include <utility>
 #include <thread>
 #include <atomic>
+#include <iostream>
 
 #include <tbb/queuing_mutex.h>
+#include <boost/filesystem.hpp>
 
 namespace bts {
 namespace utils {
@@ -238,6 +240,38 @@ public:
          offset = sizeof(SegmentHeader);
       }
    }
+   
+   void dump_to(const char* directory) const {
+      using namespace std;
+      boost::system::error_code _;
+      if (!boost::filesystem::exists(directory, _)) {
+         boost::filesystem::create_directories(directory, _);
+      }
+
+      if (!boost::filesystem::is_directory(directory)) {
+         LOG(md,Error, "Error dumping memory to {}", std::string(directory));
+         return;
+      }
+
+      boost::filesystem::path root(directory);
+
+      // OK to hold the lock for a while as we're typically crashing when this is called
+      tbb::queuing_mutex::scoped_lock lock(_mutex);
+
+      for (size_t x = 0; x < _segments.size(); ++x) {
+         char segment_name[512];
+         SharedList<THeader, TEntry>::get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), _name, x);
+
+         // create/overwrite output file
+         std::ofstream writer( (root/segment_name).string(), ios::out | ios::app | ios::binary);
+         if (writer.is_open()) {
+            // writer just loves signed types
+            writer.write(
+               (const char*)_segments[x].address, 
+               (std::streamsize)_segments[x].size);
+         }
+      }
+   }
 
 private:
    BTS_NO_COPY(SharedListReader);
@@ -281,7 +315,7 @@ private:
 
    const std::string _name;
    THeader* _header; // null value indicates no memory segment
-   tbb::queuing_mutex _mutex;
+   mutable tbb::queuing_mutex _mutex;
    std::unique_ptr<LowLevelSharedMemory> _shm;
    std::vector<Segment> _segments;
 };

src/sdk/README.md

@@ -67,7 +67,7 @@ $ cd spark_sdk
 
 ### Development
 
-Algo Orders typically implement the IAlgoOrder interface to respond to events and use the IAlgoServices interface to create and control child orders as well as manage their own life cycles. See [examples/order_plugin_example.cpp](examples/order_plugin_example.cpp) for a fully fledged example
+Algo Orders typically implement the [IAlgoOrder](include/spark/lib/orderman/orders/algo/IAlgoOrder.h) interface to respond to events and use the [IAlgoServices](include/spark/lib/orderman/orders/algo/IAlgoServices.h) interface to create and control child orders as well as manage their own life cycles. See [examples/order_plugin_example.cpp](examples/order_plugin_example.cpp) for a fully fledged example
 
 Look at [examples/md_publisher_example.cpp](examples/md_publisher_example.cpp) to look at how to build a custom market data producer
 

src/server/sparkd/cmds/Benchmark.cpp

@@ -34,6 +34,10 @@
 #include <mutex>
 #include <sys/time.h>
 
+#include <unordered_set>
+#include <set>
+#include <google/btree/btree_set.h>
+
 #include <date/date.h>
 #include <tbb/concurrent_queue.h>
 namespace bts {
@@ -655,6 +659,73 @@ void int_price_less(benchmark::State& state) {
 }
 BENCHMARK(int_price_less)->Range(8, 8 << 11);
 
+
+void set_hashset_insert(benchmark::State& state) {
+   std::unordered_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+   }
+}
+void set_stdset_insert(benchmark::State& state) {
+   std::set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+   }
+}
+void set_btreeset_insert(benchmark::State& state) {
+   btree::btree_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+   }
+}
+
+void set_hashset_iterate(benchmark::State& state) {
+   std::unordered_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+      for (const auto& p : s) {
+         benchmark::DoNotOptimize(p);
+      }
+   }
+}
+void set_stdset_iterate(benchmark::State& state) {
+   std::set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+      for (const auto& p : s) {
+         benchmark::DoNotOptimize(p);
+      }
+   }
+}
+void set_btreeset_iterate(benchmark::State& state) {
+   btree::btree_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+      for (const auto& p : s) {
+         benchmark::DoNotOptimize(p);
+      }
+   }
+}
+
+BENCHMARK(set_hashset_insert);
+BENCHMARK(set_stdset_insert);
+BENCHMARK(set_btreeset_insert);
+BENCHMARK(set_hashset_iterate);
+BENCHMARK(set_stdset_iterate);
+BENCHMARK(set_btreeset_iterate);
+
 std::string Benchmark::help() {
    return "Run micro-benchmarks. Use --benchmark_filter=regex to run only "
           "specific benchmarks.";

src/server/sparkd/cmds/Run.cpp

@@ -59,7 +59,7 @@ int Run::execute(std::vector<std::string> const &) {
    persistence::PersistenceServiceServer settings_saver(sc, "settings");
    user::AuthServiceServer login_server(sc);
 
-   md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO"));
+   md::MBOProcessor mbo;
    auto* mbowriter = config::get<config::toggles::PlacementInQueue>().enabled() ? &mbo.writer() : nullptr;
    md::run_market_datapublishers(mbowriter);
 

src/server/sparkd/exchange_sim/book/OrderMatcher.cpp

@@ -2,6 +2,8 @@
 #include "lib/utils/bts_assert.h"
 #include "server/sparkd/exchange_sim/book/Book.h"
 #include "server/sparkd/exchange_sim/book/Order.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/ExchangeSimCfg.pb.h"
 
 #include <set>
 
@@ -10,11 +12,12 @@ namespace ex_sim {
 
 struct OrderMatcher::PImpl {
    PImpl()
-       : bids(OrderCmp(false)), asks(OrderCmp(true)) {}
+       : bids(OrderCmp(false)), asks(OrderCmp(true)), max_orders(config::get<config::ExchangeSim>().orders_per_level()) {}
 
    OrdersSide bids;
    OrdersSide asks;
    std::vector<Trade> pending_trades;
+   size_t max_orders;
 };
 
 OrderMatcher::OrderMatcher()
@@ -53,24 +56,24 @@ void OrderMatcher::add_new(Order const &o, std::vector<Order> *out) {
    match(out);
 }
 
-std::pair<OrderMatcher::OrdersSide::iterator, OrderMatcher::OrdersSide::iterator>
-OrderMatcher::find_level(OrdersSide &side, double level)
+OrderMatcher::Level OrderMatcher::find_level(OrdersSide &side, double level)
 {
-   std::pair<OrdersSide::iterator, OrdersSide::iterator> result {side.end(), side.end()};
+   auto first = side.end(), second = side.end();
+   size_t count = 0;
 
    for (auto it = side.begin() ; it != side.end(); it++) {
-
       if (price_equals(it->price(),level)) {
-         result.first = it;
+         first = it;
          for (;it != side.end(); it++) {
+            ++count;
             if (!price_equals(it->price(),level))
                break;
          }
-         result.second = it;
+         second = it;
          break;
       }
    }
-   return result;
+   return { first, second, count };
 }
 
 void OrderMatcher::add_or_merge(Order const &o, std::vector<Order> *out) {
@@ -82,12 +85,12 @@ void OrderMatcher::add_or_merge(Order const &o, std::vector<Order> *out) {
       side = &me->asks;
 
    auto range = find_level(*side,o.price());
-
    // If the last order at this level is the same owner we can update it
    if (range.first != range.second) {
       auto last = range.second;
       last--;
-      if (last->owner() == o.owner()) {
+      // Keep enough orders to make MBO look good but don't grow forever
+      if (last->owner() == o.owner() && range.size > me->max_orders) {
          auto no_const = const_cast<Order *>(&(*last));
          no_const->set_open_quantity(last->open_quantity() + o.quantity());
          match(out);
@@ -139,11 +142,10 @@ void OrderMatcher::cancel(std::string const &clordid) {
 void OrderMatcher::try_reduce_quantity_at_level(
     api::Side side, Price level, Quantity by_quantity,
     std::string const &order_owner) {
-   std::pair<OrdersSide::iterator, OrdersSide::iterator> range;
-   if (side == api::SIDE_BUY)
-      range = find_level(me->bids,level.value());
-   else
-      range = find_level(me->asks,level.value());
+   auto range = 
+      side == api::SIDE_BUY ? 
+         find_level(me->bids,level.value()) :
+         find_level(me->asks,level.value());
    
    std::vector<std::string> orders_to_cancel;
    for (auto it = range.first; it != range.second && by_quantity.value();

src/server/sparkd/exchange_sim/book/OrderMatcher.h

@@ -9,6 +9,7 @@
 
 #include <memory>
 #include <set>
+#include <tuple>
 
 namespace bts {
 namespace ex_sim {
@@ -48,8 +49,12 @@ class OrderMatcher {
    std::unique_ptr<PImpl> me;
    void match(std::vector<Order> *out);
    void match(Order *bid, Order *ask);
-   std::pair<OrderMatcher::OrdersSide::iterator, OrderMatcher::OrdersSide::iterator>
-   find_level(OrdersSide &side, double level);
+   struct Level {
+      OrderMatcher::OrdersSide::iterator first;
+      OrderMatcher::OrdersSide::iterator second;
+      size_t size;
+   };
+   Level find_level(OrdersSide &side, double level);
 };
 
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -120,7 +120,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
    onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
 }
 
-void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) {
+void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook& ) {
+   #if 0
    if (_mbo) {
       auto data = _index.get_security(sec.id());
       if (data == nullptr) return;
@@ -136,6 +137,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
       }
       _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot);
    }
+   #endif
 }
 
 //

src/server/sparkd/support/ShellLatencyServer.cpp

@@ -36,7 +36,7 @@ void ShellLatencyServer::report(
       reply(bts::api::VoidMessage());
 
       // look up or create the metric
-      auto key = request.host_name() + "-" + request.version();
+      auto key = request.host_name() + "-" + request.version() + "-" + request.user();
       auto iter = _metrics.find(key);
       std::shared_ptr<AutoResetMetric> metric;
       // report at least 1ms because the metric will not report anything if it's 0
@@ -44,8 +44,8 @@ void ShellLatencyServer::report(
       if (iter == _metrics.end()) {
          // create metric
          auto metric = 
-         std::make_shared<AutoResetMetric>("spark_shell_latency", 
-            std::vector<Label>({{"host",request.host_name()}, {"version",request.version()}, {"user", request.user()}}));
+            std::make_shared<AutoResetMetric>("spark_shell_latency", 
+               std::vector<Label>({{"host",request.host_name()}, {"version",request.version()}, {"user", request.user()}}));
          _metrics.insert(std::make_pair(key, metric));
          metric->max(delta);
          _mc.publish(metric);

src/web/ui/src/Accounts/View.fs

@@ -10,6 +10,7 @@ open EnumUtils
 open Fable
 open System
 open bts.co2
+open Elmish.React.Common
 
 
 let accountIsSelected (initialState:BTSAccount) (scratch:BTSAccount) = scratch <> initialState
@@ -179,7 +180,6 @@ let getSecurityMap (model:Model) (positionLimit:AccountSecurityPositionLimitUnit
 
 
 let clipLimitsView model dispatch =
-  let symbols = GetSymbolArray model
   let headerRow:TableControl.row =
     {
         cells =
@@ -322,7 +322,8 @@ let clipLimitsView model dispatch =
                     | true -> "is-selected"
         header = None
       }
-    ] @ [footerRow]
+      yield footerRow
+    ]
 
   let tempSec = { SecurityScope.empty with securitySymbol =
                                             match GetFilteredSymbolArray model usedSymbolNames |> Array.tryHead with
@@ -347,6 +348,13 @@ let clipLimitsView model dispatch =
       TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
     ]
 
+let lazyClipLimitsView model dispatch = 
+  lazyView2With (fun a b -> 
+    LanguagePrimitives.PhysicalEquality a.SelectedClipLimits b.SelectedClipLimits &&
+    LanguagePrimitives.PhysicalEquality a.scratch.securityClipLimits b.scratch.securityClipLimits)
+    clipLimitsView
+    model dispatch
+
 let productLimits model dispatch =
   let headerRow:TableControl.row =
     {
@@ -534,6 +542,12 @@ let productLimits model dispatch =
       br []
     ]
 
+let lazyProductLimitsView model dispatch = 
+  lazyView2With (fun a b -> 
+    LanguagePrimitives.PhysicalEquality a.SelectedPositionLimits b.SelectedPositionLimits &&
+    LanguagePrimitives.PhysicalEquality a.scratch.productPositionLimits b.scratch.productPositionLimits)
+    productLimits
+    model dispatch
 
 let securityLimitsView model dispatch =
   let headerRow:TableControl.row =
@@ -653,8 +667,9 @@ let securityLimitsView model dispatch =
 
 let LimitsView model dispatch =
   div [ Class "limits-container"] [
-    clipLimitsView model dispatch
-    productLimits model dispatch
+    lazyClipLimitsView model dispatch
+    lazyProductLimitsView model dispatch
+
     //securityLimitsView model dispatch
 
     accountControls model dispatch

testing/manual/ladder/badges.txt

@@ -28,12 +28,12 @@ the badge will have a green border, depicting the single-order limit order type
 the badge will have a dot in the upper right hand corner (7)
 
 2. Now place another limit order at the same level:
-the badge now has a triangle in the lower right, indicating we have multiple workikng orders (9)
+the badge now has a triangle in the lower right, indicating we have multiple working orders (9)
 the green border and dot have gone away (5, 7)
 
 Now place another limit below the bid:
 
-3. Outcome similar to 1, except the background color is ligther bid background color (blue)
+3. Outcome similar to 1, except the background color is lighter bid background color (blue)
 
 Now place an a Buy Iceberg at the same level with a sufficiently large qty, say 200, as your working ask
 
@@ -44,7 +44,7 @@ Test 2
 ------
 This will test characteristics 4,10
 
-1. Create a spreader comprised 1 quoting leg and 1 heding leg of your favorite instruments and place a spread order
+1. Create a spreader comprised 1 quoting leg and 1 hedging leg of your favorite instruments and place a spread order
 2. Pull up the spreader ladder and place a sell order above the market
 3. Pull up the quoting leg's ladder and observe the badge; it may be helpful to pull up the Orderbook to find the price at which the leg is being quoted.
 	The badge should have a rectangle filled in with the Spreader algo's color

testing/manual/ladder/piq_mbo.txt

@@ -0,0 +1,57 @@
+Introduction
+-------------
+
+Ideally this test should be done with a cert connection because the simulator can be quirky.  If you ignore this advice and use sim you must not admit it to anyone.
+
+Only exchange orders will have a PIQ so you should probably just use a Limit order for this test.
+
+General Stuff to Look For:
+--------------------------
+- The MBO popup is 'live' and should update as the market moves.
+- The MBO popup should increase in size if more orders are added while it is open.
+- The MBO popup should not shrink when orders are removed. It only grows.
+- The scale of the bars is based on the largest order seen by the popup.  
+    - If a larger order shows up it will scale down the other orders.
+    - When the large order goes away it should *not* expand the other bars.  The scale only increases while the popup is open.
+    - For example:
+        - You open the popup and there are 2 orders of size [1,2]
+        - size 1 will be about half way across the available space and 2 will fill the space.
+        - An order of size 5 comes around it will fill the available space and the other orders will be 20%, 40% across respectively.
+        - If the size 5 order is canceled the [1,2] orders stay at 20%, 40% rather than growing to fill the space.
+        - If you close and re-open the popup state is reset and [1,2] will be 50%/100% again.
+- Middle clicking on a row with no data does not open the popup.
+    - This doesn't mean clicking past the end of the visible book won't show anything.  The MBO tracks much deeper than the market by price.  If there's a gap between 2 prices that's a good spot to test.
+
+
+Part 1:
+-------
+- Find a security with a non-empty market that isn't moving too fast.  Cert can be temperamental so open the market view and filter for Qty > 0.
+- Open settings and turn on the PIQ column (settings -> column settings)
+- Choose a price near the top of the book and middle click the price column to show the MBO popup.
+- Add up the sizes in the popup and make sure they agree with the total size in the bid or ask column on the ladder. 
+    - Test both sides of the market near the top and away from the top.
+- Enter an order.
+    - The PIQ column should reflect the number of orders ahead of you in the queue.  The size in the bid/ask column is how many orders were there when you entered your order so you should expect the PIQ to be the size from before you entered your order.
+    When the market moves you should expect your PIQ to go down.  If cert is working correctly your PIQ should never go up.
+    - Open the MBO popup.  Your order should be shown in a different color from the other orders at that level.
+    - Verify that your order position matches the value from the PIQ column.
+    - Test both sides of the market near the top and away from the top.
+- Moving your order to a different price or *increasing* the size of your order will move you to the back of the queue.
+    - If you ignored the instructions and you're using sim this probably won't work.  You should have listened.
+- Reducing the size of your order should not change your queue position.  You might may have to be quick to observe this.
+
+Part 2:
+-------
+- Open several securities with varying tick sizes and make sure that the MBO popup opens for all levels of the book where there is a size.
+- Why? The ladder sends the price you click on to the server and the server must identify the right price level.  As you know, the IEEE double is out to get you.
+
+Part 3:
+-------
+- The MBO popup should expand either up or down from the mouse position to maximize the available space.
+- If the number of orders exceeds the space available the mouse wheel should work to let you scroll up/down to see the rest (up to LadderKnobs.max_mbo_orders which defaults to 200)
+- Find a security with enough orders to cause scrolling by either
+  - Connect to a demo system with production md and look at a ZN spread (i.e. ZNM9 - ZNU9)
+  - Enter a bunch of orders at the level on cert.
+  - If you try this on sim (you're not a very good listener, are you?) It will cap out at ExchangeSim.orders_per_level (default:10) 
+- Make sure that scrolling works.
+

testing/manual/spreader_misc/simple_sanity.txt

@@ -3,11 +3,10 @@ This test makes sure the basic spreader functionality works for 2 legged synthet
 0. Follow the instructions from docs/spreader_common_setup_1.txt
 
 1. Enter a synthetic buy order of 1 lot at -0.30. Verify that a quoting order of 18.60 shows up in the second leg.
+
 Cancel the synthetic order and the quoting order should be cancelled too.
 
-2. Enter a synthetic buy order of 1 lot at -0.30, again. Now drag the the synthetic order down the ladder
-(ie lower bid prices) to a few random prices of your choice. Verify that the quoter leg price is adjusted
-accordingly and is correct.
+2. Enter a synthetic buy order of 1 lot at -0.30, again. Now drag the the synthetic order down the ladder (ie lower bid prices) to a few random prices of your choice. Verify that the quoter leg price is adjusted accordingly and is correct.
 
 (If you have followed the instructions in common_setup_1.txt: Leg_1 will buy at 18.300 , and leg_2 will be your quoting leg)
 
@@ -16,4 +15,5 @@ spread_price == price_mult_1 * leg_1 + price_mult_2 * leg_2
 leg_2 == price_mult_1 * leg_1 - spread_price / -1.0 * price_mult_2
 
 3. Now cross the synthetic order by dragging it all the way to -0.20 (there should be a 1 lot at that price).
+
 Verify that the synthetic order gets filled at -0.20, the quoter order at 18.50, and the hedger order at 18.30.