Fix System tests

RTM: SPARK-2363 - PIQ popup

Approved-by: Jake Norbut <jake.norbut@bluetradesys.com>

.vscode/launch.json

@@ -1,83 +1,14 @@
 {
     "version": "0.2.0",
     "configurations": [
-
-        {
-            "name": "(gdb) Launch",
-            "type": "cppdbg",
-            "request": "launch",
-            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd/sparkd",
-            "args": ["run"],
-            "stopAtEntry": false,
-            "cwd": "${workspaceRoot}/build/ninja_debug/src/server/sparkd",
-            "environment": [],
-            "externalConsole": true,
-            "MIMode": "gdb",
-            "setupCommands": [
-                {
-                    "description": "Enable pretty-printing for gdb",
-                    "text": "-enable-pretty-printing",
-                    "ignoreFailures": true
-                }
-            ]
-        },
-        {
-         "name": "Client Launch",
-         "type": "clr",
-         "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/spark.exe",
-         "args": [],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-        },
-      {
-         "name": "SDK Launch",
-         "type": "clr",
-         "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/spark_sdk.exe",         
-         "args": ["md","GLBX","Future","CL", "CLZ9", "--user=bts", "--pass="],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-      },        
-      {
-         "name": "Excel Launch",
-         "type": "clr",
-         "request": "launch",
-         "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
-         "args": ["${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-      },          
-      {
-         "name": "Client Tests",
-         "type": "clr",
-         "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
-         "args": ["test","test_filter"],
-         "cwd": "${workspaceRoot}",
-         "stopAtEntry": false,
-         "externalConsole": true,
-         "requireExactSource": false
-      },           
-      {
-            "name": "Client Attach",
-            "type": "clr",
-            "request": "attach",
-            "processId":"${command:pickProcess}"
-        },
         {
             "name": "SparkVM Launch",
             "type": "cppdbg",
             "request": "launch",
-            
             "program": "/home/bts/spark/build/ninja_debug/server/sparkd",
-            "args": ["run"],
+            "args": [
+                "run"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server/",
             "environment": [],
@@ -102,24 +33,113 @@
                     "description": "LD LIBRARY",
                     "text": "set env LD_LIBRARY_PATH=.",
                     "ignoreFailures": false
-                }                
+                }
             ],
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             }
         },
+        {
+            "name": "(gdb) Launch",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd/sparkd",
+            "args": [
+                "run"
+            ],
+            "stopAtEntry": false,
+            "cwd": "${workspaceRoot}/build/ninja_debug/src/server/sparkd",
+            "environment": [],
+            "externalConsole": true,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                }
+            ]
+        },
+        {
+            "name": "Client Launch",
+            "type": "clr",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/Debug/bin/net471/spark.exe",
+            "args": [],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "SDK Launch",
+            "type": "clr",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/Debug/bin/net471/spark_sdk.exe",
+            "args": [
+                "md",
+                "GLBX",
+                "Future",
+                "CL",
+                "CLZ9",
+                "--user=bts",
+                "--pass="
+            ],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "Excel Launch",
+            "type": "clr",
+            "request": "launch",
+            "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
+            "args": [
+                "${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"
+            ],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "Client Tests",
+            "type": "clr",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+            "args": [
+                "test",
+                "test_filter"
+            ],
+            "cwd": "${workspaceRoot}",
+            "stopAtEntry": false,
+            "externalConsole": true,
+            "requireExactSource": false
+        },
+        {
+            "name": "Client Attach",
+            "type": "clr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+        },
         {
             "name": "Spreader Tests",
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test", "[spreader]"],
+            "args": [
+                "test",
+                "[spreader]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],
@@ -139,7 +159,9 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
@@ -151,7 +173,10 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test", "[admin_service]"],
+            "args": [
+                "test",
+                "[admin_service]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],
@@ -171,7 +196,9 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
@@ -183,7 +210,10 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test", "[limiter]"],
+            "args": [
+                "test",
+                "[limiter]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],
@@ -203,7 +233,9 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
@@ -215,7 +247,10 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/server/sparkd",
-            "args": ["test", "[mbo]"],
+            "args": [
+                "test",
+                "[mbo]"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server/",
             "environment": [],
@@ -240,14 +275,16 @@
             "pipeTransport": {
                 "pipeCwd": "${workspaceRoot}",
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             }
         },
-         {
+        {
             "name": "SparkVM attach",
             "type": "cppdbg",
             "request": "attach",
@@ -255,16 +292,18 @@
             "processId": "${command:pickRemoteProcess}",
             "MIMode": "gdb",
             "pipeTransport": {
-               "pipeCwd": "${workspaceRoot}",
-               "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-               "pipeArgs": ["bts@sparkvm"],
-               "debuggerPath": "/usr/bin/gdb"
+                "pipeCwd": "${workspaceRoot}",
+                "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
+                "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
-               "/home/bts/spark/": "${workspaceRoot}"
+                "/home/bts/spark/": "${workspaceRoot}"
             }
-         },
-         {
+        },
+        {
             "name": "Python",
             "type": "python",
             "request": "launch",
@@ -274,7 +313,9 @@
             "cwd": "${workspaceRoot}",
             "env": {},
             "envFile": "${workspaceRoot}/.env",
-            "args": [ "DEBUG" ],
+            "args": [
+                "DEBUG"
+            ],
             "debugOptions": [
                 "WaitOnAbnormalExit",
                 "WaitOnNormalExit",
@@ -288,7 +329,9 @@
             "processId": "${command:pickRemoteProcess}",
             "pipeTransport": {
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/home/bts/vsdbg/vsdbg",
                 "pipeCwd": "${workspaceRoot}",
                 "quoteArgs": true
@@ -303,12 +346,18 @@
             "type": "coreclr",
             "request": "launch",
             "program": "/usr/bin/dotnet",
-            "args": ["sparkcmd.dll", "test", "test_join_create_and_takeout_market"],
+            "args": [
+                "sparkcmd.dll",
+                "test",
+                "test_join_create_and_takeout_market"
+            ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server",
             "pipeTransport": {
                 "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@sparkvm"],
+                "pipeArgs": [
+                    "bts@sparkvm"
+                ],
                 "debuggerPath": "/home/bts/vsdbg/vsdbg",
                 "pipeCwd": "${workspaceRoot}",
                 "quoteArgs": true

doc/user/content/release_notes/_index.md

@@ -4,6 +4,15 @@ weight = 3
 sort_by = "weight"
 insert_anchor_links = "right"
 +++
+## 2.0.2
+### User Visible Changes
+  - Middle click the ladder price column to show the orders for that price level 
+  - Middle click the bid or ask column in the ladder no longer sends an order
+
+### Internal changes
+  - Improved market by order performance under heavy load
+  - Fixed a concurrency bug in SIM
+
 ## 2.0.0
 ### User Visible Changes
   - Support for attaching and detaching algos to existing orders

src/client/spark/Modules/Trading/Ladder/LadderModel.cs

@@ -2,6 +2,7 @@
 using bts.api;
 using bts.api.core;
 using bts.api.orderman;
+using bts.md;
 using mbts.co2;
 using System;
 using System.Threading.Tasks;
@@ -10,12 +11,14 @@ namespace mbts.spark {
    class LadderModel {
       readonly OrdersModel _order_gateway;
 
-      public LadderModel(SecurityMD source, OrdersModel ordergw, Func<OrderSecurity, bool> order_filter = null) {
+      public LadderModel(SecurityMD source, OrdersModel ordergw, MBOService mbo, Func<OrderSecurity, bool> order_filter = null) {
          security_md = source;
          _order_gateway = ordergw;
          orders = ordergw.on_security_working_orders_with_piq(source.security.Id);
-         if (order_filter != null)
+         this.mbo_service = mbo;
+         if (order_filter != null) {
             orders = orders.Filter(order_filter, o => o.order.Key);
+		 }
       }
 
       public enum CancelType { All, Bids, Asks }
@@ -26,6 +29,8 @@ namespace mbts.spark {
       public OrdersModel order_gateway => _order_gateway;
       public IObservable<TableUpdate<OrderSecurity, uint>> orders { get; }
 
+      public MBOService mbo_service { get; }
+
       public Task cancel_orders(CancelType ct) {
          switch (ct) {
             case CancelType.All: return _order_gateway.cancel_security_orders(security.Id);

src/client/spark/Modules/Trading/Ladder/LadderWindowFactory.cs

@@ -1,4 +1,5 @@
 using bts.api;
+using bts.md;
 using mbts;
 using mbts.config;
 using mbts.persist;
@@ -21,17 +22,26 @@ namespace bts.shell {
       readonly LadderConfig _ladder_conf;
       readonly IBus _bus;
       readonly IWindowManager _wm;
+      readonly MBOService _mbo;
       readonly UserSettingsHub _settings_hub;
       readonly OrdersModel _orders_model;
       IReadOnlyDictionary<uint, SecurityMD> _securities = new Dictionary<uint, SecurityMD>();
 
-      public LadderWindowFactory(IShellModule owner, IBus bus, IWindowManager wm, UserSettingsHub settings_hub, OrdersModel orders_model)
+      public LadderWindowFactory(
+         IShellModule owner, 
+         IBus bus, 
+         IWindowManager wm, 
+         UserSettingsHub settings_hub, 
+         OrdersModel orders_model,
+         MBOService mbo)
+
          : base(owner, "SparkLadder") {
          _bus = bus;
          _wm = wm;
          _settings_hub = settings_hub;
          _orders_model = orders_model;
          _ladder_conf = new LadderConfig(Configuration.get<bts.config.LadderKnobs>(), Configuration.get<bts.config.MultiThreadedWindows>().ladder);
+         _mbo = mbo;
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s),
             bus.subscribe(SparkTags.ShowLadderTag).OnDispatcher().Subscribe(s => show_windows(new[] { s })),
@@ -54,7 +64,7 @@ namespace bts.shell {
       IShellWindow create_ladder_window(SecurityMD md, LadderSettings state_settings) {
          var id = Interlocked.Increment(ref _ladder_count) % Environment.ProcessorCount;
          var view = new Lazy<UIElement>(() => {
-            var model = new LadderModel(md, _orders_model);
+            var model = new LadderModel(md, _orders_model, _mbo);
             var settings_manager = new WindowSettingsManager<LadderSettings>(
                _settings_hub, state_settings == null || state_settings.use_defaults ? null : state_settings,
                us => sanitize_ladder_settings(us.ladder),

src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs

@@ -169,6 +169,11 @@ namespace mbts.spark {
          RaisePropertyChanged(nameof(text_lines));
       }
 
+      /// <summary>
+      /// Add Order performs an add the first time it's called via the constructor. When it's called, directly
+      /// it really serves as an update/replace. Also note that since the min_piq is a separate piece of state representing
+      /// the lowest piq of the orders the badge represents, we must reevaluate if that has changed on said update.
+      /// </summary>
       public void add_order((OrderRecordImmutable order, uint? p, AlgoData? algo, AlgoDefinitionMetadata? meta) new_op) {
          _orders[new_op.order.Key] = new_op;
          // reevaluate min piq if this order's piq is less

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -116,7 +116,7 @@ namespace mbts.spark {
          //_window_height = backing(nameof(window_height), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 600);
          //_window_width = backing(nameof(window_width), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 420);
          position_click_command = new DelegateCommand(() => size_calculator.value = DoubleHelper.safe_convert_to_uint(Math.Abs(position)));
-         send_order_command = new DelegateCommand<(double price, Side? side, bool size_from_alternate)>(send_order);
+         send_order_command = new DelegateCommand<(double price, Side? side, MouseButton button)>(send_order);
          cancel_command = new AsyncCommand<LadderModel.CancelType>(_model.cancel_orders, true);
          show_book_command = new DelegateCommand(_show_book);
          show_ticket_command = new DelegateCommand(_show_ticket);
@@ -232,7 +232,7 @@ namespace mbts.spark {
       // we return only entries who have piqs and whose piqs are not uint.Max
       // uint.Max happens when the server is restarted but the client is still up
       // maybe long term we don't show this data while the exchange connection is down
-      public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value));  
+      public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value));
       public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic;
 
       Property<string> _status;
@@ -348,13 +348,15 @@ namespace mbts.spark {
          }
       }
 
-      public async void send_order((double price, Side? side, bool size_from_alternate) args) {
+      public async void send_order((double price, Side? side, MouseButton buton) args) {
+         if (args.buton == MouseButton.Middle) return;
+         var size_from_alternate = args.buton == MouseButton.Right;
          if (order_properties_vm == null)
             return;
          var s = _settings_manager.current_settings();
-         if (DoubleHelper.IsNaNOrInfinity(args.price) || !args.side.HasValue || HasErrors || order_properties_vm.HasErrors || (args.size_from_alternate && s.alternate_order_size == 0))
+         if (DoubleHelper.IsNaNOrInfinity(args.price) || !args.side.HasValue || HasErrors || order_properties_vm.HasErrors || (size_from_alternate && s.alternate_order_size == 0))
             return;
-         else if (!args.size_from_alternate && (!size_calculator.value.HasValue || size_calculator.value.Value == 0)) {
+         else if (!size_from_alternate && (!size_calculator.value.HasValue || size_calculator.value.Value == 0)) {
             size_calculator.flash_error();
          } else {
             var order = order_properties_vm.create(args.side.Value);
@@ -371,7 +373,7 @@ namespace mbts.spark {
                }
             }
                
-            uint size = args.size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
+            uint size = size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
             await _model.order_gateway.send_order(order, args.price, size);
             if (auto_clear)
                size_calculator.value = 0;

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -1,8 +1,10 @@
 using bts;
 using bts.api;
 using bts.api.md.auxiliary;
+using bts.md;
 using bts.collections;
 using bts.utils;
+using bts.config;
 using mbts.co2;
 using mbts.ui;
 using mbts.ui.controls;
@@ -17,11 +19,13 @@ using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Threading.Tasks;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Input;
 using System.Windows.Media;
 using System.Windows.Media.Animation;
+using mbts.config;
 
 namespace mbts.spark {
    partial class LadderGridHost : Control {
@@ -320,25 +324,56 @@ namespace mbts.spark {
          Focus();
          var point = e.GetPosition(this);
          var column = _column_for_point(point);
-         if (column.column_type == ColumnType.Price && e.ChangedButton == MouseButton.Middle)
-            _scroll_to_center();
+         if (column.column_type == ColumnType.Price && e.ChangedButton == MouseButton.Middle) {
+            show_mbo_popup(point).ignore();
+         }
          else if (_settings != null) {
             var price = price_for_point(point);
             Side? bs = null;
             if (column.column_type == ColumnType.Bid) bs = Side.Buy;
             if (column.column_type == ColumnType.Ask) bs = Side.Sell;
-            var pbs = (price, bs, e.ChangedButton == MouseButton.Right);
+            var pbs = (price, bs, e.ChangedButton);
             if (Command != null && Command.CanExecute(pbs)) {
                if (bs.HasValue) {
                   using (_tactile_adorner.show_busy(_row_for_point(point), bs.Value == Side.Buy ? ColumnType.Bid : ColumnType.Ask))
                      await Command.ExecuteAsync(pbs);
-               } else
+               }
+               else {
                   Command.Execute(pbs);
+               }
             }
          }
          base.OnMouseDown(e);
       }
 
+      private async Task show_mbo_popup(Point point) {
+         var query = MBOPopupModel.default_query.with(
+            security_id: _vm.model.security.Id,
+            price_level: price_for_point(point)
+         );
+         var model = new MBOPopupModel(_vm.model.mbo_service, _vm.model.security, query);
+         // Load the first set of orders so we know if there are any before showing the popup
+         var result = await model.load_orders();
+         if (result.success && result.value.orders.Count > 0) {
+            var view = new MBOPopupView(model, result.value);
+            var popup = new AutoClosePopup {
+               PlacementTarget = this,
+               PlacementRectangle = new Rect(point, point),
+               IsOpen = true,
+               Child = new Border {
+                  BorderBrush = Brushes.Black,
+                  BorderThickness = new Thickness(0.5),
+                  Child = new ScrollViewer {
+                     VerticalScrollBarVisibility = ScrollBarVisibility.Hidden,
+                     HorizontalScrollBarVisibility = ScrollBarVisibility.Disabled,
+                     Content = view,
+                  }
+               }
+            };
+            view.RequestClose += (o, _) => popup.IsOpen = false;
+         }
+      }
+
       protected override void OnMouseDoubleClick(MouseButtonEventArgs e) {
          var column = _column_for_point(e.GetPosition(this));
          if (column.column_type == ColumnType.Price)

src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs

@@ -72,8 +72,7 @@ namespace mbts.spark {
                   panel.Children.Remove(item);
                }
             }
-         } //else
-            //Log.Ui.warn($"Failed to remove order ({order.id}): order not in {nameof(LadderOrdersHost)}'s {nameof(_orders)}.");
+         }
       }
 
       public void clear_orders() {
@@ -192,11 +191,8 @@ namespace mbts.spark {
                Margin = Badge.DefaultMargin,
                Orientation = Orientation.Horizontal,
                MaxWidth = _col_width
-               //MaxHeight = _row_height,
-               //ClipToBounds = true
             };
 
-            //var view_box = new Viewbox { Child = panel, Width = _col_width, Height = _row_height };
             _position_panel(panel, price);
             _stacks[price] = new PriceCellComponents(panel);
             Children.Add(panel);

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -3,6 +3,7 @@ using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
+using bts.md;
 using bts.shell;
 using bts.shell.orderbook;
 using bts.spark;
@@ -64,7 +65,14 @@ namespace mbts.spark {
          _detachable_selected_orders = new HashSet<uint>();
       FilterData _cached_filter_data;
 
-      public OrderBookVM(IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model, IWindowSettingsManager<OrderBookGridSettings> settings_manager, bool rows_are_initially_expanded = false) {
+      public OrderBookVM(
+         IObservable<string> owner_title, 
+         IBus bus, 
+         INotificationService notification_service, 
+         OrdersModel model, 
+         IWindowSettingsManager<OrderBookGridSettings> settings_manager, 
+         bool rows_are_initially_expanded = false) {
+
          _bus = bus;
          _model = model;
          _selected_params_mon = new SelectedOrderParamsMon(model.dynamic_helper);
@@ -874,7 +882,10 @@ namespace mbts.spark {
                return;
             Disposable.replace(ref _ladder_vm,
                _security_md.HasValue ?
-                  LadderVM.create_readonly(_owner._bus, new LadderModel(security_md.Value, _owner._model, new Func<OrderSecurity, bool>(osec => _selected_order_tree.Contains(osec.order.Key)))) :
+                  LadderVM.create_readonly(
+                     _owner._bus, 
+                     new LadderModel(security_md.Value, _owner._model, _owner._model.mbo_service,
+                        new Func<OrderSecurity, bool>(osec => _selected_order_tree.Contains(osec.order.Key)))) :
                   null);
             RaisePropertyChanged(nameof(ladder_vm));
          }

src/client/spark/Modules/Trading/TradingShellModule.cs

@@ -21,9 +21,17 @@ namespace bts.shell {
       AlgoStudioWindowFactory _aswf;
 
       [ImportingConstructor]
-      public TradingShellModule(IBus bus, IWindowManager wm, INotificationService notification_service, INotificationServiceDataCache notification_service_data_cache,
-         UserSettingsHub settings_hub, OrdersModel orders_model, PAndLModel pandl_model, ErrorReporter error_reporter) {
-         _lwf = new LadderWindowFactory(this, bus, wm, settings_hub, orders_model);
+      public TradingShellModule(
+            IBus bus, 
+            IWindowManager wm, 
+            INotificationService notification_service, 
+            INotificationServiceDataCache notification_service_data_cache,
+            UserSettingsHub settings_hub, 
+            OrdersModel orders_model, 
+            PAndLModel pandl_model,
+            md.MBOService mbo,
+            ErrorReporter error_reporter) {
+         _lwf = new LadderWindowFactory(this, bus, wm, settings_hub, orders_model, mbo);
          _obwf = new OrderBookWindowFactory(this, bus, notification_service, notification_service_data_cache, orders_model, settings_hub);
          _fwf = new FillsWindowFactory(this, bus, settings_hub, orders_model);
          _plwf = new PAndLWindowFactory(this, bus, settings_hub, pandl_model);

src/client/spark/Orders/OrdersModel.cs

@@ -182,6 +182,7 @@ namespace mbts.spark {
       public IObservable<TableUpdate<(AlgoData algo, AlgoDefinitionMetadata? meta), AlgoKey>> studio_algos_with_meta { get; }
 
       public AdminModel admin_model { get; }
+      public MBOService mbo_service => _mbo_service;
       public IAlgoStudioModel algo_model { get; }
 
       public IObservable<TableUpdate<OrderSecurity, uint>> on_security_working_orders_with_piq(uint security) =>

src/client/spark/view/Spark/Ladder/MBO/MBOPopupModel.cs

@@ -0,0 +1,133 @@
+using bts.api;
+using bts.api.core;
+using bts.md;
+using mbts.config;
+using mbts.ui.utility.text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   public struct MBOPopupData {
+      public readonly Side side;
+      public readonly IReadOnlyList<OrderPositionImmutable> orders;
+
+      public MBOPopupData(Side side, IReadOnlyList<OrderPositionImmutable> orders) {
+         this.side = side;
+         this.orders = orders;
+      }
+   }
+   public struct MBOPopupDrawParams {
+      public MBOPopupData data;
+      public double width => 300.0;
+      public double height;
+      public FormattedText header;
+
+      public static readonly Thickness margin = new Thickness(10, 5, 5, 5);
+
+      //columns
+      public double row_height => header.Height + 2;
+      public double bar_height => 0.6 * row_height;
+
+      public List<FormattedText> row_numbers;
+      public double numbers_width;
+      public List<FormattedText> row_labels;
+      public double scale;
+   }
+
+   public class MBOPopupModel {
+      const double font_size = 9.0;
+      readonly MBOService _mbo;
+      readonly SecurityImmutable _security;
+      readonly MBOSecurityQueryImmutable _query;
+      readonly Dictionary<string, FormattedText> _cache;
+      int _max_order_size = 1;
+      double _height = 0;
+      double _number_width = 0;
+
+
+      public MBOPopupModel(MBOService mbo, SecurityImmutable security, MBOSecurityQueryImmutable query) {
+         _mbo = mbo;
+         _security = security;
+         _query = query;
+
+         // prime the cache with numbers on the left hand side
+         _cache =
+            Enumerable.Range(1, 1+(int)default_query.Limit)
+            .AsParallel()
+            .Select(i => TextUtils.GetFormattedText(i.ToString(), font_size))
+            .ToDictionary(ft => ft.Text);
+      }
+
+      FormattedText format_text(string s, bool bold=false) {
+         // cache FormattedText for the life of the popup
+         if (_cache.TryGetValue(s, out FormattedText ft)) {
+            return ft;
+         }
+         else
+         ft = bold ? TextUtils.GetFormattedTextBold(s, font_size) : TextUtils.GetFormattedText(s, font_size);
+         _cache.TryAdd(s, ft);
+         return ft;
+      }
+
+      public async Task<Result<MBOPopupData>> load_orders() {
+         if (_mbo == null) return new Result<MBOPopupData>(ApiError.NoServer);
+         var data = await _mbo.get_orders_for_security_async(_query);
+         if (data.success) {
+            var side = data.value.side();
+            IEnumerable<OrderPosition> orders = new OrderPosition[0];
+            if (side == Side.Buy) orders = data.value.Bid;
+            if (side == Side.Sell) orders = data.value.Ask;
+            return new Result<MBOPopupData>(new MBOPopupData(side, orders.Select(o => (OrderPositionImmutable)o).ToList()));
+         }
+         else {
+            return new Result<MBOPopupData>(data.code);
+         }
+      }
+
+      string buy_sell_text(Side side) {
+         switch (side) {
+            case Side.Buy: return "BUY";
+            case Side.Sell: return "SELL";
+            default:
+            case Side.Undefined: return "";
+         }
+      }
+
+      public MBOPopupDrawParams compute_layout(MBOPopupData p) {
+         var pdp = new MBOPopupDrawParams { data = p };
+         pdp.header = format_text(
+            $"{buy_sell_text(p.side)} {_security.security_desc()} @ {_security.ContractSpec.format(_query.PriceLevel)}", true);
+
+         var margin = MBOPopupDrawParams.margin;
+         var text_margin = margin.Left + margin.Right;
+
+         pdp.row_numbers = p.orders.Select((o, i) => format_text((i+1).ToString())).ToList();
+         pdp.numbers_width = _number_width = Math.Max(_number_width, (int)pdp.row_numbers.Max(r => r.Width) + text_margin);
+         pdp.row_labels = p.orders.Select(o => format_text(o.Size.ToString())).ToList();
+         var max_label_width = pdp.row_labels.Max(l => l.Width) + text_margin;
+
+         var available_width = pdp.width - pdp.numbers_width - max_label_width;
+         _max_order_size = Math.Max(p.orders.Max(o => o.Size), _max_order_size);
+         pdp.scale = available_width / _max_order_size;
+
+         pdp.height = _height =
+            Math.Max(_height,
+            pdp.header.Height + margin.Top + margin.Bottom + (pdp.row_height * p.orders.Count) + margin.Bottom);
+
+         return pdp;
+      }
+
+      #region static
+      public static readonly MBOSecurityQueryImmutable default_query = (MBOSecurityQueryImmutable) 
+         new MBOSecurityQuery {
+            Limit = Configuration.get<bts.config.LadderKnobs>().max_mbo_orders
+         };
+      #endregion
+   }
+
+}

src/client/spark/view/Spark/Ladder/MBO/MBOPopupView.cs

@@ -0,0 +1,86 @@
+using System.Windows;
+using System.Windows.Media;
+using System.Collections.Generic;
+using bts.api.orderman;
+using System.Linq;
+using bts.md;
+using bts;
+using mbts.ui;
+using System.Windows.Input;
+using System.Threading.Tasks;
+using System;
+using mbts.ui.utility.text;
+using bts.api;
+using System.Windows.Threading;
+
+namespace mbts.spark {
+   internal class MBOPopupView : UIElement, IRequestViewClose {
+      readonly Brush _background = BTSColors.LadderSpecial.default_column_background;
+      readonly MBOPopupModel _model;
+      MBOPopupDrawParams _dp;
+
+      public MBOPopupView(MBOPopupModel model, MBOPopupData p) {
+         _model = model;
+         _dp = model.compute_layout(p);
+         IsVisibleChanged += (o, _) => poll_orders();
+      }
+
+      private async void poll_orders() {
+         while (IsVisible) {
+            var result = await _model.load_orders();
+            if (result.success && result.value.orders.Count > 0) {
+               Log.Ui.diag($"MBOpopup got {result.value.orders.Count} orders");
+               _dp = _model.compute_layout(result.value);
+               InvalidateMeasure();
+               InvalidateVisual();
+            }
+            else RequestClose?.Invoke(this, null);
+            await Task.Delay(250);
+         }
+      }
+
+      public event EventHandler RequestClose;
+
+      protected override Size MeasureCore(Size availableSize) {
+         return new Size(_dp.width, _dp.height);
+      }
+
+      static Thickness margin => MBOPopupDrawParams.margin;
+      protected override void OnRender(DrawingContext dc) {
+         base.OnRender(dc);
+         // draw background
+         dc.DrawRectangle(_background, null, new Rect(0, 0, _dp.width, _dp.height));
+         // draw header
+         dc.DrawText(_dp.header, new Point(margin.Left, margin.Top));
+         // draw grid
+         draw_grid(dc, _dp, _dp.header.Height + margin.Top + margin.Bottom);
+      }
+
+      static Brush brush_for(Side side, OrderPositionImmutable order) {
+         if (order.OrderId != 0) return Brushes.Black;
+         switch (side) {
+            case Side.Buy: return BTSColors.MarketData.bid_brush;
+            case Side.Sell: return BTSColors.MarketData.ask_brush;
+            default:
+               return Brushes.DarkGray;
+         }
+      }
+
+      static void draw_grid(DrawingContext dc, MBOPopupDrawParams dp, double offset) {
+         for (int x = 0; x < dp.data.orders.Count(); ++x) {
+            // draw numbers
+            var number = dp.row_numbers[x];
+            var left_offset = dp.numbers_width - number.Width - 5;
+            dc.DrawText(number, new Point(left_offset, offset));
+            // draw rectangles
+            var order = dp.data.orders[x];
+            var rect = new Rect(dp.numbers_width, offset + (dp.row_height-dp.bar_height)/2, order.Size * dp.scale, dp.bar_height);
+            dc.DrawRectangle(brush_for(dp.data.side, order), null,rect);
+            // draw sizes
+            var label = dp.row_labels[x];
+            dc.DrawText(label, new Point(rect.Right+margin.Right, offset));
+            offset += dp.row_height;
+         }
+      }
+   }
+}

src/client/spark/view/Spark/Ladder/MBO/PositionDataExtensions.cs

@@ -0,0 +1,19 @@
+using bts.api;
+using bts.md;
+
+namespace mbts.spark {
+   static class PositionDataExtensions {
+      public static Side side(this PositionData pd) {
+         var bidlen = pd.Bid.Count;
+         var asklen = pd.Ask.Count;
+         if (bidlen > 0 && asklen == 0) {
+            return Side.Buy;
+         }
+         else if (asklen > 0 && bidlen == 0) {
+            return Side.Sell;
+         }
+         return Side.Undefined;
+      }
+   }
+
+}

src/client/spark/view/Utility/AutoClosePopup.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Input;
+
+namespace mbts.ui.controls {
+   class AutoClosePopup : System.Windows.Controls.Primitives.Popup {
+      public AutoClosePopup() {
+         StaysOpen = false;
+         CommandBindings.Add(new CommandBinding(
+            ApplicationCommands.Close, 
+            (o,_) => { IsOpen = false; }
+         ));
+         InputBindings.Add(new KeyBinding(ApplicationCommands.Close, Key.Escape, ModifierKeys.None));
+      }
+
+      public new FrameworkElement Child {
+         get => base.Child as FrameworkElement;
+         set {
+            var old_child = Child;
+            base.Child = value;
+            hook_child_events(value);
+         }
+      }
+
+      private void hook_child_events(FrameworkElement fe) {
+         MouseDown += OnMouseDown;
+         fe.MouseDown += OnMouseDown;
+         fe.PreviewMouseDown += OnMouseDown;
+         if (fe is IRequestViewClose vc) {
+            vc.RequestClose += (o,_) => IsOpen = false;
+         }
+      }
+
+      private void OnMouseDown(object sender, MouseButtonEventArgs e) {
+         if (sender == this && e.MiddleButton == MouseButtonState.Pressed) IsOpen = false;
+         else {
+            if (!e.Handled) {
+               IsOpen = false;
+            }
+         }
+      }
+   }
+}

src/client/spark_core/TestRunner/TestAttributes.cs

@@ -17,7 +17,7 @@ namespace bts.test {
    }
    [AttributeUsage(AttributeTargets.Method)]
    public class TestAttribute : Attribute {
-      public uint Timeout = 10;
+      public uint Timeout = 10; //timeout in seconds
    }
 
    // Use constructor for setup

src/client/spark_sdk/spark_sdk.csproj

@@ -61,7 +61,7 @@
      <None Include="$(OutputPath)\net471\spark_excel_example_2018.xlsm" pack="true" PackagePath=".\excel\spark_excel_example_2018.xlsm" />
      <None Include="$(OutputPath)\net471\spark_excel-addin64-packed.xll" pack="true" PackagePath=".\excel\spark_excel-addin64-packed.xll" />
      <None Include="$(OutputPath)\net471\spark_excel-addin64.dna" pack="true" PackagePath=".\excel\spark_excel-addin64.dna" />
-     <None Include="../../../doc/user/content/bts_spark_sdk/Client/**" pack="true" PackagePath=".\doc\" />
+     <None Include="doc/**" pack="true" PackagePath=".\doc\" />
   </ItemGroup>
 
   <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences" Condition= " '$(SparkRoot)' != '' ">

src/client/spark_test/System/LimitsTests.cs

@@ -24,7 +24,10 @@ namespace bts.tests
       AdminService _admin_service;
       IDisposable _order_subscription;
       Update<OrderRecordImmutable, uint> _orders;
-      public LimitsTests (){
+      public LimitsTests (){}
+
+      private async Task setup_limits_tests() {
+         await TestUtils.wait_for_system_ready();
          _context = new ServiceContext();
 
          _order_service = _context.locate<OrderService>();
@@ -80,10 +83,9 @@ namespace bts.tests
          return securities;
       }
 
-   [Test(Timeout = 300)]
+   [Test(Timeout = 300000)]
       public async Task test_position_limits() {
-         bool system_ready = await TestUtils.test_for_system_ready(_context, TimeSpan.FromMinutes(5));
-         Assert.IsTrue(system_ready, "System is not ready");
+         await setup_limits_tests();
 
          string test_symbol = "CL";
 
@@ -212,8 +214,10 @@ namespace bts.tests
       }
 
 
-      [Test(Timeout = 10)]
+      [Test(Timeout = 300000)]
       public async Task test_clip_limits() {
+         await setup_limits_tests();
+
          //Get and set limit values from the admin service
 
          uint test_limit_value = 10u;
@@ -247,6 +251,8 @@ namespace bts.tests
             Quantity = test_limit_value
          });
 
+         Assert.IsTrue(no_limit_order_result.value.OrderKey > 0, "failed to enter order");
+
          var set_success = await AdminUtilities.set_clip_limit(_admin_service, test_limit_value, account_data, test_symbol);
          Assert.IsTrue(set_success, "Failed to set limit");
 
@@ -255,12 +261,15 @@ namespace bts.tests
             Price = 5,
             Quantity = test_limit_value
          });
+         Assert.IsTrue(under_limit_order_result.value.OrderKey > 0, "failed to enter order");
 
          Result<Response> over_limit_order_result = await _order_service.create_order_async(new Create() {
             NewOrder = order_def,
             Price = 5,
             Quantity = test_limit_value + 1
          });
+         Assert.IsTrue(over_limit_order_result.value.OrderKey > 0, "failed to enter order");
+
 
          under_limit_order_key = under_limit_order_result.value.OrderKey;
          over_limit_order_key = over_limit_order_result.value.OrderKey;

src/client/spark_test/System/SpreaderTests.cs

@@ -28,7 +28,10 @@ namespace bts.tests
       SyntheticSecurityService _synthetic_service;
       IDisposable _order_subscription;
       Update<OrderRecordImmutable, uint> _orders;
-      public SpreaderTests (){
+      public SpreaderTests (){}
+
+      public async Task setup_spreader_tests () {
+         await TestUtils.wait_for_system_ready();
          _context = new ServiceContext();
 
          _order_service = _context.locate<OrderService>();
@@ -52,10 +55,9 @@ namespace bts.tests
          Disposable.dispose(ref _order_subscription);
       }
 
-      [Test(Timeout = 30)]
+      [Test(Timeout = 300000)]
       public async Task spreader_simple_sanity() {
-         bool system_ready = await TestUtils.test_for_system_ready(_context, TimeSpan.FromMinutes(5));
-         Assert.IsTrue(system_ready, "System is not ready");
+         await setup_spreader_tests();
 
          string test_symbol = "SI";// don't use ES, the prices will be rounded to the same number and the result will be off
          var test_user = (name: "bts", id: 1u);

src/client/spark_test/Utils/TestUtils.cs

@@ -2,6 +2,7 @@
 using bts.api.admin;
 using bts.api.core;
 using bts.api.orderman;
+using bts.metrics;
 using bts.collections;
 using System;
 using System.Collections.Generic;
@@ -80,42 +81,110 @@ namespace bts.tests.utils {
          return tr.result;
       }
 
-
-      public static async Task<bool> test_for_system_ready(ServiceContext context, TimeSpan? timeout = null) {
+      public static async Task wait_for_system_ready(TimeSpan? timeout = null) {
          if(timeout == null) {
-            timeout = TimeSpan.FromSeconds(30);
+            timeout = TimeSpan.FromSeconds(120000);
          }
 
          System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
          sw.Start();
+         // bool shell_latency_service_ready = await wait_for_shell_latency_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         // Console.WriteLine("Shell Latency Service Ready: " + shell_latency_service_ready);
+         // bts.test.Assert.IsTrue(shell_latency_service_ready, "System is not ready");
 
-         bool admin_ready = await test_for_admin_service_ready(context, timeout.Value.Subtract(sw.Elapsed));
-         Console.WriteLine("Admin Service Ready: " + admin_ready);
-         bool md_ready = await test_for_md_service_ready(context, timeout.Value.Subtract(sw.Elapsed));
+         bool md_ready = await wait_for_md_service_ready(timeout.Value.Subtract(sw.Elapsed));
          Console.WriteLine("MD Service Ready: " + md_ready);
+         bts.test.Assert.IsTrue(md_ready, "System is not ready");
 
-         return admin_ready && md_ready;
+         bool orders_ready = await wait_for_orders_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         Console.WriteLine("Order Service Ready: " + orders_ready);
+         bts.test.Assert.IsTrue(orders_ready, "System is not ready");
+
+         // bool admin_ready = await wait_for_admin_service_ready(timeout.Value.Subtract(sw.Elapsed));
+         // Console.WriteLine("Admin Service Ready: " + admin_ready);
+         // bts.test.Assert.IsTrue(admin_ready, "System is not ready");
+
+
+         Console.WriteLine("System Ready after: " + sw.Elapsed.TotalSeconds + " seconds");
+
+         bts.test.Assert.IsTrue(orders_ready, "System is not ready");
       }
 
-      public static async Task<bool> test_for_admin_service_ready(ServiceContext context, TimeSpan timeout) {
-         System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
-         sw.Start();
+      public static async Task<bool> wait_for_shell_latency_service_ready(TimeSpan timeout) {
+         System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
 
-         var admin_service = context.locate<AdminService>();
+         Update<TickImmutable, int> service_value = null;
+         while(service_value == null && watch.Elapsed < timeout) {
+            var context = new ServiceContext();
+            var shell_latency_service = context.locate<ShellLatencyService>();
 
-         Update<AdminDataImmutable, int> admin_value = null;
-         var account_sub = admin_service.subscribei_admin_data().Subscribe(update => admin_value = update);
+            var clock_sub = shell_latency_service.subscribei_clock().Subscribe(update => service_value = update);
 
-         while(admin_value == null && sw.Elapsed < timeout) {
-            Console.WriteLine("Waiting for admin service...");
-            await Task.Delay(100);
+            Console.WriteLine("Waiting for shell latency service...");
+            await Task.Delay(15000);
+
+            clock_sub.Dispose();
+            context.Dispose();
          }
+         return service_value != null;
+      }
 
-         return admin_value != null;
+      public static async Task<bool> wait_for_orders_service_ready(TimeSpan timeout) {
+         System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
+
+         var odef = OrderDefinitionUtilities.create_limit_order_def(1, Side.Buy, ("bts", 1), ("ACC A", 1));
+
+         Update<OrderRecordImmutable, uint> service_value = null;
+         while(service_value == null && watch.Elapsed < timeout) {
+            var context = new ServiceContext();
+            var order_service = context.locate<OrderService>();
+
+            Result<Response> oret = await OrderUtilities.send_order(order_service, odef, 0,1);
+
+            var sub = order_service.subscribei_orders().Subscribe(async update => {
+                  if(service_value == null) {
+                     await OrderUtilities.cancel_orders(order_service, null, oret.value.OrderKey);
+                     service_value = update;
+                  }
+               });
+
+            if(service_value == null) {
+               Console.WriteLine("Waiting for order service.");
+               await Task.Delay(10000);
+            }
+
+            sub.Dispose();
+            context.Dispose();
+         }
+         return service_value != null;
+      }
+
+      public static async Task<bool> wait_for_admin_service_ready(TimeSpan timeout) {
+         System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
+         watch.Start();
+
+         Update<AdminDataImmutable, int> service_value = null;
+         while(service_value == null && watch.Elapsed < timeout) {
+            var context = new ServiceContext();
+            var admin_service = context.locate<AdminService>();
+
+            var sub = admin_service.subscribei_admin_data().Subscribe(update => service_value = update);
+
+            Console.WriteLine("Waiting for admin service...");
+            await Task.Delay(15000);
+
+            sub.Dispose();
+            context.Dispose();
+            //mbts.config.Configuration.initialize_for_test();
+
+         }
+         return service_value != null;
       }
 
 
-      public static async Task<bool> test_for_md_service_ready(ServiceContext context, TimeSpan timeout) {
+      public static async Task<bool> wait_for_md_service_ready(TimeSpan timeout) {
          var md_request = new co2.MDRequest() {
             Host = "system_test",
             Channels = {
@@ -130,7 +199,7 @@ namespace bts.tests.utils {
                }
             }
          };
-
+         var context = new ServiceContext();
          var md_relay = context.locate<co2.MarketDataRelay>();
          Result<co2.MDReply>? md_result = await await_with_timeout(
                                                                md_relay.requestmd_async(md_request),

src/client/spark_test/system_test_scripts/delete_dbs.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ -d ~/sparkdata/ ]
+then
+   sparkdata_dir=~/sparkdata
+else
+   sparkdata_dir=/sparkdata
+fi
+
+if [ -d "$sparkdata_dir/data" ]
+then
+   [ -d $sparkdata_dir/data/databases/orders ] && rm -rf $sparkdata_dir/data/databases/orders
+   [ -d $sparkdata_dir/data/databases/fills ] && rm -rf $sparkdata_dir/data/databases/fills
+   [ -d $sparkdata_dir/data/databases/ex_sim_orders ] && rm -rf $sparkdata_dir/data/databases/ex_sim_orders
+
+   [ -d $sparkdata_dir/data/jetfix ] && rm -rf $sparkdata_dir/data/jetfix
+   [ -d $sparkdata_dir/data/exchange_sim ] && rm -rf $sparkdata_dir/data/exchange_sim
+
+fi

src/client/spark_test/system_test_scripts/spreader_simple_sanity.sh

@@ -7,16 +7,13 @@ else
    sparkdata_dir=/sparkdata
 fi
 
+./delete_dbs.sh
+
 config=$sparkdata_dir/conf/spark.conf
 bak=$sparkdata_dir/conf/spark.conf.bak
 
-rm -r $sparkdata_dir/data/databases/ 2>/dev/null
-
 [ ! -d "$sparkdata_dir/conf" ] && mkdir $sparkdata_dir/conf/
-
-if  [ -f "$conf" ];then
-   mv $conf $bak 2>/dev/null
-fi
+[ -f "$conf" ] && mv $conf $bak
 
 cat << EOF > $sparkdata_dir/conf/spark.conf
    ExchangeSim {
@@ -38,17 +35,11 @@ nohup ./sparkd run &
 pid=$!
 echo "sparkd pid: " $pid
 
-echo "sleeping for 120 seconds..."
-sleep 120
-
 dotnet sparkcmd.dll test spreader_simple_sanity
 exit_status=$?
 
 pkill -P $$
 
-if  [ -f "$bak" ];then
-   mv $bak $conf
-fi
-
+[ -f "$bak" ] && mv $bak $conf
 [ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
 exit $exit_status

src/client/spark_test/system_test_scripts/test_clip_limits.sh

@@ -7,17 +7,13 @@ else
    sparkdata_dir=/sparkdata
 fi
 
+./delete_dbs.sh
+
 config=$sparkdata_dir/conf/spark.conf
 bak=$sparkdata_dir/conf/spark.conf.bak
 
-
-rm -r $sparkdata_dir/data/databases/ 2>/dev/null
-
 [ ! -d "$sparkdata_dir/conf" ] && mkdir $sparkdata_dir/conf/
-
-if  [ -f "$conf" ];then
-   mv $conf $bak 2>/dev/null
-fi
+[ -f "$conf" ] && mv $conf $bak
 
 cat << EOF > $sparkdata_dir/conf/spark.conf
    ExchangeSim {
@@ -38,17 +34,11 @@ nohup ./sparkd run &
 pid=$!
 echo "sparkd pid: " $pid
 
-echo "sleeping for 120 seconds..."
-sleep 120
-
 dotnet sparkcmd.dll test test_clip_limits
 exit_status=$?
 
 pkill -P $$
-
-if  [ -f "$bak" ];then
-   mv $bak $conf
-fi
+[ -f "$bak" ] && mv $bak $conf
 
 [ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
 exit $exit_status

src/client/spark_test/system_test_scripts/test_position_limits.sh

@@ -7,16 +7,13 @@ else
    sparkdata_dir=/sparkdata
 fi
 
+./delete_dbs.sh
+
 config=$sparkdata_dir/conf/spark.conf
 bak=$sparkdata_dir/conf/spark.conf.bak
 
-rm -r $sparkdata_dir/data/databases/ 2>/dev/null
-
 [ ! -d "$sparkdata_dir/conf" ] && mkdir $sparkdata_dir/conf/
-
-if  [ -f "$conf" ];then
-   mv $conf $bak 2>/dev/null
-fi
+[ -f "$conf" ] && mv $conf $bak
 
 cat << EOF > $sparkdata_dir/conf/spark.conf
    ExchangeSim {
@@ -38,16 +35,12 @@ nohup ./sparkd run &
 pid=$!
 echo "sparkd pid: " $pid
 
-echo "sleeping for 120 seconds..."
-sleep 120
 dotnet sparkcmd.dll test test_position_limits
 exit_status=$?
 
 pkill -P $$
 
-if  [ -f "$bak" ];then
-   mv $bak $conf
-fi
+[ -f "$bak" ] && mv $bak $conf
 
 [ $exit_status -eq 0 ] && echo "Test Finished" || echo "Test failed"
 exit $exit_status

src/client/sparkcmd/Cmd/MBOCommand.cs

@@ -23,26 +23,14 @@ namespace bts {
          return 0;
       }
       
-      static void dump_mbo(UInt32 key, api.Side side) {
+      static void dump_mbo(uint key, api.Side side) {
          using (var ctx = new ServiceContext()) {
             var mbo = ctx.locate<MBOService>();
             var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key });
 
             Console.WriteLine($"Order data for {data.SecurityId}");
-            IEnumerable<OrderPosition> orders = data.Bid;
-            switch(side) {
-               case api.Side.Undefined:
-                  orders = data.Ask.Concat(data.Bid);
-               break;
-               case api.Side.Buy:
-                  orders = data.Bid;
-               break;
-               case api.Side.Sell:
-                  orders = data.Ask;
-               break;
-            }
             var price = 0.0;
-            foreach(var e in orders) {
+            foreach(var e in data.Bid.Concat(data.Ask)) {
                if (e.Price != price) {
                   price = e.Price;
                   Console.WriteLine("================");

src/client/sparkcmd/Cmd/ServiceTail.cs

@@ -45,6 +45,9 @@ namespace bts {
          if (o.json) {
             _format = PrintFormat.Json;
          }
+         if (o.time) {
+            _timestamp = true;
+         }
 
          tail(split[0], split[1]);
          return 0;

src/lib/api/services/MBOService.proto

@@ -13,16 +13,18 @@ message OrderPIQ {
 message OrderPosition {
   uint64 exchange_id = 1;
   uint32 order_id = 2;
-  float price = 10;
+  double price = 10;
   int32 size = 11;
 }
 message PositionData {
   uint32 security_id = 1;
   repeated OrderPosition bid = 10;
-  repeated OrderPosition ask = 20;
+  repeated OrderPosition ask = 11;
 }
 message MBOSecurityQuery {
   uint32 security_id = 1;
+  double price_level = 2;
+  uint32 limit = 3;
 }
 
 service MBOService {

src/lib/config/protos/ExchangeSimCfg.proto

@@ -9,5 +9,6 @@ message ExchangeSim {
         string replay_file = 12 [default = "futures.coconut"];
         bool local_relay = 13;
     }
+    optional uint32 orders_per_level = 20 [default = 10];
 }
 

src/lib/config/protos/UI.proto

@@ -69,6 +69,7 @@ message LadderKnobs {
   optional uint32 security_timeout_secs = 5 [default = 60];
   optional bool open_algo_popup_on_select = 6 [default = false];
   optional bool use_algo_param_grid = 7 [default = true];
+  optional uint32 max_mbo_orders = 8 [default = 200];
 }
 
 message OrderBookOptions {

src/lib/md/MBOProcessor.cpp

@@ -1,15 +1,21 @@
 #include "MBOProcessor.h"
-#include  "lib/utils/Dispatcher.h"
 #include "lib/utils/ConcurrentPool.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/toggles.pb.h"
+#include "lib/utils/thread_manager.h"
 
 #include <map>
 #include <functional>
 #include <vector>
 #include <sstream>
+#include <thread>
+#include <atomic>
 
+#ifndef _WIN32
+#include <google/btree/btree_set.h>
+#endif
 #include <tbb/concurrent_hash_map.h>
+#include <tbb/concurrent_queue.h>
 
 
 namespace bts {
@@ -57,34 +63,126 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
 }
 
 typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
+struct Incremental {
+   MBOWorkingState* state;
+   inc::Update update;
+};
 struct MBOProcessorImpl {
    utils::ConcurrentPool<MBOSnapshot> pool;
+   tbb::concurrent_queue<Incremental> incremental_input;
    MapType snapshots;
-   utils::Dispatcher& dispatcher;
-   utils::DispatcherTimer timer;
-   MBOProcessorImpl(utils::Dispatcher& d) :
+   bool running;
+   utils::TimeStamp last_pool_print;
+   std::thread thread;
+
+   MBOProcessorImpl() :
       pool(2048,512),
       snapshots(),
-      dispatcher(d),
-      timer(d, 600*1000, [&]() { print_pool_stats(pool); })
-      {
-         if (config::get<config::toggles::MBOStats>().enabled()) {
-            timer.start();
-         }
+      running(true),
+      last_pool_print(utils::TimeStamp::invalid()),
+      thread(threading::make_thread("MBO", [=]() { process_incremental(); }))
+   {
+      if (config::get<config::toggles::MBOStats>().enabled()) {
+         last_pool_print = utils::TimeStamp::now();
       }
+
+   }
+
+   ~MBOProcessorImpl() {
+      running = false;
+      thread.join();
+   }
+
+   void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
+      bts_assert(*snapshot != nullptr);
+      MapType::accessor accessor;
+      snapshots.insert(accessor, key);
+      std::swap(accessor->second, *snapshot);
+      if (*snapshot != nullptr) {
+         // Note: relase() must happen inside accessor lifetime to ensure correctness
+         pool.release(snapshot);
+      }
+   }
+
+   void publish_incremental(MBOWorkingState& state) {
+      // determine if we need to publish
+      auto key = SecurityKey(state.handle.key());
+      if (key.value() != invalid_security_key) {
+         auto snap = pool.allocate();
+         snap->bids.reserve(state.bid.size());
+         snap->asks.reserve(state.ask.size());
+         copy_to_snapshot(state, snap);
+         publish_snapshot(key, &snap);
+      }
+   }
+
+   void process_incremental() {
+      #ifndef _WIN32
+      // drain queue and update state
+      // insert changed state pointers into a set
+      // after draining publish the set in small batches
+      // re-check input queue regularly 
+
+      btree::btree_set<MBOWorkingState*> dirty;
+      std::vector<MBOWorkingState*> clean;
+
+      while(running) {
+         Incremental temp;
+
+         // Drain entire input queue quickly
+         while(incremental_input.try_pop(temp)) {
+            bts_assert_d(temp.state != nullptr);
+            if (temp.state != nullptr) {
+               apply_incremental(*temp.state, temp.update);
+               dirty.insert(temp.state);
+            }
+         }
+
+         // publish a small batch of snapshots (then check queue again)
+         for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
+            publish_incremental(**iter);
+            clean.push_back(*iter);
+            if (clean.size() >= 128) break;
+         }
+
+         // Remove 'clean' from set
+         for (const auto& iter : clean) {
+            dirty.erase(iter);
+         }
+         clean.clear();
+
+         // If the queue is still empty sleep briefly
+         if (incremental_input.empty()) { 
+            if (last_pool_print != TimeStamp::invalid()) {
+               auto now = TimeStamp::now();
+               if ((now - last_pool_print).seconds() > 600) {
+                  last_pool_print =  now;
+                  print_pool_stats(pool);
+               }
+            }
+            std::this_thread::sleep_for(
+               std::chrono::milliseconds(1)
+            );
+         }
+
+      }
+      #endif
+   }
+
 };
 
 //
 // Processor
 
-MBOProcessor::MBOProcessor(utils::Dispatcher* d) :
-   _me(std::make_shared<MBOProcessorImpl>(d == nullptr ? utils::Dispatcher::background() : *d)),
-   _producer(_me),
-   _consumer(_me)
-{}
+MBOProcessor::MBOProcessor() :
+   _me(std::make_shared<MBOProcessorImpl>()),
+   _writer(_me),
+   _reader(_me)
+{
+}
 
-MBOWriter& MBOProcessor::writer() { return _producer;}
-MBOReader& MBOProcessor::reader() { return _consumer; }
+MBOWriter& MBOProcessor::writer() { return _writer;}
+MBOReader& MBOProcessor::reader() { return _reader; }
 
 //
 // Writer
@@ -98,18 +196,11 @@ void MBOWriter::release_snapshot(MBOSnapshot** s) {
    _me->pool.release(s);
 }
 void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
-   bts_assert(*snapshot != nullptr);
-   MapType::accessor accessor;
-   _me->snapshots.insert(accessor, key);
-   std::swap(accessor->second, *snapshot);
-   if (*snapshot != nullptr) {
-      // Note: relase() must happen inside accessor lifetime to ensure correctness
-      _me->pool.release(snapshot);
-   }
+   _me->publish_snapshot(key,snapshot);
 }
 
+#if 0
 bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) {
-   // bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS");
    #define REQUIRE(a,b,y) \
       if ((a) != (b)) { \
          LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \
@@ -137,25 +228,11 @@ bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot*
    #undef REQUIRE
    return true;
 }
+#endif
 
 void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) {
    LOG(md,Diag, "sec:{} : {}", state.handle.key(), update);
-
-   _me->dispatcher.post([this,&state,update]() {
-      // fast part
-      apply_incremental(state,update);
-      // slow part
-      publish_incremental(state);
-   });
-}
-void MBOWriter::publish_incremental(MBOWorkingState& state) {
-   // determine if we need to publish
-   auto key = SecurityKey(state.handle.key());
-   if (key.value() != invalid_security_key) {
-      auto snap = allocate_snapshot(state.bid.size(), state.ask.size());
-      copy_to_snapshot(state, snap);
-      publish_snapshot(key, &snap);
-   }
+   _me->incremental_input.push({&state, update});
 }
 
 void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) {
@@ -267,4 +344,4 @@ ostream& operator<<(ostream& os, const inc::Update& update) {
    return os;
 }
 
-}
+}

src/lib/md/MBOProcessor.h

@@ -2,7 +2,6 @@
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/domain_model/Security.h"
 #include "lib/utils/TimeStamp.h"
-#include "lib/utils/Dispatcher.h"
 #include "IntPrice.h"
 #include "ExchangeOrderKey.h"
 #include "MDProducerHandle.h"
@@ -152,8 +151,6 @@ class MBOWriter {
 
    // Enqueue the update to the mbo work queue.
    void enqueue_incremental(MBOWorkingState&, const inc::Update&);
-   // Update working state and publish directly.
-   void publish_incremental(MBOWorkingState&);
 };
 
 class MBOReader {
@@ -169,11 +166,11 @@ class MBOReader {
 
 class MBOProcessor {
    std::shared_ptr<MBOProcessorImpl> _me;
-   MBOWriter _producer;
-   MBOReader _consumer;
+   MBOWriter _writer;
+   MBOReader _reader;
 
 public:
-   MBOProcessor(utils::Dispatcher* _=nullptr);
+   MBOProcessor();
    MBOWriter& writer();
    MBOReader& reader();
 };
@@ -184,3 +181,4 @@ public:
 namespace std {
 std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&);
 }
+

src/lib/md/MBOServiceServer.cpp

@@ -4,6 +4,7 @@
 #include "lib/api/services/OrderService.client.pb.h"
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/TimeStamp.h"
+#include "lib/domain_model/Security.h"
 #include "MDClient.h"
 #include "ExchangeOrderKey.h"
 #include "MBOProcessor.h"
@@ -307,27 +308,41 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
       reverse.insert(std::make_pair(pair.second.exchange_id(), pair.first));
    }
 
+   size_t count = 0;
+   auto tick = handle.get_minimum_tick();
    if(handle.is_initialized()) {
       auto mbo = handle.get_mbo();
       if (mbo) {
          for (const auto& o : mbo->bids) {
-            auto* entry = response.add_bid();
-            entry->set_exchange_id(o.id.value());
-            entry->set_price(o.price.price());
-            entry->set_size(o.size);
-            auto iter = reverse.find(o.id);
-            if (iter != reverse.end()) {
-               entry->set_order_id(iter->second.value());
+            double price = o.price.price();
+            bool isnan = os::is_nan(request.price_level());
+            if (isnan || securities::test_price_equality(request.price_level(), price, tick)) {
+               ++count;
+               if (request.limit() && count > request.limit()) break;
+               auto* entry = response.add_bid();
+               entry->set_exchange_id(o.id.value());
+               if (isnan) entry->set_price(price);
+               entry->set_size(o.size);
+               auto iter = reverse.find(o.id);
+               if (iter != reverse.end()) {
+                  entry->set_order_id(iter->second.value());
+               }
             }
          }
          for (const auto& o : mbo->asks) {
-            auto* entry = response.add_ask();
-            entry->set_exchange_id(o.id.value());
-            entry->set_price(o.price.price());
-            entry->set_size(o.size);
-            auto iter = reverse.find(o.id);
-            if (iter != reverse.end()) {
-               entry->set_order_id(iter->second.value());
+            double price = o.price.price();
+            bool isnan = os::is_nan(request.price_level());
+            if (isnan || securities::test_price_equality(request.price_level(), price, tick)) {
+               ++count;
+               if (request.limit() && count > request.limit()) break;
+               auto* entry = response.add_ask();
+               entry->set_exchange_id(o.id.value());
+               if (isnan) entry->set_price(price);
+               entry->set_size(o.size);
+               auto iter = reverse.find(o.id);
+               if (iter != reverse.end()) {
+                  entry->set_order_id(iter->second.value());
+               }
             }
          }
       }

src/lib/md/MDProducer.cpp

@@ -169,6 +169,7 @@ MDProducerChannel MDProducer::get_channel(const char* symbol, SecurityType type,
 }
 
 void MDProducer::heartbeat() {
+   std::lock_guard<std::mutex> lock(*_mutex);
    for (auto& impl : _producers) {
       impl->heartbeat();
    }

src/server/sparkd/cmds/Benchmark.cpp

@@ -34,6 +34,10 @@
 #include <mutex>
 #include <sys/time.h>
 
+#include <unordered_set>
+#include <set>
+#include <google/btree/btree_set.h>
+
 #include <date/date.h>
 #include <tbb/concurrent_queue.h>
 namespace bts {
@@ -655,6 +659,73 @@ void int_price_less(benchmark::State& state) {
 }
 BENCHMARK(int_price_less)->Range(8, 8 << 11);
 
+
+void set_hashset_insert(benchmark::State& state) {
+   std::unordered_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+   }
+}
+void set_stdset_insert(benchmark::State& state) {
+   std::set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+   }
+}
+void set_btreeset_insert(benchmark::State& state) {
+   btree::btree_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+   }
+}
+
+void set_hashset_iterate(benchmark::State& state) {
+   std::unordered_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+      for (const auto& p : s) {
+         benchmark::DoNotOptimize(p);
+      }
+   }
+}
+void set_stdset_iterate(benchmark::State& state) {
+   std::set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+      for (const auto& p : s) {
+         benchmark::DoNotOptimize(p);
+      }
+   }
+}
+void set_btreeset_iterate(benchmark::State& state) {
+   btree::btree_set<uintptr_t> s;
+   size_t x = 0;
+   for (auto _ : state) {
+      x = std::rand();
+      s.insert(x);
+      for (const auto& p : s) {
+         benchmark::DoNotOptimize(p);
+      }
+   }
+}
+
+BENCHMARK(set_hashset_insert);
+BENCHMARK(set_stdset_insert);
+BENCHMARK(set_btreeset_insert);
+BENCHMARK(set_hashset_iterate);
+BENCHMARK(set_stdset_iterate);
+BENCHMARK(set_btreeset_iterate);
+
 std::string Benchmark::help() {
    return "Run micro-benchmarks. Use --benchmark_filter=regex to run only "
           "specific benchmarks.";

src/server/sparkd/cmds/Run.cpp

@@ -59,7 +59,7 @@ int Run::execute(std::vector<std::string> const &) {
    persistence::PersistenceServiceServer settings_saver(sc, "settings");
    user::AuthServiceServer login_server(sc);
 
-   md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO"));
+   md::MBOProcessor mbo;
    auto* mbowriter = config::get<config::toggles::PlacementInQueue>().enabled() ? &mbo.writer() : nullptr;
    md::run_market_datapublishers(mbowriter);
 

src/server/sparkd/exchange_sim/book/OrderMatcher.cpp

@@ -2,6 +2,8 @@
 #include "lib/utils/bts_assert.h"
 #include "server/sparkd/exchange_sim/book/Book.h"
 #include "server/sparkd/exchange_sim/book/Order.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/ExchangeSimCfg.pb.h"
 
 #include <set>
 
@@ -10,11 +12,12 @@ namespace ex_sim {
 
 struct OrderMatcher::PImpl {
    PImpl()
-       : bids(OrderCmp(false)), asks(OrderCmp(true)) {}
+       : bids(OrderCmp(false)), asks(OrderCmp(true)), max_orders(config::get<config::ExchangeSim>().orders_per_level()) {}
 
    OrdersSide bids;
    OrdersSide asks;
    std::vector<Trade> pending_trades;
+   size_t max_orders;
 };
 
 OrderMatcher::OrderMatcher()
@@ -53,24 +56,24 @@ void OrderMatcher::add_new(Order const &o, std::vector<Order> *out) {
    match(out);
 }
 
-std::pair<OrderMatcher::OrdersSide::iterator, OrderMatcher::OrdersSide::iterator>
-OrderMatcher::find_level(OrdersSide &side, double level)
+OrderMatcher::Level OrderMatcher::find_level(OrdersSide &side, double level)
 {
-   std::pair<OrdersSide::iterator, OrdersSide::iterator> result {side.end(), side.end()};
+   auto first = side.end(), second = side.end();
+   size_t count = 0;
 
    for (auto it = side.begin() ; it != side.end(); it++) {
-
       if (price_equals(it->price(),level)) {
-         result.first = it;
+         first = it;
          for (;it != side.end(); it++) {
+            ++count;
             if (!price_equals(it->price(),level))
                break;
          }
-         result.second = it;
+         second = it;
          break;
       }
    }
-   return result;
+   return { first, second, count };
 }
 
 void OrderMatcher::add_or_merge(Order const &o, std::vector<Order> *out) {
@@ -82,12 +85,12 @@ void OrderMatcher::add_or_merge(Order const &o, std::vector<Order> *out) {
       side = &me->asks;
 
    auto range = find_level(*side,o.price());
-
    // If the last order at this level is the same owner we can update it
    if (range.first != range.second) {
       auto last = range.second;
       last--;
-      if (last->owner() == o.owner()) {
+      // Keep enough orders to make MBO look good but don't grow forever
+      if (last->owner() == o.owner() && range.size > me->max_orders) {
          auto no_const = const_cast<Order *>(&(*last));
          no_const->set_open_quantity(last->open_quantity() + o.quantity());
          match(out);
@@ -139,11 +142,10 @@ void OrderMatcher::cancel(std::string const &clordid) {
 void OrderMatcher::try_reduce_quantity_at_level(
     api::Side side, Price level, Quantity by_quantity,
     std::string const &order_owner) {
-   std::pair<OrdersSide::iterator, OrdersSide::iterator> range;
-   if (side == api::SIDE_BUY)
-      range = find_level(me->bids,level.value());
-   else
-      range = find_level(me->asks,level.value());
+   auto range = 
+      side == api::SIDE_BUY ? 
+         find_level(me->bids,level.value()) :
+         find_level(me->asks,level.value());
    
    std::vector<std::string> orders_to_cancel;
    for (auto it = range.first; it != range.second && by_quantity.value();

src/server/sparkd/exchange_sim/book/OrderMatcher.h

@@ -9,6 +9,7 @@
 
 #include <memory>
 #include <set>
+#include <tuple>
 
 namespace bts {
 namespace ex_sim {
@@ -48,8 +49,12 @@ class OrderMatcher {
    std::unique_ptr<PImpl> me;
    void match(std::vector<Order> *out);
    void match(Order *bid, Order *ask);
-   std::pair<OrderMatcher::OrdersSide::iterator, OrderMatcher::OrdersSide::iterator>
-   find_level(OrdersSide &side, double level);
+   struct Level {
+      OrderMatcher::OrdersSide::iterator first;
+      OrderMatcher::OrdersSide::iterator second;
+      size_t size;
+   };
+   Level find_level(OrdersSide &side, double level);
 };
 
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -120,7 +120,8 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
    onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
 }
 
-void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) {
+void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook& ) {
+   #if 0
    if (_mbo) {
       auto data = _index.get_security(sec.id());
       if (data == nullptr) return;
@@ -136,6 +137,7 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
       }
       _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot);
    }
+   #endif
 }
 
 //

testing/manual/ladder/badges.txt

@@ -28,12 +28,12 @@ the badge will have a green border, depicting the single-order limit order type
 the badge will have a dot in the upper right hand corner (7)
 
 2. Now place another limit order at the same level:
-the badge now has a triangle in the lower right, indicating we have multiple workikng orders (9)
+the badge now has a triangle in the lower right, indicating we have multiple working orders (9)
 the green border and dot have gone away (5, 7)
 
 Now place another limit below the bid:
 
-3. Outcome similar to 1, except the background color is ligther bid background color (blue)
+3. Outcome similar to 1, except the background color is lighter bid background color (blue)
 
 Now place an a Buy Iceberg at the same level with a sufficiently large qty, say 200, as your working ask
 
@@ -44,7 +44,7 @@ Test 2
 ------
 This will test characteristics 4,10
 
-1. Create a spreader comprised 1 quoting leg and 1 heding leg of your favorite instruments and place a spread order
+1. Create a spreader comprised 1 quoting leg and 1 hedging leg of your favorite instruments and place a spread order
 2. Pull up the spreader ladder and place a sell order above the market
 3. Pull up the quoting leg's ladder and observe the badge; it may be helpful to pull up the Orderbook to find the price at which the leg is being quoted.
 	The badge should have a rectangle filled in with the Spreader algo's color

testing/manual/ladder/piq_mbo.txt

@@ -0,0 +1,57 @@
+Introduction
+-------------
+
+Ideally this test should be done with a cert connection because the simulator can be quirky.  If you ignore this advice and use sim you must not admit it to anyone.
+
+Only exchange orders will have a PIQ so you should probably just use a Limit order for this test.
+
+General Stuff to Look For:
+--------------------------
+- The MBO popup is 'live' and should update as the market moves.
+- The MBO popup should increase in size if more orders are added while it is open.
+- The MBO popup should not shrink when orders are removed. It only grows.
+- The scale of the bars is based on the largest order seen by the popup.  
+    - If a larger order shows up it will scale down the other orders.
+    - When the large order goes away it should *not* expand the other bars.  The scale only increases while the popup is open.
+    - For example:
+        - You open the popup and there are 2 orders of size [1,2]
+        - size 1 will be about half way across the available space and 2 will fill the space.
+        - An order of size 5 comes around it will fill the available space and the other orders will be 20%, 40% across respectively.
+        - If the size 5 order is canceled the [1,2] orders stay at 20%, 40% rather than growing to fill the space.
+        - If you close and re-open the popup state is reset and [1,2] will be 50%/100% again.
+- Middle clicking on a row with no data does not open the popup.
+    - This doesn't mean clicking past the end of the visible book won't show anything.  The MBO tracks much deeper than the market by price.  If there's a gap between 2 prices that's a good spot to test.
+
+
+Part 1:
+-------
+- Find a security with a non-empty market that isn't moving too fast.  Cert can be temperamental so open the market view and filter for Qty > 0.
+- Open settings and turn on the PIQ column (settings -> column settings)
+- Choose a price near the top of the book and middle click the price column to show the MBO popup.
+- Add up the sizes in the popup and make sure they agree with the total size in the bid or ask column on the ladder. 
+    - Test both sides of the market near the top and away from the top.
+- Enter an order.
+    - The PIQ column should reflect the number of orders ahead of you in the queue.  The size in the bid/ask column is how many orders were there when you entered your order so you should expect the PIQ to be the size from before you entered your order.
+    When the market moves you should expect your PIQ to go down.  If cert is working correctly your PIQ should never go up.
+    - Open the MBO popup.  Your order should be shown in a different color from the other orders at that level.
+    - Verify that your order position matches the value from the PIQ column.
+    - Test both sides of the market near the top and away from the top.
+- Moving your order to a different price or *increasing* the size of your order will move you to the back of the queue.
+    - If you ignored the instructions and you're using sim this probably won't work.  You should have listened.
+- Reducing the size of your order should not change your queue position.  You might may have to be quick to observe this.
+
+Part 2:
+-------
+- Open several securities with varying tick sizes and make sure that the MBO popup opens for all levels of the book where there is a size.
+- Why? The ladder sends the price you click on to the server and the server must identify the right price level.  As you know, the IEEE double is out to get you.
+
+Part 3:
+-------
+- The MBO popup should expand either up or down from the mouse position to maximize the available space.
+- If the number of orders exceeds the space available the mouse wheel should work to let you scroll up/down to see the rest (up to LadderKnobs.max_mbo_orders which defaults to 200)
+- Find a security with enough orders to cause scrolling by either
+  - Connect to a demo system with production md and look at a ZN spread (i.e. ZNM9 - ZNU9)
+  - Enter a bunch of orders at the level on cert.
+  - If you try this on sim (you're not a very good listener, are you?) It will cap out at ExchangeSim.orders_per_level (default:10) 
+- Make sure that scrolling works.
+