always bypass entitlement take 2

SPARK-1736 - client-based ICMP ping

.vscode/launch.json

@@ -42,7 +42,18 @@
          "stopAtEntry": false,
          "externalConsole": true,
          "requireExactSource": false
-      },     
+      },        
+      {
+         "name": "Excel Launch",
+         "type": "clr",
+         "request": "launch",
+         "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
+         "args": ["${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64.xll"],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },          
       {
          "name": "Client Tests",
          "type": "clr",
@@ -54,7 +65,7 @@
          "externalConsole": true,
          "requireExactSource": false
       },           
-		{
+      {
             "name": "Client Attach",
             "type": "clr",
             "request": "attach",
@@ -264,63 +275,5 @@
                 "RedirectOutput"
             ]
         },
-        {
-            "name": "edge-asp051",
-            "type": "cppdbg",
-            "request": "launch",
-            "program": "/home/bts/spark/sparkd",
-            "args": ["run"],
-            "stopAtEntry": false,
-            "cwd": "/home/bts/spark/",
-            "environment": [],
-            "externalConsole": true,
-            "MIMode": "gdb",
-            "preLaunchTask": "",
-            "setupCommands": [
-                {
-                    "description": "Enable pretty-printing for gdb",
-                    "text": "enable pretty-printer",
-                    "ignoreFailures": true
-                }
-            ],
-            "pipeTransport": {
-                "pipeCwd": "${workspaceRoot}",
-                "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["bts@edge-asp051"],
-                "debuggerPath": "/usr/bin/gdb"
-            },
-            "sourceFileMap": {
-               "/bts/": "${workspaceRoot}"
-            }
-        },
-        {
-            "name": "Docker Launch",
-            "type": "cppdbg",
-            "request": "launch",
-            "program": "/bts/build/gcc_debug/src/server/sparkd/sparkd",
-            "args": ["test"],
-            "stopAtEntry": false,
-            "cwd": "/bts/build/gcc_debug/src/server/sparkd",
-            "environment": [],
-            "externalConsole": true,
-            "MIMode": "gdb",
-            "preLaunchTask": "",
-            "setupCommands": [
-                {
-                    "description": "Enable pretty-printing for gdb",
-                    "text": "enable pretty-printer",
-                    "ignoreFailures": true
-                }
-            ],
-            "pipeTransport": {
-                "pipeCwd": "${workspaceRoot}",
-                "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
-                "pipeArgs": ["-p 2000","root@localhost"],
-                "debuggerPath": "/usr/bin/gdb"
-            },
-            "sourceFileMap": {
-                "/bts/": "${workspaceRoot}"
-            }
-        }
     ]
 }

CMakeLists.txt

@@ -55,6 +55,8 @@ else ()
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/web ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+    COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
+    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/sdk/examples/order_plugin_example.so ${CMAKE_BINARY_DIR}/server/plugins/
     COMMENT "Creating server runtime layout"
     DEPENDS sparkd data_files_dir)  
 

doc/user/bts-help-theme/base.html

@@ -26,7 +26,6 @@
 
         <link href="https://fonts.googleapis.com/css?family=Source+Sans+Pro:300,400,600" rel="stylesheet" />
         <link rel="stylesheet" href="{{ base_url }}/css/local.css?1529096332" type="text/css" />
-        <!--link rel="stylesheet" href="https://www.bluetradingsystems.com/templates/bluetradesys/scripts/colorbox-master/example5/colorbox.css" type="text/css" /-->
         <!--[if lt IE 9]>
             <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
             <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>

doc/user/bts-help-theme/toc.html

@@ -30,9 +30,8 @@
                     {% if nav_item.active%}
                     <ul>
                         {%- for toc_item in page.toc %}
-                            <li class="main {% if toc_item.active %}active{% endif %}"><a href="{{ toc_item.url|url }}">{{ toc_item.title }}</a></li>
-                            {%- for toc_item in toc_item.children %}
-                                <li><a href="{{ toc_item.url }}">{{ toc_item.title }}</a></li>
+                            {%- for toc_subitem in toc_item.children %}
+                                    <li><a href="{{ toc_subitem.url }}">{{ toc_subitem.title }}</a></li>
                             {%- endfor %}
                         {%- endfor %}
                     </ul>

doc/user/docs/index.md

@@ -1,37 +1,3 @@
-#Getting Started
 
-If you are new to BTS Spark we recommend starting with these pages to learn the basics.
+Click on the headings on the the left to expand their contents. If you need specific information quickly use the Search button in the top right of this page.
 
-If you need specific information quickly use the Search button in the top right of the navigation of use the table of contents at the left.
-
-<ol>
-   <li><a href="Logging-In-to-the-System">Login</a></li>
-   <li><a href="Launcher">Module Launcher</a></li>
-   <li><a href="Market">Market</a>
-      <ul>
-         <li><a href="Market/#market-view">Market View</a></li>
-         <li><a href="Market/#time-and-sales">Time and Sales</a></li>
-      </ul>
-   </li>
-   <li><a href="Trading">Trading</a>
-      <ul>
-         <li><a href="Trading/#order-book">Order Book</a></li>
-         <li><a href="Trading/#fills">Fills</a></li>
-         <li><a href="Trading/#pwzxhzdk0l">P&amp;L</a></li>
-         <li><a href="Trading/#ladder">Ladder</a></li>
-         <li><a href="Trading/#order-ticket">Order Ticket</a></li>
-      </ul>
-   </li>
-   <li><a href="Spreader">Spreader</a></li>
-   <li><a href="My-Windows">My Windows</a></li>
-   <li><a href="Settings">Settings</a>
-      <ul><li><a href="Settings/#workspaces">Workspaces</a></li></ul>
-   </li>
-   <li><a href="Help">Help</a>
-      <ul>
-         <li><a href="Help/#report-a-problem">Report a Problem</a></li>
-         <li><a href="Help/#about-spark">About Spark</a></li>
-         <li><a href="Help/#activity-log">Activity Log</a></li>
-      </ul>
-   </li>
-</ol>

doc/user/docs/release_notes.md

@@ -1,8 +1,42 @@
-# 1.2 
+# Release Notes
+
+## 1.3.7
+
+### Net Change pricing
+  - The ability to price synthetics based off of net change
+
+## 1.3.5
+
+### Speed Enhancements
+  - Further speed enhancements
+### Separation of implied and direct market data
+  - Viewing within a market view will now show 3 columns
+    - Direct
+    - Implied
+    - Merged
+  - Ability to configure the market data size in a ladder as the following
+    - Direct
+    - Implied
+    - Merged
+### Synthetic creation option to choose Merged or Direct
+  - The ability to choose Merged or Direct per leg when creating a synthetic
+### Volume at price
+  - Within a ladder you can now see a volume at price graph
+  - This is configurable from a ladder's settings as well
+### Customized settings in all windows
+ - All windows now contain a settings page where grid settings (font color, size, etc) are applied
+ - These settings can be saved on a per-window basis or as a Default for newly-launched windows
+
+## 1.3.1 Through 1.3.4
+
+### Speed Enhancements
+  - Further speed enhancements
+
+## 1.2
 
 ### Market View
   - Synthetic future leg working orders are now shown
-  
+
 ### Order Book
   - Many visual facets of the Order Book grid can now be customized
     - The Order Book settings page can be access from the gears icon in the Order Book toolbar or from the right-click menu
@@ -18,7 +52,7 @@
 ### PnL
   - Prices are now formatted with penny precision
   - Day PnL has been added to the Totals footer at the bottom of the window
-  
+
 ### Ladder
   - Order badges now have been improved to communicate a variety of order state
     - Background color now indicates order side
@@ -37,7 +71,7 @@
     - Default size preset values can now be changed, as well as added or removed
     - This can be configured per ladder instance (i.e. per security) by making use of the Custom tab
   - Synthetic ladders that were saved as part of a workspace are now restored on startup
-  
+
 ### Synthetics
   - Hedge Ratio parameter has been renamed to Lean Qty Factor
   - Spreader now includes a Hidden mode parameter
@@ -48,7 +82,7 @@
   - IOC/FAK order type added
   - Cover order type added
   - Cancelling a synthetic order no longer cancels any working hedge orders
-  
+
 ### General
   - A manual page has been added to the launcher Help menu
   - Various help buttons and tooltips were added throughout the application
@@ -59,3 +93,41 @@
     - Individual fills be compacted into start of day position records
     - Orders from the day before the recycle will be deleted
 
+
+##1.1
+
+###1.1.22
+- Orderbook: Orders in Error status can now be recreated
+- Orderbook: Orders containing child orders will now be collapsed by default.
+- Orderbook: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
+- Orderbook: Symbol column added to allow for filtering based on group symbol
+- Time filters now have a quick Set Last Hour hyperlink which will set the Period Begin filter to (Now - 1 hour)
+
+###1.1.20
+- Update CME market data handler for price precision extension.
+  https://www.cmegroup.com/confluence/display/EPICSANDBOX/Price+Precision+Extension
+
+###1.1.8
+- Enabled wild cards in risk management UI
+- Fixed spreader order updates
+
+###1.1.7
+- Fixed spread leg price fill
+- Fixed HO cash per point value
+
+###1.1.6
+- Fixed occasional crash when filtering fills
+- Fixed several cash per point multipliers
+- Colored time of sales row with aggressor side
+- Iceberg orders now cancel (rathern than error) when a child is canceled
+
+###1.1.0
+- Added per user self match protection, location and self match protection FIX tag settings
+- Implemented a new "Trigger Order" algo
+- Corrected tick formatting for a number of products
+- Added the hability to recreate canceled orders
+- Added leg fills to facilitate futured based PnL
+- PnL report now has PnL columns calculated based on your start of day, current day, and total position
+- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
+- Corrected cash per point values, used in pnl calculations, for a number of products
+- PnL report will now remember any customizations you make to filtering and/or sorting criteria

release_notes.txt

@@ -1,82 +0,0 @@
-1.3.5
-- Speed Enhanments
-  - Further speed enhancements
-- Net Change pricing
-  - The ability to price synthetics based off of net change
-- Separation of implied and direct market data 
-  - Viewing within a market view will now show 3 columns
-    - Direct
-    - Implied
-    - Merged
-  - Ability to configure the market data size in a ladder as the following
-    - Direct
-    - Implied
-    - Merged
-- Synthetic creation option to choose Merged or Direct
-  - The ability to choose Merged or Direct per leg when creating a synthetic
-- Volume at price
-  - Within a ladder you can now see a volume at price graph
-  - This is configurable from a ladder's settings as well
-  - - Volume at price
-  - Within a ladder you can now see a volume at price graph
-  - This is configurable from a ladder's settings as well
-- Customized settings in all windows
-  - All windows now contain a settings parge where grid settings (font color, size, etc) are applied
-  - These settings can be saved on a per-window basis or as a Default for newly-launched windows
-
-1.3.4
-- Significantly reduced client memory use
-- Improved OrderBook performance
-- Improved workspace load time
-
-1.3.3
-- Reduced server memory usage
-
-1.3.2
-- Improved OrderBook performance
-- Fix some memory leaks when switching workspaces
-
-1.3.1
-- Improved server side logging
-- Fixed leaking of market data subscriptions on market view form closing
-
-1.3.0
-- Significantly improved order execution latency
-- Added latency monitoring tools
-- Provided core binding controls to system admins so as to reduce latency jitter
-
-1.1.22
-- Orderbook: Orders in Error status can now be recreated
-- Orderbook: Orders containing child orders will now be collapsed by default.
-- Orderbook: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
-- Orderbook: Symbol column added to allow for filtering based on group symbol
-- Time filters now have a quick Set Last Hour hyperlink which will set the Period Begin filter to (Now - 1 hour)
-
-1.1.20
-- Update CME market data handler for price precision extension.
-  https://www.cmegroup.com/confluence/display/EPICSANDBOX/Price+Precision+Extension
-
-1.1.8
-- Enabled wild cards in risk management UI
-- Fixed spreader order updates
-
-1.1.7
-- Fixed spread leg price fill
-- Fixed HO cash per point value
-
-1.1.6
-- Fixed occasional crash when filtering fills
-- Fixed several cash per point multipliers
-- Colored time of sales row with aggressor side
-- Iceberg orders now cancel (rathern than error) when a child is canceled
-
-1.1.0
-- Added per user self match protection, location and self match protection FIX tag settings
-- Implemented a new "Trigger Order" algo
-- Corrected tick formatting for a number of products
-- Added the hability to recreate canceled orders
-- Added leg fills to facilitate futured based PnL
-- PnL report now has PnL columns calculated based on your start of day, current day, and total position
-- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
-- Corrected cash per point values, used in pnl calculations, for a number of products
-- PnL report will now remember any customizations you make to filtering and/or sorting criteria 

spark_client.sln

@@ -4,7 +4,7 @@ VisualStudioVersion = 15.0.27130.2024
 MinimumVisualStudioVersion = 15.0.26124.0
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libspark", "build\src\lib\libspark.vcxproj", "{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libspark", "build\src\lib\libspark.vcxproj", "{3335AAF0-489D-359D-8305-9B7A44E2C760}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
 EndProject
@@ -12,15 +12,17 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sparkcmd", "src\client\spar
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZERO_CHECK", "build\ZERO_CHECK.vcxproj", "{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZERO_CHECK", "build\ZERO_CHECK.vcxproj", "{2256015B-E7D5-3614-8F39-814CA68A4444}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
 	ProjectSection(ProjectDependencies) = postProject
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C} = {69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}
+		{3335AAF0-489D-359D-8305-9B7A44E2C760} = {3335AAF0-489D-359D-8305-9B7A44E2C760}
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_excel", "src\client\spark_excel\spark_excel.csproj", "{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -57,22 +59,22 @@ Global
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = Release|x64
 		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = Release|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Debug|x64.ActiveCfg = Debug|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Debug|x64.Build.0 = Debug|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Debug|x86.ActiveCfg = Debug|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.MinSizeRel|x64.Build.0 = MinSizeRel|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.MinSizeRel|x86.ActiveCfg = MinSizeRel|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Release|Any CPU.ActiveCfg = Release|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Release|x64.ActiveCfg = Release|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Release|x64.Build.0 = Release|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.Release|x86.ActiveCfg = Release|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64
-		{69D73CF1-9D7A-38CA-9817-0D0C3E3BCB3C}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Debug|x64.ActiveCfg = Debug|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Debug|x64.Build.0 = Debug|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Debug|x86.ActiveCfg = Debug|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.MinSizeRel|x64.Build.0 = MinSizeRel|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.MinSizeRel|x86.ActiveCfg = MinSizeRel|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Release|Any CPU.ActiveCfg = Release|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Release|x64.ActiveCfg = Release|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Release|x64.Build.0 = Release|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.Release|x86.ActiveCfg = Release|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64
+		{3335AAF0-489D-359D-8305-9B7A44E2C760}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
 		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
@@ -137,22 +139,22 @@ Global
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = Release|x64
 		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = Release|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Debug|Any CPU.ActiveCfg = Debug|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Debug|x64.ActiveCfg = Debug|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Debug|x64.Build.0 = Debug|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Debug|x86.ActiveCfg = Debug|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.MinSizeRel|x64.Build.0 = MinSizeRel|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Release|Any CPU.ActiveCfg = Release|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Release|x64.ActiveCfg = Release|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Release|x64.Build.0 = Release|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.Release|x86.ActiveCfg = Release|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64
-		{33E7E6C5-04E3-36A8-A751-639BE5E72B9B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Debug|x64.ActiveCfg = Debug|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Debug|x64.Build.0 = Debug|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Debug|x86.ActiveCfg = Debug|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.MinSizeRel|x64.Build.0 = MinSizeRel|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.MinSizeRel|x86.ActiveCfg = MinSizeRel|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Release|Any CPU.ActiveCfg = Release|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Release|x64.ActiveCfg = Release|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Release|x64.Build.0 = Release|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.Release|x86.ActiveCfg = Release|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64
+		{2256015B-E7D5-3614-8F39-814CA68A4444}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
 		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
@@ -197,6 +199,26 @@ Global
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
 		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Debug|x64.ActiveCfg = Debug|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Debug|x64.Build.0 = Debug|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Debug|x86.ActiveCfg = Debug|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.MinSizeRel|Any CPU.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.MinSizeRel|Any CPU.Build.0 = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.MinSizeRel|x64.Build.0 = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.MinSizeRel|x86.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.MinSizeRel|x86.Build.0 = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Release|Any CPU.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Release|x64.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Release|x64.Build.0 = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.Release|x86.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|Any CPU.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|Any CPU.Build.0 = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.ActiveCfg = Release|x64
+		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/client/Directory.Build.Props

@@ -31,6 +31,9 @@
     <AvailableItemName Include="Proto3">
         <Targets>CompileProto3</Targets>
     </AvailableItemName>
+    <AvailableItemName Include="Proto3Excel">
+        <Targets>CompileProto3Excel</Targets>
+    </AvailableItemName>    
 </ItemGroup>
 
 <PropertyGroup Label="ProtoBuf">
@@ -48,9 +51,14 @@
         <OutCS>%(OutBase)%(FileName).cs</OutCS>
         <OutCSClient>%(OutBase)%(FileName).client.pb.cs</OutCSClient>
         <OutCSServer>%(OutBase)%(FileName).server.pb.cs</OutCSServer>
-        <OutCSImmutable>%(OutBase)%(FileName).immutable.pb.cs</OutCSImmutable>
+        <OutCSImmutable>%(OutBase)%(FileName).immutable.pb.cs</OutCSImmutable>        
         <CompileOptions></CompileOptions>
     </Proto3>
+    <Proto3Excel>
+        <OutBase>$(IntermediateOutputPath)\gen\%(relativedir)</OutBase>
+        <OutCSExcel>%(OutBase)%(FileName).excel.pb.cs</OutCSExcel>
+        <CompileOptions></CompileOptions>
+    </Proto3Excel>    
 </ItemDefinitionGroup>
 
 </Project>

src/client/Directory.Build.Targets

@@ -57,12 +57,38 @@
     </ItemGroup>
 </Target>
 
+<Target Name="CompileProto3Excel" AfterTargets="GetAssemblyVersion"
+    Condition="'@(Proto3Excel)'!=''"
+    Inputs="$(protoc);
+    %(Proto3Excel.Identity);$(ProtocPluginBase)\plugin.py;
+    $(ProtocPluginBase)\templates\excel\excel.pb.cs;"
+    Outputs="%(Proto3Excel.OutCSExcel)">
+    <MakeDir Directories="%(Proto3Excel.OutBase)" />
+
+    <Message Importance="High" Text="Generating excel service %(Proto3Excel.FileName)" />
+    <Exec Command="
+       set PATH=$(spark_build_python_path);%path% &amp;
+       set PYTHONPATH=$(Externals)\python &amp;
+       $(protoc) ^
+       %(Proto3Excel.FullPath) ^
+       --error_format=msvs ^
+       --proto_path=$(protoinc);$(SourceRoot) ^
+       --plugin=&quot;protoc-gen-service=$(ProtocPluginBase)\plugin.cmd&quot; ^
+       --service_out=lang=excel,strip_out_path=1:%(Proto3Excel.OutBase) ^
+       "
+      WorkingDirectory="%(relativedir)" />
+    <ItemGroup>
+       <Compile Include="%(Proto3Excel.OutCSExcel)" />
+    </ItemGroup>
+</Target>
+
 <Target Name="AfterClean" AfterTargets="Clean">
     <Delete Files="%(Proto3.OutCS)" />
     <Delete Files="%(Proto3.OutCSClient)" />
     <Delete Files="%(Proto3.OutCSServer)" />
     <Delete Files="%(Proto3.OutCSImmutable)" />
     <Delete Files="$(GenDir)\proto2\*" />
+    <Delete Files="%(Proto3Excel.OutCSExcel)" />
 </Target>
 
 <Target Name="Test" DependsOnTargets="Build">

src/client/spark/Daemons/DeleteSyntheticDaemon.cs

@@ -12,11 +12,13 @@ namespace bts {
    [Export, Export(typeof(IShellDaemon))]
    class DeleteSyntheticDaemon : IShellDaemon, IDisposable {
       readonly OrdersModel _orders_model;
+      readonly ErrorReporter _error_reporter;
       IDisposable _dispose;
 
       [ImportingConstructor]
-      public DeleteSyntheticDaemon(IBus bus, OrdersModel orders_model) {
+      public DeleteSyntheticDaemon(IBus bus, OrdersModel orders_model, ErrorReporter error_reporter) {
          _orders_model = orders_model;
+         _error_reporter = error_reporter;
          _dispose = bus.subscribe(SparkTags.SyntheticDeleteTag)
             .OnDispatcher()
             .Subscribe(_delete_synthetic);
@@ -28,6 +30,7 @@ namespace bts {
       async void _delete_synthetic(uint id) {
          var (orders, orders_success) = await _orders_model.on_orders.FirstAsyncTask(SparkUtils.DefaultTimeOut);
          if (!orders_success) {
+            _error_reporter.report_error($"Unable to delete synthetic security ({id}): unable to determine whether or not security ({id}) has working orders (deletion is not permitted while there are working orders).");
             Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because the orders lookup timed out (must confirm no working orders).");
          } else if (orders.state.Values.Any(o => o.Def.SecurityKey == id && o.is_non_terminal())) {
             Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because non-terminal-orders were detected.");

src/client/spark/Daemons/MDDaemon.cs

@@ -13,11 +13,13 @@ using System.Linq;
 using SecurityKey = System.UInt32;
 using System.Collections.Concurrent;
 using System.Threading;
+using bts.api.user;
 
 namespace bts {
    [Export(typeof(IInitializable))]
    class MDSource : IObserver<List<SecurityAndHandle>>, IInitializable, IDisposable {
       readonly Lazy<IBus> _bus;
+      readonly Lazy<AuthToken> _token;
       readonly Lazy<SyntheticSecurityService> _ss;
       readonly MDConsumer _consumer = new MDConsumer();
       readonly Co2RelayControl _relay;
@@ -27,14 +29,20 @@ namespace bts {
       List<IDisposable> _dispose = new List<IDisposable>();
 
       [ImportingConstructor]
-      public MDSource(Lazy<IBus> bus, Lazy<SyntheticSecurityService> ss) {
+      public MDSource(
+         Lazy<IBus> bus,
+         Lazy<AuthToken> token, 
+         Lazy<SyntheticSecurityService> ss) {
          _bus = bus;
+         _token = token;
          _ss = ss;
          _relay = new Co2RelayControl(_consumer);
          _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore();
          _dispose.Add(_consumer.on_security.Subscribe(this));
       }
+
       public void initialize() {
+         _relay.set_token(_token.Value);
          _dispose.Add(
             _ss.Value.subscribei_metadata()
             .Select(upd => upd.all.ToDictionary(i => i.SecurityKey, i => i.Strategy))

src/client/spark/Daemons/ShellLatencyDaemon.cs

@@ -1,11 +1,13 @@
 using bts.api.core;
 using bts.metrics;
+using mbts;
 using mbts.config;
 using mbts.ui;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
 using System.Linq;
+using System.Net.NetworkInformation;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -18,6 +20,7 @@ namespace bts.perf {
       readonly string _version;
       readonly string _username;
       IDisposable _subs;
+      Ping _pinger = new Ping();
 
       [ImportingConstructor]
       public ShellLatencyDaemon(ShellLatencyService service, [Import("username")] string user) {
@@ -25,7 +28,12 @@ namespace bts.perf {
          _version = Configuration.get<bts.config.System>().version;
          _username = user;
          _hostname = Environment.MachineName;
-         _subs = _service.subscribei_clock().Subscribe(on_tick);
+
+         _forever_ping_host(_pinger);
+         _subs = Disposable.Create(
+            _pinger,
+            _service.subscribei_clock().Subscribe(on_tick)
+         );
       }
 
       void on_tick(Update<TickImmutable, int> update) {
@@ -40,8 +48,60 @@ namespace bts.perf {
          }
       }
 
+      async void _forever_ping_host(Ping pinger) {
+         var config = Configuration.get<bts.config.ShellPingOptions>();
+         string server_ip = Configuration.get<bts.config.System>().config_ip;
+         int delay = config.workloop_milliseconds_interval;
+         int _current_missed_ping_count = 0;
+
+         if (config.shell_ping_is_enabled && pinger != null) {
+            while (true) {
+               try {
+                  await Task.Delay(delay);
+                  var response = await pinger.SendPingAsync(server_ip, config.timeout_milliseconds_threshold);
+                  if (response.Status == IPStatus.Success) {
+                     _current_missed_ping_count = 0;
+                     if (response.RoundtripTime > config.logging_milliseconds_threshold) {
+                        Log.Ui.warn($"Ping response not recieved in a timely manner:\n" +
+                           $"\tAddress:\t{response.Address}\n" +
+                           $"\tRoundTrip:\t{response.RoundtripTime} ms");
+                     }
+                     // potentially reset delay to initial work period (no-op otherwise)
+                     delay = config.workloop_milliseconds_interval;
+                  } else {
+                     _current_missed_ping_count++;
+                     delay = _get_exp_delay(_current_missed_ping_count, config.workloop_milliseconds_interval, config.maximum_backoff_milliseconds);
+                     Log.Ui.warn($"Ping was not successful. Ping response: {response.Status}. Missed ping count: {_current_missed_ping_count}. Backing off for {delay} milliseconds.");
+                  }
+               } catch (PingException e) {
+                  _current_missed_ping_count++;
+                  delay = _get_exp_delay(_current_missed_ping_count, config.workloop_milliseconds_interval, config.maximum_backoff_milliseconds);
+                  Log.Ui.error($"Ping exception. {e}\nBacking off for {delay} milliseconds.");
+               }
+            } // end while
+         }
+
+         // since we're returning milliseconds here up to a 1 minute delay
+         // we don't need to go crazy with the power, else we will get
+         // an ArgumentOutOfRangeExecption.
+         // Int.MaxValue = 2^31, so we need to be below that
+         // Also, even if we're under the 2^31 int.maxvalue, we don't want to be multed over it
+         // so that is why we have checked() here.
+         int _get_exp_delay(int retry_num, int min_interval, int max_delay) {
+            // add some jitter up to half the max delay so we don't
+            // have a ping spike from all clients at once
+            var jitter_amount = new Random().Next(max_delay / 2);
+            int power_factor = (int)Math.Pow(2, Math.Min(retry_num, 32) - 1);
+            try {
+               return Math.Min(checked(min_interval * power_factor + jitter_amount), max_delay + jitter_amount);
+            } catch (OverflowException) {
+               return max_delay + jitter_amount;
+            }
+         }
+      }
+
       public void Dispose() {
-         throw new NotImplementedException();
+         Disposable.dispose(ref _subs);
       }
    }
 }

src/client/spark/Modules/Market/MarketView/MarketViewView.xaml

@@ -16,29 +16,48 @@
     </ResourceDictionary>
   </UserControl.Resources>
   <AdornerDecorator>
-    <c:DIYGrid
+    <Grid>
+      <Grid.RowDefinitions>
+        <RowDefinition />
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
+      <c:DIYGrid
       x:Name="_grid"
+      Grid.Row="0"
       Columns="{Binding columns}"
       CurrentRow="{Binding current_row}"
       Rows="{Binding rows_view}"
       SelectionHighlight="{Binding grid_settings_manager.selection_highlight, Mode=OneWay}"
     >
-      <c:DIYGrid.InputBindings>
-        <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
-        <KeyBinding Key="Return" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
-      </c:DIYGrid.InputBindings>
-      <c:DIYGrid.CommandBindings>
-        <CommandBinding
+        <c:DIYGrid.InputBindings>
+          <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
+          <KeyBinding Key="Return" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
+        </c:DIYGrid.InputBindings>
+        <c:DIYGrid.CommandBindings>
+          <CommandBinding
           Command="{x:Static c:DIYGrid.DefaultRowCommand}"
           Executed="{x:Static c:DIYGrid.DefaultRowActionExecuted}"
           CanExecute="{x:Static c:DIYGrid.DefaultRowActionCanExecute}"
         />
-      </c:DIYGrid.CommandBindings>
-      <c:DIYGrid.RowTemplate>
-        <DataTemplate>
-          <usp:MarketViewRowView usp:GridAnimation.RowAnimation="{Binding animations}"/>
-        </DataTemplate>
-      </c:DIYGrid.RowTemplate>
-    </c:DIYGrid>
+        </c:DIYGrid.CommandBindings>
+        <c:DIYGrid.RowTemplate>
+          <DataTemplate>
+            <usp:MarketViewRowView usp:GridAnimation.RowAnimation="{Binding animations}"/>
+          </DataTemplate>
+        </c:DIYGrid.RowTemplate>
+      </c:DIYGrid>
+      <StatusBar Grid.Row="1" DockPanel.Dock="Bottom">
+        <StatusBarItem HorizontalAlignment="Right">
+          <Grid>
+            <Grid.ColumnDefinitions>
+              <ColumnDefinition />
+              <ColumnDefinition />
+            </Grid.ColumnDefinitions>
+            <TextBlock Grid.Column="0" Text="Securities: "/>
+            <TextBlock Grid.Column="1" Text="{Binding ElementName=_grid, Path=Rows.Count}"/>
+          </Grid>
+        </StatusBarItem>
+      </StatusBar>
+    </Grid>
   </AdornerDecorator>
 </UserControl>

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs

@@ -167,9 +167,6 @@ namespace mbts.spark {
          public AggressorSide aggressor_side { get; }
          public ICommand show_ladder { get; }
 
-         ObservableCollection<RowAnimation> DIYGrid.IRowVM.row_animations { get; } = new ObservableCollection<RowAnimation>();
-         IReadOnlyList<DIYGrid.IGridCellStore> DIYGrid.IRowVM.cell_stores => new DIYGrid.IGridCellStore[0];
-
          public string this[TimeAndSalesView.Columns col] {
             get {
                switch (col) {

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml

@@ -18,10 +18,18 @@
   </UserControl.Resources>
   <AdornerDecorator>
     <Grid>
-      <c:DIYGrid Columns="{Binding columns}" 
-                 Rows="{Binding rows_view}" CurrentRow="{Binding current_row}"
-                 SelectionHighlight="{Binding grid_settings_manager.selection_highlight, Mode=OneWay}"
-                 >
+      <Grid.RowDefinitions>
+        <RowDefinition />
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
+      <c:DIYGrid 
+        x:Name="_grid"
+        Grid.Row="0"
+        Columns="{Binding columns}" 
+        Rows="{Binding rows_view}"
+        CurrentRow="{Binding current_row}"
+        SelectionHighlight="{Binding grid_settings_manager.selection_highlight, Mode=OneWay}"
+        >
         <c:DIYGrid.InputBindings>
           <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
           <KeyBinding Key="Return" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
@@ -39,6 +47,17 @@
           </DataTemplate>
         </c:DIYGrid.RowTemplate>
       </c:DIYGrid>
+      <StatusBar Grid.Row="1" DockPanel.Dock="Bottom">
+        <StatusBarItem HorizontalAlignment="Right">
+          <Grid>
+            <Grid.ColumnDefinitions>
+              <ColumnDefinition />
+              <ColumnDefinition />
+            </Grid.ColumnDefinitions>
+            <TextBlock Grid.Column="0" Text=" "/>
+          </Grid>
+        </StatusBarItem>
+      </StatusBar>      
     </Grid>
   </AdornerDecorator>
 </UserControl>

src/client/spark/Modules/Spreader/SyntheticManagementView.xaml

@@ -16,11 +16,16 @@
   </UserControl.Resources>
   <AdornerDecorator>
     <Grid>
+      <Grid.RowDefinitions>
+        <RowDefinition />
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
       <c:DIYGrid
-          x:Name="_grid"
-          Columns="{Binding columns}"
-          CurrentRow="{Binding current_row}"
-          Rows="{Binding rows_view}"
+        x:Name="_grid"
+        Grid.Row="0"
+        Columns="{Binding columns}"
+        CurrentRow="{Binding current_row}"
+        Rows="{Binding rows_view}"
         >
         <c:DIYGrid.InputBindings>
           <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static c:DIYGrid.DefaultRowCommand}"/>
@@ -39,6 +44,18 @@
           </DataTemplate>
         </c:DIYGrid.RowTemplate>
       </c:DIYGrid>
+      <StatusBar Grid.Row="1" DockPanel.Dock="Bottom">
+        <StatusBarItem HorizontalAlignment="Right">
+          <Grid>
+            <Grid.ColumnDefinitions>
+              <ColumnDefinition />
+              <ColumnDefinition />
+            </Grid.ColumnDefinitions>
+            <TextBlock Grid.Column="0" Text="Synthetics: "/>
+            <TextBlock Grid.Column="1" Text="{Binding ElementName=_grid, Path=Rows.Count}"/>
+          </Grid>
+        </StatusBarItem>
+      </StatusBar>      
     </Grid>
   </AdornerDecorator>  
 </UserControl>

src/client/spark/Modules/Support/SupportShellModule.cs

@@ -38,10 +38,10 @@ namespace bts.shell {
          //SparkCommands.ShowAuditLogs,
          //SparkCommands.ShowDiagnostics,
          SparkCommands.ShowReportIssue.complete("Created from launcher"),
-         SparkCommands.ShowAboutSpark,
          // SparkCommands.ShowReleaaseNotes,
          SparkCommands.ShowActivityLog,
          SparkCommands.ShowManual,
+         SparkCommands.ShowAboutSpark,
       };
 
       public Task<IShellWindow> create(Stream load_stream) {

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -207,9 +207,6 @@ namespace mbts.spark {
          public string fill_key => _fill.FillKey;
          public string order_key => _order.Key.ToString();
 
-         ObservableCollection<RowAnimation> IRowVM.row_animations { get; } = new ObservableCollection<RowAnimation>();
-         IReadOnlyList<IGridCellStore> IRowVM.cell_stores => new IGridCellStore[0];
-
          public bool Equals(Row other) => other != null && other._fill.Equals(_fill) && other._order.Equals(_order);
          public override bool Equals(object obj) => Equals(obj as Row);
          public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();

src/client/spark/Modules/Trading/Fills/FillsView.xaml

@@ -19,8 +19,13 @@
   </UserControl.Resources>
   <AdornerDecorator>
     <Grid>
+      <Grid.RowDefinitions>
+        <RowDefinition />
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
       <c:DIYGrid
         x:Name="_grid"
+        Grid.Row="0"
         Columns="{Binding columns}"
         CurrentRow="{Binding current_row}"
         Rows="{Binding rows_view}"
@@ -42,6 +47,18 @@
           </DataTemplate>
         </c:DIYGrid.RowTemplate>
       </c:DIYGrid>
+      <StatusBar Grid.Row="1" DockPanel.Dock="Bottom">
+        <StatusBarItem HorizontalAlignment="Right">
+          <Grid>
+            <Grid.ColumnDefinitions>
+              <ColumnDefinition />
+              <ColumnDefinition />
+            </Grid.ColumnDefinitions>
+            <TextBlock Grid.Column="0" Text="Fills: "/>
+            <TextBlock Grid.Column="1" Text="{Binding ElementName=_grid, Path=Rows.Count}"/>
+          </Grid>
+        </StatusBarItem>
+      </StatusBar>
     </Grid>
   </AdornerDecorator>
 </UserControl>

src/client/spark/Modules/Trading/OrderBook/OrderBookCell.cs

@@ -1,168 +0,0 @@
-using bts.utils;
-using mbts.ui;
-using mbts.ui.controls;
-using mbts.utils;
-using System;
-using System.Collections.Generic;
-using System.Collections.ObjectModel;
-using System.ComponentModel;
-
-namespace mbts.spark {
-   partial class OrderBookRowVM {
-      public interface IOrderBookCell : DIYGrid.IGridCell, IComparable { }
-
-      public interface IOrderBookCell<T> : IOrderBookCell {
-         T value { get; }
-         IOrderBookCellStore<T> store { get; }
-      }
-
-      public interface IOrderBookCellStore : DIYGrid.IGridCellStore {
-         OrderBookRowVM row { get; }
-         new ICVOrderBookColumn column { get; }
-         string prop_name { get; }
-         new IOrderBookCell current { get; }
-         //
-         IOrderBookCellStore select_other(OrderBookRowVM row);
-      }
-
-      public interface IOrderBookCellStore<T> : IOrderBookCellStore, IComparer<T>, IEqualityComparer<T> {
-         new IOrderBookCell<T> current { get; }
-         new IOrderBookCellStore<T> select_other(OrderBookRowVM row);
-         //
-         T get_value(OrderBookRowVM row);
-         string format(T value);
-      }
-
-      public static class CellFactory {
-         public static IOrderBookCellStore<T> create<T>(OrderBookRowVM row, ICVOrderBookColumn column, string prop_name, Func<OrderBookRowVM, T> get_value, Func<OrderBookRowVM, IOrderBookCellStore<T>> select_other, Func<T, string> format = null) =>
-            new OrderBookCellStore<T>(row, column, prop_name, get_value, select_other, format, new Comparison<T>(Comparer<T>.Default.Compare), EqualityComparer<T>.Default);
-
-         public static IOrderBookCellStore<T> create<T>(OrderBookRowVM row, ICVOrderBookColumn column, string prop_name, Func<OrderBookRowVM, T> get_value, Func<OrderBookRowVM, IOrderBookCellStore<T>> select_other, Comparison<T> comparison, Func<T, string> format = null) =>
-            new OrderBookCellStore<T>(row, column, prop_name, get_value, select_other, format, comparison, EqualityComparer<T>.Default);
-
-         public static IOrderBookCellStore<T?> create_nullable<T>(OrderBookRowVM row, ICVOrderBookColumn column, string prop_name, Func<OrderBookRowVM, T?> get_value, Func<OrderBookRowVM, IOrderBookCellStore<T?>> select_other, Func<T?, string> format = null)
-            where T : struct => new OrderBookCellStore<T?>(row, column, prop_name, get_value, select_other, format, CompareUtils.nullable_comparison<T>(Comparer<T>.Default.Compare), EqualityComparer.CreateNullable<T>(EqualityComparer<T>.Default.Equals));
-
-         public static IOrderBookCellStore<T?> create_nullable<T>(OrderBookRowVM row, ICVOrderBookColumn column, string prop_name, Func<OrderBookRowVM, T?> get_value, Func<OrderBookRowVM, IOrderBookCellStore<T?>> select_other, Func<T, T, int> compare, Func<T?, string> format = null)
-            where T : struct =>
-            new OrderBookCellStore<T?>(row, column, prop_name, get_value, select_other, format, CompareUtils.nullable_comparison(compare), EqualityComparer.CreateNullable<T>(EqualityComparer<T>.Default.Equals));
-
-         class OrderBookCellStore<T> : IOrderBookCellStore<T>, INotifyPropertyChanged {
-            readonly OrderBookRowVM _row;
-            readonly ICVOrderBookColumn _column;
-            readonly string _prop_name;
-            readonly Func<OrderBookRowVM, T> _get_value;
-            readonly Func<OrderBookRowVM, IOrderBookCellStore<T>> _select_other;
-            readonly Func<T, string> _format;
-            readonly Comparison<T> _comparison;
-            readonly IEqualityComparer<T> _eq_comparer;
-            OrderBookCell<T> _current;
-
-            // Property changed handler to avoid WPF binding leak
-            // https://blog.jetbrains.com/dotnet/2014/09/04/fighting-common-wpf-memory-leaks-with-dotmemory/
-            public event PropertyChangedEventHandler PropertyChanged = (_,__) => { };
-
-            public OrderBookCellStore(
-              OrderBookRowVM row,
-              ICVOrderBookColumn column,
-              string prop_name,
-              Func<OrderBookRowVM, T> get_value,
-              Func<OrderBookRowVM, IOrderBookCellStore<T>> select_other,
-              Func<T, string> format,
-              Comparison<T> comparison,
-              IEqualityComparer<T> eq_comparer) {
-               //
-               _row = row;
-               _column = column;
-               _prop_name = prop_name;
-               _get_value = get_value;
-               _select_other = select_other;
-               _format = format ?? new Func<T, string>(v => v?.ToString() ?? string.Empty);
-               _comparison = comparison;
-               _eq_comparer = eq_comparer;
-               _current = new OrderBookCell<T>(this, _get_value(_row));
-            }
-
-            public OrderBookRowVM row => _row;
-            public ICVOrderBookColumn column => _column;
-            public string prop_name => _prop_name;
-            public IOrderBookCell<T> current => _current;
-
-            IOrderBookCell IOrderBookCellStore.current => _current;
-            DIYGrid.IGridColumn DIYGrid.IGridCellStore.column => _column;
-            DIYGrid.IGridCell DIYGrid.IGridCellStore.current => _current;
-
-            public ObservableCollection<CellAnimation> animations { get; } = new ObservableCollection<CellAnimation>();
-
-            public int Compare(T x, T y) => _comparison(x, y);
-            public bool Equals(T x, T y) => _eq_comparer.Equals(x, y);
-            public string format(T value) => _format(value);
-            public int GetHashCode(T obj) => _eq_comparer.GetHashCode(obj);
-            public T get_value(OrderBookRowVM row) => _get_value(row);
-            public IOrderBookCellStore<T> select_other(OrderBookRowVM row) => _select_other(row);
-            IOrderBookCellStore IOrderBookCellStore.select_other(OrderBookRowVM row) => select_other(row);
-
-            public bool refresh() {
-               T v = _get_value(_row);
-               if (!_eq_comparer.Equals(v, _current.value)) {
-                  _current = new OrderBookCell<T>(this, v);
-                  _row.RaisePropertyChanged(_prop_name);
-                  return true;
-               } else return false;
-            }
-
-            public void regenerate_cell() {
-               _current = new OrderBookCell<T>(this, _current.value);
-               _row.RaisePropertyChanged(_prop_name);
-            }
-
-            public void invalidate_text() => regenerate_cell();
-         }
-
-         class OrderBookCell<T> : IOrderBookCell<T> {
-            string _text;
-
-            public OrderBookCell(OrderBookCellStore<T> store, T value) {
-               this.store = store;
-               this.value = value;
-            }
-
-            public string text => _text ?? (_text = store.format(value));
-            public T value { get; }
-            public IOrderBookCellStore<T> store { get; }
-
-            public int CompareTo(OrderBookCell<T> other) {
-               if (other == null)
-                  return 1;
-               var lin = lineage(store.row);
-               var olin = lineage(other.store.row);
-               int c = 0;
-               using (var en = lin.GetEnumerator())
-               using (var oen = olin.GetEnumerator()) {
-                  while (en.MoveNext() && oen.MoveNext()) {
-                     if (en.Current.order_key.value != oen.Current.order_key.value) {
-                        c = store.Compare(
-                           store.select_other(en.Current).current.value,
-                           store.select_other(oen.Current).current.value);
-                        if (c != 0)
-                           return c;
-                        else {
-                           c = en.Current.order_key.value.CompareTo(
-                              oen.Current.order_key.value);
-                           return store.column.SortDirection.HasValue && store.column.SortDirection.Value == ListSortDirection.Ascending ? c : -c;
-                        }
-                     }
-                  }
-               }
-               return store.column.SortDirection.HasValue && store.column.SortDirection.Value == ListSortDirection.Ascending ?
-                  lin.Count.CompareTo(olin.Count) :
-                  olin.Count.CompareTo(lin.Count);
-            }
-
-            public int CompareTo(object obj) => CompareTo(obj as OrderBookCell<T>);
-
-            public override string ToString() => text;
-         }
-      }
-   }
-}

src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs

@@ -1,4 +1,5 @@
-using bts.api;
+using bts;
+using bts.api;
 using bts.api.orderman;
 using bts.utils;
 using mbts.spark.filters;
@@ -7,6 +8,7 @@ using mbts.ui.controls;
 using mbts.ui.converters;
 using mbts.ui.utility;
 using mbts.ui.utility.text;
+using mbts.utils;
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
@@ -19,37 +21,53 @@ using System.Windows.Controls.Primitives;
 using System.Windows.Data;
 using System.Windows.Media;
 using static bts.shell.OrderBookWindowFactory;
-using static mbts.spark.OrderBookRowVM;
 using static mbts.spark.OrderBookView;
 using static mbts.ui.controls.DIYGrid;
 using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
 
 namespace mbts.spark {
    // This interface is used to hide ColumnBase.SortDirection, and offer a SortDirection member that does NOT require a DependencyProperty lookup.
-   // In OrderBookRowVM, each "cell(property)" has a corresponding TreeComparable property that the column uses for sorting
-   //    - each columns' sort_member_path points to the TreeComparable.
-   // The TreeComparable class needs to know which direction it is sorting so it can force child-orders to always be beneath their parents.
-   //    - when comparing with their parent, it must return 1 if Ascending, -1 if Descending.
-   // Since the TreeComparables in each row must access their respective column SortDirections in their CompareTo methods, those DP lookups could add significant overhead in large collections.
-   // I considered adding ListSortDirection members directly to OrderBookVM, and letting the TreeComparables refer to that property,
-   // but decided it would be better to let the Columns own it so we wouldn't have to worry about keeping the 2 props in sync.
-   // TODO: Maybe binding to the SortDirection would be less smelly?
-   //    - initially, I tried to bind to the SortDirection of each column, unfortunately, the binding target was not updated before the ICollectionView sorting took place.
-   //      I decided it was not worth changing the DIYGrid.Sorting function to force the SortDirection binding targets to update immediately, before the ICollectionView sorting.
-   interface ICVOrderBookColumn : IKeyedColumn<Columns>, IGridColumn<OrderBookRowVM> {
+   interface ICVOrderBookColumn : IKeyedColumn<Columns>, IGridColumn {
       string sort_member_path { get; }
-      ListSortDirection? SortDirection { get; }
-      IDataGridColumnFilter<OrderBookRowVM> filter { get; }
+      ListSortDirection? SortDirection { get; set; }
+      IDataGridColumnFilter<OrderRecordTuple> filter { get; }
+      IComparer<OrderBookRowVM> comparer { get; }
+   }
+
+   class OrderBookSortingData : IEquatable<OrderBookSortingData> {
+      public readonly IReadOnlyList<SortingCriteria> sorting_criterion;
+      public readonly IComparer<OrderBookRowVM> comparer;
+      public readonly string desc;
+
+      public OrderBookSortingData(IEnumerable<ICVOrderBookColumn> column_sorting_hierarchy) {
+         var comp_builder = new ComparerBuilder<OrderBookRowVM>();
+         foreach (var col in column_sorting_hierarchy)
+            comp_builder.add(col.SortDirection.Value == ListSortDirection.Ascending ? col.comparer : col.comparer.reverse());
+         if (!column_sorting_hierarchy.Any(col => col.key == Columns.OrderKey)) {
+            // the resultant comparer must to be 100% deterministic, as SortedList (the building block of the SortedTree) uses a BinarySearch to find/remove items and does NOT check for item-equality.
+            comp_builder.add(Comparer<OrderBookRowVM>.Create((l, r) => l.order_key.CompareTo(r.order_key)));
+         }
+         comparer = comp_builder.to_comparer();
+         sorting_criterion = column_sorting_hierarchy.Select(col => new SortingCriteria(col.id, col.SortDirection.GetValueOrDefault() == ListSortDirection.Ascending)).ToArray();
+         desc = string.Join(", ", sorting_criterion);
+      }
+
+      public bool Equals(OrderBookSortingData other) => other != null && EqualityComparer.safe_sequence_equals(sorting_criterion, other.sorting_criterion);
+      public override bool Equals(object obj) => base.Equals(obj);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(sorting_criterion);
+      public override string ToString() => desc;
    }
 
    partial class OrderBookVM {
-      public class OrderBookColumnCollection : ObservableCollection<ColumnBase>, IValueFactory<FilterStates>, IDisposable {
+      public class OrderBookColumnCollection : ObservableCollection<ColumnBase>, IValueFactory<(FilterStates filters, IReadOnlyList<SortingCriteria> sorting)>, IDisposable {
+         readonly List<ICVOrderBookColumn> _active_comparers = new List<ICVOrderBookColumn>();
+
          public OrderBookColumnCollection(IBus bus, OrdersModel model, IObservable<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)> order_securities) {
             // security
             Add(security_column = new SecurityColumn(
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, SecurityImmutable, SecurityCriteria>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, SecurityImmutable, SecurityCriteria>(
                   order_securities,
-                  r => r.security.value.GetValueOrDefault(SecurityDataHelper.Defaults.SecurityImmutable),
+                  ort => ort.security_md.HasValue ? ort.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable,
                   SecurityCriteria.from_securities,
                   SecurityCriteria.get_to_securities(bus),
                   nameof(OrderBookRowVM.security),
@@ -61,172 +79,222 @@ namespace mbts.spark {
                Columns.Symbol,
                DefaultColumnHeaders.Symbol,
                nameof(OrderBookRowVM.symbol),
-               r => r.symbol.store,
+               r => r.symbol,
                DefaultColumnWidths.Symbol,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, string>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, string>(
                   mbts.co2.UserSymbol.get_all().Select(u => u.symbol).Distinct().ToList(),
-                  r => r.symbol.value,
-                  value_path(nameof(OrderBookRowVM.symbol))
+                  ort => ort.security_md.HasValue ? ort.security_md.Value.security.Symbol : string.Empty,
+                  nameof(OrderBookRowVM.symbol)
                ),
-               DataGridFilterKnownTemplates.DiscreteValues));
+               DataGridFilterKnownTemplates.DiscreteValues,
+               _wrap_comparer(r => r.symbol)));
             // time_created
-            Add(time_created_column = new TimeCreatedColumn(
-               new TimeFilterVM<OrderBookRowVM>(r => r.time_created.value, value_path(nameof(OrderBookRowVM.time_created)))));
+            Add(time_created_column = new SimpleTextColumn<TimeFilterTuple>(
+               Columns.Time,
+               "Time",
+               nameof(OrderBookRowVM.time_created),
+               r => r.time_created_text,
+               DefaultColumnWidths.Time,
+               new TimeFilterVM<OrderRecordTuple>(
+                  ort => ort.order.created(),
+                  nameof(OrderBookRowVM.time_created)),
+               DataGridFilterKnownTemplates.TimeFilter,
+               _wrap_comparer(r => r.time_created)));
             // last_change
-            Add(last_change_column = new LastChangeTimeColumn(
-               new TimeFilterVM<OrderBookRowVM>(r => r.last_change.value, value_path(nameof(OrderBookRowVM.last_change)))));
+            Add(last_change_column = new SimpleTextColumn<TimeFilterTuple>(
+               Columns.LastChange,
+               "Last Change",
+               nameof(OrderBookRowVM.last_change),
+               r => r.last_change_text,
+               DefaultColumnWidths.Time,
+               new TimeFilterVM<OrderRecordTuple>(
+                  ort => ort.order.last_change(),
+                  nameof(OrderBookRowVM.last_change)),
+               DataGridFilterKnownTemplates.TimeFilter,
+               _wrap_comparer(r => r.last_change)));
             // user
             Add(user_column = new SimpleTextColumn<string[]>(
                Columns.User,
                DefaultColumnHeaders.User,
                nameof(OrderBookRowVM.user),
-               r => r.user.store,
+               r => r.user,
                DefaultColumnWidths.User,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, string>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, string>(
                   model.factory.subscribe_users().Select(d => d.Values),
-                  r => r.user.value,
-                  value_path(nameof(OrderBookRowVM.user))),
-               DataGridFilterKnownTemplates.DiscreteValues));
+                  ort => ort.order.user(),
+                  nameof(OrderBookRowVM.user)),
+               DataGridFilterKnownTemplates.DiscreteValues,
+               _wrap_comparer(r => r.user)));
             // account
             Add(account_column = new SimpleTextColumn<string[]>(
                Columns.Account,
                DefaultColumnHeaders.Account,
                nameof(OrderBookRowVM.account),
-               r => r.account.store,
+               r => r.account,
                DefaultColumnWidths.Account,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, string, string[]>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, string, string[]>(
                   model.factory.subscribe_accounts().Select(d => d.Values),
-                  r => r.account.value,
+                  ort => ort.order.account(),
                   vals => vals.ToArray(),
                   vals => Observable.Return(vals),
-                  value_path(nameof(OrderBookRowVM.account))),
-               DataGridFilterKnownTemplates.DiscreteValues));
+                  nameof(OrderBookRowVM.account)),
+               DataGridFilterKnownTemplates.DiscreteValues,
+               _wrap_comparer(r => r.account)));
             // exchange
             Add(exchange_column = new SimpleTextColumn<Venue[]>(
                Columns.Exchange,
                DefaultColumnHeaders.Exchange,
                nameof(OrderBookRowVM.exchange),
-               r => r.exchange.store,
+               r => r.exchange.ToString(),
                DefaultColumnWidths.Exchange,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, Venue>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, Venue>(
                   EnumHelper.get_values_except(Venue.Unset).ToList(),
-                  r => r.exchange.value,
-                  value_path(nameof(OrderBookRowVM.exchange))),
-               DataGridFilterKnownTemplates.EnumValues));
+                  ort => ort.security_md.HasValue ? ort.security_md.Value.security.Exchange : Venue.Unset,
+                  nameof(OrderBookRowVM.exchange)),
+               DataGridFilterKnownTemplates.EnumValues,
+               _wrap_comparer(r => r.exchange)));
             // time in force
             Add(time_in_force_column = new SimpleTextColumn<TimeInForce[]>(
                Columns.TimeInForce,
                DefaultColumnHeaders.TimeInForce,
                nameof(OrderBookRowVM.time_in_force),
-               r => r.time_in_force.store,
+               r => r.time_in_force.ToString(),
                DefaultColumnWidths.TimeInForce,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, TimeInForce>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, TimeInForce>(
                   EnumHelper.get_values<TimeInForce>().ToList(),
-                  r => r.time_in_force.value,
-                  value_path(nameof(OrderBookRowVM.time_in_force))),
-               DataGridFilterKnownTemplates.EnumValues));
+                  ort => ort.order.time_in_force(),
+                  nameof(OrderBookRowVM.time_in_force)),
+               DataGridFilterKnownTemplates.EnumValues,
+               _wrap_comparer(r => r.time_in_force)));
             // order type
             Add(order_type_column = new OrderBookOrderTypeColumn(
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, OrderType>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, OrderType>(
                   utilities.OrderFactory.all_order_types,
-                  r => r.order_type.value,
-                  value_path(nameof(OrderBookRowVM.order_type)))));
+                  ort => ort.order.order_type(),
+                  nameof(OrderBookRowVM.order_type))));
             // order sub type
             Add(sub_type_column = new SimpleTextColumn<OrderSubType[]>(
                Columns.SubType,
                "Sub Type",
                nameof(OrderBookRowVM.order_sub_type),
-               r => r.order_sub_type.store,
+               r => r.order_sub_type_text,
                80,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, OrderSubType>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, OrderSubType>(
                   EnumHelper.get_values<OrderSubType>().ToList(),
-                  r => r.order_sub_type.value,
-                  value_path(nameof(OrderBookRowVM.order_sub_type))),
-               SubTypeFilterTemplateKey));
+                  ort => ort.order.sub_type(),
+                  nameof(OrderBookRowVM.order_sub_type)),
+               SubTypeFilterTemplateKey,
+               _wrap_comparer(r => r.order_sub_type)));
             // side
             Add(side_column = new SideColumn(
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, Side>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, Side>(
                   new[] { Side.Buy, Side.Sell },
-                  r => r.side.value,
-                  value_path(nameof(OrderBookRowVM.side)))));
+                  ort=> ort.order.side(),
+                  nameof(OrderBookRowVM.side))));
             // quantity
             Add(qty_column = new SizeColumn(
-               new UIntMinMax<OrderBookRowVM>(r => r.quantity.value, value_path(nameof(OrderBookRowVM.quantity)))));
-            // price
-            Add(price_column = new PriceColumn(
-               new DoubleMinMax<OrderBookRowVM>(r => r.price.value, value_path(nameof(OrderBookRowVM.price)))));
+               new UIntMinMax<OrderRecordTuple>(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity))));
             // filled quantity
             Add(filled_qty_column = new FilledColumn(
-               new UIntMinMax<OrderBookRowVM>(r => r.filled_quantity.value, value_path(nameof(OrderBookRowVM.filled_quantity)))));
+               new UIntMinMax<OrderRecordTuple>(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity))));
+            // price
+            Add(price_column = new SimpleTextColumn<DoubleMinMaxState>(
+               Columns.Price,
+               "Price",
+               nameof(OrderBookRowVM.price),
+               r => r.price_text,
+               DefaultColumnWidths.Price,
+               new DoubleMinMax<OrderRecordTuple>(
+                  ort => ort.order.price(),
+                  nameof(OrderBookRowVM.price)),
+               DataGridFilterKnownTemplates.DoubleMinMax,
+               _wrap_comparer(r => r.price)));
             // avg filled price
-            Add(avg_price_column = new AvgFilledColumn(
-               new DoubleMinMax<OrderBookRowVM>(r => r.avg_price.value, value_path(nameof(OrderBookRowVM.avg_price)))));
+            Add(avg_price_column = new SimpleTextColumn<DoubleMinMaxState>(
+               Columns.AvgFilled,
+               "Avg Price",
+               nameof(OrderBookRowVM.avg_price),
+               r => r.avg_price_text,
+               DefaultColumnWidths.Price,
+               new DoubleMinMax<OrderRecordTuple>(
+                  ort => ort.order.avg_price(),
+                  nameof(OrderBookRowVM.avg_price)),
+               DataGridFilterKnownTemplates.DoubleMinMax,
+               _wrap_comparer(r => r.avg_price)));
             // last filled price
-            Add(last_price_column = new LastFilledColumn(
-               new DoubleMinMax<OrderBookRowVM>(r => r.last_price.value, value_path(nameof(OrderBookRowVM.last_price)))));
+            Add(last_price_column = new SimpleTextColumn<DoubleMinMaxState>(
+               Columns.LastPrice,
+               "Last Filled",
+               nameof(OrderBookRowVM.last_price),
+               r => r.last_price_text,
+               DefaultColumnWidths.Price,
+               new DoubleMinMax<OrderRecordTuple>(
+                  ort => ort.order.last_price(),
+                  nameof(OrderBookRowVM.last_price)),
+               DataGridFilterKnownTemplates.DoubleMinMax,
+               _wrap_comparer(r => r.last_price)));
             // status
             Add(status_column = new SimpleTextColumn<OrderStatus[]>(
                Columns.Status,
                "Status",
                nameof(OrderBookRowVM.status),
-               r => r.status.store,
+               r => r.status.ToString(),
                80,
-               DiscreteValuesFilterVM.Create<OrderBookRowVM, OrderStatus>(
+               DiscreteValuesFilterVM.Create<OrderRecordTuple, OrderStatus>(
                   EnumHelper.get_values<OrderStatus>().ToList(),
-                  r => r.status.value,
-                  value_path(nameof(OrderBookRowVM.status))),
-               DataGridFilterKnownTemplates.EnumValues));
+                  ort => ort.order.status(),
+                  nameof(OrderBookRowVM.status)),
+               DataGridFilterKnownTemplates.EnumValues,
+               _wrap_comparer(r => r.status)));
             // last message
             Add(last_message_column = new OrderBookLastMessageColumn(
-               new RegexFilterVM<OrderBookRowVM>(
-                  r => r.last_msg.value,
-                  value_path(nameof(OrderBookRowVM.last_msg)))));
+               new RegexFilterVM<OrderRecordTuple>(
+                  ort => ort.order.last_msg(),
+                  nameof(OrderBookRowVM.last_msg))));
             // security key
             Add(security_key_column = new SimpleTextColumn<string>(
                Columns.SecurityKey,
                DefaultColumnHeaders.SecurityKey,
                nameof(OrderBookRowVM.security_key),
-               r => r.security_key.store,
+               r => r.security_key.ToString(),
                DefaultColumnWidths.SecurityKey,
-               new RegexFilterVM<OrderBookRowVM>(
-                  r => r.security_key.text,
-                  value_path(nameof(OrderBookRowVM.security_key))),
-               DataGridFilterKnownTemplates.Regex));
+               new RegexFilterVM<OrderRecordTuple>(
+                  ort => ort.order.security_key().ToString(),
+                  nameof(OrderBookRowVM.security_key)),
+               DataGridFilterKnownTemplates.Regex,
+               _wrap_comparer(r => r.security_key)));
             // order key
             Add(order_key_column = new SimpleTextColumn<string>(
                Columns.OrderKey,
                DefaultColumnHeaders.OrderKey,
                nameof(OrderBookRowVM.order_key),
-               r => r.order_key.store,
+               r => r.order_key.ToString(),
                DefaultColumnWidths.OrderKey,
-               new RegexFilterVM<OrderBookRowVM>(
-                  r => r.order_key.text,
-                  value_path(nameof(OrderBookRowVM.order_key))),
-               DataGridFilterKnownTemplates.Regex));
+               new RegexFilterVM<OrderRecordTuple>(
+                  ort => ort.order.Key.ToString(),
+                  nameof(OrderBookRowVM.order_key)),
+               DataGridFilterKnownTemplates.Regex,
+               _wrap_comparer(r => r.order_key)));
             // parent key (this is for diagnostic purposes, remove this column once OrderBook seems stable again)
             Add(parent_key_column = new SimpleTextColumn<string>(
                Columns.ParentKey,
                "Parent Key",
                nameof(OrderBookRowVM.parent_order_key),
-               r => r.parent_order_key.store,
+               r => r.parent_order_key.ToString(),
                DefaultColumnWidths.ParentKey,
-               new RegexFilterVM<OrderBookRowVM>(
-                  r => r.parent_order_key.text,
-                  value_path(nameof(OrderBookRowVM.parent_order_key))),
-               DataGridFilterKnownTemplates.Regex));
-
-            string value_path(string cell_name) =>
-               string.Join(".", cell_name, nameof(OrderBookRowVM.IOrderBookCell<uint>.value)); //uint is an arbitrary 'T', just want to get compile check on 'value' path.
+               new RegexFilterVM<OrderRecordTuple>(
+                  ort => ort.order.ParentKey.ToString(),
+                  nameof(OrderBookRowVM.parent_order_key)),
+               DataGridFilterKnownTemplates.Regex,
+               _wrap_comparer(r => r.parent_order_key)));
          }
 
-         List<IDataGridColumnFilter<OrderBookRowVM>> _active_filters = new List<IDataGridColumnFilter<OrderBookRowVM>>(0);
-
          public event Action filter_changed;
 
          public SecurityColumn security_column { get; }
          public SimpleTextColumn<string[]> symbol_column { get; }
-         public TimeCreatedColumn time_created_column { get; }
-         public LastChangeTimeColumn last_change_column { get; }
+         public SimpleTextColumn<TimeFilterTuple> time_created_column { get; }
+         public SimpleTextColumn<TimeFilterTuple> last_change_column { get; }
          public SimpleTextColumn<string[]> user_column { get; }
          public SimpleTextColumn<string[]> account_column { get; }
          public SimpleTextColumn<Venue[]> exchange_column { get; }
@@ -235,19 +303,17 @@ namespace mbts.spark {
          public SimpleTextColumn<OrderSubType[]> sub_type_column { get; }
          public SideColumn side_column { get; }
          public SizeColumn qty_column { get; }
-         public PriceColumn price_column { get; }
+         public SimpleTextColumn<DoubleMinMaxState> price_column { get; }
          public FilledColumn filled_qty_column { get; }
-         public LastFilledColumn last_price_column { get; }
-         public AvgFilledColumn avg_price_column { get; }
+         public SimpleTextColumn<DoubleMinMaxState> last_price_column { get; }
+         public SimpleTextColumn<DoubleMinMaxState> avg_price_column { get; }
          public SimpleTextColumn<OrderStatus[]> status_column { get; }
          public OrderBookLastMessageColumn last_message_column { get; }
          public SimpleTextColumn<string> security_key_column { get; }
          public SimpleTextColumn<string> order_key_column { get; }
          public SimpleTextColumn<string> parent_key_column { get; }
 
-         public bool filter(OrderBookRowVM row) => _active_filters.All(f => f.filter(row));
-
-         public FilterStates create() =>
+         public (FilterStates filters, IReadOnlyList<SortingCriteria> sorting) create() => (
             new FilterStates {
                order_key_regex = order_key_column.filter.get_state(),
                time_created = time_created_column.filter.get_state(),
@@ -268,13 +334,11 @@ namespace mbts.spark {
                last_change = last_change_column.filter.get_state(),
                security_criteria = security_column.filter.get_state(),
                symbols = symbol_column.filter.get_state()
-            };
+            },
+            get_sorting_data().sorting_criterion);
 
-         public void load_from(FilterStates fs) {
-            if (fs != null) {
-               foreach (var c in this.Cast<ICVOrderBookColumn>())
-                  c.filter.filter_changed -= _set_active_filters;
-               //
+         public void load_from((FilterStates filters, IReadOnlyList<SortingCriteria> sorting) state) {
+            if (state.filters is FilterStates fs) {
                security_column.filter.load_state(fs.security_criteria);
                symbol_column.filter.load_state(fs.symbols);
                time_created_column.filter.load_state(fs.time_created);
@@ -294,45 +358,127 @@ namespace mbts.spark {
                status_column.filter.load_state(fs.statuses);
                last_message_column.filter.load_state(fs.exchange_message_regex);
                last_change_column.filter.load_state(fs.last_change);
-               //
-               _set_active_filters();
-               foreach (var c in this.Cast<ICVOrderBookColumn>())
-                  c.filter.filter_changed += _set_active_filters;
+            }
+            _active_comparers.Clear();
+            foreach (var col in this)
+               col.SortDirection = null;
+            if (state.sorting is IReadOnlyList<SortingCriteria> sorting) {
+               foreach (var sc in sorting) {
+                  this.OfType<ICVOrderBookColumn>().ApplyToFirst(col => col.id.iequals(sc.col_name),
+                     col => {
+                        col.SortDirection = sc.is_ascending ? ListSortDirection.Ascending : ListSortDirection.Descending;
+                        _active_comparers.Add(col);
+                     });
+               }
+            }
+            if (_active_comparers.Count == 0) {
+               // safeguard against empty sort, fallback to time-created
+               time_created_column.SortDirection = ListSortDirection.Descending;
+               _active_comparers.Add(time_created_column);
+            }
+         }
+
+         public Func<OrderRecordTuple, bool> get_filter() {
+            var active_filters = this.OfType<ICVOrderBookColumn>().Where(c => c.filter.active.GetValueOrDefault()).Select(c => c.filter).ToArray();
+            return active_filters.Length == 0 ? null : new Func<OrderRecordTuple, bool>(r => active_filters.All(f => f.filter(r)));
+         }
+
+         public OrderBookSortingData get_sorting_data() {
+            var desc_parts = new List<string>();
+            var comp_builder = new ComparerBuilder<OrderBookRowVM>();
+            // this should be unneccesary, safe_active_comparers is just an extra guard against unexpected results from future code changes
+            // (get_comparer_data() should NEVER return a null/non-deterministic IComparer, fallback to sorting by Time Created in descending order if necessary)
+            var safe_active_comparers = _active_comparers.Where(c => c.SortDirection.HasValue).ToList();
+            if (safe_active_comparers.Count == 0) {
+               time_created_column.SortDirection = ListSortDirection.Descending;
+               _active_comparers.Clear();
+               _active_comparers.Add(time_created_column);
+               safe_active_comparers = _active_comparers;
+            }
+            return new OrderBookSortingData(safe_active_comparers);
+         }
+
+         public void on_sorting(ICVOrderBookColumn col, bool clear_current_sort) {
+            col.SortDirection = col.SortDirection.HasValue && col.SortDirection.Value == ListSortDirection.Ascending ? ListSortDirection.Descending : ListSortDirection.Ascending;
+            if (_active_comparers.Contains(col, EqualityComparer.Create<ICVOrderBookColumn, Columns>(r => r.key))) {
+               if (clear_current_sort) _reset_active_comparer(col);
+            } else {
+               if (clear_current_sort) _reset_active_comparer(col);
+               else _active_comparers.Add(col);
             }
          }
 
          public void Dispose() {
-            foreach (var c in this.Cast<ICVOrderBookColumn>()) {
-               c.filter.filter_changed -= _set_active_filters;
-               (c.filter as IDisposable)?.Dispose();
+            foreach (var col in this.OfType<ICVOrderBookColumn>())
+               (col.filter as IDisposable)?.Dispose();
+            Clear();
+         }
+
+         /// <see cref="OrderBookColumnCollection"/> ONLY supports sorting/filtering behavior for columns that implement <see cref="ICVOrderBookColumn"/>.
+         /// This is because OrderBook, unlike the other windows (as of 10/30/2018), does not use <see cref="ICollectionView"/> as the sorting/filtering model.
+         protected override void InsertItem(int index, ColumnBase item) {
+            if (item is ICVOrderBookColumn col) {
+               try {
+                  _hook(col);
+                  base.InsertItem(index, item);
+               } catch {
+                  _unhook(col);
+               } finally { }
+            } else {
+               Log.Ui.warn($"{nameof(OrderBookColumnCollection)} added a column that is not of type {nameof(ICVOrderBookColumn)}");
+               base.InsertItem(index, item);
             }
          }
 
-         void _set_active_filters() {
-            _active_filters = this
-               .Cast<ICVOrderBookColumn>()
-               .Select(c => c.filter)
-               .Where(f => f.active.GetValueOrDefault())
-               .ToList();
-            filter_changed?.Invoke();
+         protected override void RemoveItem(int index) {
+            if (Count > index && this[index] is ICVOrderBookColumn col)
+               _unhook(col);
+            base.RemoveItem(index);
+         }
+         protected override void ClearItems() {
+            var ob_columns = this.OfType<ICVOrderBookColumn>().ToList();
+            Log.Ui.info($"{nameof(OrderBookColumnCollection)} is unhooking all columns of type {nameof(ICVOrderBookColumn)} ({ob_columns.Count}/{Count}).");
+            foreach (var col in ob_columns)
+               _unhook(col);
+            base.ClearItems();
+         }
+
+         void _hook(ICVOrderBookColumn new_item) => new_item.filter.filter_changed += _on_column_filter_changed;
+         void _unhook(ICVOrderBookColumn old_item) => old_item.filter.filter_changed -= _on_column_filter_changed;
+         void _on_column_filter_changed() => filter_changed?.Invoke();
+
+         void _reset_active_comparer(ICVOrderBookColumn column) {
+            foreach (var col in _active_comparers) {
+               if (!col.Equals(column)) col.SortDirection = null;
+            }
+            _active_comparers.Clear();
+            _active_comparers.Add(column);
          }
       }
 
+      static IComparer<OrderBookRowVM> _wrap_comparer<T>(Func<OrderBookRowVM, T> get) =>
+         Comparer<T>.Default.wrap(get);
+
+      static IComparer<OrderBookRowVM> _wrap_comparer<T>(Func<OrderBookRowVM, T> get, IComparer<T> comparer) =>
+         comparer.wrap(get);
+
       public abstract class OrderbookColumnBase<T> : ColumnBase<Columns>, IKeyedColumn<Columns>, IFilterableColumn, ICVOrderBookColumn {
          ListSortDirection? _cached_sort_direction;
          readonly string _filter_template;
-         protected readonly Func<OrderBookRowVM, IOrderBookCellStore> _get_cell_store;
+         protected readonly Func<OrderBookRowVM, string> _get_text;
          GlyphContext _root_glyph_context, _child_glyph_context;
 
-         protected OrderbookColumnBase(Columns col_key, string header, Func<OrderBookRowVM, IOrderBookCellStore> get_cell_store, string sort_prop, double width, IDataGridColumnPersistentFilter<OrderBookRowVM, T> filter, string filter_template) : base(col_key) {
+         protected OrderbookColumnBase(Columns col_key, string header, Func<OrderBookRowVM, string> get_text, string sort_prop, double width,
+            IDataGridColumnPersistentFilter<OrderRecordTuple, T> filter, string filter_template, IComparer<OrderBookRowVM> comparer) : base(col_key) {
             id = key.ToString();
             Width = width;
             Header = header;
-            _get_cell_store = get_cell_store;
+            _get_text = get_text;
             sort_member_path = sort_prop;
             this.filter = filter;
             _filter_template = filter_template;
             _cached_sort_direction = SortDirection;
+            this.comparer = comparer;
             font_params_changed += refresh_glyph_contexts;
             refresh_glyph_contexts();
             void refresh_glyph_contexts() {
@@ -341,14 +487,13 @@ namespace mbts.spark {
             }
          }
 
-         public IDataGridColumnPersistentFilter<OrderBookRowVM, T> filter { get; }
+         public IDataGridColumnPersistentFilter<OrderRecordTuple, T> filter { get; }
+         public IComparer<OrderBookRowVM> comparer { get; }
 
-         IDataGridColumnFilter<OrderBookRowVM> ICVOrderBookColumn.filter => filter;
+         IDataGridColumnFilter<OrderRecordTuple> ICVOrderBookColumn.filter => filter;
          IDataGridColumnFilter IFilterableColumn.filter => filter;
          string IFilterableColumn.filter_template => _filter_template;
-         ListSortDirection? ICVOrderBookColumn.SortDirection => _cached_sort_direction;
-
-         public IGridCellStore get_cell_store(OrderBookRowVM row) => _get_cell_store(row);
+         ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; }
 
          protected override void OnSortDirectionChanged(ListSortDirection? new_value) {
             _cached_sort_direction = new_value;
@@ -358,99 +503,51 @@ namespace mbts.spark {
       }
 
       public class SimpleTextColumn<TFilterState> : OrderbookColumnBase<TFilterState>, IRenderColumn<OrderBookRowVM> {
-         public SimpleTextColumn(Columns key, string header, string sort_prop, Func<OrderBookRowVM, IOrderBookCellStore> get_cell_store, double width, IDataGridColumnPersistentFilter<OrderBookRowVM, TFilterState> filter, string filter_template)
-            : base(key, header, get_cell_store, sort_prop, width, filter, filter_template) { }
+         public SimpleTextColumn(Columns key, string header, string sort_prop, Func<OrderBookRowVM, string> get_text, double width,
+            IDataGridColumnPersistentFilter<OrderRecordTuple, TFilterState> filter, string filter_template, IComparer<OrderBookRowVM> comparer)
+            : base(key, header, get_text, sort_prop, width, filter, filter_template, comparer) { }
 
-         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) {
-            var cell = _get_cell_store(row);
-            if (!string.IsNullOrEmpty(cell.current.text))
-               CellRender.draw_text(dc, _get_glyph_context(row), font_size, cell.current.text, position, RowHeight,
-                  font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, cell.animations);
-         }
+         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
+            CellRender.draw_text(dc, _get_glyph_context(row), font_size, _get_text(row), position, RowHeight,
+               font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding);
       }
 
       #region Explicit Column Defs
       public class SideColumn : OrderbookColumnBase<Side[]>, IRenderColumn<OrderBookRowVM> {
-         public SideColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, Side[]> filter)
-            : base(Columns.Side, "B/S", r => r.side.store, nameof(OrderBookRowVM.side), DefaultColumnWidths.Side, filter, DataGridFilterKnownTemplates.EnumValues) { }
+         public SideColumn(IDataGridColumnPersistentFilter<OrderRecordTuple, Side[]> filter)
+            : base(Columns.Side, "B/S", r => r.side.ToString(), nameof(OrderBookRowVM.side), DefaultColumnWidths.Side, filter, DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.side)) { }
 
          public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.side.text, position, RowHeight,
-               RenderUtils.get_side_brush(row), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.side.store.animations);
+            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.side.ToString(), position, RowHeight,
+               RenderUtils.get_side_brush(row), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding);
       }
 
       public class SizeColumn : OrderbookColumnBase<UIntMinMaxState>, IRenderColumn<OrderBookRowVM> {
-         public SizeColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, UIntMinMaxState> filter)
-            : base(Columns.Size, "Qty", r => r.quantity.store, nameof(OrderBookRowVM.quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax) {
+         public SizeColumn(IDataGridColumnPersistentFilter<OrderRecordTuple, UIntMinMaxState> filter)
+            : base(Columns.Size, "Qty", r => r.quantity.ToString(), nameof(OrderBookRowVM.quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.quantity)) {
             sort_member_path = nameof(OrderBookRowVM.quantity);
          }
 
          public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.quantity.text, position, RowHeight,
-               RenderUtils.get_side_brush(row), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.quantity.store.animations);
+            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.quantity.ToString(), position, RowHeight,
+               RenderUtils.get_side_brush(row), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding);
       }
 
       public class FilledColumn : OrderbookColumnBase<UIntMinMaxState>, IRenderColumn<OrderBookRowVM> {
-         public FilledColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, UIntMinMaxState> filter)
-            : base(Columns.Filled, "Filled Qty", r => r.filled_quantity.store, nameof(OrderBookRowVM.filled_quantity), DefaultColumnWidths.Filled, filter, DataGridFilterKnownTemplates.UIntMinMax) {
+         public FilledColumn(IDataGridColumnPersistentFilter<OrderRecordTuple, UIntMinMaxState> filter)
+            : base(Columns.Filled, "Filled Qty", r => r.filled_quantity.ToString(), nameof(OrderBookRowVM.filled_quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.filled_quantity)) {
             sort_member_path = nameof(OrderBookRowVM.filled_quantity);
          }
 
          public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.filled_quantity.text, position, RowHeight,
-               RenderUtils.get_side_brush(row), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.filled_quantity.store.animations);
-      }
-
-      public class PriceColumn : OrderbookColumnBase<DoubleMinMaxState>, IRenderColumn<OrderBookRowVM> {
-         public PriceColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, DoubleMinMaxState> filter)
-            : base(Columns.Price, "Price", r => r.price.store, nameof(OrderBookRowVM.price), DefaultColumnWidths.Price, filter, DataGridFilterKnownTemplates.DoubleMinMax) { }
-
-         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.price.text,
-               position, RowHeight, font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.price.store.animations);
-      }
-
-      public class AvgFilledColumn : OrderbookColumnBase<DoubleMinMaxState>, IRenderColumn<OrderBookRowVM> {
-         public AvgFilledColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, DoubleMinMaxState> filter)
-            : base(Columns.AvgFilled, "Avg Filled", r => r.avg_price.store, nameof(OrderBookRowVM.avg_price), DefaultColumnWidths.AveragePrice, filter, DataGridFilterKnownTemplates.DoubleMinMax) { }
-
-         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) {
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.avg_price.text,
-               position, RowHeight, font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.avg_price.store.animations);
-         }
-      }
-
-      public class LastFilledColumn : OrderbookColumnBase<DoubleMinMaxState>, IRenderColumn<OrderBookRowVM> {
-         public LastFilledColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, DoubleMinMaxState> filter)
-            : base(Columns.LastPrice, "Last Filled", r => r.last_price.store, nameof(OrderBookRowVM.last_price), DefaultColumnWidths.LastPrice, filter, DataGridFilterKnownTemplates.DoubleMinMax) { }
-
-         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.last_price.text,
-               position, RowHeight, font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.last_price.store.animations);
-      }
-
-      public class TimeCreatedColumn : OrderbookColumnBase<TimeFilterTuple>, IRenderColumn<OrderBookRowVM> {
-         public TimeCreatedColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, TimeFilterTuple> filter)
-            : base(Columns.Time, "Time", r => r.time_created.store, nameof(OrderBookRowVM.time_created), DefaultColumnWidths.Time, filter, DataGridFilterKnownTemplates.TimeFilter) { }
-
-         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.time_created.text,
-               position, RowHeight, font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.time_created.store.animations);
-      }
-
-      public class LastChangeTimeColumn : OrderbookColumnBase<TimeFilterTuple>, IRenderColumn<OrderBookRowVM> {
-         public LastChangeTimeColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, TimeFilterTuple> filter)
-            : base(Columns.LastChange, "Last Change", r => r.last_change.store, nameof(OrderBookRowVM.last_change), DefaultColumnWidths.Time, filter, DataGridFilterKnownTemplates.TimeFilter) { }
-
-         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
-            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.last_change.text,
-               position, RowHeight, font_brush, null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding, row.last_change.store.animations);
+            CellRender.draw_text(dc, _get_glyph_context(row), font_size, row.filled_quantity.ToString(), position, RowHeight,
+               RenderUtils.get_side_brush(row), null, horizontal_alignment, VerticalAlignment.Center, DefaultCellPadding);
       }
 
       public class OrderBookOrderTypeColumn : OrderTypeColumn<Columns>, ICVOrderBookColumn {
          ListSortDirection? _cached_sort_direction;
-         public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, OrderType[]> filter)
-            : base(Columns.OrderType, new Binding($"{nameof(OrderBookRowVM.order_type)}.{nameof(IOrderBookCell<OrderType>.value)}"), new Binding(nameof(OrderBookRowVM.order_metadata)), filter) {
+         public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter<OrderRecordTuple, OrderType[]> filter)
+            : base(Columns.OrderType, new Binding(nameof(OrderBookRowVM.order_type)), new Binding(nameof(OrderBookRowVM.order_metadata)), filter) {
             id = key.ToString();
             this.filter = filter;
             sort_member_path = nameof(OrderBookRowVM.order_type);
@@ -458,12 +555,11 @@ namespace mbts.spark {
             _cached_sort_direction = SortDirection;
          }
 
-         public new IDataGridColumnPersistentFilter<OrderBookRowVM, OrderType[]> filter { get; }
+         public new IDataGridColumnPersistentFilter<OrderRecordTuple, OrderType[]> filter { get; }
+         public IComparer<OrderBookRowVM> comparer => _wrap_comparer(r => r.order_type);
 
-         IDataGridColumnFilter<OrderBookRowVM> ICVOrderBookColumn.filter => filter;
-         ListSortDirection? ICVOrderBookColumn.SortDirection => _cached_sort_direction;
-
-         public IGridCellStore get_cell_store(OrderBookRowVM row) => row.order_type.store;
+         IDataGridColumnFilter<OrderRecordTuple> ICVOrderBookColumn.filter => filter;
+         ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; }
 
          protected override void OnSortDirectionChanged(ListSortDirection? new_value) {
             _cached_sort_direction = new_value;
@@ -474,8 +570,8 @@ namespace mbts.spark {
       public class OrderBookLastMessageColumn : TextColumn<Columns>, ICVOrderBookColumn {
          ListSortDirection? _cached_sort_direction;
 
-         public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, string> filter)
-            : base(Columns.LastMessage, new Binding($"{nameof(OrderBookRowVM.last_msg)}.{nameof(OrderBookRowVM.IOrderBookCell<string>.value)}"), filter, DataGridFilterKnownTemplates.Regex) {
+         public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter<OrderRecordTuple, string> filter)
+            : base(Columns.LastMessage, new Binding(nameof(OrderBookRowVM.last_msg)), filter, DataGridFilterKnownTemplates.Regex) {
             id = key.ToString();
             this.filter = filter;
             Header = "Last Message";
@@ -484,12 +580,11 @@ namespace mbts.spark {
             tooltip_binding = new Binding(nameof(OrderBookRowVM.last_msg));
          }
 
-         public new IDataGridColumnPersistentFilter<OrderBookRowVM, string> filter { get; }
+         public new IDataGridColumnPersistentFilter<OrderRecordTuple, string> filter { get; }
+         public IComparer<OrderBookRowVM> comparer => _wrap_comparer(r => r.last_msg);
 
-         IDataGridColumnFilter<OrderBookRowVM> ICVOrderBookColumn.filter => filter;
-         ListSortDirection? ICVOrderBookColumn.SortDirection => _cached_sort_direction;
-
-         public IGridCellStore get_cell_store(OrderBookRowVM row) => row.last_msg.store;
+         IDataGridColumnFilter<OrderRecordTuple> ICVOrderBookColumn.filter => filter;
+         ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; }
 
          protected override void OnSortDirectionChanged(ListSortDirection? new_value) {
             _cached_sort_direction = new_value;
@@ -504,7 +599,7 @@ namespace mbts.spark {
 
          ListSortDirection? _cached_sort_direction;
 
-         public SecurityColumn(IDataGridColumnPersistentFilter<OrderBookRowVM, SecurityCriteria> filter) : base(Columns.Security) {
+         public SecurityColumn(IDataGridColumnPersistentFilter<OrderRecordTuple, SecurityCriteria> filter) : base(Columns.Security) {
             id = key.ToString();
             this.filter = filter;
             Header = DefaultColumnHeaders.Security;
@@ -513,14 +608,13 @@ namespace mbts.spark {
             _cached_sort_direction = SortDirection;
          }
 
-         public IDataGridColumnPersistentFilter<OrderBookRowVM, SecurityCriteria> filter { get; }
+         public IDataGridColumnPersistentFilter<OrderRecordTuple, SecurityCriteria> filter { get; }
+         public IComparer<OrderBookRowVM> comparer => _wrap_comparer(r => r.security, SecurityHelper.Comparer.InstanceImmutable.to_nullable());
 
-         IDataGridColumnFilter<OrderBookRowVM> ICVOrderBookColumn.filter => filter;
+         IDataGridColumnFilter<OrderRecordTuple> ICVOrderBookColumn.filter => filter;
          IDataGridColumnFilter IFilterableColumn.filter => filter;
          string IFilterableColumn.filter_template => DataGridFilterKnownTemplates.DiscreteValues;
-         ListSortDirection? ICVOrderBookColumn.SortDirection => _cached_sort_direction;
-
-         public IGridCellStore get_cell_store(OrderBookRowVM row) => row.security.store;
+         ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; }
 
          public override FrameworkElement view_factory() {
             // expander
@@ -533,7 +627,7 @@ namespace mbts.spark {
             var security_view = new TextBlock { Foreground = font_brush };
             security_view.SetBinding(TextBlock.FontFamilyProperty, new Binding(nameof(font_family)) { Source = this });
             security_view.SetBinding(TextBlock.FontSizeProperty, new Binding(nameof(font_size)) { Source = this });
-            security_view.SetBinding(TextBlock.TextProperty, $"{nameof(OrderBookRowVM.security)}.{nameof(IGridCell.text)}");
+            security_view.SetBinding(TextBlock.TextProperty, nameof(OrderBookRowVM.security_text));
             security_view.SetBinding(TextBlock.HorizontalAlignmentProperty, new Binding(nameof(horizontal_alignment)) { Source = this });
             security_view.SetBinding(TextBlock.ForegroundProperty, new Binding(nameof(font_brush)) { Source = this });
             security_view.SetBinding(TextBlock.FontFamilyProperty, new Binding(nameof(font_family)) { Source = this });
@@ -547,7 +641,7 @@ namespace mbts.spark {
             grid.Children.Add(security_view);
             //
             var border = new Border { Child = grid };
-            CellRender.set_background_animation_bindings(border, $"{nameof(OrderBookRowVM.security)}.{nameof(OrderBookRowVM.security.store)}");
+            CellRender.set_background_animation_bindings(border, nameof(OrderBookRowVM.security));
             return border;
          }
 
@@ -581,7 +675,7 @@ namespace mbts.spark {
             return mb;
          }
 
-         public static Brush get_side_brush(OrderBookRowVM r) => r.side.value == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask;
+         public static Brush get_side_brush(OrderBookRowVM r) => r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask;
          public static BindingBase get_font_weight_binding() => new Binding(nameof(OrderBookRowVM.is_child)) { Converter = _font_weight_converter };
       }
    }

src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs

@@ -0,0 +1,211 @@
+using bts;
+using bts.api.core;
+using bts.collections;
+using bts.utils;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.ComponentModel;
+using System.Linq;
+using System.Reactive.Linq;
+
+namespace mbts.spark {
+   class OrderBookRowCollection : ObservableCollection<OrderBookRowVM> {
+      static readonly IEqualityComparer<INode<OrderBookRowVM>> _node_equality_comparer = EqualityComparer.Create<INode<OrderBookRowVM>, uint>(n => n.value.order_key);
+      readonly Func<OrderRecordTuple, OrderBookRowVM> _row_factory;
+      SortedTree<OrderBookRowVM, uint> _tree;
+      IComparer<OrderBookRowVM> _comparer;
+      Func<OrderRecordTuple, bool> _filter;
+      bool _is_refresh_pending;
+      int _deferment_count = 0;
+      /// When processing changed-items, the <see cref="SortedTree{T, TKey}.TryReplace(TKey, Func{T, T})"/> method is used to update the changed row.  It is not expected to fail (return false),
+      /// but if it does, we don't want to overwhelm the logs with a bunch of useless noise in case it starts happening repeatedly, hence the int fields below.
+      /// <see cref="_max_replace_misses_logged"/>, <see cref="_n_replace_misses_logged"/>, <see cref="update(IIncrementalTableUpdateTransform{OrderRecordTuple, uint})"/>.
+      const int _max_replace_misses_logged = 10;
+      int _n_replace_misses_logged;
+
+      public OrderBookRowCollection(Func<OrderRecordTuple, OrderBookRowVM> row_factory, IComparer<OrderBookRowVM> comparer) {
+         _row_factory = row_factory;
+         _tree = _mk_tree(_comparer = comparer);
+      }
+
+      public Func<OrderRecordTuple, bool> filter {
+         get => _filter;
+         set {
+            _filter = value;
+            _refresh_branches(_tree.RootNodes);
+            reset();
+         }
+      }
+      public IComparer<OrderBookRowVM> comparer {
+         get => _comparer;
+         set {
+            _tree = _tree.Resort(_comparer = value);
+            reset(); 
+         }
+      }
+      public int total_row_count { get; private set; }
+      IEnumerable<OrderBookRowVM> _visible_rows => _tree.Where(r => r.is_visible);
+
+      bool _is_refresh_deferred => _deferment_count > 0;
+
+      public IEnumerable<INode<OrderBookRowVM>> get_inner_nodes(OrderBookRowVM row) =>
+         _tree.EnumerateInnerNodes(row);
+
+      public IEnumerable<OrderBookRowVM> get_descendent_rows(OrderBookRowVM row) =>
+         _tree.EnumerateInnerValues(row);
+
+      public void collapse_all() {
+         foreach (var n in _tree.RootNodes) {
+            collapse_node(n);
+         }
+         if (_is_refresh_deferred) {
+            _is_refresh_pending = true;
+         } else {
+            Clear();
+            foreach (var n in _tree.RootNodes) {
+               if (n.value.is_visible)
+                  Add(n.value);
+            }
+         }
+         // local methods
+         void collapse_node(INode<OrderBookRowVM> node) {
+            node.value.collapse_internal();
+            foreach (var cn in node.inner) {
+               cn.value.is_visible = false;
+               collapse_node(cn);
+            }
+         }
+      }
+
+      public void expand_all() {
+         foreach (var n in _tree.RootNodes) {
+            expand_node(n);
+         }
+         if (_is_refresh_deferred) {
+            _is_refresh_pending = true;
+         } else {
+            Clear();
+            foreach (var n in _tree.RootNodes) {
+               if (n.value.is_visible) {
+                  Add(n.value);
+                  foreach (var r in n.inner_values())
+                     Add(r);
+               }
+            }
+         }
+         // local methods
+         void expand_node(INode<OrderBookRowVM> node) {
+            node.value.expand_internal();
+            foreach (var cn in node.inner) {
+               cn.value.is_visible = node.value.is_visible;
+               expand_node(cn);
+            }
+         }
+      }
+
+      /// <summary>
+      /// Refreshes the collection by calling <see cref="GlobalExtensions.sync_with{T}(IList{T}, IEnumerable{T})"/> with all visible rows.
+      /// </summary>
+      public void refresh() {
+         if (_is_refresh_deferred) {
+            _is_refresh_pending = true;
+         } else {
+            _is_refresh_pending = false;
+            this.sync_with(_visible_rows);
+         }
+      }
+
+      /// <summary>
+      /// Resets the collection by calling <see cref="Collection{T}.Clear"/>, then adding all visible rows.
+      /// </summary>
+      public void reset(bool force = false) {
+         if (_is_refresh_deferred && !force) {
+            _is_refresh_pending = true;
+         } else {
+            Clear();
+            this.AddRange(_visible_rows);
+            _is_refresh_pending = false;
+         }
+      }
+
+      public IDisposable defer_refresh(bool force_refresh_after = false) {
+         // use a counter instead of a simple bool flag to gracefully handle recursive calls to defer_refresh(),
+         // by only calling refresh() after all IDisposables returned by defer_refresh() have been disposed.
+         _deferment_count++;
+         _is_refresh_pending |= force_refresh_after;
+         return Disposable.Create(() => {
+            _deferment_count--;
+            if (_is_refresh_pending)
+               refresh();
+         });
+      }
+
+      public void update(IIncrementalTableUpdateTransform<OrderRecordTuple, uint> table) {
+         IEnumerable<INode<OrderBookRowVM>> impacted_branch_roots;
+         int n_removed = table.removed.Count();
+         if (n_removed == total_row_count) {
+            // full refresh (most likely this is the first update or a server restart)
+            _tree = _mk_tree(_comparer, table.added.Select(_row_factory));
+            impacted_branch_roots = _tree.RootNodes;
+         } else {
+            foreach (OrderRecordTuple ort in table.removed)
+               _tree.Remove(ort.order.Key);
+            foreach (Change<OrderRecordTuple> change in table.changes) {
+               if (!_tree.TryReplace(change.New.order.Key, r => { r.update(change.New); return r; })) {
+                  // uh oh, replace failed!  Let's log this, but not a million times (this seems like the kind of error that, if it happens once, could happen a lot).
+                  if (_n_replace_misses_logged++ < _max_replace_misses_logged)
+                     Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderRecordTuple)} in update (old:{change.Old.key}, new:{change.New.key}).  This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook.");
+               }
+            }
+            foreach (OrderRecordTuple ort in table.added)
+               _tree.Insert(_row_factory(ort));
+            impacted_branch_roots =
+              table.added.Concat(table.changed).Select(ort => ort.order.Key).Select(_tree.GetRootNode)
+              .Concat(
+                 table.removed
+                 .Where(ort => ort.order.has_parent())
+                 .Select(ort => ort.order.ParentKey)
+                 .Distinct() // since these are ParentKeys, there could be duplicates.
+                 .Select(_tree.GetRootNode)
+                 .Where(n => n != null)
+              ).Distinct(_node_equality_comparer);
+         }
+         _refresh_branches(impacted_branch_roots);
+         refresh();
+         int old_row_count = total_row_count;
+         total_row_count += table.added.Count() - n_removed;
+         if (old_row_count != total_row_count)
+            OnPropertyChanged(new PropertyChangedEventArgs(nameof(total_row_count)));
+      }
+
+      static SortedTree<OrderBookRowVM, uint> _mk_tree(IComparer<OrderBookRowVM> comp, IEnumerable<OrderBookRowVM> init = null) =>
+         new SortedTree<OrderBookRowVM, uint>(comp, r => r.order_key, r => r.parent_order_key, init);
+
+      void _refresh_branches(IEnumerable<INode<OrderBookRowVM>> roots) {
+         if (filter == null) {
+            foreach (var n in roots)
+               refresh_tree_properties(0, true, n);
+         } else {
+            foreach (var n in roots)
+               refresh_tree_properties(0, get_branch_passes_filter(n), n);
+         }
+         // local methods
+         void refresh_tree_properties(int depth_level, bool is_visible, INode<OrderBookRowVM> node) {
+            node.value.depth_level = depth_level;
+            node.value.is_visible = is_visible;
+            using (var en = node.inner.GetEnumerator()) {
+               if (en.MoveNext()) {
+                  node.value.is_parent = true;
+                  depth_level++;
+                  do refresh_tree_properties(depth_level, node.value.is_visible && node.value.is_expanded, en.Current);
+                  while (en.MoveNext());
+               }
+            }
+         }
+         bool get_branch_passes_filter(INode<OrderBookRowVM> n) =>
+            filter(n.value.order_tuple) || n.inner.Any(get_branch_passes_filter);
+      }
+   }
+}

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -6,407 +6,135 @@ using mbts.ui;
 using mbts.ui.controls;
 using mbts.ui.shell;
 using System;
-using System.Collections.Generic;
-using System.Collections.ObjectModel;
-using System.Reactive.Linq;
 using System.Windows.Input;
-using static mbts.spark.OrderBookView;
-using OrderStatus = bts.api.orderman.OrderStatus;
 
 namespace mbts.spark {
-   partial class OrderBookRowVM : DisposableViewModelBase, IEquatable<OrderBookRowVM>, DIYGrid.IRowVM<OrderRecordTuple>, DIYGrid.IActionableRowVM {
-      //public readonly PredicateAnimator<OrderBookRowVM, OrderRecordTuple> animator;
-
+   partial class OrderBookRowVM : DisposableViewModelBase, IEquatable<OrderBookRowVM>, DIYGrid.IRowVM, DIYGrid.IActionableRowVM {
       const string _unknown_security_price_format_string = "0.#########";
       const int _unknown_security_price_num_fractional_digits = 9;
-      readonly OrderBookVM _book_vm;
-      readonly OrdersModel _model;
-      readonly Property<bool> _visible, _expanded, _passes_filter;
-      readonly DelegateProperty<int> _depth_level;
-      readonly HashSet<OrderBookRowVM> _child_rows = new HashSet<OrderBookRowVM>();
-      readonly ObservantCommand _show_order_modify_cmd;
-      OrderRecordImmutable _order;
-      SecurityMD? _security_md;
-      string _symbol;
-      OrderBookRowVM _parent;
-      OrderBookVM.BranchFilterSource _branch_filter_source;
-      // Core
-      IOrderBookCellStore<uint> _order_key_store_backing;
-      IOrderBookCellStore<uint> _parent_order_key_store_backing;
-      IOrderBookCellStore<uint> _security_key_store_backing;
-      IOrderBookCellStore<SecurityImmutable?> _security_store_backing;
-      IOrderBookCellStore<Venue> _exchange_store_backing;
-      IOrderBookCellStore<string> _symbol_store_backing;
-      // Def
-      IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store_backing;
-      IOrderBookCellStore<Side> _side_store_backing;
-      IOrderBookCellStore<string> _user_store_backing;
-      IOrderBookCellStore<string> _account_store_backing;
-      IOrderBookCellStore<TimeInForce> _time_in_force_store_backing;
-      IOrderBookCellStore<DateTime> _time_created_store_backing;
-      // State
-      IOrderBookCellStore<DateTime> _last_change_store_backing;
-      IOrderBookCellStore<OrderStatus> _status_store_backing;
-      IOrderBookCellStore<double> _price_store_backing;
-      IOrderBookCellStore<uint> _quantity_store_backing;
-      IOrderBookCellStore<uint> _filled_quantity_store_backing;
-      IOrderBookCellStore<double> _last_price_store_backing;
-      IOrderBookCellStore<double> _avg_price_store_backing;
-      IOrderBookCellStore<string> _last_msg_store_backing;
-      IOrderBookCellStore<OrderSubType> _order_sub_type_store_backing;
-      // Core
-      IOrderBookCellStore<uint> _order_key_store => _order_key_store_backing ?? (_order_key_store_backing = CellFactory.create(this, _book_vm.columns.order_key_column, nameof(order_key), r => r._order.Key, r => r._order_key_store));
-      IOrderBookCellStore<uint> _parent_order_key_store => _parent_order_key_store_backing ?? (_parent_order_key_store_backing = CellFactory.create(this, _book_vm.columns.parent_key_column, nameof(parent_order_key), r => r._order.ParentKey, r => r._parent_order_key_store));
-      IOrderBookCellStore<uint> _security_key_store => _security_key_store_backing ?? (_security_key_store_backing = CellFactory.create(this, _book_vm.columns.security_key_column, nameof(security_key), r => r._order.security_key(), r => r._security_key_store));
-      IOrderBookCellStore<SecurityImmutable?> _security_store => _security_store_backing ?? (_security_store_backing = CellFactory.create_nullable(this, _book_vm.columns.security_column, nameof(security), r => r._security, r => r._security_store, v => v.display_name()));
-      IOrderBookCellStore<Venue> _exchange_store => _exchange_store_backing ?? (_exchange_store_backing = CellFactory.create(this, _book_vm.columns.exchange_column, nameof(exchange), r => r._security.HasValue ? _security.Value.Exchange : Venue.Unset, r => r._exchange_store));
-      IOrderBookCellStore<string> _symbol_store => _symbol_store_backing ?? (_symbol_store_backing = CellFactory.create(this, _book_vm.columns.symbol_column, nameof(symbol), r => r._symbol, r => r._symbol_store, v => !String.IsNullOrEmpty(v) ? v : "unknown symbol"));
-      // Def
-      IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store => _order_type_store_backing ?? (_order_type_store_backing = CellFactory.create(this, _book_vm.columns.order_type_column, nameof(order_type), r => r._order.order_type(), r => r._order_type_store));
-      IOrderBookCellStore<Side> _side_store => _side_store_backing ?? (_side_store_backing = CellFactory.create(this, _book_vm.columns.side_column, nameof(side), r => r._order.side(), r => r._side_store));
-      IOrderBookCellStore<string> _user_store => _user_store_backing ?? (_user_store_backing = CellFactory.create(this, _book_vm.columns.user_column, nameof(user), r => _order.user(), r => r._user_store));
-      IOrderBookCellStore<string> _account_store => _account_store_backing ?? (_account_store_backing = CellFactory.create(this, _book_vm.columns.account_column, nameof(account), r => r._order.account(), r => r._account_store));
-      IOrderBookCellStore<TimeInForce> _time_in_force_store => _time_in_force_store_backing ?? (_time_in_force_store_backing = CellFactory.create(this, _book_vm.columns.time_in_force_column, nameof(time_in_force), r => r._order.time_in_force(), r => r._time_in_force_store));
-      IOrderBookCellStore<DateTime> _time_created_store => _time_created_store_backing ?? (_time_created_store_backing = CellFactory.create(this, _book_vm.columns.time_created_column, nameof(time_created), r => r._order.created(), r => r._time_created_store, format_date));
-      // State
-      IOrderBookCellStore<DateTime> _last_change_store => _last_change_store_backing ?? (_last_change_store_backing = CellFactory.create(this, _book_vm.columns.last_change_column, nameof(last_change), r => _order.last_change(), r => r._last_change_store, format_date));
-      IOrderBookCellStore<OrderStatus> _status_store => _status_store_backing ?? (_status_store_backing = CellFactory.create(this, _book_vm.columns.status_column, nameof(status), r => _order.status(), r => r._status_store));
-      IOrderBookCellStore<double> _price_store => _price_store_backing ?? (_price_store_backing = CellFactory.create(this, _book_vm.columns.price_column, nameof(price), r => _order.price(), r => r._price_store, format_price));
-      IOrderBookCellStore<uint> _quantity_store => _quantity_store_backing ?? (_quantity_store_backing = CellFactory.create(this, _book_vm.columns.qty_column, nameof(quantity), r => _order.quantity(), r => r._quantity_store));
-      IOrderBookCellStore<uint> _filled_quantity_store => _filled_quantity_store_backing ?? (_filled_quantity_store_backing = CellFactory.create(this, _book_vm.columns.filled_qty_column, nameof(filled_quantity), r => _order.filled(), r => r._filled_quantity_store));
-      IOrderBookCellStore<double> _last_price_store => _last_price_store_backing ?? (_last_price_store_backing = CellFactory.create(this, _book_vm.columns.last_price_column, nameof(last_price), r => _order.last_price(), r => r._last_price_store, format_price));
-      IOrderBookCellStore<double> _avg_price_store => _avg_price_store_backing ?? (_avg_price_store_backing = CellFactory.create(this, _book_vm.columns.avg_price_column, nameof(avg_price), r => _order.avg_price(), r => r._avg_price_store, format_price));
-      IOrderBookCellStore<string> _last_msg_store => _last_msg_store_backing ?? (_last_msg_store_backing = CellFactory.create(this, _book_vm.columns.last_message_column, nameof(last_msg), r => _order.last_msg(), r => r._last_msg_store));
-      IOrderBookCellStore<OrderSubType> _order_sub_type_store => _order_sub_type_store_backing ?? (_order_sub_type_store_backing = CellFactory.create(this, _book_vm.columns.sub_type_column, nameof(order_sub_type), r => r._order.sub_type(), r => r._order_sub_type_store, format_sub_type));
+      readonly OrderBookVM _owner;
+      IRaiseCanExecuteChangedCommand _show_order_modify_cmd, _toggle_expanded_cmd;
+      bool _is_parent, _is_expanded;
+      int _depth_level;
+      SecurityMD? _security_md => order_tuple.security_md;
+     
+      public OrderBookRowVM(OrderBookVM owner, OrderRecordTuple order_tuple) {
+         _owner = owner;
+         this.order_tuple = order_tuple;
+         is_visible = !is_child;
+      }
 
-      static string format_date(DateTime v) => $"{v:M/d} {v:h:mm:ss.fff tt}";
-      string format_price(double v) => _security.HasValue ?
-         _security.Value.ContractSpec.format(v) :
-         Math.Round(v, _unknown_security_price_num_fractional_digits, MidpointRounding.AwayFromZero).ToString(_unknown_security_price_format_string);
-      static string format_sub_type(OrderSubType v) {
+      public event Action invalidated;
+
+      // Core
+      public OrderRecordTuple order_tuple { get; private set; }
+      public OrderRecordImmutable order => order_tuple.order;
+      public OrderMetadata order_metadata => _owner.get_metadata(this);
+      public uint order_key => order.Key;
+      public uint parent_order_key => order.ParentKey;
+      public uint security_key => order.security_key();
+      public SecurityImmutable? security => _security_md.security();
+      public string security_text => security.display_name();
+      public Venue exchange => _security_md.exchange();
+      public string symbol => _security_md.symbol();
+      // Def
+      public CustomOrderDefinition.CustomOneofCase order_type => order.order_type();
+      public OrderSubType order_sub_type => order.sub_type();
+      public string order_sub_type_text => _fmt_order_sub_type(order_sub_type);
+      public Side side => order.side();
+      public string user => order.user();
+      public string account => order.account();
+      public TimeInForce time_in_force => order.time_in_force();
+      public DateTime time_created => order.created();
+      public string time_created_text => _fmt_date(time_created);
+      // State
+      public DateTime last_change => order.last_change();
+      public string last_change_text => _fmt_date(last_change);
+      public OrderStatus status => order.status();
+      public double price => order.price();
+      public double last_price => order.last_price();
+      public double avg_price => order.avg_price();
+      public string price_text => _fmt_price(price);
+      public string last_price_text => _fmt_price(last_price);
+      public string avg_price_text => _fmt_price(avg_price);
+      public uint quantity => order.quantity();
+      public uint filled_quantity => order.filled();
+      public string last_msg => order.last_msg();
+      // Tree
+      public bool is_parent { get => _is_parent; set => Set(ref _is_parent, value); }
+      public bool is_child => order.has_parent();
+      public bool is_visible { get; set; }
+      public bool is_expanded {
+         get => _is_expanded;
+         set {
+            if (Set(ref _is_expanded, value))
+               _owner.on_expanded_changed(this);
+         }
+      }
+      public int depth_level { get => _depth_level; set => Set(ref _depth_level, value); }
+      // Commands
+      public ICommand toggle_command => _toggle_expanded_cmd ?? (
+         _toggle_expanded_cmd = new DelegateCommand(() => is_expanded = !is_expanded));
+
+      ICommand DIYGrid.IActionableRowVM.default_command => _show_order_modify_cmd ??
+         (_show_order_modify_cmd = new DelegateCommand(() => _owner.show_order_modify(this), () => status == OrderStatus.Working));
+
+      // These fns present a way for the orderbook to avoid noisy, intermittent collection changes during "Expand/Collapse All" ops,
+      // when incremental changes are ignored in favor of a full reset, after 'is_expanded' is updated for all rows.
+      internal void expand_internal() => Set(ref _is_expanded, true);
+      internal void collapse_internal() => Set(ref _is_expanded, false);
+
+      public void update(OrderRecordTuple order_tuple) {
+         Log.assert(order_tuple.order.Key == this.order.Key, "OrderBookRowVM: update_order was called with an order with the wrong key.");
+         bool had_security = _security_md.HasValue;
+         // _show_order_modify_cmd is lazy (instantiated in the default_command getter); it might be null, don't not force instantiation
+         bool? could_modify_execute = _show_order_modify_cmd?.CanExecute(null);
+         OrderRecordImmutable old_order = order;
+         this.order_tuple = order_tuple;
+         // most cells are "render-cells," which don't use databindings - the only way to update such cells is to raise the invalidated event
+         bool invalidate = !(
+            old_order.last_change() == order.last_change() &&
+            old_order.filled() == order.filled() &&
+            old_order.last_price() == order.last_price() &&
+            old_order.avg_price().Equals(order.avg_price()) &&
+            old_order.price().Equals(order.price()) &&
+            old_order.quantity() == order.quantity() &&
+            old_order.last_msg() == order.last_msg());
+         // however, the order-type and security cells DO use databindings - if these cells have changed, the PropertyChanged event must be raised regardless of whether or not invalidated is raised
+         if (old_order.order_type() != order.order_type())
+            RaisePropertyChanged(nameof(order_type), nameof(order_metadata));
+         if (!had_security && _security_md.HasValue) {
+            RaisePropertyChanged(nameof(security));
+            // need to raise invalidated to refresh the price cells (the security.ContractSpec determines how they are formatted - Ticks!)
+            invalidate = true;
+         }
+         if (old_order.status() != order.status()) {
+            invalidate = true;
+            RaisePropertyChanged(nameof(status));
+         }
+         if (could_modify_execute.HasValue && could_modify_execute.Value != _show_order_modify_cmd.CanExecute(null))
+            _show_order_modify_cmd.RaiseCanExecuteChanged();
+         if (invalidate)
+            raise_invalidated();
+      }
+
+      public void raise_invalidated() => invalidated?.Invoke();
+
+      public bool Equals(OrderBookRowVM other) => other != null && other.order.Key == order.Key;
+      public override bool Equals(object obj) => Equals(obj as OrderBookRowVM);
+      public override int GetHashCode() => (int)order.Key;
+      public override string ToString() => $"{order_key} {last_change} {parent_order_key}";
+
+      static string _fmt_date(DateTime v) => $"{v:M/d} {v:h:mm:ss.fff tt}";
+      static string _fmt_order_sub_type(OrderSubType v) {
          switch (v) {
             case OrderSubType.HedgingLeg: return "H";
             case OrderSubType.QuotingLeg: return "Q";
             default: return string.Empty;
          }
       }
-
-      public OrderBookRowVM(IBus bus, OrderBookVM source_vm, OrdersModel model, OrderRecordImmutable record, SecurityMD? s) {
-         _book_vm = source_vm;
-         _model = model;
-         _order = record;
-         _security_md = s;
-         _symbol = _security_md.HasValue ? _security_md.Value.security.Symbol : null;
-         //animator = new PredicateAnimator<OrderBookRowVM,OrderRecordTuple>(this);
-         _visible = backing(nameof(is_visible), true);
-         _expanded = backing(nameof(is_expanded), false);
-         _depth_level = backing(nameof(depth_level), () => {
-            int i = 0;
-            var curr = this;
-            while (curr._parent != null) {
-               i++;
-               curr = curr._parent;
-            }
-            return i;
-         });
-         toggle_command = new DelegateCommand(() => is_expanded = !is_expanded);
-         _show_order_modify_cmd = new ObservantCommand(() => bus.post(SparkTags.ShowModifyOrderTag, _order.Key));
-         _show_order_modify_cmd.OnNext(status.value == OrderStatus.Working);
-         row_passes_filter = _book_vm.columns.filter(this);
-         _passes_filter = backing(nameof(passes_filter), row_passes_filter);
-      }
-
-      public event Action invalidated;
-      public event Action<(OrderRecordTuple old_val, OrderRecordTuple new_val)> updated;
-
-      public OrderRecordImmutable order => _order;
-      public OrderMetadata order_metadata => _model.factory.get_metadata(order_type.value);
-      public OrderBookRowVM parent => _parent;
-      public OrderRecordTuple current_data => new OrderRecordTuple(_order, _security_md);
-      // Core
-      public IOrderBookCell<uint> order_key => _order_key_store.current;
-      public IOrderBookCell<uint> parent_order_key => _parent_order_key_store.current;
-      public IOrderBookCell<uint> security_key => _security_key_store.current;
-      public IOrderBookCell<SecurityImmutable?> security => _security_store.current;
-      public IOrderBookCell<Venue> exchange => _exchange_store.current;
-      public IOrderBookCell<string> symbol => _symbol_store.current;
-      // Def
-      public IOrderBookCell<CustomOrderDefinition.CustomOneofCase> order_type => _order_type_store.current;
-      public IOrderBookCell<Side> side => _side_store.current;
-      public IOrderBookCell<string> user => _user_store.current;
-      public IOrderBookCell<string> account => _account_store.current;
-      public IOrderBookCell<TimeInForce> time_in_force => _time_in_force_store.current;
-      public IOrderBookCell<DateTime> time_created => _time_created_store.current;
-      // State
-      public IOrderBookCell<DateTime> last_change => _last_change_store.current;
-      public IOrderBookCell<OrderStatus> status => _status_store.current;
-      public IOrderBookCell<double> price => _price_store.current;
-      public IOrderBookCell<uint> quantity => _quantity_store.current;
-      public IOrderBookCell<uint> filled_quantity => _filled_quantity_store.current;
-      public IOrderBookCell<double> last_price => _last_price_store.current;
-      public IOrderBookCell<double> avg_price => _avg_price_store.current;
-      public IOrderBookCell<string> last_msg => _last_msg_store.current;
-      public IOrderBookCell<OrderSubType> order_sub_type => _order_sub_type_store.current;
-      // Commands
-      public ICommand toggle_command { get; }
-      ICommand ui.controls.DIYGrid.IActionableRowVM.default_command => _show_order_modify_cmd;
-      // Accessories
-      public bool is_parent => _child_rows.Count > 0;
-      public bool is_child => _order.has_parent();
-      public IReadOnlyCollection<OrderBookRowVM> child_orders => _child_rows;
-      public OrderBookVM.BranchFilterSource branch_filter_source => _branch_filter_source;
-      public bool passes_filter => _passes_filter.Value;
-      public bool row_passes_filter { get; private set; }
-      public bool is_visible {
-         get => _visible.Value;
-         set {
-            _visible.set(value);
-            if (is_expanded) {
-               foreach (var child_row in _child_rows)
-                  child_row.is_visible = value;
-            }
-         }
-      }
-      public bool is_expanded {
-         get => _expanded.Value;
-         set {
-            _expanded.set(value);
-            foreach (var child_row in child_orders)
-               child_row.is_visible = value;
-         }
-      }
-      public int depth_level => _depth_level.Value;
-
-      SecurityImmutable? _security => _security_md?.security;
-      // '_dynamic_render_cells' are presented by 'RenderColumns,' and may change value during the lifetime of this row
-      // Note: by instantiating all these cells inside a CellPropertyGroup, we can update all of them with a single call.
-      // this reduces the risk of accidentally failing to update one of the cells, esp if we make changes to them down the road.
-      IEnumerable<IOrderBookCellStore> _dynamic_render_cells {
-         get {
-            yield return _last_change_store;
-            yield return _price_store;
-            yield return _quantity_store;
-            yield return _filled_quantity_store;
-            yield return _avg_price_store;
-            yield return _last_price_store;
-            yield return _status_store;
-            yield return _last_msg_store;
-         }
-      }
-
-      IEnumerable<IOrderBookCellStore> _security_dependent_cells {
-         get {
-            yield return _security_store;
-            yield return _exchange_store;
-            yield return _symbol_store;
-         }
-      }
-
-      IEnumerable<IOrderBookCellStore> _price_cells {
-         get {
-            yield return _price_store;
-            yield return _avg_price_store;
-            yield return _last_price_store;
-            yield return _symbol_store;
-         }
-      }
-
-      public ObservableCollection<RowAnimation> row_animations { get; } = new ObservableCollection<RowAnimation>();
-
-      DIYGrid.IGridCellStore[] _cell_stores_backing;
-      public IReadOnlyList<DIYGrid.IGridCellStore> cell_stores => _cell_stores_backing ?? (_cell_stores_backing = new DIYGrid.IGridCellStore[] {
-         _security_store,
-         _security_key_store,
-         _parent_order_key_store,
-         _order_key_store,
-         _symbol_store,
-         _time_created_store,
-         _user_store,
-         _account_store,
-         _exchange_store,
-         _time_in_force_store,
-         _order_type_store,
-         _order_sub_type_store,
-         _side_store,
-         _last_change_store,
-         _price_store,
-         _quantity_store,
-         _filled_quantity_store,
-         _avg_price_store,
-         _last_price_store,
-         _status_store,
-         _last_msg_store
-      });
-
-      public static Stack<OrderBookRowVM> lineage(OrderBookRowVM node) {
-         var rv = new Stack<OrderBookRowVM>();
-         rv.Push(node);
-         while (node._parent != null)
-            rv.Push((node = node._parent));
-         return rv;
-      }
-
-      public void join_branch_filter_source(OrderBookVM.BranchFilterSource new_branch_filter_source) {
-         if (_branch_filter_source != null) {
-            _branch_filter_source.passes_filter_changed -= _passes_filter.set;
-         }
-         _branch_filter_source = new_branch_filter_source;
-         _branch_filter_source.set_row_passes_filter(_order.Key, row_passes_filter);
-         foreach (var child_row in _child_rows)
-            child_row.join_branch_filter_source(_branch_filter_source);
-         _passes_filter.set(_branch_filter_source.passes_filter);
-         _branch_filter_source.passes_filter_changed += _passes_filter.set;
-      }
-
-      /// <summary>
-      /// Attaches the child-row to this row's branch-filter-source, and recursively refreshes sort/filter properties for the child and it's descendents.
-      /// </summary>
-      public void add_child_order(OrderBookRowVM child_row) {
-         if (_child_rows.Add(child_row) && _child_rows.Count == 1)
-            RaisePropertyChanged(nameof(is_parent));
-         child_row._parent = this;
-         // child orders that arrived in an earlier update use a temporary branch_filter_source
-         // need to tell _book_vm to remove the now-stale branch_filter_source.
-         if (child_row._branch_filter_source != null)
-            _book_vm.remove_branch_filter_root(child_row._order.Key);
-         // recursively updates 'live_sorting_props,' and 'is_visible' for child_row and descendents
-         child_row._on_lineage_changed();
-      }
-
-      public void remove_child_order(OrderBookRowVM child_row) {
-         if (_child_rows.Remove(child_row) && _child_rows.Count == 0)
-            RaisePropertyChanged(nameof(is_parent));
-      }
-
-      public void remove_parent(OrderBookVM.BranchFilterSource new_branch_filter_source) {
-         _parent = null;
-         join_branch_filter_source(new_branch_filter_source);
-         refresh_row_passes_filter();
-         foreach (var p in _book_vm.grid_settings_manager.live_sorting_props)
-            RaisePropertyChanged(p);
-         _depth_level.refresh();
-         foreach (OrderBookRowVM child_row in _child_rows)
-            child_row._on_lineage_changed();
-      }
-
-      public void update_order(OrderRecordImmutable order) {
-         Log.assert(order.ParentKey == _order.ParentKey, "Parent Keys can change?");
-         Log.assert(order.Key == _order.Key, "Keys can change?");
-         var old_data = current_data;
-         _order = order;
-         if (_update_partial(_dynamic_render_cells))
-            raise_invalidated();
-         _show_order_modify_cmd.OnNext(status.value == OrderStatus.Working);
-         refresh_row_passes_filter();
-         updated?.Invoke((old_data, current_data));
-      }
-
-      public void update_security(SecurityMD security_md) {
-         var old_data = current_data;
-         var old_spec = _security.spec_or_default();
-         _security_md = security_md;
-         _symbol = _security_md.HasValue ? _security_md.Value.security.Symbol : null;
-         if (!Equals(old_spec, _security.spec_or_default())) {
-            _update_partial(_security_dependent_cells);
-            foreach (var cell_store in _price_cells)
-               cell_store.regenerate_cell();
-            raise_invalidated();
-         } else if (_update_partial(_security_dependent_cells)) {
-            raise_invalidated();
-         }
-         refresh_row_passes_filter();
-         updated?.Invoke((old_data, current_data));
-      }
-
-      public void refresh_row_passes_filter() {
-         row_passes_filter = _book_vm.columns.filter(this);
-         if (_branch_filter_source != null) {
-            _branch_filter_source.set_row_passes_filter(_order.Key, row_passes_filter);
-         } else {
-            _passes_filter.set(row_passes_filter);
-         }
-      }
-
-      public void raise_invalidated() => invalidated?.Invoke();
-
-      public bool Equals(OrderBookRowVM other) => other != null && other._order.Key == _order.Key;
-      public override bool Equals(object obj) => Equals(obj as OrderBookRowVM);
-      public override int GetHashCode() => (int)_order.Key;
-      public override string ToString() => $"{order_key} {last_change} {parent_order_key}";
-
-      protected override void Dispose(bool disposing) {
-         // idempotent
-         if (_branch_filter_source != null) {
-            _branch_filter_source.passes_filter_changed -= _passes_filter.set;
-            _branch_filter_source = null;
-         }
-      }
-
-      static IEnumerable<OrderBookRowVM> _descendents(OrderBookRowVM root) {
-         foreach (var child in root.child_orders) {
-            yield return child;
-            foreach (var grand_child in _descendents(child))
-               yield return grand_child;
-         }
-      }
-
-      static IOrderBookCellStore _cell_store_from_column(OrderBookRowVM row, Columns column) {
-         switch (column) {
-            case Columns.OrderKey: return row._order_key_store;
-            case Columns.ParentKey: return row._parent_order_key_store;
-            case Columns.SecurityKey: return row._security_key_store;
-            case Columns.Security: return row._security_store;
-            case Columns.Symbol: return row._symbol_store;
-            case Columns.Exchange: return row._exchange_store;
-            case Columns.OrderType: return row._order_type_store;
-            case Columns.Side: return row._side_store;
-            case Columns.User: return row._user_store;
-            case Columns.Account: return row._account_store;
-            case Columns.TimeInForce: return row._time_in_force_store;
-            case Columns.Time: return row._time_created_store;
-            case Columns.LastChange: return row._last_change_store;
-            case Columns.Status: return row._status_store;
-            case Columns.Price: return row._price_store;
-            case Columns.Size: return row._quantity_store;
-            case Columns.Filled: return row._filled_quantity_store;
-            case Columns.LastPrice: return row._last_price_store;
-            case Columns.AvgFilled: return row._avg_price_store;
-            case Columns.LastMessage: return row._last_msg_store;
-            case Columns.SubType: return row._order_sub_type_store;
-            default: throw new ArgumentException(nameof(column));
-         }
-      }
-
-      // Note: this method assumes that lineage exists (i.e. _parent is not null).
-      void _on_lineage_changed() {
-         _depth_level.refresh();
-         // parent 'branch_filter_source' can be null:
-         // if the parent is itself an orphan - meaning this is the middle of an update -
-         // the final branch-filter-root will be determined after all new orders have been added
-         if (_parent._branch_filter_source != null)
-            join_branch_filter_source(_parent._branch_filter_source);
-         _visible.set(_parent.is_visible && _parent.is_expanded);
-         // refresh the sort by calling regenerate_cell() for each live-sorting-columns corresponding cell-store
-         foreach (Columns column in _book_vm.live_sorting_columns)
-            _cell_store_from_column(this, column).regenerate_cell();
-         // finally, repeat for each child-row
-         foreach (var child_row in _child_rows)
-            child_row._on_lineage_changed();
-      }
-
-      bool _update_partial(IEnumerable<IOrderBookCellStore> target_cell_stores) {
-         bool updated = false;
-         foreach (var cell_store in target_cell_stores) {
-            if (cell_store.refresh()) {
-               updated = true;
-               if (_book_vm.live_sorting_columns.Contains(cell_store.column.key)) {
-                  // refresh the sort by calling regenerate_cell() for each descendent-row
-                  foreach (OrderBookRowVM dr in _descendents(this))
-                     cell_store.select_other(dr).regenerate_cell();
-               }
-            }
-         }
-         return updated;
-      }
+      string _fmt_price(double v) => security.HasValue ?
+         security.Value.ContractSpec.format(v) :
+         Math.Round(v, _unknown_security_price_num_fractional_digits, MidpointRounding.AwayFromZero).ToString(_unknown_security_price_format_string);
    }
 }

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -1,9 +1,9 @@
 using bts;
 using bts.api;
-using bts.api.core;
 using bts.api.orderman;
 using bts.shell;
-using mbts.collections;
+using bts.spark;
+using bts.utils;
 using mbts.persist;
 using mbts.ui;
 using mbts.ui.controls;
@@ -11,219 +11,61 @@ using mbts.ui.settings;
 using mbts.ui.shell;
 using mbts.ui.utility;
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
-using System.ComponentModel;
 using System.Linq;
 using System.Reactive.Linq;
-using System.Reactive.Subjects;
 using System.Windows.Controls;
-using System.Windows.Data;
 using System.Windows.Input;
 using static bts.shell.OrderBookWindowFactory;
 using static mbts.spark.OrderBookView;
 using _GridSettingsManager = mbts.ui.GridSettingsManager<mbts.spark.OrderBookRowVM, mbts.spark.OrderRecordTuple, bts.shell.INotificationServiceDataCache, mbts.OrderRecordTuplePredicate, bts.shell.OrderBookWindowFactory.OrderBookGridSettings>;
 
-namespace mbts.spark {
-   partial class OrderBookVM : DisposableViewModelBase, IHandleRowSelection {
-      readonly LiveSortingColumKeys _live_sorting_columns;
+namespace mbts.spark { 
+   partial class OrderBookVM : ViewModelBase, IDisposable, IHandleRowSelection, IHandleGridSorting, IWillLetYouKnowWhenIDispose {
+      readonly IBus _bus;
+      readonly OrdersModel _model;
       readonly Property<OrderBookRowVM> _current_row;
-      readonly Subject<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)> _order_securities_subject =
-         new Subject<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)>();
+      readonly OrderBookRowCollection _row_collection;
+      readonly ObservableCollection<OrderBookRowVM> _selected_rows = new ObservableCollection<OrderBookRowVM>();
+      readonly OrderBookColumnCollection _columns;
       readonly IWindowSettingsManager<OrderBookGridSettings> _settings_manager;
+      readonly List<IDisposable> _resources;
       OrderBookStartupData _state;
 
-      Dictionary<uint, OrderBookRowVM> _order_rows_cache = new Dictionary<uint, OrderBookRowVM>();
-      Dictionary<uint, BranchFilterSource> _branch_filter_roots = new Dictionary<uint, BranchFilterSource>();
-      ObservableCollection<OrderBookRowVM> _order_rows = new ObservableCollection<OrderBookRowVM>();
-      ObservableCollection<OrderBookRowVM> _selected_rows = new ObservableCollection<OrderBookRowVM>();
-      SetMap<uint, uint> _parent_to_children = new SetMap<uint, uint>();
-      SetMap<uint, uint> _security_orders = new SetMap<uint, uint>();
-
       public OrderBookVM(IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model, IWindowSettingsManager<OrderBookGridSettings> settings_manager) {
+         _bus = bus;
+         _model = model;
          _settings_manager = settings_manager;
-         var order_securities_subject = new Subject<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)>();
-         columns = new OrderBookColumnCollection(bus, model, order_securities_subject);
-         rows_view = CollectionViewSource.GetDefaultView(_order_rows);
-         rows_view.Filter = obj => obj is OrderBookRowVM vm && vm.is_visible && vm.passes_filter;
-         columns.filter_changed += () => {
-            foreach (var vm in _order_rows)
-               vm.refresh_row_passes_filter();
-         };
-         var cvls = (ICollectionViewLiveShaping)rows_view;
-         cvls.IsLiveFiltering = true;
-         cvls.IsLiveSorting = true;
-         cvls.LiveFilteringProperties.AddRange(nameof(OrderBookRowVM.is_visible), nameof(OrderBookRowVM.passes_filter));
+         _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(u => (u.added.Values.Select(smd => smd.security), u.removed.Values.Select(smd => smd.security))));
+         _row_collection = new OrderBookRowCollection(ort => new OrderBookRowVM(this, ort), _columns.get_sorting_data().comparer) { filter = _columns.get_filter() };
          _current_row = backing<OrderBookRowVM>(nameof(current_row));
-
-         collapse_all_rows_command = new DelegateCommand(
-            () => {
-                foreach (var r in _order_rows.Where(r => r.is_visible && r.passes_filter))
-                    r.is_expanded = false;
-            });
-         expand_all_rows_command = new DelegateCommand(
-            () => {
-                foreach (var r in _order_rows.Where(r => r.is_visible && r.passes_filter))
-                    r.is_expanded = true;
-            });
-
-         grid_settings_manager = _create_grid_settings_manager(owner_title, bus, notification_service, model, _order_rows, rows_view, columns);
-         grid_settings_manager.load_from(OrderBookStartupData.DefaultGrid);
-         _live_sorting_columns = new LiveSortingColumKeys(grid_settings_manager.live_sorting_props);
-
-         SetMap<uint, uint> missing_security_orders = new SetMap<uint, uint>();
-         IReadOnlyDictionary<uint, SecurityMD> security_dict = new Dictionary<uint, SecurityMD>();
-
-         _dispose = Disposable.Create(
-            bus.subscribe(SparkTags.SecuritiesTag).OnDispatcher().Subscribe(on_securities),
-            model.on_orders.OnDispatcher().Subscribe(on_orders),
+         _columns.filter_changed += on_filter_changed;
+         collapse_all_rows_command = new DelegateCommand(_row_collection.collapse_all);
+         expand_all_rows_command = new DelegateCommand(_row_collection.expand_all);
+         grid_settings_manager = _create_grid_settings_manager(owner_title, bus, notification_service, model, rows, columns);
+         _resources = new List<IDisposable> {
+            model.on_order_tuples_table.OnDispatcher().Subscribe(_row_collection.update),
             settings_manager.subscribe_current().OnDispatcher().Subscribe(grid_settings_manager),
             grid_settings_manager,
-            columns,
-            order_securities_subject);
-
-         void on_orders(TableUpdate<OrderRecordImmutable, uint> table) {
-            // clear everything (most likely the server restarted)
-            if (table.removed.Count() == _order_rows.Count && !table.added.Any()) {
-               _selected_rows.Clear();
-               _order_rows.DisposeAll();
-               _order_rows.Clear();
-               _order_rows_cache.Clear();
-               _branch_filter_roots.Clear();
-               _parent_to_children.clear();
-               _security_orders.clear();
-               missing_security_orders.clear();
-               Log.Ui.warn("OrderBook received reset message.");
-               return;
-            }
-            var added_securities = new List<SecurityImmutable>(1);
-            var removed_securities = new List<SecurityImmutable>(1);
-
-            foreach (var o in table.removed) {
-               if (_order_rows_cache.TryPopValue(o.Key, out OrderBookRowVM row)) {
-                  row.Dispose();
-                  _order_rows.Remove(row);
-                  if (_security_orders.removes_key(o.Def.SecurityKey, o.Key)) {
-                     if (row.security.value.HasValue)
-                        removed_securities.Add(row.security.value.Value);
-                  }
-                  if (o.has_parent()) {
-                     _parent_to_children.remove(o.ParentKey, o.Key);
-                     if (row.parent != null) {
-                        row.parent.remove_child_order(row);
-                     } else {
-                        _branch_filter_roots.Remove(o.Key);
-                     }
-                  } else {
-                     _branch_filter_roots.Remove(o.Key);
-                  }
-                  // the child_rows will most-likely be removed in this update, but we can't be sure,
-                  // until that happens they must serve as tempororay branch-filter-roots
-                  foreach (var child_row in row.child_orders) {
-                     var bfs = new BranchFilterSource();
-                     child_row.remove_parent(bfs);
-                     _branch_filter_roots.Add(child_row.order.Key, bfs);
-                  }
-                  missing_security_orders.remove(o.Key);
-               }
-            }
-            foreach (var ch in table.changes)
-               _order_rows_cache[ch.New.Key].update_order(ch.New);
-            var orphans = new Dictionary<uint, OrderBookRowVM>();
-            foreach (var o in table.added) {
-               SecurityMD? security_md;
-               if (security_dict.TryGetValue(o.Def.SecurityKey, out SecurityMD smd)) {
-                  security_md = smd;
-                  if (_security_orders.adds_key(o.Def.SecurityKey, o.Key))
-                     added_securities.Add(security_md.Value.security);
-               } else {
-                  missing_security_orders.add(o.Def.SecurityKey, o.Key);
-                  security_md = null;
-                  _security_orders.add(o.Def.SecurityKey, o.Key);
-               }
-               OrderBookRowVM row = new OrderBookRowVM(bus, this, model, o, security_md);
-               if (!o.has_parent()) {
-                  // if it doesn't have a parent, it must be a root; create a new BranchFilterSource
-                  var bfs = new BranchFilterSource();
-                  _branch_filter_roots.Add(o.Key, bfs);
-                  row.join_branch_filter_source(bfs);
-               } else {
-                  _parent_to_children.add(o.ParentKey, o.Key);
-                  if (_order_rows_cache.TryGetValue(o.ParentKey, out OrderBookRowVM pvm)) {
-                     pvm.add_child_order(row);
-                  } else {
-                     // (**) look for orphans' parents again after all new orders have been added to _order_vm_cache
-                     // don't create a temporary BranchFilterSource root just yet. (**)
-                     orphans.Add(o.Key, row);
-                     _order_rows_cache.Add(o.Key, row);
-                     try_add_children(row);
-                     continue;
-                  }
-               }
-               _order_rows_cache.Add(o.Key, row);
-               _order_rows.Add(row);
-               try_add_children(row);
-            }
-            // (**) since all new rows have been added by now, any remaining orphans will be assigned their own branch-filter-roots
-            // (usually, rows are tempororily added to orphans bc their parent is just behind them in the list of added orders) (**)
-            foreach (OrderBookRowVM row in orphans.Values) {
-               var bfs = new BranchFilterSource();
-               row.join_branch_filter_source(bfs);
-               _branch_filter_roots.Add(row.order.Key, bfs);
-               _order_rows.Add(row);
-            }
-            if (added_securities.Count > 0 || removed_securities.Count > 0)
-               order_securities_subject.OnNext((added_securities, removed_securities));
-
-            void try_add_children(OrderBookRowVM row) {
-               if (_parent_to_children.try_lookup(row.order.Key, out ISet<uint> children))
-                  foreach (uint child_key in children) {
-                     OrderBookRowVM child_vm = _order_rows_cache[child_key];
-                     // (**) if children arrived in an earlier update, this will notify them that their parent has arrived (**)
-                     //    - their tempororary branch-filter-roots will be removed, and they will join that of their parent's
-                     //    - for the live-filter-properties, their corresponding cells will be regenerated and they will be re-inserted beneath their new parent.
-                     row.add_child_order(child_vm);
-                     if (orphans.Remove(child_key))
-                        _order_rows.Add(child_vm);
-                     else {
-                        _order_rows.Remove(child_vm);
-                        _order_rows.Add(child_vm);
-                     }
-                  }
-            }
-         }
-
-         // I assume we don't need to worry about orders "losing" or "changing" their security
-         // once the order vm discovers its security (by id), it will never change.
-         void on_securities(IReadOnlyDictionary<uint, SecurityMD> securities) {
-            security_dict = securities;
-            var added_securities = new List<SecurityImmutable>(); // assume securities won't be removed
-            foreach (var missing_sec_orders in missing_security_orders.ToList()) {
-               if (securities.TryGetValue(missing_sec_orders.Key, out SecurityMD smd)) {
-                  missing_security_orders.remove(missing_sec_orders.Key);
-                  added_securities.Add(smd.security);
-                  foreach (var order_key in missing_sec_orders) {
-                     var vm = _order_rows_cache[order_key];
-                     vm.update_security(smd);
-                  }
-               }
-            }
-            if (added_securities.Count > 0)
-               order_securities_subject.OnNext((added_securities, new List<SecurityImmutable>(0)));
-         }
+            Disposable.Create(() => {
+               _columns.filter_changed -= on_filter_changed;
+               Disposable.dispose(_row_collection);
+            }),
+            _columns
+         };
+         void on_filter_changed() => _row_collection.filter = _columns.get_filter();
       }
 
       public event Action<OrderRecordImmutable?> selected_order_changed;
+      public event Action<bool> OnDisposed;
 
-      public OrderBookColumnCollection columns { get; }
-      public ISet<Columns> live_sorting_columns => _live_sorting_columns;
-      public ICollectionView rows_view { get; }
+      public ObservableCollection<DIYGrid.ColumnBase> columns => _columns;
+      public ObservableCollection<OrderBookRowVM> rows => _row_collection;
       public OrderBookRowVM current_row { get => _current_row.Value; set => _current_row.Value = value ?? _selected_rows.FirstOrDefault(); }
       public OrderRecordImmutable? current_order => current_row?.order;
       public IEnumerable<OrderRecordImmutable> selected_orders => _selected_rows.Select(r => r.order);
-
       public _GridSettingsManager grid_settings_manager { get; }
       public ICommand collapse_all_rows_command { get; }
       public ICommand expand_all_rows_command { get; }
@@ -234,29 +76,93 @@ namespace mbts.spark {
       }
 
       public OrderBookStartupData create() {
-         if (_settings_manager.instance_settings == null)
-            return new OrderBookStartupData(columns.create(), use_defaults: true);
+         var (filters, sorting) = _columns.create();
          var gs = grid_settings_manager.create();
-         return new OrderBookStartupData(columns.create(), gs.general, gs.predicate_animations, use_defaults: false);
+         return new OrderBookStartupData(filters, gs.general.with(sorting_criterion: sorting), gs.predicate_animations, use_defaults: _settings_manager.instance_settings == null);
       }
 
       public void load_from(OrderBookStartupData state) {
          if (state == null) return;
          _state = state;
          var gs = state.grid_settings_wrapper();
-         if (gs != null) grid_settings_manager.load_from(gs);
-         columns.load_from(state.filters);
+         if (gs != null) {
+            grid_settings_manager.load_from(gs);
+         }
+         (var old_filter, var old_sort) = _columns.create();
+#pragma warning disable CS0612 // Type or member is obsolete
+         _columns.load_from((state.filters, (state.grid ?? OrderBookStartupData.Default.grid).sorting_criterion));
+#pragma warning restore CS0612 // Type or member is obsolete
+         (var new_filter, var new_sort) = _columns.create();
+         bool reset_filter = !new_filter.Equals(old_filter);
+         bool reset_sorting = !new_sort.Equals(old_sort);
+         if (reset_filter) {
+            if (reset_sorting) {
+               // both filter and sort changed; use defer_refresh to avoid refreshing the collection twice
+               using (_row_collection.defer_refresh()) {
+                  _row_collection.filter = _columns.get_filter();
+                  _row_collection.comparer = _columns.get_sorting_data().comparer;
+                  /// <see cref="OrderBookRowCollection.reset(bool)"/> is an optimization; a way to hedge our bets. Interally it calls Clear(), then AddAll(_visible_rows).
+                  /// Otherwise, <see cref="OrderBookRowCollection.refresh(bool)"/> would be called when we exit/dispose the defer_refresh() scope.
+                  /// The latter method calls sync_with(_visible_rows) internally, which won't perform as well as the former if _visible_rows has changed significantly.
+                  _row_collection.reset(true);
+               }
+            } else {
+               // only filter changed; row_collection will refresh accordingly
+               _row_collection.filter = _columns.get_filter();
+            }
+         } else if (reset_sorting) {
+            // only sorting changed; row_collection will refresh accordingly
+            _row_collection.comparer = _columns.get_sorting_data().comparer;
+         }
       }
-    
-      internal void remove_branch_filter_root(uint key) =>
-         _branch_filter_roots.Remove(key);
 
-      protected override void Dispose(bool disposing) =>
-         Disposable.dispose(_order_rows);
+      public void on_sorting(object sender, DIYGridSortingEventArgs e) {
+         if (e.Column is ICVOrderBookColumn col) {
+            e.Handled = true;
+            _columns.on_sorting(col, !KeyboardExtensions.IsShiftDown());
+            var sorting_data = _columns.get_sorting_data();
+            Log.Ui.info($"{nameof(OrderBookRowVM)} begin resort: {sorting_data}");
+            _row_collection.comparer = sorting_data.comparer;
+         } else {
+            Log.Ui.warn($"{nameof(OrderBookVM)} failed to process sorting event: Column was not of type {nameof(ICVOrderBookColumn)}");
+         }
+      }
+
+      internal void show_order_modify(OrderBookRowVM row) =>
+         _bus.post(SparkTags.ShowModifyOrderTag, row.order_key);
+
+      internal void on_expanded_changed(OrderBookRowVM row) {
+         using (_row_collection.defer_refresh(true)) {
+            if (row.is_expanded) {
+               _on_expanded(row);
+            } else {
+               _on_collapsed(row);
+            }
+         }
+      }
+
+      void _on_expanded(OrderBookRowVM row) {
+         foreach (var node in _row_collection.get_inner_nodes(row)) {
+            node.value.is_visible = true;
+            if (node.value.is_expanded)
+               _on_expanded(node.value);
+         }
+      }
+
+      void _on_collapsed(OrderBookRowVM row) {
+         foreach (var dr in _row_collection.get_descendent_rows(row))
+            dr.is_visible = false;
+      }
+
+      public OrderMetadata get_metadata(OrderBookRowVM row) => _model.factory.get_metadata(row.order_type);
+
+      public void Dispose() {
+         Disposable.dispose(_resources);
+         OnDisposed?.Invoke(true);
+      }
 
       static _GridSettingsManager _create_grid_settings_manager(
-         IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model,
-         ObservableCollection<OrderBookRowVM> rows, ICollectionView rows_view, ObservableCollection<DIYGrid.ColumnBase> columns) {
+         IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model, ObservableCollection<OrderBookRowVM> rows, ObservableCollection<DIYGrid.ColumnBase> columns) {
          // make font-weight immutable for all columns (the font-weight rule is hard-coded: Regular for root-orders, Light for all others)
          // TODO: Remove the immutable font-weight flags when cell animation is ready
          var col_settings_flags = EnumHelper.get_values<Columns>().ToDictionary(
@@ -267,7 +173,7 @@ namespace mbts.spark {
             col_settings_flags[c.ToString()] = GridColumnSettingsFeatureFlags.ImmutableFontColor | GridColumnSettingsFeatureFlags.ImmutableFontWeight;
          }
          return new _GridSettingsManager(
-            owner_title, bus, notification_service, model, rows, rows_view, columns,
+            owner_title, bus, notification_service, model, rows, columns,
             () => {
                var pvm = PredicateBuilderVMWrapper.create<OrderRecordTuple, OrderRecordTuplePredicate, OrderRecordTuplePredicate.PredicateTypeOneofCase>(
                   OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case);
@@ -309,84 +215,5 @@ namespace mbts.spark {
             default: return default;
          }
       }
-
-      internal class BranchFilterSource {
-         readonly HashSet<uint> _orders_pass_filter = new HashSet<uint>();
-
-         public event Action<bool> passes_filter_changed;
-
-         public bool passes_filter => _orders_pass_filter.Count > 0;
-
-         public void set_row_passes_filter(uint key, bool value) {
-            if (value) {
-               if (_orders_pass_filter.Add(key) && _orders_pass_filter.Count == 1)
-                  passes_filter_changed?.Invoke(value);
-            } else if (_orders_pass_filter.Remove(key) && _orders_pass_filter.Count == 0) {
-               passes_filter_changed?.Invoke(value);
-            }
-         }
-      }
-
-      class LiveSortingColumKeys : HashSet<Columns> {
-         public LiveSortingColumKeys(ObservableCollection<string> live_sorting_props) {
-            _add_items(live_sorting_props);
-            live_sorting_props.CollectionChanged += (s, e) => {
-               switch (e.Action) {
-                  case NotifyCollectionChangedAction.Add: _add_items(e.NewItems); break;
-                  case NotifyCollectionChangedAction.Remove: _remove_items(e.OldItems); break;
-                  case NotifyCollectionChangedAction.Replace:
-                     _remove_items(e.OldItems);
-                     _add_items(e.NewItems);
-                     break;
-                  case NotifyCollectionChangedAction.Reset:
-                     Clear();
-                     _add_items(live_sorting_props);
-                     break;
-               }
-            };
-         }
-
-         public static bool try_parse(string prop_name, out Columns result) {
-            switch (prop_name) {
-               case nameof(OrderBookRowVM.security): result = Columns.Security; break;
-               case nameof(OrderBookRowVM.symbol): result = Columns.Symbol; break;
-               case nameof(OrderBookRowVM.time_created): result = Columns.Time; break;
-               case nameof(OrderBookRowVM.last_change): result = Columns.LastChange; break;
-               case nameof(OrderBookRowVM.user): result = Columns.User; break;
-               case nameof(OrderBookRowVM.account): result = Columns.Account; break;
-               case nameof(OrderBookRowVM.exchange): result = Columns.Exchange; break;
-               case nameof(OrderBookRowVM.time_in_force): result = Columns.TimeInForce; break;
-               case nameof(OrderBookRowVM.order_type): result = Columns.OrderType; break;
-               case nameof(OrderBookRowVM.order_sub_type): result = Columns.SubType; break;
-               case nameof(OrderBookRowVM.side): result = Columns.Side; break;
-               case nameof(OrderBookRowVM.quantity): result = Columns.Size; break;
-               case nameof(OrderBookRowVM.filled_quantity): result = Columns.Filled; break;
-               case nameof(OrderBookRowVM.price): result = Columns.Price; break;
-               case nameof(OrderBookRowVM.avg_price): result = Columns.AvgFilled; break;
-               case nameof(OrderBookRowVM.last_price): result = Columns.LastPrice; break;
-               case nameof(OrderBookRowVM.status): result = Columns.Status; break;
-               case nameof(OrderBookRowVM.last_msg): result = Columns.LastMessage; break;
-               case nameof(OrderBookRowVM.security_key): result = Columns.SecurityKey; break;
-               case nameof(OrderBookRowVM.order_key): result = Columns.OrderKey; break;
-               case nameof(OrderBookRowVM.parent_order_key): result = Columns.ParentKey; break;
-               default: result = default; return false;
-            }
-            return true;
-         }
-
-         void _add_items(IList prop_names) {
-            foreach (string pn in prop_names) {
-               if (try_parse(pn, out Columns c))
-                  Add(c);
-            }
-         }
-
-         void _remove_items(IList prop_names) {
-            foreach (string pn in prop_names) {
-               if (try_parse(pn, out Columns c))
-                  Remove(c);
-            }
-         }
-      }
    }
-}
+}

src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml

@@ -48,7 +48,7 @@
           <MultiDataTrigger>
             <MultiDataTrigger.Conditions>
               <Condition Binding="{Binding IsSelected, RelativeSource={RelativeSource Self}}" Value="False"/>
-              <Condition Binding="{Binding status.value}" Value="{uc:Enum Working, Type=order:OrderStatus}"/>
+              <Condition Binding="{Binding status}" Value="{uc:Enum Working, Type=order:OrderStatus}"/>
             </MultiDataTrigger.Conditions>
             <MultiDataTrigger.Setters>
               <Setter Property="Background" Value="{x:Static Member=working_order_background_brush, MemberType={uc:Nested uc:BTSColors.Orders}}"/>
@@ -62,12 +62,17 @@
     </ResourceDictionary>
   </UserControl.Resources>
   <AdornerDecorator>
-    <c:DIYGrid
+    <Grid>
+      <Grid.RowDefinitions>
+        <RowDefinition />
+        <RowDefinition Height="Auto"/>
+      </Grid.RowDefinitions>
+      <c:DIYGrid
       x:Name="_grid"
-      Grid.Row="2"
+      Grid.Row="0"
       Columns="{Binding columns}"
       CurrentRow="{Binding current_row, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}"
-      Rows="{Binding rows_view}"
+      Rows="{Binding rows}"
       RowStyle="{StaticResource {x:Static sp:OrderBookView.RowStyleKey}}"
       RowTemplate="{StaticResource {x:Static sp:OrderBookView.RowTemplateKey}}"
       SelectionHighlight="{Binding grid_settings_manager.selection_highlight, Mode=OneWay}"
@@ -148,5 +153,22 @@
         />
       </c:DIYGrid.CommandBindings>
     </c:DIYGrid>
+      <StatusBar Grid.Row="1" DockPanel.Dock="Bottom">
+        <StatusBarItem HorizontalAlignment="Right">
+          <Grid>
+            <Grid.ColumnDefinitions>
+              <ColumnDefinition />
+              <ColumnDefinition />
+              <ColumnDefinition />
+              <ColumnDefinition />
+            </Grid.ColumnDefinitions>
+            <TextBlock Grid.Column="0" Text="Orders: "/>
+            <TextBlock Grid.Column="1" Text="{Binding ElementName=_grid, Path=Rows.Count}"/>
+            <TextBlock Grid.Column="2" Text=" of "/>
+            <TextBlock Grid.Column="3" Text="{Binding rows_view.SourceCollection.Count}"/>
+          </Grid>
+        </StatusBarItem>
+      </StatusBar>
+    </Grid>
   </AdornerDecorator>
 </UserControl>

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -121,7 +121,7 @@ namespace mbts.spark {
             _vm.expand_all_rows_command);
 
          _show_grid_settings_command = SparkCommands.window_settings_command(new DialogCommand(() => {
-            var vm = new WindowSettingsDialogVM<OrderBookGridSettings, OrderBookGridSettingsVM>(
+            var gs_vm = new WindowSettingsDialogVM<OrderBookGridSettings, OrderBookGridSettingsVM>(
                _settings_manager,
                us => us.order_book_grid,
                UserSettings.FileExtensions.GRID,
@@ -137,11 +137,11 @@ namespace mbts.spark {
                }
             );
             // TODO: bind to main window's Title
-            var dialog = new SimpleGridSettingsDialog { DataContext = vm };
-            dialog.SetBinding(Window.TitleProperty, new Binding($"{nameof(vm.instance_vm)}.{nameof(OrderBookGridSettingsVM.owner_title)}") {
+            var dialog = new SimpleGridSettingsDialog { DataContext = gs_vm };
+            dialog.SetBinding(Window.TitleProperty, new Binding($"{nameof(gs_vm.instance_vm)}.{nameof(OrderBookGridSettingsVM.owner_title)}") {
                Converter = ValueConverter.create<string, string>(str => (string.IsNullOrEmpty(str) ? _default_title : str) + " Settings")
             });
-            vm.RequestClose += (s, e) => WindowUtilities.force_close(dialog);
+            gs_vm.RequestClose += (s, e) => WindowUtilities.force_close(dialog);
             return dialog;
          }));
 

src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs

@@ -12,7 +12,6 @@ using mbts.ui.settings;
 using ProtoBuf;
 using System;
 using System.Collections.Generic;
-using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
 using System.Windows;
@@ -37,7 +36,8 @@ namespace bts.shell {
 
       public override Task<IShellWindow> load(OrderBookStartupData state) {
          var settings_manager = new WindowSettingsManager<OrderBookGridSettings>(_settings_hub, state.grid_settings_wrapper(),
-            us => us.order_book_grid, (us, s) => us.with(order_book_grid: s));
+            us => us.order_book_grid ?? OrderBookStartupData.DefaultGrid,
+            (us, s) => us.with(order_book_grid: s));
          var w = new OrderBookWindow(_model, this, _bus, _notification_service, state, SparkCommands.ShowOrders.icon, settings_manager.as_non_shared());
          return Task.FromResult<IShellWindow>(w);
       }
@@ -149,7 +149,7 @@ namespace bts.shell {
       }
 
       [ProtoContract]
-      public class FilterStates {
+      public class FilterStates : IEquatable<FilterStates> {
          [ProtoMember(1)]
          public SecurityCriteria security_criteria;
          [ProtoMember(2)]
@@ -190,6 +190,28 @@ namespace bts.shell {
          public api.orderman.OrderSubType[] order_sub_types;
          [ProtoMember(20)]
          public string[] symbols;
+
+         public bool Equals(FilterStates other) =>
+            other != null &&
+            security_criteria.safe_equals(other.security_criteria) &&
+            order_key_regex == other.order_key_regex &&
+            time_created.Equals(other.time_created) &&
+            EqualityComparer.safe_sequence_equals(users, other.users) &&
+            EqualityComparer.safe_sequence_equals(accounts, other.accounts) &&
+            EqualityComparer.safe_sequence_equals(exchanges, other.exchanges) &&
+            EqualityComparer.safe_sequence_equals(tifs, other.tifs) &&
+            EqualityComparer.safe_sequence_equals(order_types, other.order_types) &&
+            EqualityComparer.safe_sequence_equals(sides, other.sides) &&
+            qty.safe_equals(other.qty) &&
+            price.safe_equals(other.price) &&
+            qty_filled.safe_equals(other.qty_filled) &&
+            avg_price.safe_equals(other.avg_price) &&
+            last_price.safe_equals(other.last_price) &&
+            EqualityComparer.safe_sequence_equals(statuses, other.statuses) &&
+            exchange_message_regex == other.exchange_message_regex &&
+            last_change.Equals(other.last_change) &&
+            EqualityComparer.safe_sequence_equals(order_sub_types, other.order_sub_types) &&
+            EqualityComparer.safe_sequence_equals(symbols, other.symbols);
       }
 
       [ProtoContract]
@@ -225,4 +247,4 @@ namespace bts.shell {
                EqualityComparer.safe_aggregate_hash(predicate_animations));
       }
    }
-}
+}

src/client/spark/Orders/OrderModificationControl.xaml.cs

@@ -24,7 +24,8 @@ namespace mbts.ui.spark {
       public DataTemplate LeanOrderTemplate { get; set; }
       public DataTemplate JoinOrderTemplate { get; set; }
       public DataTemplate TopOrderTemplate { get; set; }
-      public DataTemplate CoverOrderTemplate { get; set; }      
+      public DataTemplate CoverOrderTemplate { get; set; }
+      public DataTemplate OffsetOrderTemplate { get; set; }
       public DataTemplate TriggerOrderTemplate { get; set; }
       public DataTemplate IcebergOrderTemplate { get; set; }
       public DataTemplate SpreaderTemplate { get; set; }
@@ -42,6 +43,7 @@ namespace mbts.ui.spark {
                case CustomOrderDefinition.CustomOneofCase.Join: return JoinOrderTemplate;
                case CustomOrderDefinition.CustomOneofCase.Top: return TopOrderTemplate;
                case CustomOrderDefinition.CustomOneofCase.Cover: return CoverOrderTemplate;
+               case CustomOrderDefinition.CustomOneofCase.Offset: return OffsetOrderTemplate;
                case CustomOrderDefinition.CustomOneofCase.Trigger: return TriggerOrderTemplate;
                case CustomOrderDefinition.CustomOneofCase.Iceberg: return IcebergOrderTemplate;
                case CustomOrderDefinition.CustomOneofCase.Spreader: return SpreaderTemplate;
@@ -51,4 +53,5 @@ namespace mbts.ui.spark {
          return null;
       }
    }
+
 }

src/client/spark/Orders/OrderRecordTuple.cs

@@ -42,5 +42,6 @@ namespace mbts.spark {
 
       public override bool Equals(object obj) => Equals(obj as OrderRecordTuple);
       public override int GetHashCode() => key.GetHashCode();
+      public override string ToString() => $"order:{order.Key}, sec:{security_md.display_name()}";
    }
 }

src/client/spark/Orders/OrderRecordTupleSource.cs

@@ -7,80 +7,94 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
-using System.Threading;
 
 namespace mbts.spark {
-   class OrderRecordTupleSource : IObservable<IIncrementalUpdate<OrderRecordTuple, uint>>, IDisposable {
-      // returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] OrderRecordTupleSource.
-      // (initializes when the first subscription is made, and destroys when the last subscription is disposed).
-      public static IObservable<IIncrementalUpdate<OrderRecordTuple, uint>> create_observable(
-         IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities,
-         IObservable<TableUpdate<OrderRecordImmutable, uint>> orders) {
-         OrderRecordTupleSource source = null;
-         int ref_count = 0;
-         return Observable.Create<IIncrementalUpdate<OrderRecordTuple, uint>>(observer => {
-            if (Interlocked.Increment(ref ref_count) == 1) {
-               SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak."));
-               source = new OrderRecordTupleSource(securities, orders);
-            }
-            IDisposable sub = source.Subscribe(observer);
-            return Disposable.Create(() => {
-               Disposable.dispose_once(ref sub);
-               if (Interlocked.Decrement(ref ref_count) == 0)
-                  Disposable.dispose_once(ref source);
-            });
-         });
-         string fmt_assert_msg(string msg) => $"{nameof(OrderRecordTupleSource)}.{nameof(create_observable)}: {msg}";
-      }
-
+   class OrderRecordTupleSource : IDisposable {
+      readonly object _lock = new object();
+      // subjects
       readonly Subject<IIncrementalUpdate<OrderRecordTuple, uint>> _subject = new Subject<IIncrementalUpdate<OrderRecordTuple, uint>>();
+      readonly Subject<IIncrementalTableUpdateTransform<OrderRecordTuple, uint>> _table_subject = new Subject<IIncrementalTableUpdateTransform<OrderRecordTuple, uint>>();
+      readonly Subject<IIncrementalUpdate<SecurityMD, uint>> _order_securities_subject = new Subject<IIncrementalUpdate<SecurityMD, uint>>();
+      // data cache
       readonly Dictionary<uint, OrderRecordTuple> _state = new Dictionary<uint, OrderRecordTuple>();
       readonly SetMap<uint, uint> _security_orders = new SetMap<uint, uint>();
       IReadOnlyDictionary<uint, OrderRecordImmutable> _orders_cache = new Dictionary<uint, OrderRecordImmutable>(0);
       IReadOnlyDictionary<uint, SecurityMD> _security_md_cache = new Dictionary<uint, SecurityMD>(0);
       IDisposable _dispose;
-      object _lock = new object();
 
-      OrderRecordTupleSource(IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities, IObservable<TableUpdate<OrderRecordImmutable, uint>> orders) =>
+      public OrderRecordTupleSource(IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities, IObservable<TableUpdate<OrderRecordImmutable, uint>> orders) =>
          _dispose = Disposable.Create(
             securities.Subscribe(_update_securities),
-            orders.AddRemoveOnly().Subscribe(_update_orders));
+            orders.Subscribe(_update_orders));
 
-      public IDisposable Subscribe(IObserver<IIncrementalUpdate<OrderRecordTuple, uint>> observer) {
-         lock (_lock) {
-            observer.OnNext(new IncrementalUpdate<OrderRecordTuple, uint>(new Dictionary<uint, OrderRecordTuple>(_state), new Dictionary<uint, OrderRecordTuple>(0)));
-            return _subject.Subscribe(observer);
-         }
-      }
+      public IObservable<IIncrementalUpdate<OrderRecordTuple, uint>> subscribe_incremental_update() =>
+         Observable.Create<IIncrementalUpdate<OrderRecordTuple, uint>>(observer => {
+            lock (_lock) {
+               observer.OnNext(new IncrementalUpdate<OrderRecordTuple, uint>(new Dictionary<uint, OrderRecordTuple>(_state), new Dictionary<uint, OrderRecordTuple>(0)));
+               return _subject.Subscribe(observer);
+            }
+         });
+
+      public IObservable<IIncrementalTableUpdateTransform<OrderRecordTuple, uint>> subscribe_incremental_table_update() =>
+         Observable.Create<IIncrementalTableUpdateTransform<OrderRecordTuple, uint>>(observer => {
+            lock (_lock) {
+               observer.OnNext(IncrementalTableUpdateTranform<OrderRecordTuple, uint>.init(_state.Values));
+               return _table_subject.Subscribe(observer);
+            }
+         });
+
+      public IObservable<IIncrementalUpdate<SecurityMD, uint>> subscribe_order_securities() =>
+         Observable.Create<IIncrementalUpdate<SecurityMD, uint>>(observer => {
+            lock (_lock) {
+               var init = new Dictionary<uint, SecurityMD>();
+               foreach (uint sec_key in _security_orders.Keys) {
+                  if (_security_md_cache.TryGetValue(sec_key, out SecurityMD smd))
+                     init.Add(sec_key, smd);
+               }
+               observer.OnNext(new IncrementalUpdate<SecurityMD, uint>(init, new Dictionary<uint, SecurityMD>(0)));
+               return _order_securities_subject.Subscribe(observer);
+            }
+         });
 
       public void Dispose() => Disposable.dispose(ref _dispose);
 
-      void _update_orders(LegacyTableUpdate<OrderRecordImmutable, uint> orders) {
+      void _update_orders(TableUpdate<OrderRecordImmutable, uint> orders) {
          lock (_lock) {
             Dictionary<uint, OrderRecordTuple> removed = new Dictionary<uint, OrderRecordTuple>(), added = new Dictionary<uint, OrderRecordTuple>();
+            Dictionary<uint, SecurityMD> added_securities = new Dictionary<uint, SecurityMD>(), removed_securities = new Dictionary<uint, SecurityMD>();
             foreach (OrderRecordImmutable order in orders.removed) {
                uint sec_key = order.security_key();
                OrderRecordTuple old_ft = new OrderRecordTuple(order, _get_security_md(sec_key));
                removed.Add(old_ft.order.Key, old_ft);
-               _security_orders.remove(sec_key, order.Key);
+               if (_security_orders.removes_key(sec_key, order.Key) && old_ft.security_md.HasValue)
+                  removed_securities.Add(sec_key, old_ft.security_md.Value);
+            }
+            foreach (Change<OrderRecordImmutable> ch in orders.changes) {
+               // orders cannot change securities; ch.New can use the same security_md as ch.Old
+               var sec_md = _get_security_md(ch.Old.security_key());
+               removed.Add(ch.Old.Key, new OrderRecordTuple(ch.Old, sec_md));
+               added.Add(ch.New.Key, new OrderRecordTuple(ch.New, sec_md));
             }
             foreach (OrderRecordImmutable order in orders.added) {
                uint sec_key = order.security_key();
                OrderRecordTuple new_ort = new OrderRecordTuple(order, _get_security_md(sec_key));
                added.Add(new_ort.order.Key, new_ort);
-               _security_orders.add(sec_key, order.Key);
+               if (_security_orders.adds_key(sec_key, order.Key) && new_ort.security_md.HasValue)
+                  added_securities.Add(sec_key, new_ort.security_md.Value);
             }
             _orders_cache = orders.state;
-            _apply_changes(added, removed);
+            _apply_changes(added, removed, added_securities, removed_securities);
          }
       }
 
       void _update_securities(IReadOnlyDictionary<uint, SecurityMD> securities) {
          lock (_lock) {
             Dictionary<uint, OrderRecordTuple> added = new Dictionary<uint, OrderRecordTuple>(), removed = new Dictionary<uint, OrderRecordTuple>();
+            Dictionary<uint, SecurityMD> added_securities = new Dictionary<uint, SecurityMD>(), removed_securities = new Dictionary<uint, SecurityMD>();
             _security_md_cache.Keys.difference(securities.Keys, out IEnumerable<uint> added_keys, out IEnumerable<uint> removed_keys);
             foreach (var k in removed_keys) {
                if (_security_orders.try_lookup(k, out ISet<uint> order_keys)) {
+                  removed_securities.Add(k, _security_md_cache[k]);
                   IEnumerable<OrderRecordImmutable> orders = order_keys.Select(fk => _orders_cache[fk]);
                   SecurityMD old_sec_md = _security_md_cache[k];
                   foreach (OrderRecordImmutable order in orders) {
@@ -93,6 +107,7 @@ namespace mbts.spark {
             }
             foreach (var k in added_keys) {
                if (_security_orders.try_lookup(k, out ISet<uint> order_keys)) {
+                  added_securities.Add(k, securities[k]);
                   IEnumerable<OrderRecordImmutable> orders = order_keys.Select(fk => _orders_cache[fk]);
                   SecurityMD new_sec_md = securities[k];
                   foreach (OrderRecordImmutable order in orders) {
@@ -104,7 +119,7 @@ namespace mbts.spark {
                }
             }
             _security_md_cache = securities;
-            _apply_changes(added, removed);
+            _apply_changes(added, removed, added_securities, removed_securities);
          }
       }
 
@@ -114,12 +129,15 @@ namespace mbts.spark {
       OrderRecordImmutable? _get_order(uint key) =>
          _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?();
 
-      void _apply_changes(Dictionary<uint, OrderRecordTuple> added, Dictionary<uint, OrderRecordTuple> removed) {
+      void _apply_changes(Dictionary<uint, OrderRecordTuple> added, Dictionary<uint, OrderRecordTuple> removed, Dictionary<uint, SecurityMD> added_securities, Dictionary<uint, SecurityMD> removed_securities) {
          foreach (var kv in removed)
             _state.Remove(kv.Key);
          foreach (var kv in added)
             _state.Add(kv.Key, kv.Value);
-         _subject.OnNext(new IncrementalUpdate<OrderRecordTuple, uint>(added, removed));
+         var update = new IncrementalUpdate<OrderRecordTuple, uint>(added, removed);
+         _subject.OnNext(update);
+         _table_subject.OnNext(update.as_table());
+         _order_securities_subject.OnNext(new IncrementalUpdate<SecurityMD, uint>(added_securities, removed_securities));
       }
    }
 }

src/client/spark/Orders/OrderTypeVMs/DynamicOrderHelper.cs

@@ -0,0 +1,54 @@
+using bts.api;
+using bts.api.orderman;
+using mbts.ui;
+using mbts.ui.utility;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.ComponentModel;
+using System.ComponentModel.Composition;
+using System.Linq;
+using System.Reactive.Linq;
+using static bts.api.orderman.CustomOrderDefinition;
+
+namespace mbts.spark
+{
+   [Export]   
+   public class DynamicOrderHelper
+   {
+      readonly RegisteredDynamicOrderTypes _dynamic_types;
+      [ImportingConstructor]
+      private DynamicOrderHelper(OrderService order_service)
+      {
+         _dynamic_types = order_service.get_registered_dynamic_order_types(new VoidMessage());
+      }
+
+      public void add_dynamic_order_defs(List<CustomOrderDefinitionImmutable> defs,
+                                         Func<CustomOrderDefinition.CustomOneofCase, CustomOrderDefinition> factory)
+      {
+         foreach (var d in _dynamic_types.DynamicOrderDesc)
+         {
+            var dorder = factory(CustomOrderDefinition.CustomOneofCase.Dynamic);
+            dorder.Dynamic.DynamicOrderType = d.DynamicOrderType;
+            //Add only the descriptor here for reflection later
+            foreach(var p in d.ParamDesc)
+               dorder.Dynamic.OrderParams.Add( p.ParamName, new DynamicOrderParam() );
+            defs.Add(new CustomOrderDefinitionImmutable(dorder));
+         }
+      }
+
+      internal IDictionary<string, DynamicOrderParamVM> get_parameters(string dyntype)
+      {
+         Dictionary<string, DynamicOrderParamVM> par = new Dictionary<string, DynamicOrderParamVM>();
+         var order = _dynamic_types.DynamicOrderDesc.FirstOrDefault(x => x.DynamicOrderType == dyntype);
+         if (order != null)
+         {
+            foreach (var p in order.ParamDesc)
+               par[p.ParamName] = new DynamicOrderParamVM() { param = new DynamicOrderParam(), descriptor = p };
+         }
+         return par;
+      }
+   }
+
+}

src/client/spark/Orders/OrderTypeVMs/DynamicOrderParamVM.cs

@@ -0,0 +1,21 @@
+using bts.api;
+using bts.api.orderman;
+using mbts.ui;
+using mbts.ui.utility;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+
+
+namespace mbts.spark
+{
+
+   class DynamicOrderParamVM : mbts.ui.shell.ViewModelBase
+   {
+      public DynamicOrderParamVM() { }
+      public DynamicOrderParamDescriptor descriptor { get; set; }
+      public DynamicOrderParam param { get; set; }
+
+   }
+
+}

src/client/spark/Orders/OrderTypeVMs/DynamicOrderTemplateSelector.cs

@@ -0,0 +1,51 @@
+using bts.api.orderman;
+using mbts.spark;
+using mbts.ui.converters;
+using mbts.ui.utilities;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Data;
+
+namespace mbts.ui.spark
+{
+   class DynamicOrderParamTemplateSelector : DataTemplateSelector
+   {
+      public DataTemplate BoolTemplate { get; set; }
+      public DataTemplate DoubleTemplate { get; set; }
+      public DataTemplate IntTemplate { get; set; }
+      public DataTemplate UIntTemplate { get; set; }
+      public DataTemplate StringTemplate { get; set; }
+      public DataTemplate OrderDefTemplate { get; set; }
+
+      public DataTemplate EnumTemplate { get; set; }
+
+      public override DataTemplate SelectTemplate(object item, DependencyObject container)
+      {
+         var vm = item as DynamicOrderParamVM;
+         if (vm != null)
+         {
+            switch (vm.descriptor.ParamType.DescriptorCase)
+            {
+               case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+                  {
+                     switch (vm.descriptor.ParamType.Primitive)
+                     {
+                        case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: return BoolTemplate;
+                        case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: return DoubleTemplate;
+                        case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: return IntTemplate;
+                        case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: return UIntTemplate;
+                        case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: return StringTemplate;
+                        case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: return OrderDefTemplate;
+                        default: return null;
+                     }
+                  }
+               case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated: return EnumTemplate;
+            }
+         }
+         return null;
+      }
+   }   
+}

src/client/spark/Orders/OrderTypeVMs/DynamicOrderVM.cs

@@ -7,47 +7,13 @@ using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.ComponentModel;
+using System.ComponentModel.Composition;
 using System.Linq;
 using System.Reactive.Linq;
 using static bts.api.orderman.CustomOrderDefinition;
 
 namespace mbts.spark
 {
-   public static class DynamicOrderHelper
-   {
-      static Lazy<bts.config.dynamic_orders.DynamicOrderList> _list = new Lazy<bts.config.dynamic_orders.DynamicOrderList>(
-         () => mbts.config.Configuration.get<bts.config.dynamic_orders.DynamicOrderList>(), true);
-      public static void add_dynamic_order_defs(List<CustomOrderDefinitionImmutable> defs, 
-                                                Func<CustomOrderDefinition.CustomOneofCase, CustomOrderDefinition> factory)
-      {
-         var dyntypes = _list.Value;
-         foreach (var d in dyntypes.dynamic_orders)
-         {
-            var dorder = factory(CustomOrderDefinition.CustomOneofCase.Dynamic);
-            dorder.Dynamic.DynamicOrderType = d.type_name;
-            foreach (var p in d.parameter_keys)
-               dorder.Dynamic.Parameters.Add(p, string.Empty);
-            defs.Add(new CustomOrderDefinitionImmutable(dorder));
-         }
-      }
-
-      internal static List<NameValueVM> get_parameters(string dyntype)
-      {
-         List<NameValueVM> par = new List<NameValueVM>();
-         var order = _list.Value.dynamic_orders.FirstOrDefault(x => x.type_name == dyntype);
-         if (order != null)
-            par.AddRange(order.parameter_keys.Select(x => new NameValueVM() { name = x, value = string.Empty }));
-         return par;
-      }
-   }
-
-
-   class NameValueVM : mbts.ui.shell.ViewModelBase
-   {
-      public NameValueVM() { }
-      public string name { get; set; }
-      public string value { get; set; }
-   }
 
    static partial class OrderTypeVMFactory
    {
@@ -75,7 +41,8 @@ namespace mbts.spark
          IDisposable _dispose, _child_order_errors_sub;
 
          string _dyn_type;
-         List<NameValueVM> _parameters;
+         IDictionary<string, DynamicOrderParamVM> _parameters;
+         IEnumerable<string> _order_def_tags;
 
          public DynamicOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source, string dyn_type, string id, string display_name)
             : this(model, security_md_source, dyn_type, security_md_source.Select(smd =>
@@ -87,7 +54,9 @@ namespace mbts.spark
          {
             _security_md_source = security_md_source;
             _dyn_type = dyn_type;
-            _parameters = DynamicOrderHelper.get_parameters(dyn_type);
+            _parameters = model.dynamic_helper.get_parameters(dyn_type);
+            _order_def_tags = _parameters.Values.Where(x => x.descriptor.ParamType.DescriptorCase == DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive &&
+                                                       x.descriptor.ParamType.Primitive == DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef).Select(xx=>xx.descriptor.ParamName);
             _dispose = Disposable.Create(
                security_md_source.OnDispatcher()
                .Subscribe(smd => _security = smd.security),
@@ -111,7 +80,12 @@ namespace mbts.spark
 
          public override string type_name => _dyn_type;
 
-         public IEnumerable<NameValueVM> parameters => _parameters;
+         public IEnumerable<DynamicOrderParamVM> parameters => _parameters.Values.Where(x =>
+        {
+           return x.descriptor.ParamType.DescriptorCase == DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated ||
+                  (x.descriptor.ParamType.DescriptorCase == DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive &&
+                   x.descriptor.ParamType.Primitive != DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef);
+        });
 
          public override void assign_to(CustomOrderDefinition value)
          {
@@ -122,16 +96,25 @@ namespace mbts.spark
             {
                var i = factory.dynamic_order();
                i.DynamicOrderType = _dyn_type;
-               set_parameter_dictionary(i.Parameters);
-               child_order.assign_to(i.Child);
+               set_parameter_dictionary(i.OrderParams);
                value.Dynamic = i;
             }
          }
 
-         void set_parameter_dictionary(Google.Protobuf.Collections.MapField<string, string> par)
+         void set_parameter_dictionary(Google.Protobuf.Collections.MapField<string, DynamicOrderParam> par)
          {
-            foreach(var vm in _parameters)
-               par[vm.name] = vm.value ?? string.Empty;
+            foreach(var vm in _parameters.Values)
+               par[vm.descriptor.ParamName] = vm.param;
+
+            if (child_order != null)
+            {
+               var newkid = new DynamicOrderParam();
+               var cust = new CustomOrderDefinition();
+               newkid.AsOrderDef = cust;               
+               child_order.assign_to(newkid.AsOrderDef);
+               foreach(var child in _order_def_tags)
+                  par[child] = newkid;
+            }
          }
 
          public ICollection<IOrderTypeVM> child_order_types
@@ -204,8 +187,7 @@ namespace mbts.spark
             if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors))
                set_error(nameof(child_order), string.Empty, true);
             else clear_error(nameof(child_order));
-
          }
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/JoinOrderVM.cs

@@ -22,33 +22,14 @@ namespace mbts.spark
             ExceptionAssert.InvalidOperation.That(child_order_types.Contains(selected_child_order_type), $"Child order type {selected_child_order_type} is not in child_order_types");
             this.selected_child_order_type = new InlineUserProperty<OrderTemplateHeader>(selected_child_order_type, true);
             this.child_order_types = new InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>>(child_order_types, true);
-
-            var hedgetype = CustomOneofCase.Lean;
-            if(!child_order_types.Any(t => t.key.type == hedgetype))
-               hedgetype = CustomOneofCase.Limit;
-            
-            //ExceptionAssert.InvalidOperation.That(child_order_types.Contains(hedgetype), $"Hedge order type {hedgetype} is not in hedge_order_types");
-            this.selected_hedge_order_type = new InlineUserProperty<OrderTemplateHeader>(new OrderTemplateHeader(hedgetype), true);
-            this.hedge_order_types = new InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>>(child_order_types, true);
-
-            tick_offset_on_deploy = new InlineUserProperty<uint>(0, false);
+         
             join_threshold = new InlineUserProperty<bts.spark.QuantityMeasure>(
                               new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1}, false);
-            enable_hedge = new InlineUserProperty<bool>(false, false);
-            hedge_ticks = new InlineUserProperty<int>(1, false);
          }
          public readonly InlineUserProperty<OrderTemplateHeader> selected_child_order_type;
          public readonly InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>> child_order_types;
-         public readonly InlineUserProperty<OrderTemplateHeader> selected_hedge_order_type;
-         public readonly InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>> hedge_order_types;
-
-
-         public readonly InlineUserProperty<uint> tick_offset_on_deploy;
-         public readonly InlineUserProperty<bts.spark.QuantityMeasure> join_threshold;
-                  
-         public readonly InlineUserProperty<bool> enable_hedge;
-         public readonly InlineUserProperty<int> hedge_ticks;
 
+         public readonly InlineUserProperty<bts.spark.QuantityMeasure> join_threshold;               
       }
 
       class JoinOrderVM : OrderTypeVMBase<JoinOrderDefinition>, IHierarchicalOrderTypeVM, IDisposable 
@@ -61,23 +42,7 @@ namespace mbts.spark
          IOrderTypeVM _child_order;
          IDisposable _dispose, _child_order_errors_sub;
 
-         ObservableAutoWrapper<IOrderTypeVM, OrderTemplateHeader> _hedge_order_choices;
-         IOrderTypeVM _hedge_order;
-         IDisposable _hedge_order_errors_sub;
-
-         public uint tick_offset_on_deploy { get; set; } = 0;
          public bts.spark.QuantityMeasure join_threshold { get; set; } = new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 };
-         bool _enable_hedge = false;
-         public bool enable_hedge
-         {
-            get { return _enable_hedge; }
-            set
-            {
-               _enable_hedge = value;
-               RaisePropertyChanged(nameof(enable_hedge));
-            }
-         } 
-         public int hedge_ticks { get; set; } = 1;
 
          public JoinOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name)
             : this(model, security_md_source, security_md_source.Select(smd =>
@@ -100,30 +65,11 @@ namespace mbts.spark
                   {
                      _select_child_order(p.selected_child_order_type.value.key.type);
                   }
-                  if (_hedge_order_choices != null && (_hedge_order == null || p.selected_hedge_order_type.force_override))
-                  {
-                     _select_hedge_order(p.selected_hedge_order_type.value.key.type);
-                  }
-                  if (p.tick_offset_on_deploy.force_override) 
-                  {
-                     tick_offset_on_deploy = p.tick_offset_on_deploy.value;
-                     RaisePropertyChanged(nameof(tick_offset_on_deploy));
-                  }
                   if (p.join_threshold.force_override)
                   {
                      join_threshold = p.join_threshold.value;
                      RaisePropertyChanged(nameof(join_threshold));
                   }
-                  if (p.enable_hedge.force_override)
-                  {
-                     enable_hedge = p.enable_hedge.value;
-                     RaisePropertyChanged(nameof(enable_hedge));
-                  }
-                  if (p.hedge_ticks.force_override)
-                  {
-                     hedge_ticks = p.hedge_ticks.value;
-                     RaisePropertyChanged(nameof(hedge_ticks));
-                  }
                   _inline_props = p;
                   _validate();
                })
@@ -135,19 +81,12 @@ namespace mbts.spark
          {
             if (_child_order == null)
                _select_child_order(model.factory.default_order_type(_security.Value, true));
-            if (_hedge_order == null && enable_hedge)
-               _select_hedge_order(model.factory.default_order_type(_security.Value, true));
             _validate();
             if (!HasErrors) 
             {
                var i = factory.join_order();
-               i.TickOffsetOnDeploy = tick_offset_on_deploy;
                i.JoinThreshold = join_threshold;
-               i.EnableHedge = enable_hedge;
-               i.HedgeTicks = hedge_ticks;
                child_order.assign_to(i.Child);
-               if(enable_hedge && hedge_order != null)
-                  hedge_order.assign_to(i.Hedge);
                value.Join = i;
             }
          }         
@@ -161,16 +100,6 @@ namespace mbts.spark
                return _child_order_choices;
             }
          }
-
-         public ICollection<IOrderTypeVM> hedge_order_types
-         {
-            get
-            {
-               if (_hedge_order_choices == null)
-                  _hedge_order_choices = new ObservableAutoWrapper<IOrderTypeVM, OrderTemplateHeader>(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source));
-               return _hedge_order_choices;
-            }
-         }         
  
          public IOrderTypeVM child_order 
          {
@@ -196,45 +125,17 @@ namespace mbts.spark
             }
          }
 
-         public IOrderTypeVM hedge_order
-         {
-            get
-            {
-               init();
-               return _hedge_order;
-            }
-            set
-            {
-               Disposable.dispose(ref _hedge_order_errors_sub);
-               _hedge_order = value;
-               _hedge_order?.init();
-               if (_hedge_order is INotifyDataErrorInfo ndei)
-               {
-                  ndei.ErrorsChanged += on_errors_changed;
-                  void on_errors_changed(object s, DataErrorsChangedEventArgs e) =>
-                     _validate();
-                  _hedge_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed);
-               }
-               _validate();
-               RaisePropertyChanged(nameof(hedge_order));
-            }
-         }
-
          public void Dispose() 
          {
             Disposable.dispose(ref _dispose);
             if (_child_order_choices != null)
                _child_order_choices.OfType<IDisposable>().DisposeAll();
-            if (_hedge_order_choices != null)
-               _hedge_order_choices.OfType<IDisposable>().DisposeAll();
          }
 
          public override void init() 
          {
             if (_child_order == null && _security.HasValue)
                _select_child_order();
-            if (_hedge_order == null && _security.HasValue && enable_hedge)
-               _select_hedge_order();
             _validate();
          }
 
@@ -252,35 +153,13 @@ namespace mbts.spark
             if (_child_order == null)
                _child_order = child_order_types.FirstOrDefault();
             RaisePropertyChanged(nameof(child_order));
-         }
-         
-         void _select_hedge_order()
-         {
-            CustomOneofCase ot = _inline_props != null && _inline_props.selected_hedge_order_type.has_value ?
-               _inline_props.selected_hedge_order_type.value.key.type :
-               model.factory.default_order_type(_security.Value, true);
-            _select_hedge_order(ot);
-         }
-
-         void _select_hedge_order(CustomOneofCase ot)
-         {
-            _hedge_order = hedge_order_types.FirstOrDefault(vm => vm.order_type == ot);
-            if (_hedge_order == null)
-               _hedge_order = hedge_order_types.FirstOrDefault();
-            RaisePropertyChanged(nameof(hedge_order));
          }         
-         
 
          void _validate() 
          {
             if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors))
                set_error(nameof(child_order), string.Empty, true);
             else clear_error(nameof(child_order));
-
-            if ( (_hedge_order == null && enable_hedge) || (_hedge_order is INotifyDataErrorInfo hdei && hdei.HasErrors))
-               set_error(nameof(hedge_order), string.Empty, true);
-            else clear_error(nameof(hedge_order));
-
          }
       }
    }

src/client/spark/Orders/OrderTypeVMs/OffsetOrderVM.cs

@@ -0,0 +1,164 @@
+using bts.api;
+using bts.api.orderman;
+using mbts.ui;
+using mbts.ui.utility;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.ComponentModel;
+using System.Linq;
+using System.Reactive.Linq;
+using static bts.api.orderman.CustomOrderDefinition;
+
+namespace mbts.spark 
+{
+   static partial class OrderTypeVMFactory 
+   {
+      class InlineOffsetOrderProperties 
+      {
+         public InlineOffsetOrderProperties (OrderTemplateHeader selected_child_order_type, IReadOnlyCollection<OrderTemplateHeader> child_order_types) 
+         {
+            ExceptionAssert.InvalidOperation.That(child_order_types.Contains(selected_child_order_type), $"Child order type {selected_child_order_type} is not in child_order_types");
+            this.selected_child_order_type = new InlineUserProperty<OrderTemplateHeader>(selected_child_order_type, true);
+            this.child_order_types = new InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>>(child_order_types, true);
+            tick_offset_on_deploy = new InlineUserProperty<uint>(0, false);
+         }
+         public readonly InlineUserProperty<OrderTemplateHeader> selected_child_order_type;
+         public readonly InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>> child_order_types;
+         
+         public readonly InlineUserProperty<uint> tick_offset_on_deploy;
+      }
+
+      class OffsetOrderVM : OrderTypeVMBase<OffsetOrderDefinition>, IHierarchicalOrderTypeVM, IDisposable 
+      {
+         readonly IObservable<SecurityMD> _security_md_source;
+         SecurityImmutable? _security;
+         ObservableCollection<OrderTemplateHeader> _order_types = new ObservableCollection<OrderTemplateHeader>();
+         ObservableAutoWrapper<IOrderTypeVM, OrderTemplateHeader> _child_order_choices;
+         InlineOffsetOrderProperties _inline_props;
+         IOrderTypeVM _child_order;
+         IDisposable _dispose, _child_order_errors_sub;
+         
+         public uint tick_offset_on_deploy { get; set; } = 0;
+
+         public OffsetOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name)
+            : this(model, security_md_source, security_md_source.Select(smd =>
+            new InlineOffsetOrderProperties(new OrderTemplateHeader(model.factory.default_order_type(smd.security)), model.factory.default_order_types(smd.security).Select(t => new OrderTemplateHeader(t)).ToList())), id, display_name) { }
+
+         public OffsetOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source,
+                            IObservable<InlineOffsetOrderProperties> inline_properties, string id, string display_name) : base(model, id, display_name) 
+         {
+            _security_md_source = security_md_source;
+            _dispose = Disposable.Create(
+               security_md_source.OnDispatcher()
+               .Subscribe(smd => _security = smd.security),
+               inline_properties.OnDispatcher()
+               .Subscribe(p => {
+                  if (_order_types.Count == 0 || p.child_order_types.force_override) 
+                  {
+                     _order_types.SetItems(p.child_order_types.value);
+                  }
+                  if (_child_order_choices != null && (_child_order == null || p.selected_child_order_type.force_override)) 
+                  {
+                     _select_child_order(p.selected_child_order_type.value.key.type);
+                  }                  
+                  if (p.tick_offset_on_deploy.force_override) 
+                  {
+                     tick_offset_on_deploy = p.tick_offset_on_deploy.value;
+                     RaisePropertyChanged(nameof(tick_offset_on_deploy));
+                  }
+                  _inline_props = p;
+                  _validate();
+               })
+            );
+            _validate();
+         }
+         
+         public override void assign_to(CustomOrderDefinition value) 
+         {
+            if (_child_order == null)
+               _select_child_order(model.factory.default_order_type(_security.Value, true));
+            _validate();
+            if (!HasErrors) 
+            {
+               var i = factory.offset_order();
+               i.TickOffsetOnDeploy = tick_offset_on_deploy;
+               child_order.assign_to(i.Child);
+               value.Offset = i;
+            }
+         }         
+
+         public ICollection<IOrderTypeVM> child_order_types 
+         {
+            get 
+            {
+               if (_child_order_choices == null) 
+                  _child_order_choices = new ObservableAutoWrapper<IOrderTypeVM, OrderTemplateHeader>(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source));
+               return _child_order_choices;
+            }
+         }
+ 
+         public IOrderTypeVM child_order 
+         {
+            get 
+            {
+               init();
+               return _child_order;
+            }
+            set 
+            {
+               Disposable.dispose(ref _child_order_errors_sub);
+               _child_order = value;
+               _child_order?.init();
+               if (_child_order is INotifyDataErrorInfo ndei) 
+               {
+                  ndei.ErrorsChanged += on_errors_changed;
+                  void on_errors_changed(object s, DataErrorsChangedEventArgs e) =>
+                     _validate();
+                  _child_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed);
+               }
+               _validate();
+               RaisePropertyChanged(nameof(child_order));
+            }
+         }         
+
+         public void Dispose() 
+         {
+            Disposable.dispose(ref _dispose);
+            if (_child_order_choices != null)
+               _child_order_choices.OfType<IDisposable>().DisposeAll();
+         }
+
+         public override void init() 
+         {
+            if (_child_order == null && _security.HasValue)
+               _select_child_order();
+            _validate();
+         }
+
+         void _select_child_order() 
+         {
+            CustomOneofCase ot = _inline_props != null && _inline_props.selected_child_order_type.has_value ?
+               _inline_props.selected_child_order_type.value.key.type :
+               model.factory.default_order_type(_security.Value, true);
+            _select_child_order(ot);
+         }
+
+         void _select_child_order(CustomOneofCase ot) 
+         {
+            _child_order = child_order_types.FirstOrDefault(vm => vm.order_type == ot);
+            if (_child_order == null)
+               _child_order = child_order_types.FirstOrDefault();
+            RaisePropertyChanged(nameof(child_order));
+         }
+
+         void _validate() 
+         {
+            if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors))
+               set_error(nameof(child_order), string.Empty, true);
+            else clear_error(nameof(child_order));
+         }
+      }
+   }
+}

src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs

@@ -20,6 +20,7 @@ namespace mbts.spark {
       public static IOrderTypeVM join(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name) => new JoinOrderVM(model, security_md_source, id, display_name);
       public static IOrderTypeVM top(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name) => new TopOrderVM(model, security_md_source, id, display_name);
       public static IOrderTypeVM cover(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name) => new CoverOrderVM(model, security_md_source, id, display_name);
+      public static IOrderTypeVM offset(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name) => new OffsetOrderVM(model, security_md_source, id, display_name);
       public static IOrderTypeVM trigger(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name) => new TriggerOrderVM(model, security_md_source, id, display_name);
       public static IOrderTypeVM iceberg(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name) => new IcebergOrderVM(model, security_md_source, id, display_name);
       public static IOrderTypeVM spreader(OrdersModel model, IObservable<SecurityImmutable> security_source, string id, string display_name) => new SpreaderOrderVM(model, security_source, id, display_name);
@@ -38,10 +39,11 @@ namespace mbts.spark {
             case CustomOneofCase.Join: return join(model, security_md_source, id, display_name);
             case CustomOneofCase.Top: return top(model, security_md_source, id, display_name);
             case CustomOneofCase.Cover: return cover(model, security_md_source, id, display_name);
+            case CustomOneofCase.Offset: return offset(model, security_md_source, id, display_name);
             case CustomOneofCase.Trigger: return trigger(model, security_md_source, id, display_name);
             case CustomOneofCase.Iceberg: return iceberg(model, security_md_source, id, display_name);
             case CustomOneofCase.Spreader: return spreader(model, security_md_source.Select(smd => smd.security), id, display_name);
-            case CustomOneofCase.Dynamic: return dynamic(model, security_md_source, dynamic_type, id, display_name);
+            case CustomOneofCase.Dynamic: return dynamic(model, security_md_source, dynamic_type, id, /*display_name*/ dynamic_type);
             default: throw new NotImplementedException();
          }
       }

src/client/spark/Orders/OrderTypeVMs/TopOrderVM.cs

@@ -23,30 +23,15 @@ namespace mbts.spark
             this.selected_child_order_type = new InlineUserProperty<OrderTemplateHeader>(selected_child_order_type, true);
             this.child_order_types = new InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>>(child_order_types, true);
 
-            var hedgetype = CustomOneofCase.Lean;
-            if(!child_order_types.Any(ct => ct.key.type == hedgetype))
-               hedgetype = CustomOneofCase.Limit;
-            this.selected_hedge_order_type = new InlineUserProperty<OrderTemplateHeader>(new OrderTemplateHeader(hedgetype), true);
-            this.hedge_order_types = new InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>>(child_order_types, true);
-
             watch_level_tick_offset = new InlineUserProperty<uint>(0, false);
             top_threshold = new InlineUserProperty<bts.spark.QuantityMeasure>(
                               new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 }, false);
-            enable_hedge = new InlineUserProperty<bool>(false, false);
-            hedge_ticks = new InlineUserProperty<int>(1, false);
          }
          public readonly InlineUserProperty<OrderTemplateHeader> selected_child_order_type;
          public readonly InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>> child_order_types;
-         public readonly InlineUserProperty<OrderTemplateHeader> selected_hedge_order_type;
-         public readonly InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>> hedge_order_types;
-
 
          public readonly InlineUserProperty<uint> watch_level_tick_offset;
          public readonly InlineUserProperty<bts.spark.QuantityMeasure> top_threshold;
-
-         public readonly InlineUserProperty<bool> enable_hedge;
-         public readonly InlineUserProperty<int> hedge_ticks;
-
       }
 
       class TopOrderVM : OrderTypeVMBase<TopOrderDefinition>, IHierarchicalOrderTypeVM, IDisposable
@@ -59,23 +44,8 @@ namespace mbts.spark
          IOrderTypeVM _child_order;
          IDisposable _dispose, _child_order_errors_sub;
 
-         ObservableAutoWrapper<IOrderTypeVM, OrderTemplateHeader> _hedge_order_choices;
-         IOrderTypeVM _hedge_order;
-         IDisposable _hedge_order_errors_sub;
-
          public uint watch_level_tick_offset { get; set; } = 0;
          public bts.spark.QuantityMeasure top_threshold { get; set; } = new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 };
-         bool _enable_hedge = false;
-         public bool enable_hedge
-         {
-            get { return _enable_hedge; }
-            set
-            {
-               _enable_hedge = value;
-               RaisePropertyChanged(nameof(enable_hedge));
-            }
-         }
-         public int hedge_ticks { get; set; } = 1;
 
          public TopOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name)
             : this(model, security_md_source, security_md_source.Select(smd =>
@@ -100,10 +70,6 @@ namespace mbts.spark
                   {
                      _select_child_order(p.selected_child_order_type.value.key.type);
                   }
-                  if (_hedge_order_choices != null && (_hedge_order == null || p.selected_hedge_order_type.force_override))
-                  {
-                     _select_hedge_order(p.selected_hedge_order_type.value.key.type);
-                  }
                   if (p.watch_level_tick_offset.force_override)
                   {
                      watch_level_tick_offset = p.watch_level_tick_offset.value;
@@ -114,16 +80,6 @@ namespace mbts.spark
                      top_threshold = p.top_threshold.value;
                      RaisePropertyChanged(nameof(top_threshold));
                   }
-                  if (p.enable_hedge.force_override)
-                  {
-                     enable_hedge = p.enable_hedge.value;
-                     RaisePropertyChanged(nameof(enable_hedge));
-                  }
-                  if (p.hedge_ticks.force_override)
-                  {
-                     hedge_ticks = p.hedge_ticks.value;
-                     RaisePropertyChanged(nameof(hedge_ticks));
-                  }
                   _inline_props = p;
                   _validate();
                })
@@ -135,19 +91,13 @@ namespace mbts.spark
          {
             if (_child_order == null)
                _select_child_order(model.factory.default_order_type(_security.Value, true));
-            if (_hedge_order == null && enable_hedge)
-               _select_hedge_order(model.factory.default_order_type(_security.Value, true));
             _validate();
             if (!HasErrors)
             {
                var i = factory.top_order();
                i.WatchLevelTickOffset = watch_level_tick_offset;
                i.TopThreshold = top_threshold;
-               i.EnableHedge = enable_hedge;
-               i.HedgeTicks = hedge_ticks;
                child_order.assign_to(i.Child);
-               if (enable_hedge && hedge_order != null)
-                  hedge_order.assign_to(i.Hedge);
                value.Top = i;
             }
          }
@@ -162,16 +112,6 @@ namespace mbts.spark
             }
          }
 
-         public ICollection<IOrderTypeVM> hedge_order_types
-         {
-            get
-            {
-               if (_hedge_order_choices == null)
-                  _hedge_order_choices = new ObservableAutoWrapper<IOrderTypeVM, OrderTemplateHeader>(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source));
-               return _hedge_order_choices;
-            }
-         }
-
          public IOrderTypeVM child_order
          {
             get
@@ -196,45 +136,18 @@ namespace mbts.spark
             }
          }
 
-         public IOrderTypeVM hedge_order
-         {
-            get
-            {
-               init();
-               return _hedge_order;
-            }
-            set
-            {
-               Disposable.dispose(ref _hedge_order_errors_sub);
-               _hedge_order = value;
-               _hedge_order?.init();
-               if (_hedge_order is INotifyDataErrorInfo ndei)
-               {
-                  ndei.ErrorsChanged += on_errors_changed;
-                  void on_errors_changed(object s, DataErrorsChangedEventArgs e) =>
-                     _validate();
-                  _hedge_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed);
-               }
-               _validate();
-               RaisePropertyChanged(nameof(hedge_order));
-            }
-         }
 
          public void Dispose()
          {
             Disposable.dispose(ref _dispose);
             if (_child_order_choices != null)
                _child_order_choices.OfType<IDisposable>().DisposeAll();
-            if (_hedge_order_choices != null)
-               _hedge_order_choices.OfType<IDisposable>().DisposeAll();
          }
 
          public override void init()
          {
             if (_child_order == null && _security.HasValue)
                _select_child_order();
-            if (_hedge_order == null && _security.HasValue && enable_hedge)
-               _select_hedge_order();
             _validate();
          }
 
@@ -254,33 +167,11 @@ namespace mbts.spark
             RaisePropertyChanged(nameof(child_order));
          }
 
-         void _select_hedge_order()
-         {
-            CustomOneofCase ot = _inline_props != null && _inline_props.selected_hedge_order_type.has_value ?
-               _inline_props.selected_hedge_order_type.value.key.type :
-               model.factory.default_order_type(_security.Value, true);
-            _select_hedge_order(ot);
-         }
-
-         void _select_hedge_order(CustomOneofCase ot)
-         {
-            _hedge_order = hedge_order_types.FirstOrDefault(vm => vm.order_type == ot);
-            if (_hedge_order == null)
-               _hedge_order = hedge_order_types.FirstOrDefault();
-            RaisePropertyChanged(nameof(hedge_order));
-         }
-
-
          void _validate()
          {
             if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors))
                set_error(nameof(child_order), string.Empty, true);
             else clear_error(nameof(child_order));
-
-            if ((_hedge_order == null && enable_hedge) || (_hedge_order is INotifyDataErrorInfo hdei && hdei.HasErrors))
-               set_error(nameof(hedge_order), string.Empty, true);
-            else clear_error(nameof(hedge_order));
-
          }
       }
    }

src/client/spark/Orders/OrdersModel.cs

@@ -26,19 +26,22 @@ namespace mbts.spark {
       readonly PositionService _position_service;
       readonly SpreaderModel _spreader;
       readonly ErrorReporter _error;
+      readonly DynamicOrderHelper _dynamic_helper;
       readonly OrderActivitySummarizer _activity;
       readonly TableUpdateDemuxer<OrderRecordImmutable, uint, uint> _security_orders_demuxer;
       readonly TableUpdateDemuxer<PositionUnitImmutable, int, uint> _security_positions_demuxer;
       IDisposable _dispose;
 
       [ImportingConstructor]
-      private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, SpreaderModel spreader, ErrorReporter error, AdminModel admin_model) {
+      private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, 
+                          SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model) {
          _bus = bus;
          _factory = factory;
          _service = service;
          _position_service = position_service;
          _spreader = spreader;
          _error = error;
+         _dynamic_helper = dynamic_helper;
          var scheduler = ThreadPoolScheduler.Instance;
          var interval = TimeSpan.FromMilliseconds(100);
          on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
@@ -48,7 +51,10 @@ namespace mbts.spark {
                ImmutableDictionary<uint,OrderRecordImmutable>.Empty, 
                (workers, table) => _update_working_orders(workers, table))
                .Replay(1).RefCount();
-         on_order_tuples = OrderRecordTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders);
+         var order_record_tuple_source = new OrderRecordTupleSource(bus.subscribe(SparkTags.SecuritiesTag), on_orders);
+         on_order_tuples = order_record_tuple_source.subscribe_incremental_update();
+         on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update();
+         on_order_securities = order_record_tuple_source.subscribe_order_securities();
          on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
          _activity = new OrderActivitySummarizer(bus, on_orders);
          on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills);
@@ -57,6 +63,7 @@ namespace mbts.spark {
          _dispose = Disposable.Create(
             on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer),
             on_positions.AddRemoveOnly().Subscribe(_security_positions_demuxer),
+            order_record_tuple_source,
             _bus.subscribe(SparkTags.CreateOrder).Subscribe(a => send_order(a.definition, a.price, a.size).ignore()),
             _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())),
             _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()),
@@ -65,11 +72,14 @@ namespace mbts.spark {
          );
       }
 
-      #region Output
+      #region Output      
       public OrderFactory factory => _factory;
       public SpreaderModel spreader => _spreader;
+      public DynamicOrderHelper dynamic_helper => _dynamic_helper;
       public IObservable<TableUpdate<OrderRecordImmutable, uint>> on_orders { get; }
       public IObservable<IIncrementalUpdate<OrderRecordTuple, uint>> on_order_tuples { get; }
+      public IObservable<IIncrementalTableUpdateTransform<OrderRecordTuple, uint>> on_order_tuples_table { get; }
+      public IObservable<IIncrementalUpdate<SecurityMD, uint>> on_order_securities { get; }
       public IObservable<TableUpdate<FillImmutable, string>> on_fills { get; }
       public IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> on_fill_tuples { get; }
       public IObservable<IReadOnlyDictionary<uint,OrderRecordImmutable>> working_orders { get; }
@@ -335,4 +345,4 @@ namespace mbts.spark {
          public TableUpdateTransform<OrderRecordImmutable, uint> last_update { get; set; } = new TableUpdateTransform<OrderRecordImmutable, uint>();
       }
    }
-}
+}

src/client/spark/core/Collections/SetMap.cs

@@ -29,6 +29,7 @@ namespace mbts.collections {
       public int Count => _dictionary.Count;
       public object SyncRoot => null;
       public bool IsSynchronized => false;
+      public IEnumerable<TKey> Keys => _dictionary.Keys;
       public ISet<TElement> this[TKey key] => _dictionary[key];
 
       public void CopyTo(Array array, int index) =>

src/client/spark/core/GlobalExtensions/GenericCollectionExtensions.cs

@@ -427,95 +427,6 @@ namespace System.Collections.Generic {
       }
    }
 
-   public static class DictionaryExtensions
-   {
-      #region TryGetValue
-      public static bool TryGetValue<K, VK1, VK2, V>(this Dictionary<K, Dictionary<VK1, Dictionary<VK2, V>>> target, K k1, VK1 k2, VK2 k3, out V value) {
-         if (target.TryGetValue(k1, out Dictionary<VK1, Dictionary<VK2, V>> d1) && d1.TryGetValue(k2, out Dictionary<VK2, V> d2))
-            return d2.TryGetValue(k3, out value);
-         value = default;
-         return false;
-      }
-
-      public static bool TryGetValue<K, VK, V, VKD>(this IDictionary<K, VKD> target, K k1, VK k2, out V value) where VKD : IDictionary<VK, V> {
-         if (target.TryGetValue(k1, out VKD d))
-            return d.TryGetValue(k2, out value);
-         value = default;
-         return false;
-      }
-      #endregion
-
-      public static void AddOrUpdate<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> add_value_factory, Action<TKey, TValue> update_value) {
-         if (!dictionary.TryGetValue(key, out TValue value))
-            dictionary.Add(key, add_value_factory(key));
-         else update_value(key, value);
-      }
-
-      public static void AddOrUpdate<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> add_value_factory, Func<TKey, TValue, TValue> update_value) {
-         if (!dictionary.TryGetValue(key, out TValue value))
-            dictionary.Add(key, add_value_factory(key));
-         dictionary[key] = update_value(key, value);
-      }
-
-      public static TValue GetOrAdd<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> add_value_factory) {
-         if (!dictionary.TryGetValue(key, out TValue value))
-            dictionary.Add(key, value = add_value_factory(key));
-         return value;
-      }
-
-      public static void SetValue<K, VK, V, VKD>(this IDictionary<K, VKD> target, K k1, VK k2, V value) where VKD : IDictionary<VK, V> {
-         if (!target.TryGetValue(k1, out VKD d))
-            target.Add(k1, d = Activator.CreateInstance<VKD>());
-         d[k2] = value;
-      }
-
-      public static IReadOnlyDictionary<TKey, (TValue1, TValue2)> Join<TKey, TValue1, TValue2>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target) =>
-         Join(dictionary, merge_target, (v1, v2) => (v1, v2));
-
-      public static IReadOnlyDictionary<TKey, TResult> Join<TKey, TValue1, TValue2, TResult>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target, Func<TValue1, TValue2, TResult> select) {
-         var iks = dictionary.Keys.Intersect(merge_target.Keys);
-         return new ReadOnlyDictionary<TKey, TResult>(iks.ToDictionary(k => k, k => select(dictionary[k], merge_target[k])));
-      }
-
-      public static IReadOnlyDictionary<TKey, TResult> LeftJoin<TKey, TValue1, TValue2, TResult>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target,
-         Func<TValue1, TValue2, TResult> select, Func<TValue1, TResult> select_left_only) =>
-         dictionary.ToDictionary(kv => kv.Key, kv => {
-            if (merge_target.TryGetValue(kv.Key, out TValue2 v2))
-               return select(kv.Value, v2);
-            else
-               return select_left_only(kv.Value);
-         });
-
-      public static IReadOnlyDictionary<TKey, (TValue1, TValue2)> SuperJoin<TKey, TValue1, TValue2>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target) {
-         var eks1 = dictionary.Keys.Except(merge_target.Keys);
-         var eks2 = merge_target.Keys.Except(dictionary.Keys);
-         var iks = dictionary.Keys.Intersect(merge_target.Keys);
-         return new ReadOnlyDictionary<TKey, (TValue1, TValue2)>(eks1
-            .Select(k => (k, dictionary[k], default(TValue2)))
-            .Concat(eks2.Select(k => (k, default(TValue1), merge_target[k])))
-            .Concat(iks.Select(k => (k, dictionary[k], merge_target[k])))
-            .ToDictionary(vals => vals.k, vals => (vals.Item2, vals.Item3))
-         );
-      }
-
-      /// <summary>
-      /// If TryGetValue succeeds, result will be set with the corresponding value, and the key will be removed (returns true).
-      /// Otherwise, result is set to default, and nothing is removed (returns false).
-      /// </summary>
-      /// <typeparam name="TKey"></typeparam>
-      /// <typeparam name="TValue"></typeparam>
-      /// <param name="dictionary"></param>
-      /// <param name="key"></param>
-      /// <param name="result"></param>
-      /// <returns></returns>
-      public static bool TryPopValue<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, out TValue result) {
-         if (dictionary.TryGetValue(key, out result)) {
-            dictionary.Remove(key);
-            return true;
-         } else return false;
-      }
-   }
-
    public static class ListExtensions
    {
       #region InserSorted

src/client/spark/core/Utility/CompareUtils.cs

@@ -29,6 +29,12 @@ namespace mbts.utils {
       public static IComparer<TOutter> wrap<TInner, TOutter>(this IComparer<TInner> inner_comparer, Func<TOutter, TInner> selector) =>
          Comparer<TOutter>.Create((a, b) => inner_comparer.Compare(a == null ? default : selector(a), b == null ? default : selector(b)));
 
+      public static IComparer<T?> to_nullable<T>(this IComparer<T> comparer) where T : struct =>
+         Comparer<T?>.Create(nullable_comparison<T>(comparer.Compare));
+
+      public static IComparer<T> reverse<T>(this IComparer<T> comparer) =>
+         Comparer<T>.Create((a, b) => comparer.Compare(b, a));
+
       class NonGenericComparerWrapper<T> : IComparer {
          readonly IComparer<T> _comparer;
 

src/client/spark/core/Utility/ComparerBuilder.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+
+namespace mbts.utils {
+   class ComparerBuilder<T> {
+      public readonly List<IComparer<T>> _comparers_list;
+
+      public ComparerBuilder() =>
+         _comparers_list = new List<IComparer<T>>(2); 
+
+      public ComparerBuilder(IEnumerable<IComparer<T>> comparers) =>
+         _comparers_list = new List<IComparer<T>>(comparers);
+
+      public void add(IComparer<T> comparer) =>
+         _comparers_list.Add(comparer);
+
+      public void add<TValue>(Func<T, TValue> get) =>
+         add(get, Comparer<TValue>.Default);
+
+      public void add<TValue>(Func<T, TValue> get, IComparer<TValue> comparer) =>
+         add(CompareUtils.wrap(comparer, get));
+
+      public void clear() =>
+         _comparers_list.Clear();
+
+      public IComparer<T> to_comparer() => new _Comparer(_comparers_list.ToArray());
+
+      class _Comparer : IComparer<T> {
+         readonly IComparer<T>[] _comparers;
+
+         public _Comparer(IComparer<T>[] comparers) =>
+            _comparers = comparers;
+
+         public int Compare(T x, T y) {
+            foreach (var comp in _comparers) {
+               int result = comp.Compare(x, y);
+               if (result != 0)
+                  return result;
+            }
+            return 0;
+         }
+      }
+   }
+}

src/client/spark/core/Utility/TableUpdateTransform.cs

@@ -18,6 +18,13 @@ namespace mbts.utils {
       IEnumerable<T> removed { get; }
    }
 
+   interface IIncrementalTableUpdateTransform<T, TKey> where TKey : IEquatable<TKey> {
+      IEnumerable<T> added { get; }
+      IEnumerable<T> removed { get; }
+      IEnumerable<Change<T>> changes { get; }
+      IEnumerable<T> changed { get; }
+   }
+
    class IncrementalUpdate<T, TKey> : IIncrementalUpdate<T, TKey> where TKey : IEquatable<TKey> {
       public IncrementalUpdate()
          : this(new Dictionary<TKey, T>(0), new Dictionary<TKey, T>(0)) { }
@@ -76,6 +83,51 @@ namespace mbts.utils {
          new TableUpdateTransform<T, TKey>(_state, _state.Values.ToList(), new T[0]);
    }
 
+   class IncrementalTableUpdateTranform<T, TKey> : IIncrementalTableUpdateTransform<T, TKey> where TKey : IEquatable<TKey> {
+      public static IncrementalTableUpdateTranform<T, TKey> init(IEnumerable<T> initial_values) =>
+         new IncrementalTableUpdateTranform<T, TKey>(initial_values, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>());
+
+      public static IncrementalTableUpdateTranform<T, TKey> from_incremental_update(IIncrementalUpdate<T, TKey> source) {
+         // copy the smaller dictionary (removed vs added)
+         if (source.removed.Count < source.added.Count) {
+            var changes_list = new List<Change<T>>();
+            var removed_copy = source.removed.ToDictionary();
+            var added_list = new List<T>();
+            foreach (var kv in source.added) {
+               if (removed_copy.TryPopValue(kv.Key, out T oldval)) {
+                  changes_list.Add(new Change<T>(oldval, kv.Value));
+               } else {
+                  added_list.Add(kv.Value);
+               }
+            }
+            return new IncrementalTableUpdateTranform<T, TKey>(added_list, removed_copy.Values, changes_list);
+         } else {
+            var changes_list = new List<Change<T>>();
+            var added_copy = source.added.ToDictionary();
+            var removed_list = new List<T>();
+            foreach (var kv in source.removed) {
+               if (added_copy.TryPopValue(kv.Key, out T newval)) {
+                  changes_list.Add(new Change<T>(kv.Value, newval));
+               } else {
+                  removed_list.Add(kv.Value);
+               }
+            }
+            return new IncrementalTableUpdateTranform<T, TKey>(added_copy.Values, removed_list, changes_list);
+         }
+      }
+
+      IncrementalTableUpdateTranform(IEnumerable<T> added, IEnumerable<T> removed, IEnumerable<Change<T>> changes) {
+         this.added = added;
+         this.removed = removed;
+         this.changes = changes;
+      }
+
+      public IEnumerable<T> added { get; }
+      public IEnumerable<T> removed { get; }
+      public IEnumerable<Change<T>> changes { get; }
+      public IEnumerable<T> changed => changes.Select(ch => ch.New);
+   }
+
    [Browsable(false)]
    [EditorBrowsable(EditorBrowsableState.Never)]
    static class TableUpdateTransformExtensions {
@@ -135,6 +187,12 @@ namespace mbts.utils {
          .Select(u => new IncrementalUpdate<T, TKey>(u.added.Where(kv => predicate(kv.Value)).ToDictionary(), u.removed.Where(kv => predicate(kv.Value)).ToDictionary()))
          .Where(u => u.added.Count > 0 || u.removed.Count > 0);
 
+      public static IIncrementalTableUpdateTransform<T, TKey> as_table<T, TKey>(this IIncrementalUpdate<T, TKey> update) where TKey : IEquatable<TKey> =>
+         IncrementalTableUpdateTranform<T, TKey>.from_incremental_update(update);
+
+      public static IEnumerable<T> changed<T, TKey>(this IIncrementalTableUpdateTransform<T, TKey> update) where TKey : IEquatable<TKey> =>
+         update.changes.Select(ch => ch.New);
+
       class FilteredTableUpdateTransform<T, TKey> : ITableUpdateTransform<T, TKey> where TKey : IEquatable<TKey> {
          internal FilteredTableUpdateTransform(ITableUpdateTransform<T, TKey> source, Func<T, bool> predicate) {
             state = source.state;
@@ -247,6 +305,6 @@ namespace mbts.utils {
                state_clone.Add(kv.Key, kv.Value);
             return new PivotTableUpdateTransform<T, TKey, TResult, TResultKey>(state_clone, added.Values.ToList(), removed.Values.ToList(), _key_map, _selector, _result_key, _key);
          }
-      }
+      } 
    }
 }

src/client/spark/view/Controls/Adorners/DragRectAdorner.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Documents;
@@ -9,7 +10,7 @@ namespace mbts.ui.controls.adorners {
    public class DragRectAdorner : Adorner, IDisposable {
       Brush _fill = DIYGrid.DefaultSelectionHighlight;
       Pen _stroke = DIYGrid.DefaultSelectionStroke;
-      readonly ScrollViewer _scroll_viewer;
+      readonly ScrollViewerTuple _scroll_viewer_tup;
       readonly double _drift_multiplier;
       Point? _drag_start;
       double _x_drift, _y_drift;
@@ -47,21 +48,22 @@ namespace mbts.ui.controls.adorners {
       /// <param name="scroll_viewer">The element's ScrollViewer.</param>
       /// <param name="drift_multiplier">The number of pixels per scroll unit.</param>
       public DragRectAdorner(FrameworkElement adorned_element, ScrollViewer scroll_viewer = null, double drift_multiplier = 1) : base(adorned_element) {
-         _scroll_viewer = scroll_viewer;
          _drift_multiplier = drift_multiplier;
          AdornedElement.PreviewMouseDown += _on_preview_mouse_down;
          AdornedElement.PreviewMouseMove += _on_preview_mouse_move;
          AdornedElement.PreviewMouseUp += _on_preview_mouse_up;
-         if (_scroll_viewer != null)
-            _scroll_viewer.ScrollChanged += _scroll_viewer_ScrollChanged;
+         if (scroll_viewer != null) {
+            _scroll_viewer_tup = new ScrollViewerTuple(scroll_viewer);
+            _scroll_viewer_tup.scroll_viewer.ScrollChanged += _scroll_viewer_ScrollChanged;
+         }
       }
 
       public void Dispose() {
          AdornedElement.PreviewMouseDown -= _on_preview_mouse_down;
          AdornedElement.PreviewMouseMove -= _on_preview_mouse_move;
          AdornedElement.PreviewMouseUp -= _on_preview_mouse_up;
-         if (_scroll_viewer != null)
-            _scroll_viewer.ScrollChanged -= _scroll_viewer_ScrollChanged;
+         if (_scroll_viewer_tup != null)
+            _scroll_viewer_tup.scroll_viewer.ScrollChanged -= _scroll_viewer_ScrollChanged;
       }
 
       protected override void OnRender(DrawingContext drawingContext) {
@@ -82,8 +84,10 @@ namespace mbts.ui.controls.adorners {
          _y_drift = 0;
       }
 
-      void _on_preview_mouse_down(object sender, MouseButtonEventArgs e) =>
-         _drag_start = e.GetPosition(this);
+      void _on_preview_mouse_down(object sender, MouseButtonEventArgs e) {
+         if (_scroll_viewer_tup == null || (_scroll_viewer_tup.presenter != null && _scroll_viewer_tup.presenter.IsMouseOver))
+            _drag_start = e.GetPosition(this);
+      }
 
       void _on_preview_mouse_move(object sender, MouseEventArgs e) {
          if (_drag_start.HasValue) {
@@ -105,5 +109,15 @@ namespace mbts.ui.controls.adorners {
             InvalidateVisual();
          }
       }
+
+      class ScrollViewerTuple {
+         public readonly ScrollViewer scroll_viewer;
+         ScrollContentPresenter _presenter;
+
+         public ScrollViewerTuple(ScrollViewer scroll_viewer) =>
+            this.scroll_viewer = scroll_viewer;
+
+         public ScrollContentPresenter presenter => _presenter ?? (_presenter = scroll_viewer.descendants_of_type<ScrollContentPresenter>().FirstOrDefault());
+      }
    }
 }

src/client/spark/view/Controls/DIYGrid/DIYGrid.cs

@@ -20,6 +20,10 @@ using System.Windows.Input;
 using System.Windows.Media;
 
 namespace mbts.ui.controls {
+   interface IHandleGridSorting {
+      void on_sorting(object sender, DIYGridSortingEventArgs e);
+   }
+
    class DIYGridColumnCollection : ObservableCollection<DIYGrid.ColumnBase> { }
    class DIYGridSortingEventArgs : EventArgs {
       public DIYGridSortingEventArgs(DIYGrid.ColumnBase column) => Column = column;
@@ -159,7 +163,7 @@ namespace mbts.ui.controls {
       public static readonly SolidColorBrush DefaultSelectionHighlight = new SolidColorBrush(Colors.MediumSlateBlue) { Opacity = 0.4 };
       public static readonly Pen DefaultSelectionStroke = new Pen(Brushes.MediumSlateBlue, 1);
       readonly NotifyCollectionChangedEventHandler _sort_descriptions_collection_changed_cb;
-      IDisposable _selection_handler;
+      DataContextHooks _data_context_hooks;
 
       public static readonly DependencyProperty SelectionHighlightProperty =
          WPFHelper.create_dp<SolidColorBrush, DIYGrid>(
@@ -241,7 +245,8 @@ namespace mbts.ui.controls {
          }));
          Resources[InactiveSelectionHighlightKey] = new SolidColorBrush(SelectionHighlight.Color) { Opacity = 0.2 };
          Resources[ActiveSelectionHighlightKey] = SelectionHighlight;
-         DataContextChanged += (s, e) => _try_connect_selection_handler();
+         _data_context_hooks = new DataContextHooks(this, DataContext);
+         DataContextChanged += OnDataContextChanged;
       }
 
       public event DIYGridSortingEventHandler Sorting;
@@ -306,15 +311,8 @@ namespace mbts.ui.controls {
       // Private Methods
       #region Private Methods
 
-      void _try_connect_selection_handler() {
-         Disposable.dispose(ref _selection_handler);
-         if (_row_presenter != null && DataContext is IHandleRowSelection hrs) {
-            _row_presenter.SelectionChanged += on_selection_changed;
-            _selection_handler = Disposable.Create(() => _row_presenter.SelectionChanged -= on_selection_changed);
-            void on_selection_changed(object sender, SelectionChangedEventArgs e) =>
-               hrs.row_selection_changed(e);
-         }
-      }
+      void OnDataContextChanged(object sender, DependencyPropertyChangedEventArgs e) =>
+         Disposable.replace(ref _data_context_hooks, new DataContextHooks(this, e.NewValue));
 
       #region Column Handling
       private void invalidate_column_layout(object sender = null, EventArgs args = null) {
@@ -789,7 +787,7 @@ namespace mbts.ui.controls {
 
          _scroll_horizontal.ValueChanged += on_horizontal_scroll;
          _superheader_container = GetTemplateChild("_super_headers") as Panel;
-         _try_connect_selection_handler();
+         Disposable.replace(ref _data_context_hooks, new DataContextHooks(this, DataContext));
          update_superheaders();
 
           // TODO: All edit controls go in a canvas - edit api is called with an edit type.
@@ -908,6 +906,31 @@ namespace mbts.ui.controls {
       }
 
       #endregion
+
+      class DataContextHooks : IDisposable {
+         List<IDisposable> _hooks = new List<IDisposable>(3);
+
+         // this constructor provides a way to avoid repetative dp lookups ala owner.DataContext
+         public DataContextHooks(DIYGrid owner, object data_context)
+            : this(owner, data_context as IHandleRowSelection, data_context as IHandleGridSorting, data_context as IWillLetYouKnowWhenIDispose) { }
+
+         public DataContextHooks(DIYGrid owner, IHandleRowSelection selection_handler, IHandleGridSorting sorting_handler, IWillLetYouKnowWhenIDispose dispose_broadcaster) {
+            if (selection_handler != null && owner._row_presenter is ListBox rp) {
+               rp.SelectionChanged += on_selection_changed;
+               _hooks.Add(Disposable.Create(() => rp.SelectionChanged -= on_selection_changed));
+               void on_selection_changed(object sender, SelectionChangedEventArgs e) => selection_handler.row_selection_changed(e);
+            }
+            if (sorting_handler != null) {
+               owner.Sorting += sorting_handler.on_sorting;
+               _hooks.Add(Disposable.Create(() => owner.Sorting -= sorting_handler.on_sorting));
+            }
+            if (dispose_broadcaster != null)
+               _hooks.Add(dispose_broadcaster.SubscribeDisposed(this));
+         }
+
+         public void Dispose() =>
+            Disposable.dispose(_hooks);
+      }
    }
 
    static class DIYGridExtensions {

src/client/spark/view/Controls/DIYGrid/Row/RowBase.cs

@@ -18,13 +18,13 @@ namespace mbts.ui.controls {
       public interface IRowVM {
          event Action invalidated;
          void raise_invalidated();
-         ObservableCollection<RowAnimation> row_animations { get; }
-         IReadOnlyList<IGridCellStore> cell_stores { get; }
       }
 
-      public interface IRowVM<TRowData> : IRowVM {
-         event Action<(TRowData old_val, TRowData new_val)> updated;
+      public interface IAnimatableRowVM<TRowData> : IRowVM {
+         ObservableCollection<RowAnimation> row_animations { get; }
+         IReadOnlyList<IGridCellStore> cell_stores { get; }
          TRowData current_data { get; }
+         event Action<(TRowData old_val, TRowData new_val)> updated;
       }
 
       public interface IActionableRowVM {

src/client/spark/view/Controls/DIYGrid/Settings/Animation/PredicateAnimator.cs

@@ -5,7 +5,7 @@ using System.Collections.Generic;
 using System.Threading.Tasks;
 
 namespace mbts.ui {
-   class PredicateAnimator<TRow, TRowData> where TRow : DIYGrid.IRowVM<TRowData> {
+   class PredicateAnimator<TRow, TRowData> where TRow : DIYGrid.IAnimatableRowVM<TRowData> {
       readonly TRow _row;
       SetMap<IPredicate<TRowData>, AnimationDefinitionWrapper<TRow>> _predicate_animation_map = new SetMap<IPredicate<TRowData>, AnimationDefinitionWrapper<TRow>>();
       HashSet<IPredicate<TRowData>> _active_predicates = new HashSet<IPredicate<TRowData>>();

src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.shell;
 using mbts.collections;
@@ -11,7 +11,6 @@ using mbts.utils;
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
-using System.ComponentModel;
 using System.Linq;
 using System.Windows;
 using System.Windows.Media;
@@ -19,20 +18,16 @@ using static mbts.ui.controls.DIYGrid;
 
 namespace mbts.ui {
    class GridSettingsManager<TRow, TRowData, TPredicateBuilderArg, TPredicateBuilder, TSettingsData> : ViewModelBase, IValueFactory<TSettingsData>, IObserver<TSettingsData>, IDisposable
-      where TRow : IRowVM<TRowData>
+      where TRow : IRowVM
       where TPredicateBuilder : IPredicateBuilder<TRowData, TPredicateBuilderArg>, new()
       where TSettingsData : class, IGridSettingsWithNotifications<TRowData, TPredicateBuilder> {
       //
-      public readonly ObservableCollection<string> live_sorting_props;
-
       readonly IBus _bus;
       readonly OrdersModel _orders_model;
       readonly INotificationService _notification_service;
-      readonly List<IDisposable> _resources, _notification_subs  = new List<IDisposable>();
-      readonly ObservableCollection<TRow> _rows;
-      readonly ICollectionView _rows_view;
+      readonly List<IDisposable> _resources, _notification_subs = new List<IDisposable>();
       readonly ObservableCollection<ColumnBase> _columns;
-      //readonly ObservableAutoWrapper<PredicateAnimator<TRow, TRowData>, TRow> _animators;
+      readonly ObservableCollection<TRow> _rows;
       readonly Property<GridSettings> _grid_settings;
       readonly DelegateProperty<FontFamily> _font_family;
       readonly DelegateProperty<FontWeight> _font_weight;
@@ -47,8 +42,9 @@ namespace mbts.ui {
       IEnumerable<IPredicateAnimationsData<TRowData, TPredicateBuilder>> _predicate_animations_data;
 
       public GridSettingsManager(
-         IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel orders_model, ObservableCollection<TRow> rows,
-         ICollectionView rows_view, ObservableCollection<ColumnBase> columns, Func<IPropertyWrapperVM<TPredicateBuilder>> create_predicate_vm,
+         IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel orders_model,
+         ObservableCollection<TRow> rows, ObservableCollection<ColumnBase> columns,
+         Func<IPropertyWrapperVM<TPredicateBuilder>> create_predicate_vm,
          Func<string, IPropertyWrapperVM<TPredicateBuilder>> create_column_predicate_vm,
          Func<GridSettings, IEnumerable<IPredicateAnimationsData<TRowData, TPredicateBuilder>>, TSettingsData> create_settings_obj,
          TPredicateBuilderArg arg, IReadOnlyDictionary<string, GridColumnSettingsFeatureFlags> column_flags = null) {
@@ -57,7 +53,6 @@ namespace mbts.ui {
          _notification_service = notification_service;
          _orders_model = orders_model;
          _rows = rows;
-         _rows_view = rows_view;
          _columns = columns;
          _create_predicate_vm = create_predicate_vm;
          _create_column_predicate_vm = create_column_predicate_vm;
@@ -70,18 +65,10 @@ namespace mbts.ui {
          _font_size = backing(nameof(font_size), () => grid_settings.font_size, nameof(font_size));
          _font_color = backing(nameof(font_color), () => grid_settings.font_color.rgb_solid_brush().frozen(), nameof(grid_settings));
          _selection_highlight = backing(nameof(selection_highlight), () => grid_settings.selection_highlight.to_brush().frozen(), nameof(grid_settings));
-         //_animators = new ObservableAutoWrapper<PredicateAnimator<TRow, TRowData>, TRow>(_rows = rows, create_animator);
          _resources = new List<IDisposable> {
-            ObservableAutoWrapper.connect<SortDescriptionCollection, SortDescription, string>(
-               rows_view.SortDescriptions, live_sorting_props = ((ICollectionViewLiveShaping)rows_view).LiveSortingProperties, sd => sd.PropertyName),
             (launch_settings_editor = new DialogCommand(create_settings_view))
          };
-         //
-         //PredicateAnimator<TRow, TRowData> create_animator(TRow r) {
-         //   var pa = new PredicateAnimator<TRow, TRowData>(r);
-         //   pa.set_predicate_animations(_animation_defs);
-         //   return pa;
-         //}
+         // local methods
          Window create_settings_view() {
             var vm = new GridSettingsVM<TRowData, TPredicateBuilder, TSettingsData>(owner_title, _bus, _orders_model, create_predicate_vm, create_column_predicate_vm, _create_settings_obj);
             // need to refresh before launching the settings-view bc users can chanve some grid-column-settings properties directly from the grid (e.g. width, is_collapsed, etc.)
@@ -94,12 +81,7 @@ namespace mbts.ui {
                }
             }
             var window = new GridSettingsDialog { DataContext = vm };
-            vm.committed += () => {
-               TSettingsData sd = vm.get_result();
-               // IMPORTANT: don't overwrite current sorting
-               sd = _create_settings_obj(sd.general.with(sorting_criterion: rows_view.get_sorting_criteria(columns)), sd.predicate_animations);
-               load_from(sd);
-            };
+            vm.committed += () => load_from(vm.get_result());
             vm.RequestClose += (s, e) => WindowUtilities.force_close(window);
             return window;
          }
@@ -125,13 +107,12 @@ namespace mbts.ui {
       }
 
       public TSettingsData create() {
-         // IMPORTANT: don't overwrite current sorting
          _grid_settings.set(grid_settings.with(_columns.Select(c => {
             var col_settings = c.create();
             if (_column_flags.TryGetValue(c.id, out ImmutableColumnSettingStore store))
                col_settings = store.apply(col_settings);
             return col_settings;
-         }), _rows_view.get_sorting_criteria(_columns)));
+         })));
          return _create_settings_obj(grid_settings, _predicate_animations_data);
       }
 
@@ -143,11 +124,6 @@ namespace mbts.ui {
          _grid_settings.set(grid);
          foreach (ColumnBase col in _columns)
             col.set_fallback_font_params(new FontFamily(grid.font_family), grid.font_size, grid.font_color.rgb_color(), grid.font_weight);
-         if (reset_immutables) {
-#pragma warning disable CS0612 // Type or member is obsolete
-            _rows_view.set_sort_descriptions(grid.sorting_criterion, _columns);
-#pragma warning restore CS0612 // Type or member is obsolete
-         }
          _predicate_animations_data = value.predicate_animations;
          _animation_defs.clear();
          foreach (var pa in _predicate_animations_data) {
@@ -158,13 +134,7 @@ namespace mbts.ui {
             if (pa.predicate_builder is INotificationPredicateBuilder<TRowData, TPredicateBuilderArg> tpb && tpb.is_toast_enabled) {
                _notification_subs.Add(_notification_service.link(tpb.create_notification_predicate(_arg)));
             }
-            if (pa.row_animations.Count > 0 || pa.cell_animations.Count > 0) {
-               _animation_defs.add(pa.predicate_builder.create_predicate(_arg), PredicateAnimations.flat_definitions<TRow, TRowData>(pa.row_animations, pa.cell_animations, lookup_column));
-            }
          }
-         //foreach (var a in _animators) {
-         //   a.set_predicate_animations(_animation_defs);
-         //}
          // ** in the unfortunate event that we botch backwards compatability, and the columns don't bear a 1:1 relationship with the column-settings
          // don't add insult to injury by crashing **
          if (grid_settings.columns != null && _columns != null) {
@@ -186,12 +156,11 @@ namespace mbts.ui {
                if (col_settings_id_map.TryGetValue(id_key(col.id), out GridColumnSettings cs))
                   col.load_from(cs);
             }
+            // invalidate all the rows to draw them again to reflect the latest settings
+            foreach (var r in _rows) r.raise_invalidated();
             //
             string id_key(string id) => id.ToLower();
          }
-         //
-         IGridColumn<TRow> lookup_column(string col_id) =>
-            _columns.OfType<IGridColumn<TRow>>().FirstOrDefault(c => string.Equals(col_id, c.id, StringComparison.CurrentCultureIgnoreCase));
       }
 
       void IObserver<TSettingsData>.OnNext(TSettingsData sd) {
@@ -215,4 +184,4 @@ namespace mbts.ui {
       void IObserver<TSettingsData>.OnError(Exception error) { }
       void IObserver<TSettingsData>.OnCompleted() { }
    }
-}
+}

src/client/spark/view/Controls/DataGridEx/Filters/VM/DoubleMinMaxFilterVM.cs

@@ -1,4 +1,5 @@
-using ProtoBuf;
+using bts.utils;
+using ProtoBuf;
 using System;
 
 namespace mbts.spark.filters {
@@ -8,7 +9,7 @@ namespace mbts.spark.filters {
    }
 
    [ProtoContract]
-   class DoubleStatus : INullableStructProto<double> {
+   class DoubleStatus : INullableStructProto<double>, IEquatable<DoubleStatus> {
       [ProtoMember(1)]
       bool _active;
       [ProtoMember(2)]
@@ -19,10 +20,19 @@ namespace mbts.spark.filters {
          get => _active ? new double?(_value) : new double?();
          set { if (_active = value.HasValue && !double.IsNaN(value.Value)) _value = value.Value; }
       }
+
+      public bool Equals(DoubleStatus other) =>
+         other != null &&
+         _active == other._active &&
+         _value == other._value;
+
+      public override bool Equals(object obj) => Equals(obj as DoubleStatus);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(_active.GetHashCode(), _value.GetHashCode());
+      public override string ToString() => _active ? _value.ToString() : $"({_value})";
    }
 
    [ProtoContract]
-   class DoubleMinMaxState : IMinMaxState<double> {
+   class DoubleMinMaxState : IMinMaxState<double>, IEquatable<DoubleMinMaxState> {
       [ProtoMember(1)]
       readonly DoubleStatus _minimum;
       [ProtoMember(2)]
@@ -35,5 +45,13 @@ namespace mbts.spark.filters {
 
       public INullableStructProto<double> minimum => _minimum;
       public INullableStructProto<double> maximum => _maximum;
+
+      public bool Equals(DoubleMinMaxState other) =>
+         other != null &&
+         _minimum.safe_equals(other._minimum) &&
+         _maximum.safe_equals(other._maximum);
+      public override bool Equals(object obj) => Equals(obj as DoubleMinMaxState);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(_minimum?.GetHashCode() ?? 0, _maximum?.GetHashCode() ?? 0);
+      public override string ToString() => $"Min:{_minimum}, Max:{_maximum}";
    }
 }

src/client/spark/view/Controls/DataGridEx/Filters/VM/UIntMinMaxFilterVM.cs

@@ -1,4 +1,5 @@
-using ProtoBuf;
+using bts.utils;
+using ProtoBuf;
 using System;
 
 namespace mbts.spark.filters {
@@ -8,7 +9,7 @@ namespace mbts.spark.filters {
    }
 
    [ProtoContract]
-   class UIntStatus : INullableStructProto<uint> {
+   class UIntStatus : INullableStructProto<uint>, IEquatable<UIntStatus> {
       [ProtoMember(1)]
       bool _active;
       [ProtoMember(2)]
@@ -19,10 +20,19 @@ namespace mbts.spark.filters {
          get => _active ? new uint?(_value) : new uint?();
          set { if (_active = value.HasValue) _value = value.Value; }
       }
+
+      public bool Equals(UIntStatus other) =>
+         other != null &&
+         _active == other._active &&
+         _value == other._value;
+
+      public override bool Equals(object obj) => Equals(obj as UIntStatus);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(_active.GetHashCode(), _value.GetHashCode());
+      public override string ToString() => _active ? _value.ToString() : $"({_value})";
    }
 
    [ProtoContract]
-   class UIntMinMaxState : IMinMaxState<uint> {
+   class UIntMinMaxState : IMinMaxState<uint>, IEquatable<UIntMinMaxState> {
       [ProtoMember(1)]
       readonly UIntStatus _minimum;
       [ProtoMember(2)]
@@ -35,5 +45,13 @@ namespace mbts.spark.filters {
 
       public INullableStructProto<uint> minimum => _minimum;
       public INullableStructProto<uint> maximum => _maximum;
+
+      public bool Equals(UIntMinMaxState other) =>
+         other != null &&
+         _minimum.safe_equals(other._minimum) &&
+         _maximum.safe_equals(other._maximum);
+      public override bool Equals(object obj) => Equals(obj as UIntMinMaxState);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(_minimum?.GetHashCode() ?? 0, _maximum?.GetHashCode() ?? 0);
+      public override string ToString() => $"Min:{_minimum}, Max:{_maximum}";
    }
 }

src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml

@@ -268,11 +268,9 @@
                         </Grid.ColumnDefinitions>
                         <TextBlock Grid.Row="1" VerticalAlignment="Center">Join Thresh</TextBlock>
                         <uag:QuantityMeasureControl Grid.Row="1" Grid.Column="2" QuantityMeasure="{Binding join_threshold}"/>
-                        <!-- <TextBlock Grid.Row="3" VerticalAlignment="Center">Tick Offset</TextBlock>
-                        <c:TickUpDownUInt Grid.Row="3" Grid.Column="2" Value="{Binding tick_offset_on_deploy}"/> -->
-                        <TextBlock Grid.Row="5">Child</TextBlock>
+                        <TextBlock Grid.Row="3">Child</TextBlock>
                         <uag:OrderDetailsPopupControl
-                          Grid.Row="5" Grid.Column="2"
+                          Grid.Row="3" Grid.Column="2"
                           IsOpen="{Binding child_order.editable_properties_open, FallbackValue=False}"
                           ItemsSource="{Binding child_order_types}"
                           SelectedItem="{Binding child_order}"
@@ -299,8 +297,6 @@
                         </Grid.ColumnDefinitions>
                         <TextBlock Grid.Row="1" VerticalAlignment="Center">Top Thresh</TextBlock>
                         <uag:QuantityMeasureControl Grid.Row="1" Grid.Column="2" QuantityMeasure="{Binding top_threshold}"/>
-                        <!-- <TextBlock Grid.Row="9" VerticalAlignment="Center">Watch Levels</TextBlock>
-                        <c:TickUpDownUInt Grid.Row="9" Grid.Column="2" Value="{Binding watch_level_tick_offset}"/> -->
                         <TextBlock Grid.Row="3">Child</TextBlock>
                         <uag:OrderDetailsPopupControl
                           Grid.Row="3" Grid.Column="2"
@@ -351,6 +347,37 @@
                       </Grid>
                     </DataTemplate>
                   </usp:OrderModificationTemplateSelector.CoverOrderTemplate>
+                  <usp:OrderModificationTemplateSelector.OffsetOrderTemplate>
+                    <DataTemplate>
+                      <Grid>
+                        <Grid.RowDefinitions>
+                          <RowDefinition/>
+                          <RowDefinition Height="Auto"/>
+                          <RowDefinition Height="4" />
+                          <RowDefinition Height="Auto"/>
+                          <RowDefinition Height="4" />
+                          <RowDefinition Height="Auto"/>
+                          <RowDefinition Height="4" />
+                          <RowDefinition/>
+                        </Grid.RowDefinitions>
+                        <Grid.ColumnDefinitions>
+                          <ColumnDefinition Width="Auto"/>
+                          <ColumnDefinition Width="12"/>
+                          <ColumnDefinition/>
+                        </Grid.ColumnDefinitions>
+                        <TextBlock Grid.Row="1" VerticalAlignment="Center">Tick Offset</TextBlock>
+                        <c:TickUpDownUInt Grid.Row="1" Grid.Column="2" Value="{Binding tick_offset_on_deploy}"/>
+                        <TextBlock Grid.Row="3">Child</TextBlock>
+                        <uag:OrderDetailsPopupControl
+                          Grid.Row="3" Grid.Column="2"
+                          IsOpen="{Binding child_order.editable_properties_open, FallbackValue=False}"
+                          ItemsSource="{Binding child_order_types}"
+                          SelectedItem="{Binding child_order}"
+                          HasEditableProperties="{Binding child_order.has_editable_properties, FallbackValue=False}"
+                        />
+                      </Grid>
+                    </DataTemplate>
+                  </usp:OrderModificationTemplateSelector.OffsetOrderTemplate>                  
                   <usp:OrderModificationTemplateSelector.TriggerOrderTemplate>
                     <DataTemplate>
                       <Grid Grid.IsSharedSizeScope="True">
@@ -595,19 +622,88 @@
                           <ColumnDefinition/>
                         </Grid.ColumnDefinitions>
                         <ItemsControl Grid.Row = "1" ItemsSource="{Binding parameters}"  Grid.ColumnSpan="3">
-                          <ItemsControl.ItemTemplate>
-                            <DataTemplate>
-                              <Grid>
-                                <Grid.ColumnDefinitions>
-                                  <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
-                                  <ColumnDefinition Width="12"/>
-                                  <ColumnDefinition/>
-                                </Grid.ColumnDefinitions>
-                                <TextBlock Text="{Binding name}"/>
-                                <TextBox HorizontalAlignment="Stretch" Grid.Column="2" Text="{Binding value, UpdateSourceTrigger=PropertyChanged}"/>
-                              </Grid>
-                            </DataTemplate>
-                          </ItemsControl.ItemTemplate>
+                            <ItemsControl.ItemTemplateSelector>
+                              <usp:DynamicOrderParamTemplateSelector>
+                                <usp:DynamicOrderParamTemplateSelector.BoolTemplate>
+                                  <DataTemplate>
+                                    <Grid>
+                                    <Grid.ColumnDefinitions>
+                                       <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
+                                       <ColumnDefinition Width="12"/>
+                                       <ColumnDefinition/>
+                                    </Grid.ColumnDefinitions>
+                                    <TextBlock Text="{Binding descriptor.ParamName}"/>
+                                    <CheckBox HorizontalAlignment="Stretch" Grid.Column="2" IsChecked="{Binding param.AsBool, UpdateSourceTrigger=PropertyChanged}" />
+                                    </Grid>                                  
+                                  </DataTemplate>                                      
+                                </usp:DynamicOrderParamTemplateSelector.BoolTemplate>
+                                <usp:DynamicOrderParamTemplateSelector.DoubleTemplate>
+                                  <DataTemplate>
+                                    <Grid>
+                                    <Grid.ColumnDefinitions>
+                                       <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
+                                       <ColumnDefinition Width="12"/>
+                                       <ColumnDefinition/>
+                                    </Grid.ColumnDefinitions>
+                                    <TextBlock Text="{Binding descriptor.ParamName}"/>
+                                    <c:TickUpDownDouble Grid.Column="2" Value="{Binding param.AsDouble, UpdateSourceTrigger=PropertyChanged}"/>
+                                    </Grid>                                  
+                                  </DataTemplate>    
+                                </usp:DynamicOrderParamTemplateSelector.DoubleTemplate>                                
+                                <usp:DynamicOrderParamTemplateSelector.IntTemplate>
+                                  <DataTemplate>
+                                    <Grid>
+                                    <Grid.ColumnDefinitions>
+                                       <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
+                                       <ColumnDefinition Width="12"/>
+                                       <ColumnDefinition/>
+                                    </Grid.ColumnDefinitions>
+                                    <TextBlock Text="{Binding descriptor.ParamName}"/>
+                                    <c:TickUpDownInt Grid.Column="2" Value="{Binding param.AsInt32, UpdateSourceTrigger=PropertyChanged}"/>
+                                    </Grid>                                  
+                                  </DataTemplate>    
+                                </usp:DynamicOrderParamTemplateSelector.IntTemplate>                                                                
+                                <usp:DynamicOrderParamTemplateSelector.UIntTemplate>
+                                  <DataTemplate>
+                                    <Grid>
+                                    <Grid.ColumnDefinitions>
+                                       <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
+                                       <ColumnDefinition Width="12"/>
+                                       <ColumnDefinition/>
+                                    </Grid.ColumnDefinitions>
+                                    <TextBlock Text="{Binding descriptor.ParamName}"/>
+                                    <c:TickUpDownUInt Grid.Column="2" Value="{Binding param.AsUint32, UpdateSourceTrigger=PropertyChanged}"/>
+                                    </Grid>                                  
+                                  </DataTemplate>    
+                                </usp:DynamicOrderParamTemplateSelector.UIntTemplate>                                                                                                
+                                <usp:DynamicOrderParamTemplateSelector.StringTemplate>
+                                  <DataTemplate>
+                                    <Grid>
+                                    <Grid.ColumnDefinitions>
+                                       <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
+                                       <ColumnDefinition Width="12"/>
+                                       <ColumnDefinition/>
+                                    </Grid.ColumnDefinitions>
+                                    <TextBlock Text="{Binding descriptor.ParamName}"/>
+                                    <TextBox HorizontalAlignment="Stretch" Grid.Column="2" Text="{Binding param.AsString, UpdateSourceTrigger=PropertyChanged}"/>
+                                    </Grid>                                  
+                                  </DataTemplate>    
+                                </usp:DynamicOrderParamTemplateSelector.StringTemplate> 
+                                <usp:DynamicOrderParamTemplateSelector.EnumTemplate>
+                                  <DataTemplate>
+                                    <Grid>
+                                    <Grid.ColumnDefinitions>
+                                       <ColumnDefinition Width="Auto" SharedSizeGroup="c0"/>
+                                       <ColumnDefinition Width="12"/>
+                                       <ColumnDefinition/>
+                                    </Grid.ColumnDefinitions>
+                                    <TextBlock Text="{Binding descriptor.ParamName}"/>
+                                    <ComboBox Grid.Column="2" ItemsSource="{Binding descriptor.ParamType.Enumerated.Element}"  SelectedItem="{Binding param.AsEnumerated.EnumValue}" />
+                                    </Grid>                                  
+                                  </DataTemplate>    
+                                </usp:DynamicOrderParamTemplateSelector.EnumTemplate>                                                                                                                                                                 
+                              </usp:DynamicOrderParamTemplateSelector>
+                            </ItemsControl.ItemTemplateSelector>                          
                         </ItemsControl>
                         <TextBlock Grid.Row="3">Child</TextBlock>
                         <uag:OrderDetailsPopupControl

src/client/spark/view/Extensions/DisposeExtensions.cs

@@ -63,5 +63,16 @@ namespace mbts.ui
 				});
 			}
 		}
+
+      public static IDisposable SubscribeDisposed(this IWillLetYouKnowWhenIDispose dispose_broadcaster, Action callback) =>
+         dispose_broadcaster.SubscribeDisposed(_ => { callback?.Invoke(); callback = null; });
+
+      public static IDisposable SubscribeDisposed(this IWillLetYouKnowWhenIDispose dispose_broadcaster, IDisposable dispose) =>
+         dispose_broadcaster.SubscribeDisposed(_ => Disposable.dispose(ref dispose));
+
+      public static IDisposable SubscribeDisposed(this IWillLetYouKnowWhenIDispose dispose_broadcaster, Action<bool> callback) {
+         dispose_broadcaster.OnDisposed += callback;
+         return Disposable.Create(() => dispose_broadcaster.OnDisposed -= callback);
+      }
 	}
 }

src/client/spark/view/Shell/Login/LoginModel.cs

@@ -25,24 +25,17 @@ namespace mbts.ui.shell
    internal class LoginContainer
    {
       [ImportingConstructor]
-      public LoginContainer(UserService user, [Import("settings")] PersistenceService persist, [Import("autologin", AllowDefault=true)]string auto_login) {
-         if (string.IsNullOrEmpty(auto_login)) {
-            var model = LoginModel.instance;
-            model.provide_services(user, persist);
-            // await_login pumps the dispatcher but doesn't allow control to return to MEF until the username and workspace are available
-            model.await_login();
-            var u = model.get_user();
-            UserName = u.name;
-            UserId = u.id;
-            Workspace = model.get_workspace();
-            model.Dispose();
-         }
-         else {
-            UserName = auto_login;
-            Environment.SetEnvironmentVariable("BTS_USER", UserName);
-            Workspace = string.Empty;
-   			LoginModel.instance.Dispose();
-         }
+      public LoginContainer(UserService user, [Import("settings")] PersistenceService persist) {
+         var model = LoginModel.instance;
+         model.provide_services(user, persist);
+         // await_login pumps the dispatcher but doesn't allow control to return to MEF until the username and workspace are available
+         model.await_login();
+         var u = model.get_user();
+         UserName = u.name;
+         UserId = u.id;
+         Token = u.token;
+         Workspace = model.get_workspace();
+         model.Dispose();
       }
 
       [Export("username")]
@@ -50,14 +43,22 @@ namespace mbts.ui.shell
 
       [Export("userid")]
       public uint UserId { get; private set; }
+      [Export]
+      public AuthToken Token { get; private set; }
 
       [Export("workspace")]
       public string Workspace { get; private set; }
    }
 
    public struct User {
-      public string name;
-      public UInt32 id;
+      public User(string name, uint id, AuthToken token) {
+         this.name = name;
+         this.id = id;
+         this.token = token;
+      }
+      public readonly string name;
+      public readonly UInt32 id;
+      public readonly AuthToken token;
    }
 
    public class LoginModel : IDisposable
@@ -117,7 +118,7 @@ namespace mbts.ui.shell
          _enum_workspace = new PreloginWorkspaceEnumerator(persist);
          var response = await _logon.get_users_async(new GetUsersRequest());
          if (response.success) {
-            Users = response.value.Users.Select(x => new User { name = x.Login, id = x.Id }).OrderBy(u => u.name).ToArray();
+            Users = response.value.Users.Select(x => new User(x.Login,x.Id, null)).OrderBy(u => u.name).ToArray();
          }
          else {
             ErrorMessage = "Logon: " + response.error_message;
@@ -137,7 +138,7 @@ namespace mbts.ui.shell
             if (response.success) {
                if (response.value.IsValid) {
                   _workspace = workspace;
-                  set_authenticated_user(new User { name =username, id = response.value.UserId});
+                  set_authenticated_user(new User(username, response.value.UserId, response.value.Token));
                   return (true, "");
                }
                else {

src/client/spark/view/Shell/ViewModels/ShellWindowVM.cs

@@ -65,15 +65,13 @@ namespace mbts.ui.shell
       public string TitleOverride {
          get => _title_override;
          set {
-            if (_title_override != value) {
-               if (value == _model.Title) {
-                  _title_override = null;
-               }
-               else {
-                  _title_override = value;
-               }
-               RaisePropertyChanged(nameof(TitleOverride), nameof(Title), nameof(HasCustomTitle));
+            if (value == _model.Title) {
+               _title_override = null;
             }
+            else {
+               _title_override = value;
+            }
+            RaisePropertyChanged(nameof(TitleOverride), nameof(Title), nameof(HasCustomTitle));
             _edit_mode.set(false);
          }
       }

src/client/spark/view/Shell/Workspaces/WorkspaceItemControl.xaml

@@ -0,0 +1,34 @@
+<UserControl x:Class="mbts.ui.controls.WorkspaceItemControl"
+             xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" 
+             xmlns:d="http://schemas.microsoft.com/expression/blend/2008" 
+             xmlns:local="clr-namespace:mbts.ui.controls"
+             xmlns:ui="clr-namespace:mbts.ui"
+             mc:Ignorable="d"
+             ui:StateManager.VisualState="{Binding VisualState}"
+>
+
+  <VisualStateManager.VisualStateGroups>
+    <VisualStateGroup x:Name="CommonStates">
+      <VisualState x:Name="Normal" />
+      <VisualState x:Name="Overridden">
+        <Storyboard>
+          <ColorAnimation Storyboard.TargetName="_main_button"
+                          Storyboard.TargetProperty="Background.Color" 
+                          To="{x:Static ui:BTSColors.success}" 
+                          Duration="0:0:0.2" 
+                          AutoReverse="True" RepeatBehavior="3x" />
+        </Storyboard>
+      </VisualState>
+    </VisualStateGroup>
+  </VisualStateManager.VisualStateGroups>
+
+  <Button x:Name="_main_button" Command="{Binding ActivateCommand}" Style="{DynamicResource DialogListButton}">
+    <Grid>
+      <Image Source="{Binding Icon}" HorizontalAlignment="Left" Height="16" Margin="4" />
+      <TextBlock Text="{Binding Name}" VerticalAlignment="Center" Margin="28,0,0,0" x:Name="text"  />
+      <Button Style="{StaticResource RoundCloseButton}" HorizontalAlignment="Right" VerticalAlignment="Center" Command="{Binding DeleteCommand}" />
+    </Grid>
+  </Button>
+</UserControl>

src/client/spark/view/Shell/Workspaces/WorkspaceItemControl.xaml.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Data;
+using System.Windows.Documents;
+using System.Windows.Input;
+using System.Windows.Media;
+using System.Windows.Media.Imaging;
+using System.Windows.Navigation;
+using System.Windows.Shapes;
+
+namespace mbts.ui.controls {
+    /// <summary>
+    /// Interaction logic for WorkspaceItemControl.xaml
+    /// </summary>
+    public partial class WorkspaceItemControl : UserControl
+    {
+        public WorkspaceItemControl()
+        {
+            InitializeComponent();
+        }
+    }
+}

src/client/spark/view/Shell/Workspaces/WorkspaceViewModel.cs

@@ -21,6 +21,10 @@ namespace mbts.ui.shell
       public object Icon {
          get { return Icons.Window; }
       }
+
+      private string _vstate = "Normal";
+      public string VisualState { get { return _vstate; } set { _vstate = value; RaisePropertyChanged("VisualState"); } }
+
       internal WorkspaceDefinition Definition { get; private set; }
 
       public ICommand ActivateCommand {

src/client/spark/view/Shell/Workspaces/WorkspacesDialogView.xaml

@@ -59,13 +59,7 @@
                         <ItemsControl ItemsSource="{Binding Workspaces}" Style="{StaticResource CommonList}">
                             <ItemsControl.ItemTemplate>
                                 <DataTemplate>
-                                <Button Command="{Binding ActivateCommand}" Style="{DynamicResource DialogListButton}">
-                                    <Grid>
-                                        <Image Source="{Binding Icon}" HorizontalAlignment="Left" Height="16" Margin="4" />
-                                        <TextBlock Text="{Binding Name}" VerticalAlignment="Center" Margin="28,0,0,0" x:Name="text"  />
-                                        <Button Style="{StaticResource RoundCloseButton}" HorizontalAlignment="Right" VerticalAlignment="Center" Command="{Binding DeleteCommand}" />
-                                    </Grid>
-                                </Button>
+                                  <controls:WorkspaceItemControl />
                                 </DataTemplate>
                             </ItemsControl.ItemTemplate>
                         </ItemsControl>

src/client/spark/view/Shell/Workspaces/WorkspacesDialogViewModel.cs

@@ -45,8 +45,13 @@ namespace mbts.ui.shell
                {
                   clear_error("SaveName");
                }
+               var save_candidate_already_exists = Workspaces.Any(x => x.Name == SaveName);
                // Save
                _work_model.save_workspace(SaveName, ispublic: IsPublicWorkspace);
+
+               // if we just overwrote something, have the vm reflect it
+               if (save_candidate_already_exists)
+                  Workspaces.Where(x => x.Name == SaveName).ForEach(x => x.VisualState = "Overridden");
             });
          }
       }

src/client/spark/view/Spark/Core/SparkUtils.cs

@@ -59,6 +59,12 @@ namespace mbts.spark {
       public static SecurityImmutable? security(this SecurityMD? security_md) =>
          security_md.HasValue ? new SecurityImmutable?(security_md.Value.security) : new SecurityImmutable?();
 
+      public static Venue exchange(this SecurityMD? security_md) =>
+         security_md.HasValue ? security_md.Value.security.Exchange : Venue.Unset;
+
+      public static string symbol(this SecurityMD? security_md) =>
+         security_md.HasValue ? security_md.Value.security.Symbol : "unknown";
+
       public static async Task<SecurityImmutable> get_security_async(IBus bus, uint security_key) =>
          await bus.subscribe(SparkTags.SecuritiesTag).Where(map => map.ContainsKey(security_key)).Select(map => map[security_key].security).FirstAsync();
 

src/client/spark/view/Spark/PAndL/PnLSummaryRow.cs

@@ -186,8 +186,6 @@ namespace mbts.spark {
       public string weighted_average_sell_text => _weighted_average_sell.text;
 
       public void raise_invalidated() => invalidated?.Invoke();
-      ObservableCollection<RowAnimation> DIYGrid.IRowVM.row_animations { get; } = new ObservableCollection<RowAnimation>();
-      IReadOnlyList<DIYGrid.IGridCellStore> DIYGrid.IRowVM.cell_stores => new DIYGrid.IGridCellStore[0];
 
       //public IEnumerable<FillImmutable> sod_fills => _fills.Where(f => f.Timestamp.ToDateTime() >= _start_of_day);
       //public IEnumerable<FillImmutable> pos_fills => _fills.Where(f => f.Timestamp.ToDateTime() < _start_of_day);

src/client/spark/view/Spark/SparkWindow.cs

@@ -19,6 +19,7 @@ namespace bts.spark {
          TITLE_PANEL_HEIGHT = 30,
          COLUMN_HEADER_HEIGHT = 27,
          SPARK_MENU_ITEM_HEIGHT = 45,
+         GRID_FOOTER_HEIGHT = 30,
          ABSOLUTE_MIN_HEIGHT = 230;
 
       protected static readonly TimeSpan CONTENT_MEASURE_WAIT_PERIOD = TimeSpan.FromSeconds(0.3);
@@ -60,7 +61,7 @@ namespace bts.spark {
       public abstract UIElement Content { get; }
       public virtual IEnumerable<SparkCommand> Commands => Enumerable.Empty<SparkCommand>();
       public virtual double MinWidth => 400;
-      public virtual double MinHeight => Math.Max(ABSOLUTE_MIN_HEIGHT, TITLE_PANEL_HEIGHT + COLUMN_HEADER_HEIGHT + Commands.Count() * SPARK_MENU_ITEM_HEIGHT);
+      public virtual double MinHeight => Math.Max(ABSOLUTE_MIN_HEIGHT, TITLE_PANEL_HEIGHT + COLUMN_HEADER_HEIGHT + GRID_FOOTER_HEIGHT + Commands.Count() * SPARK_MENU_ITEM_HEIGHT);
 
       public ShellWindowUI GetContent() =>
          new ShellWindowUI(new SparkWindowView(_bus, Commands, Content));

src/client/spark/view/Spark/Utilities/OrderDataHelper.cs

@@ -37,6 +37,7 @@ namespace mbts.spark.utilities {
                CustomOrderDefinition.CustomOneofCase.Lean,
                CustomOrderDefinition.CustomOneofCase.Join,
                CustomOrderDefinition.CustomOneofCase.Cover,
+               CustomOrderDefinition.CustomOneofCase.Offset,
                CustomOrderDefinition.CustomOneofCase.Top,
                CustomOrderDefinition.CustomOneofCase.Trigger,
             },
@@ -47,15 +48,18 @@ namespace mbts.spark.utilities {
                CustomOrderDefinition.CustomOneofCase.Iceberg,
                CustomOrderDefinition.CustomOneofCase.TimeBox,
                CustomOrderDefinition.CustomOneofCase.Trigger,
+               CustomOrderDefinition.CustomOneofCase.Offset,
             };
       public static readonly CustomOrderDefinition.CustomOneofCase[] all_order_types = _standard_types.Union(_spreader_types).ToArray();
 
       readonly AdminModel _admin_model;
+      readonly DynamicOrderHelper _dynamic_helper;
       IDisposable _dispose;
 
       [ImportingConstructor]
-      public OrderFactory(AdminModel admin_model) {
+      public OrderFactory(AdminModel admin_model, DynamicOrderHelper dynamic_helper) {
          _admin_model = admin_model;
+         _dynamic_helper = dynamic_helper;
       }
 
       public IObservable<IReadOnlyDictionary<uint, string>> subscribe_accounts() => _admin_model.on_accounts;
@@ -106,7 +110,7 @@ namespace mbts.spark.utilities {
             if(ot != CustomOrderDefinition.CustomOneofCase.Dynamic)
                defs.Add(new CustomOrderDefinitionImmutable(custom_order(ot)));
          }
-         DynamicOrderHelper.add_dynamic_order_defs(defs, ott => custom_order(ott));
+         _dynamic_helper.add_dynamic_order_defs(defs, ott => custom_order(ott));
          return defs;
       }
 
@@ -119,7 +123,7 @@ namespace mbts.spark.utilities {
          // TODO: replace with better naming scheme (right now, the order type ref count 'n' is appended to the order-type name)
          var order_type_ref_count = defs.ToDictionary(kv => kv.Key, kv => name_ref_seed);
          var dynamic_defs = new List<CustomOrderDefinitionImmutable>();
-         DynamicOrderHelper.add_dynamic_order_defs(dynamic_defs, ot => custom_order(ot));
+         _dynamic_helper.add_dynamic_order_defs(dynamic_defs, ot => custom_order(ot));
          foreach (var ddef in dynamic_defs) {
             var header = new OrderTemplateHeader(new OrderTemplateKey(ddef.CustomCase, ddef.Dynamic.DynamicOrderType), ddef.Dynamic.DynamicOrderType );
             // if (order_type_ref_count.TryGetValue(header, out int count)) {
@@ -212,19 +216,21 @@ namespace mbts.spark.utilities {
       {
          Child = custom_order(),
          JoinThreshold = new QuantityMeasure(),
-         Hedge = custom_order(),
       };     
       public TopOrderDefinition top_order() => new TopOrderDefinition
       {
          Child = custom_order(),
          TopThreshold = new QuantityMeasure(),
-         Hedge = custom_order(),
       };
       public CoverOrderDefinition cover_order() => new CoverOrderDefinition
       {
          Child = custom_order(),
          Hedge = custom_order(),
       };
+      public OffsetOrderDefinition offset_order() => new OffsetOrderDefinition
+      {
+         Child = custom_order(),
+      };      
 
       public TriggerOrderDefinition trigger_order() => new TriggerOrderDefinition
       {
@@ -234,8 +240,7 @@ namespace mbts.spark.utilities {
       };
 
       public DynamicOrderDefinition dynamic_order() => new DynamicOrderDefinition
-      {
-         Child = custom_order(),
+      {         
       };
 
 
@@ -251,11 +256,12 @@ namespace mbts.spark.utilities {
             case CustomOrderDefinition.CustomOneofCase.StopLimit: rv.StopLimit = stop_limit_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Iceberg: rv.Iceberg = iceberg_order(); break;
             case CustomOrderDefinition.CustomOneofCase.TimeBox: rv.TimeBox = time_box_order(); break;
-            case CustomOrderDefinition.CustomOneofCase.Tick: rv.Tick = tick_order(); break;
+            case CustomOrderDefinition.CustomOneofCase.Tick: rv.Tick = tick_order(); break;            
             case CustomOrderDefinition.CustomOneofCase.Lean: rv.Lean = lean_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Join: rv.Join = join_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Top: rv.Top = top_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Cover: rv.Cover = cover_order(); break;
+            case CustomOrderDefinition.CustomOneofCase.Offset: rv.Offset = offset_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Trigger: rv.Trigger = trigger_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = spreader_order(); break;
             case CustomOrderDefinition.CustomOneofCase.Dynamic: rv.Dynamic = dynamic_order(); break;
@@ -278,6 +284,7 @@ namespace mbts.spark.utilities {
             case CustomOrderDefinition.CustomOneofCase.Join: return new OrderMetadata { KeyBinding = "J", Color = BTSColors.Orders.join_order };
             case CustomOrderDefinition.CustomOneofCase.Top: return new OrderMetadata { KeyBinding = "D", Color = BTSColors.Orders.top_order };
             case CustomOrderDefinition.CustomOneofCase.Cover: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order };
+            case CustomOrderDefinition.CustomOneofCase.Offset: return new OrderMetadata { KeyBinding = "O", Color = BTSColors.Orders.tick_order };
             case CustomOrderDefinition.CustomOneofCase.Trigger: return new OrderMetadata { KeyBinding = "T", Color = BTSColors.Orders.trigger_order };
             case CustomOrderDefinition.CustomOneofCase.Spreader: goto default;
             case CustomOrderDefinition.CustomOneofCase.Dynamic: return new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.top_order };
@@ -298,6 +305,7 @@ namespace mbts.spark.utilities {
             case OrderState.CustomOneofCase.Join: return new OrderMetadata { KeyBinding = "J", Color = BTSColors.Orders.join_order };
             case OrderState.CustomOneofCase.Top: return new OrderMetadata { KeyBinding = "D", Color = BTSColors.Orders.top_order };
             case OrderState.CustomOneofCase.Cover: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order };
+            case OrderState.CustomOneofCase.Offset: return new OrderMetadata { KeyBinding = "O", Color = BTSColors.Orders.tick_order };
             case OrderState.CustomOneofCase.Trigger: return new OrderMetadata { KeyBinding = "T", Color = BTSColors.Orders.trigger_order };
             case OrderState.CustomOneofCase.Spreader: goto default;
             case OrderState.CustomOneofCase.Dynamic: return new OrderMetadata { KeyBinding = "T", Color = BTSColors.Orders.top_order };
@@ -336,6 +344,7 @@ namespace mbts.spark.utilities {
             case nameof(JoinOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Join;
             case nameof(TopOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Top;
             case nameof(CoverOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Cover;
+            case nameof(OffsetOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Offset;
             case nameof(TriggerOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Trigger;
             case nameof(DynamicOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Dynamic;
             default: throw new ArgumentException($"Unable to match '{order_type_def?.Descriptor.Name}' to an order definition type");

src/client/spark/view/Spark/Utilities/WorkLoop.cs

@@ -51,7 +51,7 @@ namespace mbts.spark {
          Stopwatch stop_watch = new Stopwatch();
          TimeSpan elapsed_total = TimeSpan.Zero;
          ExceptionAssert.Argument.GreaterThan(excessive_cycle_ratio, 1.0, nameof(excessive_cycle_ratio));
-         double excessive_cycle_ms = interval.Milliseconds * (double)sample_cycle_size * excessive_cycle_ratio;
+         double excessive_cycle_ms = interval.TotalMilliseconds * (double)sample_cycle_size * excessive_cycle_ratio;
          int cycle_count = 0;
          DispatcherTimer timer = new DispatcherTimer(interval, priority, on_tick, Dispatcher.CurrentDispatcher);
          timer.Start();
@@ -65,7 +65,7 @@ namespace mbts.spark {
                stop_watch.Stop();
                var cycle_ms = (stop_watch.Elapsed - elapsed_total).TotalMilliseconds;
                if (cycle_ms > excessive_cycle_ms)
-                  Log.Ui.warn($"{name} DispatcherTimer took {cycle_ms}ms to complete {sample_cycle_size} cycles, target rate is 1 per {interval.Milliseconds}ms.");
+                  Log.Ui.warn($"{name} DispatcherTimer took {cycle_ms}ms to complete {sample_cycle_size} cycles, target rate is 1 per {interval.TotalMilliseconds}ms.");
                elapsed_total = stop_watch.Elapsed;
             }
             work();

src/client/spark/view/Utility/SortingCriteria.cs

@@ -21,5 +21,7 @@ namespace mbts.ui
       public bool is_ascending;
 
       public SortDescription to_sort_description() { return new SortDescription(col_name, is_ascending ? ListSortDirection.Ascending : ListSortDirection.Descending); }
+
+      public override string ToString() => $"{col_name}:{(is_ascending ? "Ascending" : "Descending")}";
    }
 }

src/client/spark_api/md/MarketDataClient.cs

@@ -12,7 +12,7 @@ using bts.api;
 using bts.api.core;
 using bts.co2;
 
-namespace bts.sdk
+namespace bts.api.md
 {
    using security_key = System.UInt32;
    public class MarketDataClient : IDisposable
@@ -30,17 +30,39 @@ namespace bts.sdk
             MDRequest.Types.Flags.Implied | 
             MDRequest.Types.Flags.LastTrade);
       
-      readonly ConcurrentDictionary<security_key, Data> _data = new ConcurrentDictionary<security_key, Data>();      
+      readonly ConcurrentDictionary<security_key, Data> _data = new ConcurrentDictionary<security_key, Data>();
+      readonly ConcurrentDictionary<security_key, string> _security_key_to_desc= new ConcurrentDictionary<security_key, string>();
+      readonly ConcurrentDictionary<string, security_key> _security_desc_to_key = new ConcurrentDictionary<string, security_key>();      
+      readonly Lazy<MDRequest> _subscription;
+      readonly Venue _venue;
+
       readonly static Lazy<MarketDataRelay> _md = new Lazy<MarketDataRelay>(() => locate(), true);
-      readonly Lazy<MDRequest> _subscription = new Lazy<MDRequest>(() => load_request());            
-      public MarketDataClient() 
+      readonly static ConcurrentDictionary<api.Venue, MarketDataClient> _venue_cache = new ConcurrentDictionary<api.Venue, api.md.MarketDataClient>();
+      static readonly List<Product> _all_products;
+      static MarketDataClient()
       {
+         _all_products = _md.Value.request_products(new VoidMessage()).Products.ToList();
+      }
+      public static bool create_or_get(api.Venue exchange, out MarketDataClient md)
+      {
+         if (!_venue_cache.TryGetValue(exchange, out md))
+         {
+            _venue_cache[exchange] = md = new MarketDataClient(exchange);
+            return true;
+         }
+         return false;
+      }      
+
+      public MarketDataClient(bts.api.Venue exchange = Venue.Unset) 
+      {
+         _venue = exchange;
+         _subscription = new Lazy<MDRequest>(() => load_request(exchange));
       }
 
-      static MDRequest load_request()
+      MDRequest load_request(bts.api.Venue exchange)
       {
          var req = new MDRequest();
-         var prods = _md.Value.request_products(new VoidMessage()).Products;
+         var prods = _all_products.Where(x=>x.Exchange == exchange || exchange == bts.api.Venue.Unset);
          foreach(var prod in prods)
             req.Channels.Add(new SecurityChannel() { Exchange = prod.Exchange, Type = prod.Type, Symbol = prod.Symbol, Flags = 0 });
          return req;
@@ -63,6 +85,16 @@ namespace bts.sdk
             return null;
       }
 
+      public bool lookup_security_desc(security_key key, out string security_desc)
+      {
+         return _security_key_to_desc.TryGetValue(key, out security_desc);
+      }
+
+      public bool lookup_security_key_from_desc(string security_desc, out security_key key)
+      {
+         return _security_desc_to_key.TryGetValue(security_desc, out key);
+      }
+
       public Security find_security(Func<Security, bool> filter)
       {
          return _data.Values.Select(x => x.data.Security).Where(xx => filter(xx)).FirstOrDefault();
@@ -203,6 +235,20 @@ namespace bts.sdk
       /// <returns></returns>
       public async Task start_pump(TimeSpan? interval = null)
       {
+         var rep = await _md.Value.request_names_async(new VoidMessage()); 
+         int count = 0;
+         do
+         {
+            foreach (var s in rep.value.Securities)
+            {
+               foreach (var ss in s.Names)
+               {
+                  _security_desc_to_key[ss.Value] = ss.Key;
+                  _security_key_to_desc[ss.Key] = ss.Value;
+               }
+            }
+         }
+         while (!rep.success && count++ < 5);
          while (true)
          {
             var reply = await _md.Value.requestmd_async(_subscription.Value);
@@ -248,6 +294,12 @@ namespace bts.sdk
          }
       }
 
+      public async void run_pump_async(TimeSpan? interval = null)
+      {
+         await this.start_pump(interval);
+      }
+
+
       readonly CancellationTokenSource _cts = new CancellationTokenSource();
       public void Dispose()
       {

src/client/spark_api/utils/Log.cs

@@ -31,7 +31,8 @@ namespace bts {
       Mercury,
       Testing,
       Latency,
-      DataFlow
+      DataFlow,
+      Plugin
       // Please keep this list in sync with logging.h/logging.cpp
    }
    readonly struct Logger {

src/client/spark_core/Co2/Co2NativeMethods.cs

@@ -80,6 +80,8 @@ namespace mbts.co2
       internal unsafe static extern IntPtr stop_relay(IntPtr relay);
       [DllImport("libspark")]
       internal unsafe static extern IntPtr add_relay_symbol(IntPtr relay, UserSymbol symbol, bts.co2.MDRequest.Types.Flags flags);
+      [DllImport("libspark")]
+      internal unsafe static extern IntPtr set_relay_token(IntPtr relay, string user_name);
 
       [DllImport("libspark")]
       internal unsafe static extern void post_on_scan_security_thread(Action callback);

src/client/spark_core/Co2/Co2RelayControl.cs

@@ -6,6 +6,7 @@ using System.Text;
 using System.Threading.Tasks;
 using bts.api;
 using bts;
+using bts.api.user;
 
 namespace mbts.co2
 {
@@ -22,6 +23,10 @@ namespace mbts.co2
          Log.assert(_relay != IntPtr.Zero);
       }
 
+      public void set_token(AuthToken token) {
+         Co2NativeMethods.set_relay_token(_relay, token.Token);
+      }
+
       public async Task preload_securities(MDRequest.Types.Flags flags=MDRequest.Types.Flags.Security) {
          var securities = UserSymbol.get_all();
          foreach (var future in securities.Where(s => s.type == SecurityType.Future)) {

src/client/spark_core/Co2/MDConsumerHandle.cs

@@ -149,7 +149,6 @@ namespace mbts.co2 {
       public DateTime close_date { get { return _cold->close_date; } }
       public double session_high { get { return _cold->session_high; } }
       public double session_low { get { return _cold->session_low; } }
-      public double open_interest { get { return _cold->open_interest; } }
       // public double session_high_bid { get {  return _cold->session_high_bid; } }
       // public double session_low_offer { get {  return _cold->session_low_offer; } }
       public double fixing_price { get { return _cold->fixing_price; } }
@@ -245,7 +244,7 @@ namespace mbts.co2 {
          }
       }
 
-      [StructLayout(LayoutKind.Sequential, Size = 384)]
+      [StructLayout(LayoutKind.Sequential, Size = 376)]
       internal struct ColdEntry {
          public int exchange_status;
          // Last trade
@@ -268,7 +267,6 @@ namespace mbts.co2 {
          //// Misc stats for current session
          public double session_high;
          public double session_low;
-         public double open_interest;
          public double session_high_bid;
          public double session_low_offer;
          public double fixing_price;

src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs

@@ -0,0 +1,81 @@
+using System.Collections.ObjectModel;
+using System.Linq;
+
+namespace System.Collections.Generic {
+   public static class DictionaryExtensions {
+      public static void AddOrUpdate<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> add_value_factory, Action<TKey, TValue> update_value) {
+         if (!dictionary.TryGetValue(key, out TValue value))
+            dictionary.Add(key, add_value_factory(key));
+         else update_value(key, value);
+      }
+
+      public static void AddOrUpdate<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> add_value_factory, Func<TKey, TValue, TValue> update_value) {
+         if (!dictionary.TryGetValue(key, out TValue value))
+            dictionary.Add(key, add_value_factory(key));
+         dictionary[key] = update_value(key, value);
+      }
+
+      public static TValue GetOrAdd<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TKey, TValue> add_value_factory) {
+         if (!dictionary.TryGetValue(key, out TValue value))
+            dictionary.Add(key, value = add_value_factory(key));
+         return value;
+      }
+
+      public static IReadOnlyDictionary<TKey, (TValue1, TValue2)> Join<TKey, TValue1, TValue2>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target) =>
+         Join(dictionary, merge_target, (v1, v2) => (v1, v2));
+
+      public static IReadOnlyDictionary<TKey, TResult> Join<TKey, TValue1, TValue2, TResult>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target, Func<TValue1, TValue2, TResult> select) {
+         var iks = dictionary.Keys.Intersect(merge_target.Keys);
+         return new ReadOnlyDictionary<TKey, TResult>(iks.ToDictionary(k => k, k => select(dictionary[k], merge_target[k])));
+      }
+
+      public static IReadOnlyDictionary<TKey, TResult> LeftJoin<TKey, TValue1, TValue2, TResult>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target,
+         Func<TValue1, TValue2, TResult> select, Func<TValue1, TResult> select_left_only) =>
+         dictionary.ToDictionary(kv => kv.Key, kv => {
+            if (merge_target.TryGetValue(kv.Key, out TValue2 v2))
+               return select(kv.Value, v2);
+            else
+               return select_left_only(kv.Value);
+         });
+
+      public static IReadOnlyDictionary<TKey, (TValue1, TValue2)> SuperJoin<TKey, TValue1, TValue2>(this IReadOnlyDictionary<TKey, TValue1> dictionary, IReadOnlyDictionary<TKey, TValue2> merge_target) {
+         var eks1 = dictionary.Keys.Except(merge_target.Keys);
+         var eks2 = merge_target.Keys.Except(dictionary.Keys);
+         var iks = dictionary.Keys.Intersect(merge_target.Keys);
+         return new ReadOnlyDictionary<TKey, (TValue1, TValue2)>(eks1
+            .Select(k => (k, dictionary[k], default(TValue2)))
+            .Concat(eks2.Select(k => (k, default(TValue1), merge_target[k])))
+            .Concat(iks.Select(k => (k, dictionary[k], merge_target[k])))
+            .ToDictionary(vals => vals.k, vals => (vals.Item2, vals.Item3))
+         );
+      }
+
+      /// <summary>
+      /// Gets the value associated with the specified key, then removes the key if the lookup was successful.
+      /// </summary>
+      /// <typeparam name="TKey">The type of keys in the dictionary.</typeparam>
+      /// <typeparam name="TValue">The type of values in the dictionary.</typeparam>
+      /// <param name="dictionary">A dictionary to try to get/remove a KeyValuePair from.</param>
+      /// <param name="key">The key of the value to get/remove.</param>
+      /// <param name="result">When this method returns, contains the value associated with the specified key, if the key is found; otherwise, the default value for the type of the value parameter.  This parameter is passed uninitialized.</param>
+      /// <returns>true if the dictionary contains an element with the specified key; otherwise, false.</returns>
+      public static bool TryPopValue<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, out TValue result) {
+         if (dictionary.TryGetValue(key, out result)) {
+            dictionary.Remove(key);
+            return true;
+         } else return false;
+      }
+
+      /// <summary>
+      /// Gets the value associated with the specified key, then calls <paramref name="should_pop"/> with the result if the lookup was successful.  If <paramref name="should_pop"/> returns true, the key is removed.
+      /// </summary>
+      /// <typeparam name="TKey">The type of keys in the dictionary.</typeparam>
+      /// <typeparam name="TValue">The type of values in the dictionary.</typeparam>
+      /// <param name="dictionary">A dictionary to try to get/remove a KeyValuePair from.</param>
+      /// <param name="key">The key of the value to get/remove.</param>
+      /// <param name="result">When this method returns, contains the value associated with the specified key, if the key is found; otherwise, the default value for the type of the value parameter.  This parameter is passed uninitialized.</param>
+      /// <returns>true if the dictionary contains an element with the specified key, AND <paramref name="should_pop"/> returns true; otherwise, false.</returns>
+      public static bool TryPopValueIf<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, Func<TValue, bool> should_pop) =>
+         dictionary.TryGetValue(key, out TValue result) && should_pop(result) && dictionary.Remove(key);
+   }
+}

src/client/spark_core/Collections/SortedList.cs

@@ -2,7 +2,6 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 
 namespace bts.collections {
    public class SortedList<T> : IEnumerable<T> {
@@ -10,11 +9,18 @@ namespace bts.collections {
       int _size = 0;
       readonly IComparer<T> _comparer;
 
+      /// <summary>
+      /// Note: comparer MUST be 100% deterministic for <see cref="Remove(T)"/> to work properly; item-equality is NOT checked.
+      /// </summary>
       public SortedList(IComparer<T> comparer) {
          _comparer = comparer;
          _values = new T[4];
          _size = 0;
       }
+
+      /// <summary>
+      /// Note: comparer MUST be 100% deterministic for <see cref="Remove(T)"/> to work properly; item-equality is NOT checked.
+      /// </summary>
       public SortedList(IEnumerable<T> items, IComparer<T> comparer) {
          _comparer = comparer;
          var capacity = items.Count();
@@ -24,6 +30,8 @@ namespace bts.collections {
          }
       }
 
+      public int Count => _size;
+
       bool Contains(T item) {
          return BinarySearch(item) >= 0;
       }
@@ -57,6 +65,16 @@ namespace bts.collections {
          return false;
       }
 
+      public void Clear() {
+         Array.Clear(_values, 0, _values.Length);
+         _size = 0;
+      }
+
+      public bool TryGetIndex(T item, out int result) {
+         result = BinarySearch(item);
+         return result > 0;
+      }
+
       public T this [int index] {
          get {
             return _values[index];

src/client/spark_core/Collections/SortedTree.cs

@@ -2,99 +2,218 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 
 namespace bts.collections {
+   public interface INode<T> {
+      T value { get; }
+      IEnumerable<INode<T>> inner { get; }
+   }
 
-   public class SortedTree<T, TKey> : IEnumerable<T> {
+   public static class NodeExtensions {
+      public static IEnumerable<T> inner_values<T>(this INode<T> node) {
+         foreach (var n in node.inner) {
+            yield return n.value;
+            foreach (var v in n.inner_values())
+               yield return v;
+         }
+      }
+   }
+
+   public class SortedTree<T, TKey> : IEnumerable<T> where TKey : IEquatable<TKey> {
       readonly IComparer<Node> _comparer;
       readonly Func<T, TKey> id;
       readonly Func<T, TKey> parent;
+      readonly TKey _root_id;
 
       Dictionary<TKey, Node> _index = new Dictionary<TKey, Node>();
+      Dictionary<TKey, SortedList<Node>> _nursery = new Dictionary<TKey, SortedList<Node>>();
       SortedList<Node> _tree;
 
+      /// <summary>
+      /// comparer MUST be 100% deterministic, else Removes won't work right (SortedList uses a simple BinarySearch to find/remove items, and does NOT check item-equality).
+      /// </summary>
+      public SortedTree(IComparer<T> comparer, Func<T, TKey> fnid, Func<T, TKey> fnparent, IEnumerable<T> items = null)
+         : this(comparer, fnid, fnparent, default, items) { }
 
-      public SortedTree(IComparer<T> comparer, Func<T, TKey> fnid, Func<T, TKey> fnparent, IEnumerable<T> items=null) {
+      /// <summary>
+      /// comparer MUST be 100% deterministic, else Removes won't work right (SortedList uses a simple BinarySearch to find/remove items, and does NOT check item-equality).
+      /// </summary>
+      public SortedTree(IComparer<T> comparer, Func<T, TKey> fnid, Func<T, TKey> fnparent, TKey root_id, IEnumerable<T> items = null) {
          _comparer = Comparer<Node>.Create((l, r) => comparer.Compare(l.value, r.value));
          id = fnid;
          parent = fnparent;
          _tree = new SortedList<Node>(_comparer);
-         if (items != null) {
-            AddAll(items);
-         }
+         _root_id = root_id;
+         Reset(items);
       }
 
+      public IEnumerable<INode<T>> RootNodes => _tree;
+
       // Insert (return: true) or overwrite (return false) a new value
       // NOTE: you should use AddAll if you are adding multiple values
       public bool Insert(T item) {
-         var item_id = id(item);
-         if (!_index.TryGetValue(item_id, out Node itemnode)) {
-            // Item is new
-            itemnode = new Node(item, _comparer);
-            _index[item_id] = itemnode;
-            if (_index.TryGetValue(parent(item), out Node pnode)) {
-               pnode.children.Add(itemnode);
-            }
-            else {
-               _tree.Add(itemnode);
-            }
+         var k = id(item);
+         if (_index.TryGetValue(k, out Node node)) {
+            _replace(node, item);
+            return false;
+         } else {
+            node = _init_node(item);
+            _insert_node(node);
+            _index.Add(k, node);
             return true;
          }
-         else {
-            // Item exists
-            if (!parent(itemnode.value).Equals(parent(item))) {
-               // Item has a new parent
-               // Remove it entirely and re-insert at a new tree position
-               Remove(item);
-               Insert(item);
-               return false;
-            }
-            else {
-               // Item has same parent. Just overwrite in place
-               itemnode.value = item;
-               return false;
-            }
-         }
-      }
-      public void Remove(T item) {
-         var item_id = id(item);
-         if (_index.TryGetValue(item_id, out Node itemnode)) {
-            if (_index.TryGetValue(parent(item), out Node parentnode)) {
-               parentnode.children.Remove(itemnode);
-            }
-            else {
-               _tree.Remove(itemnode);
-            }
-            _index.Remove(item_id);
-         }
       }
 
+      public bool TryReplace(TKey k, Func<T, T> replace) {
+         if (!_index.TryGetValue(k, out Node node)) {
+            return false;
+         } else {
+            var newval = replace(node.value);
+            Log.assert(id(newval).Equals(k));
+            _replace(node, newval);
+            return true;
+         }
+      }
+
+   
+      public void Remove(T item) => Remove(id(item));
+
+      // this is NOT idempotent, and will throw an exception if the Key is not present.
+      // (if we decide to change it to be idempotent, we should return a bool to indicate whether or not the remove succeeded)
+      public void Remove(TKey k) {
+         // must be in the index, bc this is a remove
+         var node = _index[k];
+         _index.Remove(k);
+         var pk = parent(node.value);
+         if (pk.Equals(_root_id)) {
+            _tree.Remove(node);
+         } else if (_index.TryGetValue(pk, out Node pnode)) {
+            pnode.children.Remove(node);
+         } else {
+            _remove_from_nursery(pk, node);
+         }
+         if (node.children.Count > 0) {
+            _nursery.Add(k, node.children);
+         }
+      }
+
+      /// <summary>
+      /// comparer MUST be 100% deterministic, else Removes won't work right (SortedList uses a simple BinarySearch to find/remove items, and does NOT check item-equality).
+      /// </summary>
       public SortedTree<T, TKey> Resort(IComparer<T> comparer) {
          return new SortedTree<T, TKey>(comparer, id, parent, this);
       }
 
-      // Note: This method does not handle replacements correctly. 
-      public void AddAll(IEnumerable<T> items) {
+      public void Reset(IEnumerable<T> items) {
+         Clear();
+         if (items == null) return;
          // When adding multiple items we build the index first to avoid the children before parents problem
          foreach (var item in items) {
-            //_index[id(item)] = new Node(item, _comparer);
             _index.Add(id(item), new Node(item, _comparer));
          }
-         // Assume all parents are in this collection
-         // There's not really anything else we can do
          foreach (var item in items) {
             // We are sure it is in the index because we just added it
-            var itemnode = _index[id(item)];
-            if (_index.TryGetValue(parent(item), out Node pnode)) {
-               pnode.children.Add(itemnode);
+            var node = _index[id(item)];
+            if (_nursery.TryPopValue(id(item), out SortedList<Node> nl)) {
+               if (node.children.Count == 0) {
+                  node.children = nl;
+               } else if (node.children.Count > nl.Count) {
+                  // merge nl into node.children
+                  foreach (var cn in nl)
+                     node.children.Add(cn);
+               } else {
+                  // merge node.children into nl
+                  foreach (var cn in node.children)
+                     nl.Add(cn);
+                  node.children = nl;
+               }
             }
-            else {
-               _tree.Add(itemnode);
+            TKey pk = parent(item);
+            if (pk.Equals(_root_id)) {
+               _tree.Add(node);
+            } else if (_index.TryGetValue(pk, out Node pnode)) {
+               pnode.children.Add(node);
+            } else {
+               if (!_nursery.TryGetValue(pk, out nl))
+                  _nursery.Add(pk, nl = new SortedList<Node>(_comparer));
+               nl.Add(node);
             }
          }
       }
 
+      public void Clear() {
+         foreach (Node n in _tree)
+            clear_node(n);
+         foreach (var nl in _nursery.Values) {
+            foreach (var n in nl)
+               clear_node(n);
+            nl.Clear();
+         }
+         _tree.Clear();
+         _index.Clear();
+         _nursery.Clear();
+
+         void clear_node(Node n) {
+            foreach (Node cn in n.children)
+               clear_node(cn);
+            n.children.Clear();
+         }
+      }
+
+      void _replace(Node node, T newval) {
+         TKey pk = parent(newval);
+         TKey old_pk = parent(node.value);
+         if (pk.Equals(old_pk)) {
+            SortedList<Node> node_list;
+            if (pk.Equals(_root_id))
+               node_list = _tree;
+            else if (_index.TryGetValue(pk, out Node pnode)) {
+               node_list = pnode.children;
+            } else if (!_nursery.TryGetValue(pk, out node_list)) {
+               throw new KeyNotFoundException($"Replace was called with a parent-key ({pk}) that was not found in _index or _nursery; item-key was ({id(newval)}).");
+            }
+            //var node_list = pk.Equals(_root_id) ? _tree : _index.TryGetValue(pk, out Node pnode) ? pnode.children : _nursery[pk];
+            node_list.Remove(node);
+            node.value = newval;
+            node_list.Add(node);
+         } else {
+            if (old_pk.Equals(_root_id)) {
+               _tree.Remove(node);
+            } else {
+               if (_index.TryGetValue(old_pk, out Node pnode))
+                  pnode.children.Remove(node);
+               else {
+                  _remove_from_nursery(old_pk, node);
+               }
+            }
+            node.value = newval;
+            _insert_node(node);
+         }
+      }
+ 
+      Node _init_node(T item) =>
+         _nursery.TryPopValue(id(item), out SortedList<Node> nursery_list) ? new Node(item, nursery_list) : new Node(item, _comparer);
+
+      /// <summary>
+      /// Inserts the node into the appropriate SortedList (i.e. roots, parent.inner, or nursery), but does NOT add it to <see cref="_index"/>, NOR check the nursery for any child-nodes waiting on this item.
+      /// </summary>
+      void _insert_node(Node node) {
+         TKey pk = parent(node.value);
+         if (pk.Equals(_root_id)) {
+            _tree.Add(node);
+         } else if (_index.TryGetValue(pk, out Node pnode)) {
+            pnode.children.Add(node);
+         } else {
+            if (!_nursery.TryGetValue(pk, out SortedList<Node> nursery_list)) {
+               _nursery.Add(pk, nursery_list = new SortedList<Node>(_comparer));
+            }
+            nursery_list.Add(node);
+         }
+      }
+
+      void _remove_from_nursery(TKey pk, Node node) =>
+         _nursery.TryPopValueIf(pk, nl => nl.Remove(node) && nl.Count == 0);
+
       IEnumerable<T> iterate(SortedList<Node> nodes) {
          foreach (var n in nodes) {
             yield return n.value;
@@ -114,27 +233,54 @@ namespace bts.collections {
          return null;
       }
 
+      public INode<T> GetRootNode(TKey key) =>
+         _index.TryGetValue(key, out Node node) ? GetRootNode(node) : null;
+
+      Node GetRootNode(Node node) =>
+         _index.TryGetValue(parent(node.value), out Node pnode) ? GetRootNode(pnode) : node;
 
       public IEnumerable<T> EnumerateGroup(T item) {
          var pnode = parentmost(item);
-         if (pnode != null) {
-            yield return pnode.value;
-            foreach (var child in iterate(pnode.children)) {
-               yield return child;
-            }
+         return pnode == null ? Enumerable.Empty<T>() : EnumerateNode(pnode);
+      }
+
+      public IEnumerable<T> EnumerateInnerValues(T item) =>
+         iterate(_index[id(item)].children);
+
+      public IEnumerable<INode<T>> EnumerateInnerNodes(T item) =>
+         _index[id(item)].children;
+
+      IEnumerable<T> EnumerateNode(Node node) {
+         yield return node.value;
+         foreach (var child in iterate(node.children)) {
+            yield return child;
          }
       }
       public IEnumerator<T> GetEnumerator() {
-         return iterate(_tree).GetEnumerator();
+         foreach (T item in iterate(_tree))
+            yield return item;
+         foreach (var kv in _nursery) {
+            foreach (T item in iterate(kv.Value))
+               yield return item;
+         }
       }
 
       IEnumerator IEnumerable.GetEnumerator() {
          return GetEnumerator();
       }
-      class Node {
-         public Node(T i, IComparer<Node> c) { value = i; children = new SortedList<Node>(c); }
+      class Node : INode<T> {
+         public Node(T i, IComparer<Node> c)
+            : this(i, new SortedList<Node>(c)) { }
+
+         public Node(T i, SortedList<Node> children) {
+            value = i;
+            this.children = children;
+         }
+
          public T value;
          public SortedList<Node> children;
+         T INode<T>.value => value;
+         IEnumerable<INode<T>> INode<T>.inner => children;
       }
    }
 }

src/client/spark_core/utils/AnonymousDisposable.cs

@@ -92,6 +92,13 @@ namespace mbts
             item = null;
          }
       }
+
+      public static void replace<T>(ref T current, T newval) where T : IDisposable {
+         if (current != null) {
+            current.Dispose();
+         }
+         current = newval;
+      }
    }
 
    // inspired by: https://blogs.msdn.microsoft.com/blambert/2009/07/24/a-simple-and-totally-thread-safe-implementation-of-idisposable/

src/client/spark_core/utils/GlobalExtensions.cs

@@ -19,6 +19,9 @@ namespace mbts {
          }
          return sb.ToString();
       }
+      public static bool iequals(this string str, string comparand) =>
+         string.Equals(str, comparand, StringComparison.CurrentCultureIgnoreCase);
+
       public static T choose<T>(this Random r, IEnumerable<T> range) {
          var index = r.Next(0, range.Count());
          return range.Skip(index).First();

src/client/spark_excel/ExcelAddin.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Reflection;
+
+using System.IO;
+using System.Security.Policy;
+using System.Diagnostics;
+
+using ExcelDna.Integration;
+using ExcelDna.IntelliSense;
+using ExcelDna.Logging;
+
+namespace bts.excel
+{
+   public class TestMethod
+   {
+      [ExcelFunction]
+      static public string bts_excel_test_string(string x) { return "Hello " + x; }
+      [ExcelFunction]
+      static public object bts_excel_test_int(int x) { return x; }
+      [ExcelFunction]
+      static public object bts_excel_test_long(long x) { return x; }      
+   }
+
+   [System.Runtime.InteropServices.ComVisible(true)]
+   public class ContextTeardown : ExcelComAddIn
+   {
+      private api.core.ServiceContext _context;
+      public ContextTeardown(api.core.ServiceContext context)
+      {
+         _context = context;
+      }
+      //This is called on application shutdown
+      public override void OnBeginShutdown(ref Array custom)
+      {
+         _context.Dispose();
+         _context = null;
+      }
+   }
+
+   [System.Runtime.InteropServices.ComVisible(true)]
+   public class ExcelAddin : IExcelAddIn
+   {
+      public void AutoClose() 
+      {
+         //don't do anything here, it's only called when the addin is removed, not on shutdown
+         IntelliSenseServer.Uninstall();
+      }
+      public void AutoOpen() //This is called on EXCEL startup, when the addin is loaded
+      {
+         Trace.Listeners.Add(new LogDisplayTraceListener("bts"));
+         IntelliSenseServer.Install();
+         ExcelIntegration.RegisterUnhandledExceptionHandler(ex => ex.ToString());
+
+         load_assemblies();
+         ExcelComAddInHelper.LoadComAddIn(new ContextTeardown(APIDispatch.service_context.Value));         
+      }
+
+      private void load_assemblies()
+      {
+         var config = System.Configuration.ConfigurationManager.AppSettings["load_assemblies"];
+         if (config != null)
+         {
+            var asses = config.Split(new char[] { ';', ',', ' ' }, StringSplitOptions.RemoveEmptyEntries);
+            foreach (var ass in asses)
+               Utilities.bts_load_assembly(ass);
+         }
+      }
+   }
+}

src/client/spark_excel/Properties/AssemblyInfo.cs

@@ -0,0 +1,17 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+[assembly: InternalsVisibleTo("spark")]
+[assembly: InternalsVisibleTo("spark_core")]
+[assembly: InternalsVisibleTo("sparkcmd")]
+[assembly: InternalsVisibleTo("spark_test")]
+
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("b319f835-7390-47e5-8093-461ed0dfce63")]

src/client/spark_excel/Properties/ExcelDna.Build.props

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="ExcelDnaProps"> -->
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<!--
+  If you change properties in this file, they may not come into effect until you:
+     * Rebuild the solution/project
+
+     or
+
+     * Close Visual Studio
+     * Delete .vs folder, if exists
+     * Delete ProjectName.csproj.user (or equivalent for VB, F#, etc.), if exists
+     * Delete SolutionName.suo, if exists
+     * Open your solution/project again in Visual Studio
+  -->
+
+  <!--
+    Configuration properties for building .dna files
+  -->
+  <PropertyGroup>
+    <!--
+      Enable/Disable automatic generation of platform-specific versions of .dna files
+    -->
+    <ExcelDnaCreate32BitAddIn Condition="'$(ExcelDnaCreate32BitAddIn)' == ''">false</ExcelDnaCreate32BitAddIn>
+    <ExcelDnaCreate64BitAddIn Condition="'$(ExcelDnaCreate64BitAddIn)' == ''">true</ExcelDnaCreate64BitAddIn>
+
+    <!--
+      Define the suffix used for each platform-specific file e.g. MyAddIn64.dna
+    -->
+    <ExcelDna32BitAddInSuffix Condition="'$(ExcelDna32BitAddInSuffix)' == ''"></ExcelDna32BitAddInSuffix>
+    <ExcelDna64BitAddInSuffix Condition="'$(ExcelDna64BitAddInSuffix)' == ''">64</ExcelDna64BitAddInSuffix>
+  </PropertyGroup>
+
+  <!--
+    Configuration properties for packing .dna files
+  -->
+  <PropertyGroup>
+    <!--
+      Enable/Disable packing of .dna files
+    -->
+    <RunExcelDnaPack Condition="'$(RunExcelDnaPack)' == ''">false</RunExcelDnaPack>
+
+    <!--
+      Suffix used for packed .xll files e.g. MyAddIn-packed.xll
+    -->
+    <ExcelDnaPackXllSuffix Condition="'$(ExcelDnaPackXllSuffix)' == ''">-packed</ExcelDnaPackXllSuffix>
+  </PropertyGroup>
+</Project>
+

src/client/spark_excel/extensions/EnumParser.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Linq;
+
+namespace bts.excel.extensions
+{
+   public static class EnumParser
+   {
+      public static T ToEnum<T>(this string val, T default_val = default(T)) where T : struct, IConvertible, IComparable
+      {
+         val = val ?? string.Empty;
+         T eval;
+         if (!Enum.TryParse(val, true, out eval))
+         {
+            string[] names = Enum.GetNames(typeof(T));
+            val = names.FirstOrDefault(x => x.ToLower().Contains(val));
+            if(val != null)
+               Enum.TryParse(val, true, out eval);
+               
+            if (eval.CompareTo(default(T)) == 0 && default_val.CompareTo(default(T)) != 0)
+               eval = default_val;
+         }         
+         return eval;
+      }
+   }
+}

src/client/spark_excel/extensions/OrderService_Excel.cs

@@ -0,0 +1,161 @@
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using ExcelDna.Integration;
+
+using bts.api.admin;
+using bts.excel;
+using bts.excel.extensions;
+using bts.excel.rtd;
+using Newtonsoft.Json;
+
+
+namespace bts.api.orderman
+{
+   public partial class OrderService_Excel 
+   {
+      readonly static Lazy<user.UserService> _users = new Lazy<user.UserService>(() => locate<user.UserService>(), true);
+      readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => locate<admin.AdminService>(), true);
+      
+
+      [ExcelFunction(Category = "BTS", Description = "Cancel an Order by Order ID", IsMacroType = true)]
+      public static object spark_orderservice_cancel_order(long order_id)
+      {
+         var resp = _service.Value.cancel_order(new Cancel() { OrderKey = Convert.ToUInt32(order_id) });
+         return parse_response(resp);
+      }
+
+      [ExcelFunction(Category = "BTS", Description = "Update an Order by Order ID", IsMacroType = true)]
+      public static object spark_orderservice_update_order(long order_id, double price, long quantity)
+      {
+         var resp = _service.Value.update_order(new Update() { OrderKey = Convert.ToUInt32(order_id), Price = price, Quantity = Convert.ToUInt32(quantity) });
+         return parse_response(resp);
+      }      
+
+      [ExcelFunction(Category = "BTS", Description = "Send an Order to Spark via security_key", IsMacroType = true)]
+      public static object spark_orderservice_send_order_key(
+         [ExcelArgument("Required. Only send the order if > last")] long local_order_id,
+         [ExcelArgument("Required. Security key")] long security_key,
+         [ExcelArgument("Required. Side BUY or SELL")] string side,
+         [ExcelArgument("Required. User")] string user,
+         [ExcelArgument("Required. Account")] string account,
+         [ExcelArgument("Required. Price")] double price,
+         [ExcelArgument("Required. Quantity")] long quantity,
+         [ExcelArgument("Required. OrderType (Limit, Stop, Stop_Limit, Iceberg) - Default Limit")] string order_type,
+         [ExcelArgument("OPTIONAL. Order Params JSON (\"stop_price\" : 51.88)")] string order_params = null,
+         
+         [ExcelArgument("OPTIONAL. (DAY,GTC,IOC) - Default DAY")] string time_in_force = null,
+         [ExcelArgument("OPTIONAL. (YES,NO) - Default YES")] string manual_order_indicator = null,
+         [ExcelArgument("OPTIONAL. (1,2,3,4) - Default 1")] string cti_code = null,
+         [ExcelArgument("OPTIONAL. (CUSTOMER,FIRM) - Default FIRM")] string customer_or_firm = null)
+      {
+         if (local_order_id <= _local_order_id)
+            return "local_order_id must be greater than " + _local_order_id;
+         Create cr = new Create();
+         cr.NewOrder = new OrderDefinition()
+         {
+            SecurityKey = Convert.ToUInt32(security_key),
+            Side = EnumParser.ToEnum<api.Side>(side),
+            TimeInForce = time_in_force.ToEnum<TimeInForce>(TimeInForce.Day),
+            ManualOrderIndicator = manual_order_indicator.ToEnum<ManualOrderIndicator>(ManualOrderIndicator.Yes),
+            CtiCode = cti_code.ToEnum<CTICode>(CTICode.CtiCode1),
+            CustomerOrFirm = customer_or_firm.ToEnum<CustomerOrFirm>(CustomerOrFirm.Firm),
+         };
+         cr.Price = price;
+         cr.Quantity = Convert.ToUInt32(quantity);         
+         
+         var btsuser = resolve_user(user);
+         if(btsuser == null)
+            return "Could not resolve user: " + user;
+         cr.NewOrder.User = btsuser.Name;
+         cr.NewOrder.UserId = btsuser.Id;
+         var btaccount = resolve_account(account);
+         if(btaccount == null)
+            return "Could not resolve account: " + account;
+         cr.NewOrder.Account = btaccount.Name;
+         cr.NewOrder.AccountId = btaccount.Id;
+         cr.NewOrder.CustomDef = custom_def(order_type, order_params);
+         if(cr.NewOrder.CustomDef == null || cr.NewOrder.CustomDef.CustomCase == CustomOrderDefinition.CustomOneofCase.None)
+            return "Could not parse order_type and/or order_params info";
+
+         var resp = _service.Value.create_order(cr);
+         _local_order_id = local_order_id;
+         return parse_response(resp, true);
+      }
+
+      private static object parse_response(Response resp, bool show_vertically = false)
+      {
+         object[,] response_cells = new JsonFormatter(resp).format(false);
+         if(show_vertically)
+            response_cells = bts.excel.Utilities.transpose_keep_dimension(response_cells);
+         return response_cells;
+      }
+
+      static Lazy<AdminData> _admin_data = new Lazy<AdminData>(() => _admin.Value.get_admin_data(new VoidMessage()), true);
+      static BTSUser resolve_user(string username)
+      {         
+         var user = _admin_data.Value.Users.FirstOrDefault(x => username != null ? x.Name.ToLowerInvariant() == username.ToLowerInvariant() : true);
+         return user;
+      }
+
+      static BTSAccount resolve_account(string acctname)
+      {
+         var acc = _admin_data.Value.Accounts.FirstOrDefault(x => x.Name.ToLowerInvariant() == acctname.ToLowerInvariant());
+         return acc;
+      }      
+
+      static CustomOrderDefinition custom_def(string order_type, string order_params)
+      {
+         var member = ReflectionExtensions.get_fields_or_properties(typeof(CustomOrderDefinition))
+            .Where(x => x.Name.ToLower() == order_type.ToLower()).FirstOrDefault();
+         if (member != null)
+         {
+            var mi = member.member_type();
+            var instance = APIDispatch.generate_imessage(mi, order_params, false);
+            var cust = new CustomOrderDefinition();
+            ReflectionExtensions.set_value(member, cust, instance);
+            return cust;
+         }
+         else
+            return null;
+      }
+
+      static long _local_order_id = 0;
+
+      [ExcelFunction(Category = "BTS", Description = "Send an Order to Spark via security_key", IsMacroType = true)]
+      public static object spark_orderservice_send_order(
+         [ExcelArgument("Required. Only send the order if > last")] long local_order_id,
+         [ExcelArgument("Required. Exchange (GLBX,SIM)")] string exchange,
+         [ExcelArgument("Required. Security Desc (ZNZ9 or ZNZ9-ZNH0)")] string security_desc,
+         [ExcelArgument("Required. Side BUY or SELL")] string side,
+         [ExcelArgument("Required. User")] string user,
+         [ExcelArgument("Required. Account")] string account,
+         [ExcelArgument("Required. Price")] double price,
+         [ExcelArgument("Required. Quantity")] long quantity,
+         [ExcelArgument("Required. OrderType (Limit, Stop, Stop_Limit, Iceberg) - Default Limit")] string order_type,
+         [ExcelArgument("OPTIONAL. Order Params JSON (\"stop_price\" : 51.88)")] string order_params = null,
+        
+         [ExcelArgument("OPTIONAL. (DAY,GTC,IOC) - Default DAY")] string time_in_force = null,
+         [ExcelArgument("OPTIONAL. (YES,NO) - Default YES")] string manual_order_indicator = null,
+         [ExcelArgument("OPTIONAL. (1,2,3,4) - Default 1")] string cti_code = null,
+         [ExcelArgument("OPTIONAL. (CUSTOMER,FIRM) - Default FIRM")] string customer_or_firm = null)
+      {
+         if (local_order_id <= _local_order_id)
+            return "local_order_id must be greater than " + _local_order_id;
+         Venue ven;
+         if (!Utilities.parse_venue(exchange, out ven))
+            return "Can't Parse Exchange: " + (exchange ?? string.Empty);
+
+         uint key;
+         if (MDSubscribe.market_data(ven).lookup_security_key_from_desc(security_desc, out key))
+         {
+            return spark_orderservice_send_order_key(local_order_id, (long)key, side, user, account, price, quantity, order_type, order_params,
+                                                     time_in_force, manual_order_indicator, cti_code, customer_or_firm);
+         }
+         else
+         {
+            return "Can't find security desc " + security_desc;
+         }
+      }
+   }
+}

src/client/spark_excel/extensions/ReflectionExtensions.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Reflection;
+
+namespace bts.excel.extensions
+{
+   public static class ReflectionExtensions
+   {
+      public static void set_value(this MemberInfo info, object target, object value)
+      {
+         if (info is FieldInfo)
+            ((FieldInfo)info).SetValue(target, value);
+         else if (info is PropertyInfo)
+            ((PropertyInfo)info).SetValue(target, value);
+      }
+
+      public static object get_value(this MemberInfo info, object target)
+      {
+         if (info is FieldInfo)
+            return ((FieldInfo)info).GetValue(target);
+         else if (info is PropertyInfo)
+            return ((PropertyInfo)info).GetValue(target);
+         else
+            return null;
+      }
+
+      public static MemberInfo[] get_fields_or_properties(this Type t)
+      {
+         MemberInfo[] mi = t.GetMembers(BindingFlags.Instance | BindingFlags.Public
+            | BindingFlags.NonPublic).Where(m => (m.MemberType == MemberTypes.Field || m.MemberType == MemberTypes.Property)).ToArray();
+         return mi;
+      }
+
+      public static MemberInfo[] get_fields(this Type t)
+      {
+         MemberInfo[] mi = t.GetMembers(BindingFlags.Instance | BindingFlags.Public
+            | BindingFlags.NonPublic).Where(m => (m.MemberType == MemberTypes.Field)).ToArray();
+         return mi;
+      }
+
+      public static MemberInfo[] get_properties(this Type t)
+      {
+         MemberInfo[] mi = t.GetMembers(BindingFlags.Instance | BindingFlags.Public
+            | BindingFlags.NonPublic).Where(m => (m.MemberType == MemberTypes.Property)).ToArray();
+         return mi;
+      }
+
+      public static MemberInfo get_field_or_property(this Type t, string name)
+      {
+         MemberInfo[] mi = t.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
+            .Where(m => (m.MemberType == MemberTypes.Field || m.MemberType == MemberTypes.Property))
+            .Where(m => (m.Name == name))
+            .ToArray();
+         if (mi != null && mi.Length > 0)
+            return mi[0];
+         else
+            return null;
+      }
+
+      public static MemberInfo get_static_field_or_property(this Type t, string name)
+      {
+         MemberInfo[] mi = t.GetMembers(BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic)
+            .Where(m => (m.MemberType == MemberTypes.Field || m.MemberType == MemberTypes.Property))
+            .Where(m => (m.Name == name))
+            .ToArray();
+         if (mi != null && mi.Length > 0)
+            return mi[0];
+         else
+            return null;
+      }
+
+      public static MemberInfo[] search_field_or_property(this Type t, Func<MemberInfo, bool> criteria)
+      {
+         MemberInfo[] mi = t.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
+            .Where(m => (m.MemberType == MemberTypes.Field || m.MemberType == MemberTypes.Property))
+            .Where(criteria)
+            .ToArray();
+         if (mi != null && mi.Length > 0)
+            return mi;
+         else
+            return new MemberInfo[0];
+      }
+
+
+      public static Type member_type(this MemberInfo info)
+      {
+         if (info is FieldInfo)
+            return ((FieldInfo)info).FieldType;
+         else if (info is PropertyInfo)
+            return ((PropertyInfo)info).PropertyType;
+         else
+            return null;
+      }
+
+      public static MethodInfo[] find_method(this Type t, Func<MethodInfo, bool> criteria)
+      {
+         var mi = t.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
+            .Where(criteria).ToArray();
+         if (mi != null && mi.Length > 0)
+            return mi;
+         else
+            return new MethodInfo[0];
+      }
+   }
+}

src/client/spark_excel/packages.config

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="ExcelDna.AddIn" version="0.34.6" targetFramework="net471" />
+  <package id="ExcelDna.Integration" version="0.34.6" targetFramework="net471" />
+</packages>

src/client/spark_excel/rtd/APIRTDServer.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Linq;
+
+using ExcelDna.Integration;
+using ExcelDna.Integration.Rtd;
+using bts.excel.extensions;
+using bts.api.core;
+using bts.api;
+using bts.api.md;
+using System.Reactive.Linq;
+
+namespace bts.excel.rtd
+{
+   [System.Runtime.InteropServices.ComVisible(true)]
+   public class APIRTDServer : ExcelRtdServer
+   {
+      readonly Dictionary<int, IObservable<object>> _subs = new Dictionary<int, IObservable<object>>();
+      readonly List<IDisposable> _disps = new List<IDisposable>();      
+      readonly static ConcurrentDictionary<string, object> data_store = new ConcurrentDictionary<string, object>();      
+      public APIRTDServer()
+      {
+      }
+
+      static public bool retrieve(string key, out object data)
+      {
+         return data_store.TryGetValue(key, out data);
+      }
+
+      protected override bool ServerStart() { return true; }
+      protected override void ServerTerminate() { }
+
+      protected override ExcelRtdServer.Topic CreateTopic(int topicId, IList<string> topicInfo)
+      {
+         string[] tagvals = new string[topicInfo.Count - 2];
+         Array.Copy(topicInfo.ToArray(), 2, tagvals, 0, tagvals.Length);
+         var bt = new APITopic(this, topicId, topicInfo[0], topicInfo[1], tagvals);
+         return bt;
+      }
+
+      protected override object ConnectData(Topic topic, IList<string> topicInfo, ref bool newValues)
+      {
+         //return ExcelErrorUtil.ToComError(ExcelError.ExcelErrorNA);
+         newValues = true;
+         TopicBase bt = (TopicBase)topic;
+         IObservable<object> obs = null;         
+         try
+         {
+            string error = subscribe(bt, out obs);
+            if (obs == null)
+               return error;
+            _subs[topic.TopicId] = obs;
+         }
+         catch (Exception e)
+         {
+            return "PLEASE RESUBSCRIBE! Error: " + e.ToString();
+         }
+         _disps.Add(obs.Subscribe((data) =>
+            {
+               string gstring = bt.id.ToString();
+
+               data_store[gstring] = data;
+               bt.UpdateValue(gstring);
+               bt.UpdateNotify();
+            },
+            (error) =>
+            {
+               topic.UpdateValue(error.ToString());
+            },
+            () => { }));
+         return ExcelErrorUtil.ToComError(ExcelError.ExcelErrorNA); //success!
+      }
+
+      protected override void DisconnectData(ExcelRtdServer.Topic topic)
+      {
+         IObservable<object> sub;
+         if (_subs.TryGetValue(topic.TopicId, out sub))
+         {
+            IDisposable disp = sub as IDisposable;
+            if (disp != null)
+               disp.Dispose();
+            _subs.Remove(topic.TopicId);
+         }
+      }
+
+      protected override int Heartbeat()
+      {
+         return base.Heartbeat();
+      }
+
+
+      string subscribe(TopicBase topic, out IObservable<object> stream)
+      {
+         if(topic is APITopic apitopic)
+            return subscribe(apitopic, out stream);
+         else
+         {
+            stream = null;
+            return "Unknown Topic Type";
+         }
+      }
+
+      string subscribe(APITopic topic, out IObservable<object> stream)
+      {
+         stream = null;
+         if (topic.service_type == null)
+            return "Unknown ServiceType: " + topic.service_type_string;
+         if (topic.request_type == null)
+            return "Unknown MethodName: " + topic.method_name;
+
+         var service = APIDispatch.locate_service(topic.service_type);
+         var request = APIDispatch.generate_request(topic.service_type, topic.method_name, topic.parameters[0]);
+         if (service != null) //request can be null here
+         {
+            var subscription = APIDispatch.invoke_service_call(service, topic.method_name, request);
+            stream = subscription as IObservable<object>;
+            if (stream != null)
+            {
+               dynamic dtable = ObservableUpdateExtensions.Table((dynamic)subscription);
+               stream = dtable as IObservable<object>;
+            }
+         }
+         return string.Empty;
+      }  
+   }
+}

src/client/spark_excel/rtd/APITopic.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Linq;
+
+using ExcelDna.Integration;
+using ExcelDna.Integration.Rtd;
+using bts.excel.extensions;
+using System.Reflection;
+
+namespace bts.excel.rtd
+{
+   class TopicBase : ExcelRtdServer.Topic
+   {
+      public Guid id { get; protected set; }
+
+      public TopicBase(ExcelRtdServer server, int topicid) : base(server, topicid)
+      {
+         id = Guid.NewGuid();
+      }
+   }
+
+   class APITopic : TopicBase
+   {      
+      public Type service_type { get; private set; }
+      public Type request_type { get; private set; }
+      public string method_name { get; private set; }
+      public string[] parameters { get; private set; }
+
+      public string service_type_string { get; private set; }
+      public const string topic_type = "api";
+
+      public APITopic(ExcelRtdServer server, int topicid,
+                      string serv_type, string methodname, string[] param) : base(server, topicid)
+      {
+         service_type_string = serv_type;
+         method_name = methodname;         
+         service_type = find_api_type(service_type_string, null, typeof(bts.api.core.IApiClient));
+         request_type = APIDispatch.find_request_type(service_type, method_name);
+         parameters = param;
+      }
+
+      Type find_api_type(string type, string assem_regex = null, Type assignable = null)
+      {
+         return Utilities.find_full_type_in_all_assemblies(type, assem_regex, assignable);
+      }
+
+      public override int GetHashCode()
+      {
+         var h = service_type?.GetHashCode() ^ request_type?.GetHashCode() ^ all_params().GetHashCode();
+         if (h.HasValue)
+            return h.Value;
+         else
+            return -1;
+      }
+
+      string all_params()
+      {
+         if (parameters != null)
+            return parameters.Aggregate((a, b) => a + b);
+         else
+            return string.Empty;
+      }
+
+
+
+      public override bool Equals(object obj)
+      {
+         APITopic top = (obj as APITopic);
+         if (top == null)
+            return false;
+         else
+         {
+            return top.service_type.Equals(this.service_type) &&
+               top.request_type.Equals(this.request_type) &&
+               top.all_params().Equals(this.all_params());
+         }
+      }
+   }
+}

src/client/spark_excel/rtd/DataParser.cs

@@ -0,0 +1,319 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Linq;
+using bts.excel.extensions;
+using System.Text.RegularExpressions;
+using bts.api.core;
+
+namespace bts.excel
+{
+   public class DataParser
+   {
+      private static Type analyze_type(Type type, out bool is_array)
+      {
+         Type element_type = type;
+         is_array = false;
+         if (element_type.IsArray)
+         {
+            is_array = true;
+            element_type = element_type.GetElementType();
+         }
+         return element_type;
+      }
+
+      public static bool supported_primitive(Type t)
+      {
+         bool dummy;
+         return supported_primitive(t, out dummy);
+      }
+
+      public static bool supported_primitive(Type t, out bool is_array)
+      {
+         is_array = t.IsArray;
+         if (is_array)
+            t = t.GetElementType();
+         return t.IsPrimitive || t.IsEnum || t == typeof(string) || t == typeof(DateTime);
+      }
+
+      public static bool excluded_member(MemberInfo s)
+      {
+         return s.Name.ToLower() == "extensionobject" ||
+                s.Name.ToLower() == "proto" ||
+                s.Name.ToLower() == "descriptor" ||
+
+                s.Name.Contains(".")
+            ;
+      }
+
+      private static void analyze_fields(Type element_type,
+                                         out List<MemberInfo> superduperfields, out List<MemberInfo> superfields, out List<MemberInfo> fields,
+                                         out List<string> superdupernames, out List<string> supernames, out List<string> names, out List<string> combined)
+      {
+         var members = ReflectionExtensions.get_fields_or_properties(element_type);
+         superduperfields = new List<MemberInfo>();
+         superfields = new List<MemberInfo>();
+         fields = new List<MemberInfo>();
+         superdupernames = new List<string>();
+         supernames = new List<string>();
+         names = new List<string>();
+         combined = new List<string>();
+         foreach (var r in members)
+         {
+            if (excluded_member(r))
+               continue;
+            if (supported_primitive(ReflectionExtensions.member_type(r)))
+            {
+               superduperfields.Add(r);
+               superdupernames.Add(r.Name);
+               superfields.Add(r);
+               supernames.Add(r.Name);
+               fields.Add(r);
+               names.Add(r.Name);
+               combined.Add(r.Name);
+            }
+            else
+            {
+               var sub = ReflectionExtensions.get_fields_or_properties(ReflectionExtensions.member_type(r));
+               foreach (var s in sub)
+               {
+                  if (excluded_member(s))
+                     continue;
+                  if (supported_primitive(ReflectionExtensions.member_type(s)))
+                  {
+                     superduperfields.Add(r);
+                     superdupernames.Add(r.Name);
+                     superfields.Add(r);
+                     supernames.Add(r.Name);
+                     fields.Add(s);
+                     names.Add(s.Name);
+                     combined.Add(r.Name + "." + s.Name);
+                  }
+                  else
+                  {
+                     var subsub = ReflectionExtensions.get_fields_or_properties(ReflectionExtensions.member_type(s));
+                     foreach (var ss in subsub)
+                     {
+                        if (excluded_member(ss) || !supported_primitive(ReflectionExtensions.member_type(ss)))
+                           continue;
+                        superduperfields.Add(r);
+                        superdupernames.Add(r.Name);
+                        superfields.Add(s);
+                        supernames.Add(s.Name);
+                        fields.Add(ss);
+                        names.Add(ss.Name);
+                        combined.Add(r.Name + "." + s.Name + "." + ss.Name);
+                     }
+                  }
+               }
+            }
+         }
+      }
+
+      static public object gather_data(Type type, object[,] data)
+      {
+         if (type == null)
+            return null;
+         bool is_array;
+         Type element_type = analyze_type(type, out is_array);
+
+         bool field_name_headers = true;
+         List<MemberInfo> superduperfields, superfields, fields;
+         List<string> superdupernames, supernames, names, combined;
+         analyze_fields(element_type, out superduperfields, out superfields, out fields,
+                                      out superdupernames, out supernames, out names, out combined);
+
+         int object_len = data.GetLength(1) - (field_name_headers ? 1 : 0);
+         Array objects = Array.CreateInstance(element_type, object_len);
+         for (int j = 0; j < objects.Length; j++)
+         {
+            object target = Activator.CreateInstance(element_type);
+            for (int zz = 0; zz < superduperfields.Count; zz++)
+            {
+               if (Regex.Matches(combined[zz], @"\.").Count == 2)
+                  ReflectionExtensions.set_value(superduperfields[zz], target, Activator.CreateInstance(ReflectionExtensions.member_type(superduperfields[zz])));
+            }
+            for (int z = 0; z < superfields.Count; z++)
+            {
+               if (combined[z].Contains("."))
+                  ReflectionExtensions.set_value(superfields[z], target, Activator.CreateInstance(ReflectionExtensions.member_type(superfields[z])));
+            }
+            for (int i = 0; i < data.GetLength(0); i++)
+            {
+               string name = data[i, 0].ToString();
+               int idx = combined.IndexOf(name);
+               if (idx < 0 || idx >= combined.Count)
+                  continue;
+
+               MemberInfo fi = fields[idx];
+               object val = val = data[i, j + (field_name_headers ? 1 : 0)];
+               val = read_data(ReflectionExtensions.member_type(fi), val);
+               int dotcount = Regex.Matches(combined[i], @"\.").Count;
+               if (dotcount == 2)
+               {
+                  object subsubtarget = ReflectionExtensions.get_value(superduperfields[i], target);
+                  if (subsubtarget != null)
+                  {
+                     object subtarget = ReflectionExtensions.get_value(superfields[i], subsubtarget);
+                     if (subtarget != null)
+                        ReflectionExtensions.set_value(fi, subtarget, val);
+                  }
+               }
+               else if (dotcount == 1)
+               {
+                  object subtarget = ReflectionExtensions.get_value(superfields[i], target);
+                  if (subtarget != null)
+                     ReflectionExtensions.set_value(fi, subtarget, val);
+               }
+               else if (dotcount == 0)
+                  ReflectionExtensions.set_value(fi, target, val);
+            }
+            objects.SetValue(target, j);
+         }
+
+         if (is_array)
+            return objects;
+         else if (objects.Length > 0)
+            return objects.GetValue(0);
+         else
+            return null;
+      }
+
+      public static object read_data(Type field_type, object val)
+      {
+         if (typeof(Array).IsAssignableFrom(field_type))
+            val = unstringify_array(val, field_type);
+         else if (field_type == typeof(DateTime))
+            val = DateTime.FromOADate(Convert.ToDouble(val));
+         else if (field_type.IsEnum)
+            val = Enum.Parse(field_type, val.ToString());
+         else
+            val = Convert.ChangeType(val, field_type);
+         return val;
+      }
+
+      public static Tuple<object[], object[]> key_value_tuple<T, TKey>(TableUpdate<T, TKey> source) where TKey : IEquatable<TKey>
+      {
+         return Tuple.Create(source.state.Keys.Cast<object>().ToArray(), source.state.Values.Cast<object>().ToArray());
+      }
+
+      public static object[,] scatter_data(object data, bool include_field_names = true, Type type_override = null)
+      {
+         if (data == null)
+            return null;
+
+         bool is_array;
+         Type type = type_override ?? data.GetType();
+         if (type.IsGenericType && type.GetGenericTypeDefinition().Equals(typeof(TableUpdate<,>)))
+         {
+            Tuple<object[], object[]> tup = key_value_tuple((dynamic)data);
+            return scatter_data(tup.Item2, include_field_names, tup.Item2.FirstOrDefault()?.GetType().MakeArrayType());
+         }
+         Type element_type = analyze_type(type, out is_array);
+
+         List<MemberInfo> superduperfields, superfields, fields;
+         List<string> superdupernames, supernames, names, combined;
+         analyze_fields(element_type, out superduperfields, out superfields, out fields,
+                                      out superdupernames, out supernames, out names, out combined);
+
+         Array the_data = (is_array) ? (Array)data : new object[] { data };
+         int len = the_data.GetLength(0);
+
+         var ret = new object[fields.Count, len + (include_field_names ? 1 : 0)];
+         for (int j = 0; j < ret.GetLength(1); j++)
+         {
+            for (int i = 0; i < ret.GetLength(0); i++)
+            {
+               object dat = "";
+               int dotcount = Regex.Matches(combined[i], @"\.").Count;
+               if (dotcount == 2)
+               {
+                  if (j == 0 && include_field_names)
+                     dat = combined[i];
+                  else
+                  {
+                     object subsub = ReflectionExtensions.get_value(superduperfields[i], the_data.GetValue(j - (include_field_names ? 1 : 0)));
+                     if (subsub != null)
+                     {
+                        object sub = ReflectionExtensions.get_value(superfields[i], subsub);
+                        if (sub != null)
+                           dat = ReflectionExtensions.get_value(fields[i], sub);
+                     }
+                  }
+               }
+               else if (dotcount == 1)
+               {
+                  if (j == 0 && include_field_names)
+                     dat = combined[i];
+                  else
+                  {
+                     object sub = ReflectionExtensions.get_value(superfields[i], the_data.GetValue(j - (include_field_names ? 1 : 0)));
+                     if (sub != null)
+                        dat = ReflectionExtensions.get_value(fields[i], sub);
+                  }
+               }
+               else if (dotcount == 0)
+               {
+                  if (j == 0 && include_field_names)
+                     dat = names[i];
+                  else
+                     dat = ReflectionExtensions.get_value(fields[i], the_data.GetValue(j - (include_field_names ? 1 : 0)));
+               }
+               else
+                  dat = "Unknown value: " + combined[i];
+               write_data(out ret[i, j], dat);
+            }
+         }
+         return ret;
+      }
+
+      public static void write_data(out object ret, object dat)
+      {
+         ret = dat;
+         if (dat is DateTime)
+            ret = ((DateTime)dat).ToOADate();
+         else if (dat is Enum)
+            ret = dat.ToString();
+         else if (dat is Array)
+         {
+            Array ary = (Array)dat;
+            ret = stringify_array(ary);
+         }
+      }
+
+      public static string stringify_array(Array array)
+      {
+         string stringify = "{ ";
+         int idx = 0;
+         foreach (object d in array)
+         {
+            object o;
+            write_data(out o, d);
+            stringify += o.ToString() + ((idx++ == array.Length - 1) ? string.Empty : ",");
+         }
+         stringify += " }";
+         return stringify;
+      }
+
+      public static Array unstringify_array(object val, Type field_type = null)
+      {
+         if (field_type == null)
+            field_type = typeof(string[]);
+
+         string[] data = val.ToString().Split(new char[] { ',', '{', '}' }, StringSplitOptions.RemoveEmptyEntries);
+         Type dt = field_type.GetElementType();
+         Array ary = Array.CreateInstance(dt, data.Length);
+         for (int i = 0; i < data.Length; i++)
+         {
+            object entry = null;
+            try
+            {
+               entry = read_data(dt, Convert.ChangeType(data[i], dt));
+            }
+            catch (FormatException) { }
+            ary.SetValue(entry, i);
+         }
+         return ary;
+      }
+   }
+}

src/client/spark_excel/rtd/formatting/BookMDFormatter.cs

@@ -0,0 +1,49 @@
+using System;
+
+namespace bts.excel.rtd
+{
+   class BookMDFormatter : IExcelFormatter
+   {
+      readonly co2.ChannelUpdate _update;
+      public BookMDFormatter(co2.ChannelUpdate upd)
+      {
+         _update = upd;
+      }
+
+      public static double format_md(long mantissa, int exponent)
+      {
+         return mantissa * Math.Pow(10, exponent);
+      }
+
+      public object[,] format(bool include_headers = false)
+      {
+         const int cols = 5;
+         const int rows = 30;
+         var ret = new object[rows, cols];
+         for (int i = 0; i < rows; i++)
+            for (int j = 0; j < cols; j++)
+               ret[i, j] = string.Empty;
+
+         if (_update.Direct != null)
+         {
+            int idx = 0;
+            for (int j = _update.Direct.Asks.Count - 1; j >= 0 && idx < rows; j--)
+            {
+               var ask = _update.Direct.Asks[j];
+               ret[idx, 1] = format_md(ask.Mantissa, ask.Exponent);
+               ret[idx, 2] = ask.Size;
+               idx++;
+            }
+            for (int j = 0; j < _update.Direct.Bids.Count && idx < rows; j++)
+            {
+               var bid = _update.Direct.Bids[j];
+               ret[idx, 1] = format_md(bid.Mantissa, bid.Exponent);
+               ret[idx, 0] = bid.Size;
+               idx++;
+            }
+         }
+         return ret;
+      }
+   }
+
+}

src/client/spark_excel/rtd/formatting/FormatterFactory.cs

@@ -0,0 +1,17 @@
+using System;
+
+namespace bts.excel.rtd
+{
+   class FormatterFactory
+   {
+      public static IExcelFormatter make_formatter(object data)
+      {
+         if(data is co2.ChannelUpdate upd)
+            return new BookMDFormatter(upd);
+         else
+         {
+            return new JsonFormatter(data);
+         }
+      }
+   }
+}

src/client/spark_excel/rtd/formatting/IExcelFormatter.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace bts.excel.rtd
+{
+   internal interface IExcelFormatter
+   {
+      object[,] format(bool include_headers = false);
+   }
+}

src/client/spark_excel/rtd/formatting/JsonFormatter.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using Google.Protobuf;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace bts.excel.rtd
+{
+   class JsonFormatter : IExcelFormatter
+   {
+      private object _data;
+      public JsonFormatter(object data)
+      {
+         _data = data;
+      }
+
+      #region parser
+      static IDictionary<string, object> parse_dictionary(object[,] range)
+      {
+         Dictionary<string, object> dict = new Dictionary<string, object>();
+         if (range.Rank > 1)
+         {
+            for (int i = 0; i < range.GetLength(0); i++)
+            {
+               for (int j = 1; j < range.GetLength(1); j++)
+                  dict[range.GetValue(i, 0).ToString()] = range.GetValue(i, j);
+            }
+         }
+         else
+         {
+            for (int j = 1; j < range.Length; j++)
+               dict[range.GetValue(0).ToString()] = range.GetValue(j);
+         }
+         return dict;
+      }
+
+      public static string parse_json(Type type, object[,] range)
+      {
+         var dict = parse_dictionary(range);
+         string str = JsonConvert.SerializeObject(dict);
+         return str;
+      }
+
+      public static object parse_json_object(Type type, object[,] range)
+      {
+         string json = parse_json(type, range);
+         var obj = JsonConvert.DeserializeObject(json);
+         return obj;
+      }
+      #endregion
+
+      public object[,] format(bool include_headers = false)
+      {
+         if (_data is IEnumerable iem)
+            return format_many(iem, include_headers);
+         else
+            return format_one(_data, include_headers);
+      }   
+
+      object[,] format_many(IEnumerable data, bool include_headers = false)
+      {
+         List<SortedDictionary<string, object>> res = new List<SortedDictionary<string, object>>();
+         foreach (var d in data)
+            res.Add(to_dictionary(d));
+         object[,] ret = fill_object_array(res, include_headers);
+         return ret;
+      }
+
+      object[,] format_one(object data, bool include_headers = false)
+      {                  
+         var d = to_dictionary(data);
+         object[,] ret = fill_object_array(new[] { d }, include_headers);
+         return ret;
+      }
+
+      SortedDictionary<string, object> to_dictionary(object data)
+      {
+         string json = null;
+         if(data is IMessage imsg)
+            json = Google.Protobuf.JsonFormatter.Default.Format(imsg);
+         else
+            json = JsonConvert.SerializeObject(data);
+
+         JObject jo = JObject.Parse(json);
+         SortedDictionary<string, object> results = new SortedDictionary<string, object>();
+         parse_object(jo, results);
+         return results;
+      }
+
+      void parse_object(JObject jo, SortedDictionary<string, object> results)
+      {            
+         foreach (var kvp in jo)
+         {
+            if (kvp.Value is JArray jar) //arrays we'll just leave
+            {
+               for (int i = 0; i < jar.Count; i++)
+                  results[kvp.Key + "_" + i] = jar[i].ToString();
+            }
+            else if (kvp.Value is JValue jv)
+               results[kvp.Key] = jv.ToString();
+            else if (kvp.Value.HasValues)
+               parse_object((JObject)kvp.Value, results);
+         }         
+      }
+
+      object[,] fill_object_array(IEnumerable<SortedDictionary<string, object>> data, bool include_headers = false)
+      {
+         int count = 0;
+         SortedDictionary<string, string> keys = new SortedDictionary<string, string>();
+         foreach (var d in data)
+         {
+            count++;
+            foreach (var kv in d)
+               keys[kv.Key] = null;
+         }
+         object[,] ret = new object[count + (include_headers ? 1 : 0), keys.Count];
+         int r = 0, c = 0;
+         if (include_headers)
+         {
+            foreach (var k in keys.Keys)
+               ret[r, c++] = k;
+            r++;
+         }
+         foreach (var d in data)
+         {
+            c = 0;
+            foreach (var k in keys.Keys)
+            {
+               d.TryGetValue(k, out object val);
+               ret[r, c++] = val; //It found me!
+            }
+            r++;
+         }
+         return ret;
+      }
+   }
+}

src/client/spark_excel/rtd/formatting/MiscMDFormatter.cs

@@ -0,0 +1,18 @@
+using System;
+
+namespace bts.excel.rtd
+{
+   class MiscMDFormatter : IExcelFormatter
+   {
+      readonly co2.ChannelUpdate _update;
+      public MiscMDFormatter(co2.ChannelUpdate upd)
+      {
+         _update = upd;
+      }
+
+      public object[,] format(bool include_headers = false)
+      {
+         return new object[0, 0];
+      }
+   }   
+}

src/client/spark_excel/rtd/formatting/TopMDFormatter.cs

@@ -0,0 +1,46 @@
+
+using System;
+
+namespace bts.excel.rtd
+{
+   class TopMDFormatter : IExcelFormatter
+   {
+      readonly co2.ChannelUpdate _update;
+      public TopMDFormatter(co2.ChannelUpdate upd)
+      {
+         _update = upd;
+      }
+
+      public object[,] format(bool include_headers = false)
+      {
+         const int cols = 4;
+         const int rows = 1;
+         var ret = new object[rows, cols];
+         for (int i = 0; i < rows; i++)
+            for (int j = 0; j < cols; j++)
+               ret[i, j] = string.Empty;
+         
+         if (_update.Direct != null)
+         {
+            int idx = 0;
+            for (int j = 0; j < _update.Direct.Asks.Count && idx < rows; j++)
+            {
+               var ask = _update.Direct.Asks[j];
+               ret[idx, 2] = BookMDFormatter.format_md(ask.Mantissa, ask.Exponent);
+               ret[idx, 3] = ask.Size;
+               idx++;
+            }
+            idx = 0;
+            for (int j = 0; j < _update.Direct.Bids.Count && idx < rows; j++)
+            {
+               var bid = _update.Direct.Bids[j];
+               ret[idx, 1] = BookMDFormatter.format_md(bid.Mantissa, bid.Exponent);
+               ret[idx, 0] = bid.Size;
+               idx++;
+            }
+         }
+         return ret;
+      }
+   }
+
+}

src/client/spark_excel/rtd/subscriptions/MDSubscribe.cs

@@ -0,0 +1,120 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+using ExcelDna.Integration;
+using ExcelDna.Integration.Rtd;
+using System.Reactive.Linq;
+using bts.api;
+using bts.api.md;
+using bts.api.orderman;
+using ExcelDna.Registration.Utils;
+
+namespace bts.excel.rtd
+{
+   public class MDSubscribe
+   {
+      public static MarketDataClient market_data(api.Venue exchange)
+      {
+         var newc = MarketDataClient.create_or_get(exchange, out MarketDataClient md);
+         if(newc)
+            md.run_pump_async(TimeSpan.FromMilliseconds(333));
+         return md;
+      }
+
+      static public object spark_lookup_security_desc(string exchange, long security_key)
+      {
+         Venue ven;
+         if (!Utilities.parse_venue(exchange, out ven))
+            return Observable.Return("Can't Parse Exchange: " + exchange ?? string.Empty);
+         MDSubscribe.market_data(ven).lookup_security_desc((uint)security_key, out string desc);
+         return desc ?? string.Empty;
+      }
+
+      static IObservable<object> md_observable(string contract, string symbol, string security_type, string exchange)
+      {
+         security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;
+         exchange = string.IsNullOrEmpty(exchange) ? Security.ExchangeDataOneofCase.Glbx.ToString() : exchange;
+         symbol = string.IsNullOrEmpty(symbol) ? contract.Substring(0, 2) : symbol;
+
+         Venue ven;
+         SecurityType sectype;
+         if (!Utilities.parse_venue(exchange, out ven))
+            return Observable.Return("Can't Parse Exchange: " + exchange ?? string.Empty);
+         if (!Utilities.parse_security_type(security_type, out sectype))
+            return Observable.Return("Can't Parse Security Type: " + security_type ?? string.Empty);
+         if (string.IsNullOrEmpty(symbol))
+            return Observable.Return("Can't Parse Symbol: " + symbol ?? string.Empty);
+         if (string.IsNullOrEmpty(contract))
+            return Observable.Return("Can't Parse Contract: " + contract ?? string.Empty);
+
+         Product prod = new Product() { Exchange = ven, Type = sectype, Symbol = symbol };
+         Func<Security, bool> filter = (x => x.Glbx.SecurityDesc == contract); //TODO GLBX
+
+         var stream =  market_data(ven).subscribe(prod).Where(x => filter(x.Security));
+         return stream;
+      }      
+
+      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark full depth market data")]
+      public static object spark_md_book(
+                                    [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
+                                    [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,                                    
+                                    [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
+                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
+      {
+
+         var param = new object[] { contract, symbol, security_type, exchange };
+         return ObservableRtdUtil.Observe(nameof(spark_md_book), param, 
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
+                                          {
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new BookMDFormatter(upd).format();
+                                             return x;
+                                          }));
+         // security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;
+         // exchange = string.IsNullOrEmpty(exchange) ? Security.ExchangeDataOneofCase.Glbx.ToString() : exchange;
+         // symbol = string.IsNullOrEmpty(symbol) ? contract.Substring(0, 2) : symbol;
+         // string[] topic = new[] { MDTopic.topic_type, MDTopic.book_request_type,
+         //                          contract, symbol, security_type, exchange};
+         // object val = XlCall.RTD(typeof(RTDServer).FullName, null, topic);
+         // return Subscribe.spark_unpack(val);
+      }
+
+   
+
+      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark top of book market data")]
+      public static object spark_md_top(
+                                    [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
+                                    [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,
+                                    [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
+                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
+      {
+         var param = new object[] { contract, symbol, security_type, exchange };
+         return ObservableRtdUtil.Observe(nameof(spark_md_top), param,
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
+                                          {
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new TopMDFormatter(upd).format();
+                                             return x;
+                                          }));
+      }      
+
+      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark miscellaneous market data")]
+      public static object spark_md_misc(
+                                    [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
+                                    [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,
+                                    [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
+                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
+      {
+         var param = new object[] { contract, symbol, security_type, exchange };
+         return ObservableRtdUtil.Observe(nameof(spark_md_misc), param,
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
+                                          {
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new MiscMDFormatter(upd).format();
+                                             return x;
+                                          }));
+
+      }      
+   }
+}

src/client/spark_excel/rtd/subscriptions/Subscribe.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+using ExcelDna.Integration;
+using ExcelDna.Integration.Rtd;
+using ExcelDna.Registration;
+using ExcelDna.Registration.Utils;
+using System.Reactive.Linq;
+using bts.api;
+using bts.api.md;
+using bts.api.core;
+using bts.api.orderman;
+using bts.co2;
+
+namespace bts.excel.rtd
+{
+   public class Subscribe
+   {
+      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark API Service")]
+      public static object spark_api_subscribe(string service_type, string method_name, object parameters)
+      {
+         IList<string> topic = Utilities.get_tokens(parameters);
+         topic.Insert(0, method_name);
+         topic.Insert(0, service_type);
+         object val = XlCall.RTD(typeof(APIRTDServer).FullName, null, topic.ToArray());
+         return val;
+      }
+
+      [ExcelFunction(Category = "BTS", Description = "Unpacks BTS Spark data from a GUID reference")]
+      public static object spark_unpack(object reference)
+      {
+         object data;
+         if(unpack(reference, out data))
+            return data;
+         else
+            return reference;
+      }
+
+      public static bool unpack(object reference, out object data)
+      {
+         object tempdata; data = null;
+         if (reference is string && APIRTDServer.retrieve(reference.ToString(), out tempdata))
+         {
+            if (tempdata is IExcelFormatter fdata)
+               data = fdata.format();
+            else
+               data = DataParser.scatter_data(tempdata);
+            return true;
+         }
+         return false;
+      }
+
+      //[ExcelFunction]
+      //public static object bts_subscribe_deprecated(string service_type, string request_type, object parameters)
+      //{
+      //   IList<string> topic = Utilities.get_tokens(parameters);
+      //   topic.Insert(0, request_type);
+      //   topic.Insert(0, service_type);
+      //   object val =  XlCall.RTD(typeof(APIRTDServer).FullName, null, topic.ToArray());
+
+      //   if(val is string)
+      //   {
+      //      //I can't marshal just any old object through the excel machinery, so in order to retain a 'registration free'
+      //      //model, I have to pass this string-guid from the RTDServer, up through excel and back here.
+      //      object data;
+      //      if(APIRTDServer.retrieve(val.ToString(), out data))
+      //      {
+      //         object[,] ret = DataParser.scatter_data(data);
+      //         return ret;
+      //      }
+      //   }
+      //   return val;
+      //}      
+   }
+}

src/client/spark_excel/spark_excel-addin.dna

@@ -0,0 +1,29 @@
+<DnaLibrary Name="spark_excel" RuntimeVersion="v4.0">
+  <ExternalLibrary Path="spark_excel.dll" ExplicitExports="false" LoadFromBytes="true" Pack="true" />
+  
+   <Reference Path="ExcelDna.IntelliSense.dll" Pack="true" />
+   
+   <Reference Path="nunit.framework.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="System.Reactive.Core.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="System.Reactive.Interfaces.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="System.Reactive.Providers.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="System.Reactive.Linq.dll" Pack="true" ExplicitExports="true" />		
+   <Reference Path="System.Reactive.Runtime.Remoting.dll" Pack="true" ExplicitExports="true" />		  
+  
+  <!-- 
+       The RuntimeVersion attribute above allows two settings:
+       * RuntimeVersion="v2.0" - for .NET 2.0, 3.0 and 3.5
+       * RuntimeVersion="v4.0" - for .NET 4 and 4.5
+
+       Additional referenced assemblies can be specified by adding 'Reference' tags. 
+       These libraries will not be examined and registered with Excel as add-in libraries, 
+       but will be packed into the -packed.xll file and loaded at runtime as needed.
+       For example:
+       
+       <Reference Path="Another.Library.dll" Pack="true" />
+  
+       Excel-DNA also allows the xml for ribbon UI extensions to be specified in the .dna file.
+       See the main Excel-DNA site at http://excel-dna.net for downloads of the full distribution.
+  -->
+
+</DnaLibrary>

src/client/spark_excel/spark_excel.csproj

@@ -0,0 +1,82 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+  
+    <TargetFrameworks>net471</TargetFrameworks>
+    <OutputType>Library</OutputType>
+    <!-- <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+    <IncludeSource>true</IncludeSource>
+    <IncludeSymbols>true</IncludeSymbols>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput> -->
+    
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Description />
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <Platforms>x64</Platforms>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <Version>1.0.0</Version>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.4.0" />
+    <PackageReference Include="System.Reactive" Version="3.1.1" />
+    <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
+	 <PackageReference Include="NUnit" Version="3.10.1" />  
+    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="System.Linq.Dynamic" Version="1.0.7" />
+    
+    <PackageReference Include="ExcelDna.AddIn" Version="0.34.6" />  
+    <PackageReference Include="ExcelDna.Integration" Version="0.34.6" />      
+    <PackageReference Include="ExcelDna.IntelliSense" Version="1.1.0" />      
+    <PackageReference Include="ExcelDna.Registration" Version="0.34.7" />      
+  </ItemGroup>
+  
+  <ItemGroup>    
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Configuration" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />      
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="spark_excel-AddIn.dna" />
+    <None Include="packages.config" />
+    <None Include="Properties\ExcelDna.Build.props" />
+  </ItemGroup>  
+  
+  <ItemGroup>
+    <ProjectReference Include="../spark_api/spark_api.csproj"/>
+  </ItemGroup>
+  
+  <ItemGroup>
+    <Proto3Excel Include="**/*.proto" />
+    <Proto3Excel Include="../../lib/api/services/*.proto" />
+  </ItemGroup>
+  
+  <ItemGroup>
+    <None Include="spark_excel_example.xlsm"><CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory></None>  
+  </ItemGroup>
+  
+  <!-- <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences">
+    <ItemGroup>
+      <BuildOutputInPackage Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference'))" />
+      <BuildOutputInPackage Include="$(OutputPath)..\net471\libspark.dll"/>
+    </ItemGroup>
+  </Target>  
+   -->
+  
+  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets" Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets')" /> -->
+  <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets"/>
+  <Target Name="EnsureExcelDnaTargetsImported" BeforeTargets="BeforeBuild" Condition="'$(ExcelDnaTargetsImported)' == ''">
+    <Error Condition="!Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="You are trying to build with ExcelDna, but the NuGet targets file that ExcelDna depends on is not available on this computer. This is probably because the ExcelDna package has not been committed to source control, or NuGet Package Restore is not enabled. Please enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
+    <Error Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="ExcelDna cannot be run because NuGet packages were restored prior to the build running, and the targets file was unavailable when the build started. Please build the project again to include these packages in the build. You may also need to make sure that your build server does not delete packages prior to each build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
+  </Target>  
+</Project>

src/client/spark_excel/utils/APIDispatch.cs

@@ -0,0 +1,134 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using ExcelDna.Integration;
+
+using bts.api.core;
+using bts.excel.extensions;
+
+using Google.Protobuf;
+using Google.Protobuf.Reflection;
+
+namespace bts.excel
+{
+   public class APIDispatch
+   {
+      public static Lazy<ServiceContext> service_context = new Lazy<ServiceContext>(() => new ServiceContext(), true);
+      static MethodInfo _locate_meth;
+      readonly static Dictionary<object, object> _services = new Dictionary<object, object>();
+      const string descriptor_property_name = "Descriptor";
+
+      static MethodInfo locate_method
+      {
+         get
+         {
+            if (_locate_meth == null)
+            {
+               _locate_meth = service_context.Value.GetType().find_method((_) =>
+                        {
+                           return _.Name == nameof(ServiceContext.locate) && _.IsGenericMethod;
+                        })[0];
+            }
+            return _locate_meth;
+         }
+      }
+
+      static public object api_call_internal(Type service_type, string method_name, IMessage parameters)
+      {
+         object response = null;
+         if (service_type != null && method_name != null)
+         {
+            object service = locate_service(service_type);
+            response = invoke_service_call(service, method_name, parameters);
+         }
+         return response;
+      }
+
+      //static public object invoke_service_call(object service, Type request_type, IMessage request)
+      //{
+      //   var sub_method = service.GetType().GetMethods().Where(meth =>
+      //   {
+      //      var par = meth.GetParameters().FirstOrDefault();
+      //      return (par != null && par.ParameterType.IsAssignableFrom(request_type));
+      //   }).FirstOrDefault();
+      //   object subscription = null;
+      //   if(sub_method != null)
+      //      subscription = sub_method.Invoke(service, new[] { request });
+      //   return subscription;
+      //}
+
+      static public object invoke_service_call(object service, string method_name, IMessage request)
+      {
+         var sub_method = service.GetType().GetMethod(method_name);
+         object subscription = null;
+         if (sub_method != null)
+            subscription = sub_method.Invoke(service, new[] { request });
+         return subscription;
+      }
+
+      public static Type find_request_type(Type service_type, string method_name)
+      {
+         return service_type?.GetMethod(method_name)?.GetParameters().FirstOrDefault()?.ParameterType;
+      }
+
+      static public IMessage generate_imessage(Type type, string json, bool null_for_empty = false)
+      {
+         IMessage request = null;
+         var descproperty = type?.get_static_field_or_property(descriptor_property_name);
+         if (descproperty != null)
+         {
+            MessageDescriptor desc = (MessageDescriptor)descproperty.get_value(null);
+            if (null_for_empty)
+            {
+               if (!string.IsNullOrEmpty(json) && json != "{ }" && json != "{}")
+                  request = JsonParser.Default.Parse(json, desc);
+               else
+                  request = null; //TODO this is a workaround for Eric's bug that requires 'empty' requests to pass null instead
+            }
+            else
+            {
+               if(string.IsNullOrEmpty(json))
+                  json = "{}";
+               request = JsonParser.Default.Parse(json, desc);
+            }
+         }
+         return request;
+      }
+
+      static public IMessage generate_request(Type service_type, string method_name, string request_json)
+      {
+         Type request_type = find_request_type(service_type, method_name);
+         return generate_imessage(request_type, request_json, true);
+      }
+
+      static public object locate_service(Type service_type)
+      {
+         //object svc = null;
+         //if(service_type != null && !_services.TryGetValue(service_type, out svc))
+         //{
+         var method = locate_method?.MakeGenericMethod(service_type);
+         object svc = method?.Invoke(service_context.Value, new object[] { null });
+         if (svc != null)
+            _services[svc] = null;
+         return svc;
+      }
+
+
+      [ExcelFunction]
+      static public object spark_api_call(string service_type, string method_name, object parameters)
+      {
+         Type svc = Utilities.find_full_type_in_all_assemblies(service_type, ".*api.*");
+
+         var tok = Utilities.get_tokens(parameters);
+         string json = tok.FirstOrDefault();
+         IMessage request = generate_request(svc, method_name, json);
+         object resp = api_call_internal(svc, method_name, request);
+         if (resp != null)
+            return DataParser.scatter_data(resp);
+         else
+            return ExcelErrorUtil.ToComError(ExcelError.ExcelErrorNA);
+      }
+
+   }
+}