Merge branch 'master' of bitbucket.org:bluetradesys/spark into vwap

# Conflicts:
#	src/client/spark_historical/Notebooks/roll_cover.ipynb

.gitignore

@@ -34,5 +34,7 @@ trace_data/
 /doc/user/public/*
 .ionide/*
 /.ignore
+CMakeDeps.cmake
 /build
-.clangd/
+.fake
+.ionide

.vscode/launch.json

@@ -1,30 +1,6 @@
 {
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Spark Local",
-            "type": "cppdbg",
-            "request": "launch",
-            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd",
-            "args": [
-                "run",
-                // "test",
-                // "-a",
-                // "[ilink3]"
-            ],
-            "stopAtEntry": false,
-            "cwd": "${workspaceRoot}/build/ninja_debug/server/",
-            "environment": [],
-            "externalConsole": false,
-            "MIMode": "gdb",
-            "setupCommands": [
-                {
-                    "description": "Enable pretty-printing for gdb",
-                    "text": "-enable-pretty-printing",
-                    "ignoreFailures": true
-                }
-            ]
-        },
         {
             "name": "SparkVM Launch",
             "type": "cppdbg",
@@ -68,9 +44,10 @@
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
-                "/home/bts/spark/": "${workspaceRoot}"
+                "/home/bts/spark/": "${workspaceRoot}",
+                "/home/bts/spark_host/": "${workspaceRoot}",
             }
-        },
+        },     
         {
             "name": "(gdb) Launch",
             "type": "cppdbg",
@@ -122,19 +99,87 @@
             "externalConsole": true,
             "requireExactSource": false
         },
+            {
+         "name": "sparkcmd Launch",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "vol", 
+            "--server",
+             //"--rstart", "11/22/2019",
+             //"--rend", "11/25/2019",
+             //"--rsymbol", "ES",
+             //"--rsecuritytype", "Synthetic",
+             //"--post_process_file_type", "Trade",
+             //"--upload_to_aws", 
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },  
+      {
+         "name": "sparkcmd simulation",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "historical",
+            "--rstart", "11/26/2019",
+            "--rend", "11/26/2019",
+            "--rsymbol", "ZN",
+            "--rsecuritytype", "Synthetic",
+            "--algo_security", "ZNH0;ZNZ9-ZNH0;ZNZ9",
+            "--algo_type", "RoundTrip",
+            "--algo_params", "size=10;tick_size=0.015625,0.0078125,0.015625;hedge_factors=1,1,-1",
+            "--skip_preanalysis", "true",         
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },       
         {
             "name": "Excel Launch",
             "type": "clr",
             "request": "launch",
-            "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
+            "program": "C:/Program Files/Microsoft Office/Office16/EXCEL.EXE",
             "args": [
                 "${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"
             ],
-            "cwd": "${workspaceRoot}",
+            "cwd": "${workspaceRoot}/build/Debug/bin/net471/",
             "stopAtEntry": false,
             "externalConsole": true,
             "requireExactSource": false
         },
+      {
+         "name": "Win C++ Launch",
+         "type": "cppvsdbg",
+         "request": "launch",
+         "program": "C:/Program Files/Microsoft Office/Office16/EXCEL.EXE",
+         "args": [
+            "${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"
+         ],
+         "cwd": "${workspaceRoot}/build/Debug/bin/net471/",
+         "stopAtEntry": false,
+         "environment": [],
+         "externalConsole": false
+      },        
+        {
+           "name": "Fix Launch",
+           "type": "clr",
+           "request": "launch",
+           "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+           "args": [
+              "fix",
+              "-order_session",
+           ],
+           "cwd": "${workspaceRoot}/build/Debug/bin/net471/",
+           "stopAtEntry": false,
+           "externalConsole": true,
+           "requireExactSource": false
+        },        
         {
             "name": "Client Tests",
             "type": "clr",
@@ -150,11 +195,18 @@
             "requireExactSource": false
         },
         {
-            "name": "Client Attach",
+            "name": ".NET Framework Attach",
             "type": "clr",
             "request": "attach",
             "processId": "${command:pickProcess}"
         },
+         {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickProcess}",
+            "requireExactSource": false
+         },        
         {
             "name": "Spreader Tests",
             "type": "cppdbg",
@@ -361,20 +413,24 @@
                 "quoteArgs": true
             },
             "sourceFileMap": {
-                "/home/bts/spark/": "${workspaceRoot}"
+                "/home/bts/spark_host/": "${workspaceRoot}"
             },
             "justMyCode": true
         },
         {
-            "name": ".NET Core Remote Launch",
+            "name": "sparkcmd vsdbg (c#)",
             "type": "coreclr",
             "request": "launch",
-            "program": "/usr/bin/dotnet",
+            "program": "sparkcmd",
             "args": [
-                "sparkcmd.dll",
-                "test",
-                "test_join_create_and_takeout_market"
+            "historical",
+            "--rstart", "11/20/2019",
+            "--rend", "11/20/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Spread",
+            "--post_process_db", "mbo_history",
             ],
+            "env": {"LD_LIBRARY_PATH" : "."},
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server",
             "pipeTransport": {
@@ -385,11 +441,77 @@
                 "debuggerPath": "/home/bts/vsdbg/vsdbg",
                 "pipeCwd": "${workspaceRoot}",
                 "quoteArgs": true
-            },
+            },           
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             },
             "justMyCode": false
-        },
+        },                  
+         {
+            "name": "sparkcmd gdb (c++)",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/home/bts/spark/build/ninja_debug/server/sparkcmd",
+            "args": [
+            "historical",
+            "--rstart", "10/21/2019",
+            "--rend", "10/21/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Future",
+            "--algo_security", "CLX9",
+            "--algo_type", "TWAP",
+            "--algo_params", "'size=100;profit_ticks=1;tick_size=1,1,1;;hedge_factors=1,1,-1'",
+            "--skip_preanalysis", "true",
+            ],   
+            "stopAtEntry": false,
+            "cwd": "/home/bts/spark/build/ninja_debug/server/",
+            "environment": [],
+            "externalConsole": true,
+            "MIMode": "gdb",
+            "preLaunchTask": "",
+            "logging": {
+               "engineLogging": false
+            },
+            "setupCommands": [
+               {
+                  "description": "Enable pretty-printing for gdb",
+                  "text": "enable pretty-printer",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "Skip STL",
+                  "text": "skip -gfi /usr/include/c++/7/bits/*.h",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "LD LIBRARY",
+                  "text": "set env LD_LIBRARY_PATH=.",
+                  "ignoreFailures": false
+               }
+            ],
+            "pipeTransport": {
+               "pipeCwd": "${workspaceRoot}",
+               "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+               "pipeArgs": [
+                  "bts@sparkvm"
+               ],
+               "debuggerPath": "/usr/bin/gdb"
+            },
+            "sourceFileMap": {
+               "/home/bts/spark/": "${workspaceRoot}"
+            }
+         },
+         {
+            "name": "Jupyter notebook",
+            "type": "coreclr",
+            "request": "launch",
+            "program": "jupyter",
+            "args": ["notebook", "--notebook-dir=c:\\code\\bts\\spark\\src\\client\\spark_historical\\Notebooks"],
+            "cwd": "${workspaceFolder}",
+            "stopAtEntry": false,
+            "console": "internalConsole",
+            
+         },
+         
     ]
 }

CMakeLists.txt

@@ -48,6 +48,7 @@ else ()
   add_subdirectory(doc/user)
   add_subdirectory(src/sdk)
   add_subdirectory(src/data_files)
+  add_subdirectory(src/client)
 
   add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
@@ -59,7 +60,17 @@ else ()
     COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
     COMMENT "Creating server runtime layout"
+    DEPENDS sparkd data_files_dir spark_client_dir
     DEPENDS sparkd data_files_dir)  
 
   add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+  
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.1/linux-x64/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+else()
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.1/linux-x64/publish/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+endif()
+      
 endif()

debug.log

@@ -0,0 +1 @@
+[0108/231015.447:ERROR:file_io.cc(89)] ReadExactly: expected 36, observed 0

doc/api/content/server/getting-started/download.md

@@ -68,21 +68,12 @@ To update your Spark SDK simply download an unzip a new [spark_sdk.zip](https://
 
 To preserve the old image so you can go back to it later run `docker tag spark_sdk spark_sdk:your_custom_tag` before loading the new SDK image.
 
-### Re-build the Dev Container
-
-If you are using Visual Studio Code you also need to rebuild the dev-container so it picks up the new image.
-
-#### Inside VSCode
-To re-build from inside Visual Studio Code.  Click the 'Remote Explorer' icon on the left hand side. 
-You should see your spark_sdk project listed under the Containers section.  Right click on it and select **rebuild-container**.
-
-#### Command Line
-To re-build from the command line run `docker ps -a` and find the row with the spark_sdk image. 
+<!--
+Then stop, remove and restart spark_sdk.
 ```
-CONTAINER ID        IMAGE                          COMMAND                  CREATED              STATUS                      PORTS                                NAMES
-54d96cfa3888        spark_sdk                      "/bin/sh -c 'echo Co…"   About a minute ago   Up About a minute           127.0.0.1:9000-9001->9000-9001/tcp   quizzical_lichterman
-```
-Then run `docker rm 54d96cfa3888` (use your container id instead of 54d96cfa3888)  or `docker rm quizzical_lichterman` (use your container name).
-
+c:\bts> docker stop spark_sdk
+c:\bts> docker rm spark_sdk
+c:\bts> docker run -d -v %cd%:/sdk --name spark_sdk bluetradingsytems/spark_sdk
 
+``` -->
 

doc/user/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
+set(sources /home/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

doc/user/content/release_notes/_index.md

@@ -1 +1,710 @@
-+++
title = "Release notes"
weight = 3
sort_by = "weight"
insert_anchor_links = "right"
+++
## 2.16.2
### User Visible Change
  - Synthetic creation dialog has a "fractional" checkbox next to the tick settings.  Users may uncheck this box to enter integer tick sizes, so the synthetic security's prices will be displayed without decimals, or fractional units.
## 2.16.1
### Bug Fixes
  - OrderBook's algo state parameters (in the panel on the right side) are sorted alphabetically
## 2.16.0
### Bug Fixes
  - Solved an issue that could result in incorrect ladder levels
  - Solved an issue that could lead to incorrect routing of orders in the event of certain MSGW changes
### API Changes
  - Add C# client side API samples for order entry, position monitoring and market data observation

## 2.15.10
### Bug Fixes
  - Solved an issue that could cause the user interface to fail when playing some alert sounds
### API Changes
  - Added settlement and position information to the API
  - Added the ability for user algos to report custom state to the user interface
  - Added the ability for algos to discover all active securities
## 2.15.3
### Bug Fixes
  - Fixed a bug where some CME currency spreads exhibited incorrect tick values
## 2.15.2
### User Visible Changes  
  - Excel RTD plugin can now receive ICE market data
## 2.15.1
### Bug Fixes
  - Fixed a FIX log parsing issue that resulted in exchange gateway logon problems
## 2.15.0
### User Visible Changes  
  - Added ability to hide Ladder status messages via Ladder Settings -> Left Panel tab -> Show/Hide groupbox -> Status Message
  - Synthetic cloning now allows for any parameter to be changed, including tick size and leg contracts
  - Added ability to import synthetics from a CSV file; see our manual for further details on this process
  - Reduced Iceberg reloading latency
### Bug Fixes
  - Fixed a bug where highlighted fills could lose their highlight status
  - Fixed a bug where the (Mini) OrderBook PIQ could be displaying an old value

## 2.14.3
### Bug Fixes
  - Fixed a bug regarding ICE logon messaging
  
## 2.14.2
### Bug Fixes
  - Fixed a bug regarding ICE messaging
  
## 2.14.1
### Bug Fixes
  - Fixed a bug regarding ICE logon authentication
  
## 2.14.0
### User Visible Changes  
  - All ladders can now be recentered by (shift or alt) + double clicking over the Price column in a ladder
  - Added a Max Order Quantity setting to the Safety Checks tab on the Ladder settings. This setting allows for a user-defined maximum order quantity.
  - Added a Price Display Format setting to the Market Price and Size Display tab on the Ladder settings. This setting allows for price levels in the ladder to be displayed without a decimal separator.
  - Synthetic instrument creation with 3 legs no longer defaults to butterfly quantites
### Bug Fixes
  - Fixed an issue where market transition were not observed by IAlgoOrder
  - Fixed a bug in error reporting
  - Fixed a bug in the PnL window where specifying a certain grouping order would lead to a UI failure

## 2.13.0
### User Visible Changes  
  - Added the name of the spreader to the parent algo badge (for spreader child orders), in the fills, orderbook and floating orderbook windows
  - Added support for sounds for new orders
  - Added a new quoting throttle parameter for the spreader, that throttles quotes that are moving toward the inside market but are still a configurable number of ticks away
  - Enabled the ladder to remember the left panel scroll position across front end restarts
### Bug Fixes
  - Fixed the local fills window so that it does not show the admin fills
  - Removed the deleted accounts from the ladder accounts dropdown
  - Fixed the orderbook price for iceberg of spreaders
  
## 2.12.0
### User Visible Changes
  - Added support for ICE market data and order execution
### Internal Changes
  - Lots of permission / admin changes to facilitate ICE

## 2.11.0
### User Visible Changes
  - Fixed fills form time filter to also apply to SOD fills
  - Addressed an issue that could prevent the web Admin UI from displaying over slow HTTP connections
  - Removed the option to enter manual spread fills
  - Fixed an issue that could cause erroneous fill account attribution in the event of a crash
### Internal Changes
  - Added support for tag 1031. Currently defaults to Y, can it be override through config.
  - To facilitate support, allowed the bts user to view and edit algos from all other users 
  - Improved system tests
  - Improved log messages in UI to include dispatcher thread name
  - Added support for Windows core dumps in the UI
  
## 2.10.0
### User Visible Changes
  - Fills now include information about the algo that originated the fill
  - A new, more powerful PnL form is now available
### API Changes
  - Added security state information (pre open, halted, normal, closed, etc)
### Bug Fixes
  - Brought back the "select all" check box in the mini order book
  - Fixed an issue in the market data subsyLine truncated
++++
+title = "Release notes"
+weight = 3
+sort_by = "weight"
+insert_anchor_links = "right"
++++
+## 2.15.3
+### Bug Fixes
+  - Fixed a bug where some CME currency spreads exhibited incorrect tick values
+## 2.15.2
+### User Visible Changes  
+  - Excel RTD plugin can now receive ICE market data
+## 2.15.1
+### Bug Fixes
+  - Fixed a FIX log parsing issue that resulted in exchange gateway logon problems
+## 2.15.0
+### User Visible Changes  
+  - Added ability to hide Ladder status messages via Ladder Settings -> Left Panel tab -> Show/Hide groupbox -> Status Message
+  - Synthetic cloning now allows for any parameter to be changed, including tick size and leg contracts
+  - Added ability to import synthetics from a CSV file; see our manual for further details on this process
+  - Reduced Iceberg reloading latency
+### Bug Fixes
+  - Fixed a bug where highlighted fills could lose their highlight status
+  - Fixed a bug where the (Mini) OrderBook PIQ could be displaying an old value
+
+## 2.14.3
+### Bug Fixes
+  - Fixed a bug regarding ICE logon messaging
+  
+## 2.14.2
+### Bug Fixes
+  - Fixed a bug regarding ICE messaging
+  
+## 2.14.1
+### Bug Fixes
+  - Fixed a bug regarding ICE logon authentication
+  
+## 2.14.0
+### User Visible Changes  
+  - All ladders can now be recentered by (shift or alt) + double clicking over the Price column in a ladder
+  - Added a Max Order Quantity setting to the Safety Checks tab on the Ladder settings. This setting allows for a user-defined maximum order quantity.
+  - Added a Price Display Format setting to the Market Price and Size Display tab on the Ladder settings. This setting allows for price levels in the ladder to be displayed without a decimal separator.
+  - Synthetic instrument creation with 3 legs no longer defaults to butterfly quantites
+### Bug Fixes
+  - Fixed an issue where market transition were not observed by IAlgoOrder
+  - Fixed a bug in error reporting
+  - Fixed a bug in the PnL window where specifying a certain grouping order would lead to a UI failure
+
+## 2.13.0
+### User Visible Changes  
+  - Added the name of the spreader to the parent algo badge (for spreader child orders), in the fills, orderbook and floating orderbook windows
+  - Added support for sounds for new orders
+  - Added a new quoting throttle parameter for the spreader, that throttles quotes that are moving toward the inside market but are still a configurable number of ticks away
+  - Enabled the ladder to remember the left panel scroll position across front end restarts
+### Bug Fixes
+  - Fixed the local fills window so that it does not show the admin fills
+  - Removed the deleted accounts from the ladder accounts dropdown
+  - Fixed the orderbook price for iceberg of spreaders
+  
+## 2.12.0
+### User Visible Changes
+  - Added support for ICE market data and order execution
+### Internal Changes
+  - Lots of permission / admin changes to facilitate ICE
+
+## 2.11.0
+### User Visible Changes
+  - Fixed fills form time filter to also apply to SOD fills
+  - Addressed an issue that could prevent the web Admin UI from displaying over slow HTTP connections
+  - Removed the option to enter manual spread fills
+  - Fixed an issue that could cause erroneous fill account attribution in the event of a crash
+### Internal Changes
+  - Added support for tag 1031. Currently defaults to Y, can it be override through config.
+  - To facilitate support, allowed the bts user to view and edit algos from all other users 
+  - Improved system tests
+  - Improved log messages in UI to include dispatcher thread name
+  - Added support for Windows core dumps in the UI
+  
+## 2.10.0
+### User Visible Changes
+  - Fills now include information about the algo that originated the fill
+  - A new, more powerful PnL form is now available
+### API Changes
+  - Added security state information (pre open, halted, normal, closed, etc)
+### Bug Fixes
+  - Brought back the "select all" check box in the mini order book
+  - Fixed an issue in the market data subsystem that on rare occasions could result in a startup crash
+### Internal Changes
+  - Fixed an issue that was stopping us from interpreting SOD times as local rather than GMT
+## 2.9.3
+### Internal Changes
+  - Fixed system test scripts
+## 2.9.2
+### Bug Fixes
+  - Fixed an issue with the exchange simulator that could lead to crossed markets
+## 2.9.1
+### Bug Fixes
+  - Fixed an issue with trade sound alerts, where in some cases the use of a "In Between" condition could lead to a UI failure.
+  - Fixed an issue with the display of the new raw position column in the market view window.
+  - Fixed the popup label that describes the raw position information in the ladder
+### Internal Changes
+  - Avoided shipping production configuration defaults that lead to trying to download a position file for the development sftp server.
+### Api
+  - C# API sample now uses synchronous calls.
+
+## 2.9.0
+### User Visible Changes
+  - Improved Workspace management User Interface
+  - Added the ability to change grid background colors
+  - The Spark UI Window group now has a title in the Windows task bar
+  - Added sound alerts for trade events, see Launcher -> Settings -> Alerts
+  - Allowed composite algos that are attached to existing orders to identify fills that were already present prior to attachment
+  - Added import facilities for GMI position files
+  - Added the ability for risk managers to enter manual fills
+  - Added "raw" P&L columns, that is columns that *do not* include spread legs in their accounting
+### Bug Fixes
+  - Fixed the settings action menu (Load/Save/Reset to Factory)
+  - Fixed the method used to determine if a security is expired to take into account the start of day time
+### API Changes
+  - Fixed an issue in the Client SDK that could lead to occasional failures to connect to the spark server as well as excessive memory usage.
+  - Added the ability for algo orders to distinguish fills existing prior to launch
+### Internal Changes
+  - Changed how we packaged certain third-party components
+  - Added logging of entitlement status
+  - Tagged outgoing orders with sequence number of marketdata or FIX event that triggered them
+## 2.8.3
+### Bug Fixes
+  - Fixed a bug that could cause an OrderBook's Security column filter to be blank
+  - Fixed a bug that could cause a UI failure when clearing a filter selection on a filtering widget
+## 2.8.2
+### Bug Fixes
+  - Added proper display factors for SOM spreads
+  - Fixed a bug that could cause a UI failure when trying to change settings in a synthetic ladder
+## 2.8.1
+### Bug fixes
+  - Fixed an error in the UI grids filtering widget
+## 2.8.0
+### User Visible Changes
+  - Added ProRata and Scratch algos
+  - Ladder badge now grows/shrinks to fill the containing cell
+  - Attached algo parameters are now displayed in the update panel
+  - Fixed a bug where the update panel would show '-' values when only
+    1 order was selected
+  - Fixed a bug that cause cause a failure on startup if the user
+    loaded a workspace with a Ladder that had a dynamic order type
+    selected by default
+  - Increased width of login panel to better show login error messages
+  - Added optional 'Maker/Taker' pricing to Spreader algo
+  - Algo buttons in ladder can now be made to deactivate after launching an algo or remain 'sticky'
+### Internal changes
+  - Made it easier to generate simple UI's for new order types
+### API Changes
+  - Dynamic orders can now use QuantityMeasure parameter types
+
+## 2.7.2
+### User Visible Changes
+  - Changed the login dialog to avoid leaking workspace names from other users.
+  - Added new Splitter algo. From the manual: The splitter is useful
+    for pro-rata markets where you want to vary your exposure
+    independent of the order quantity that you want to fill.  The
+    splitter will complete when the number of fills reaches or exceeds
+    the order quantity but it will put exposed_qty total size on the
+    market. Splitter favors order size over queue position.
+### Internal Changes
+  - Made spark more resilient to lack of connectivity to the CME web site for downloading of templates.
+
+## 2.7.1
+### User Visible Changes
+  - Fixed some issues with the Cover algo, where in some circumstances
+    it would erroneously cancel launched hedges on completion.
+
+## 2.7.0
+### User Visible Changes
+  - Allowed algos to process "in-flight" fills while shutting down
+  - Changed the cover algo to continue to hedge while shutting down
+### API Changes
+  - Changed the API so that algos do not fully terminate until all
+    their children are terminated. Algos that are in the process of
+    shutting down in response to a complete or cancel call will still
+    receive child updates, which they can use to calculate and report
+    an accurate fill count (during this shutdown stage algos cannot
+    create, cancel or update children). A new IAlgoServices method,
+    is_shutting_down(), can be used to determine if an algo is
+    shutting down.
+
+## 2.6.2
+### User Visible Changes
+  - Fixed an issue in the ladder where in some cases the mini order book could not be brought up from a badge
+  - Fixed a rare race condition that could lead to erroneous limit enforcement
+  - Fixed an issue with the margin limits where an error could occur if an account name with more than 12 chars was used
+
+## 2.6.1
+### User Visible Changes
+  - Fixed an issue in the ladder where an error could occur when trying to change ladder settings for the first time
+
+## 2.6.0
+### User Visible Changes
+  - In the ladder made last trade more distinctive when overlayed on the volumn column
+  - In the ladder, corrected display of volume tool tip
+  - In the ladder, changed the consolidation of last trade to consider only the price ignoring aggressor changes
+  - Added an option for the Top Algo to delete itself if within N ticks from top of book
+  - Address a rare issue that could lead to an error report on UI exit
+  - Added margin risk limits
+  - Added the "Offset" algo. Offset prices a child algo at a parameterizable distance from the top of the book
+  - Avoided switching the current account and order type on open ladders when limits are changed
+### Internal Changes
+  - Ensure that admin logins are always allowed to login, regardless of keycode status.
+
+## 2.5.0
+### User Visible Changes
+  - Added loss limit risk check
+  - Made algos, synthetics and user names private so that multiple users can share a single system privately.
+  - Allowed the mini order book defaults for column widths to be changed from the mini order book
+  - Allowed the mini order book to shrink to the minimum size that will be needed to display the selected columns
+  - Improved accuracy of the directional arrow when displaying last trades in the volumn column
+## 2.4.0
+### User Visible Settings
+  - In the ladder, changed the last trade column to accumulate successive trades at the same price
+  - Fixed a bug in the order book where the right side ladder would display the top leve order's security rather then current row's
+  - Added OK, Cancel buttons to the security picker for ladder profiles or sounds
+  - Added ladder settings for the mini order book
+  - Added a safety check to guard against unintentional market crosses
+  - Improved the Queue Holder mode in the spreader:
+     - Added a Show Quantity, allowing the spreader to keep queue position of the trailing orders as Fills (of size Show Qty) happen
+     - Improved spreader performance when in Queue Holder mode
+### Internal changes
+  - Suppressed order callbacks for completed/canceled orders. This should eliminate some bogus error logs by spreader orders.
+  - Addressed a potential concurrency issues with service initialization.
+
+
+## 2.3.0
+### User Visible Changes
+  - Introduced multiple settings profiles for the ladder
+     - This allows multiple standard configuration for the ladder, optionally with security specific variations
+  - Made Cover order more independent from its hedging chidren:
+     - Cover continues to operate even if an hedging child is canceled
+	 - When a cover is canceled it will detach from any running hedging children, instead of canceling them
+  - Changed Lean, Join and Cover algos to hide themselves in the ladder once they have completed their work
+  - Changed manual fills to appear in day PnL rather than in the SOD records
+  - Add buy/sell coloring to fill rows in fills window
+  - Streamlined the mini order book:
+    - Display of the mini order book popup is quicker
+    - Only exchange orders are displayed
+	- Update buttons are only displayed for orders that are under control of an algo with the update applying to the entire algo, not just the exchange order
+    - We can now select multiple rows to attach, detach, update or cancel multiple orders in a batch
+    - A checkbox column has been added to make clearer which rows are selected
+### API Changes
+  - Enhanced the API to allow algos to implement custom cancel handling
+  - Allowed algos to control their visibility in the ladder through the API
+### Internal Changes
+  - Added system requirements to Spark documentation
+  - Implemented keycodes
+  - Temporarily rejected stop and stop limit orders, pending bug fixes
+
+## 2.2.0
+### User Visible Changes
+  - Made our orders easier to spot in MBO display
+  - Changed middle-click on center column to recenter ladder and introduced Alt middle-click on center to bring up the MBO popup
+  - Fixed Net/Absolute radio button in risk admin UI
+  - Fixed double counting of spread legs in limits
+  - Removed expired fills from the system
+  - Improved handling of orders with undiscovered securities
+  - Added feedback to the ladder when the user attempts to middle click on bid/ask columns using an order that does not have parameters
+  - Fixed display of algo studio algo types in order book
+  - Hid net change setting in synthetic ladders
+  - Fixed a number of problems with filtering on the Fills form
+  - On the Lean algo, OnToward and Away are now calculated with respect to the opposite inside market at time of the order launch
+  - Iceberg Spreaders with queue holders are not erroneously cancelled after the first fill
+  - Added security symbol column to fills window
+  - Re-organized ladder settings dialog to make it clearer and easier to use
+
+## 2.1.4
+### User Visible Changes
+  - Fixed handling of cancel button in security picker
+
+## 2.1.3
+### User Visible Changes
+  - Fixed a simulator bug in the handling of 0 sized orders
+
+## 2.2.1
+### User Visible Changes
+  - Fixed order book form exception when display a spreader order for whichs the legs were not present yet
+
+## 2.1.1
+### User Visible Changes
+  - Added PIQ to GLBX Iceberg orders
+  - Corrected handling of stop limit orders
+
+## 2.1.0
+### User Visible Changes
+  - Added support for the GLBX Iceberg order type
+  - Allowed quick algo param edits by middle-clicking on either the bid/ask columns or a "favorite" button
+  - Added support for manual fills, either entered manually or imported from file. Manual fills are used only for P&L purposes, they have no impact on limits
+  - Allowed the LTQ and VAP values to be overlayed on the same ladder column
+  - Added day position information to the ladder
+  - Added PIQ to the mini order book
+  - Fixed a rare bug that could lead to server failure when requesting market data
+  - Fixed a bug that could cause incorrect price displays for certain types of synthetic securities
+
+## 2.0.5
+### User Visible Changes
+  - Fixed a bug in the handling of rejected QHO orders
+  - Implemented stricter position limit enforcement
+  - Fixed a bug that could cause server failure when trying to attach to already attached orders
+
+## 2.0.4
+### User Visible Changes
+  - Fixed performance regression in order manager
+
+### Internal changes
+  - Improved market data diagnostics for bad configurations
+  - Changed market data default buffer size
+
+## 2.0.3
+### Internal changes
+  - Improved backwards compatibility in UI settings
+
+
+## 2.0.2
+### User Visible Changes
+  - Middle click the ladder price column to show the orders for that price level
+  - Middle click the bid or ask column in the ladder no longer sends an order
+
+### Internal changes
+  - Improved market by order performance under heavy load
+  - Fixed a concurrency bug in SIM
+
+## 2.0.0
+### User Visible Changes
+  - Support for attaching and detaching algos to existing orders
+  - Support for display and update of all parameters in running algos
+  - Streamlining of order badge decorations in ladder
+  - New order search/filtering algorithm for order book
+  - Simpler representation of complex algo chains in order book and mini order book
+  - Visual and usability improvements to mini order book
+  - Streamlining of algo parameter dialog in ladder
+  - New spreader queue holder behavior
+  - Visual improvements to the order book
+  - New server side API
+  - Improved admin UI performance
+  - Optimized PIQ rendering
+### Internal changes
+  - Exchange simulator no longer requires daily sequence number reset
+  - Improved MD resilience to corrupted shared memory
+  - Native orders can now buffer updates/cancels when disconnected or pending first acknowledgment
+  - Improved settings manager tool output
+  - Added diagnostics in the event MD memory corruption is detected
+  - Added systems tests for join,lean, iceberg and spreader orders
+  - Made exchange order recovery more resilient in the presence of database loss
+
+## 1.6.2
+### User Visible Changes
+### Exchange simulator
+  - Exchange simulator now now produce position in queue data
+
+## 1.6.0
+### User Visible Changes
+#### New Features
+  - You can now view your order's position in queue in the ladder
+#### Bug Fixes
+  - Fix a bug in the order popup in the ladder which could lead to a crash.
+
+## 1.5.9
+### User Visible Changes
+  - Fix for stunnel connections on Windows 7
+
+## 1.5.8
+### User Visible Changes
+  - Fixed a bug where it was possible to crash the UI by entering an order through the order ticket if Enter was used to dismiss the dialog.
+  - Fixed a spreader reporting bug which could cause the server to crash if an algo was used to control certain types of synthetics
+  - Improved user message when the UI fails to login
+  - Integrated stunnel into UI to make VPN setup unnecessary in unsecured environments.
+## 1.5.7
+### User Visible Changes
+  - New videos have been added to the manual describing the [Algo Studio](./bts_spark/Trading/_index.md#algo-studio) and [Spreader](./bts_spark/Spreader/_index.md)
+  - Fixed a bug that occasionally would cause windows to be position outside the visible screen on startup
+  - Fixed a bug in the order book form that would occasionally cause the Recreate Order option to be unavailable
+  - Added a confirmation step to Cancel All, Cancel Buys and Cancel Sells commands
+  - Fixed Aggressor coloring and ladder high/low coloring in the SIM execution gateway
+  - Added initial support for ICE market data
+  - In an effort to streamline the PnL report we've made the following changes:
+    - The PnL report will by default filter for Futures contracts only
+    - If you wish to see Spread contracts again you may change Security Type column filter
+    - The PnL report will now only show rows in which there is a position (net and/or day)
+    - The Exchange, Start PnL, Total Net Pos, and Total Net PnL columns are now hidden by default
+    - These columns can be restored by toggling their visibility within the PnL's Settings form
+    - A new Traded Today column has been added to the PnL
+      - Filtering on this column will allow a user to restrict the report to only show today's position
+### Internal changes
+  - Improved organization of the shell notification architecture
+  - Added logging to help troubleshoot shell exit hang
+
+## 1.5.6
+### User Visible Changes
+  - Fixed a bug that could cause limits to be erroneously enforced
+  - Improved algo documentation
+  - Changed how the spreader uses net change
+  - Simplified trigger algo user interface
+### Internal changes
+  - Defaulted to not recording sim market data
+  - Improved documentation build
+  - Added additional performance tracing
+
+## 1.5.5
+  - Fixed Market View 'stale working orders' bug
+
+## 1.5.4
+  - Avoided logging when warming up cache
+
+## 1.5.3
+  - Fixed sounds startup issue
+
+## 1.5.2
+  - Added better logging to AlgoDefinitionService
+
+## 1.5.1
+  - Fixed missing securities in the Fills and P&L windows.
+
+## 1.5.0
+### User Visible Changes
+#### New Features
+  - This release introduces a major new feature, the Algo Studio, a
+    set of tools allowing traders to create their own algos by
+    composing existing algos to express unique strategies.
+#### User Interface Improvements
+  - Removed security key field from most grids
+  - Removed trade key field from time and sales grid
+  - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM
+  - Fixed the floating order book so that rows are no longer
+    highlighted after an order is modified
+  - Relocated quantity and price columns in floating order book immediately after the cancel column; users now can move the mouse a much shorter distance when modifying their orders
+  - The escape key now closes the floating order book
+#### Bug Fixes
+  - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM
+  - Fixed a bug in the order book where sometimes parent orders would not update their last message column.
+
+#### Algos
+  - Added OCO Algo
+    - An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill.  Composing this order with Cover simulates scalping functionality
+#### SDK
+  - Improved SDK documentation and usability
+  - Dynamic orders are now properly identified in the order book
+#### Documentation
+  - Information on the Spark API has been added to the Spark manual
+  - The documentation has been updated and reorganized to make information easier to find
+### Internal Changes
+  - Improved robustness of limit system test
+  - Fixed a bug in handling of built-in bts to user that could lead to failure to login as bts in old systems
+  - Improved market data subsystem to support security definition updates
+  - Fixed a startup race in market data that could lead to data corruption on startup
+  - Increased default order price precision to 8 digis as required by some CME products
+
+## 1.4.11
+### New Features
+  - Floating Order Book
+    - The floating order book provides a quick view of orders occupying a price level in the ladder
+    - Launch the Floating Order Book by middle-clicking a ladder badge
+    - Cancel orders by clicking on the red X in the Cancel column
+    - Quantity and Price modification
+      - Clicking on a row will select it, providing the option for changing your price and quantity
+      - Enter in a new value or use the up/down buttons to set a desired value
+      - Press the green checkbox to finalize your change and update the order
+### SDK
+  - Improved SDK documentation and usability
+### User Interface Improvements
+  - The following spreader quoting parameters have been deprecated:
+    - Hidden parameter
+    - Price Type setting of the Conditional parameter
+### Bug Fixes
+  - Fixed a limit enforcement problem that could lead to 0 sized orders being sent to the exchange
+
+## 1.4.10
+### Bug Fixes
+  - Removed duplicate entries in performance log
+
+## 1.4.9
+### SDK
+  - Added ticker events to the order API
+### Performance Improvements
+  - Significantly improved hedging speed
+### Bug Fixes
+  - Fixed a visual oddity in the order book UI when using High DPI displays
+
+## 1.4.8
+### Marketdata
+  - Additional trade details for CME ticker
+
+## 1.4.7
+### Bug Fixes
+  - Corrected FIX tag 1028, ManualOrderIndicator, when orders are triggered by algos
+
+## 1.4.6
+### Ladder Usability Improvements
+  - The ladder can now be zoomed in/out according to a user's preference
+  - The zoom slider can be enabled from Ladder Form -> Settings button -> Left Pane tab -> check Zoom Slider checkbox
+  - The zoom functionality can also be performed via hotkey and mouse wheel combinations
+    - Zoom In = CTRL + Mouse wheel up
+    - Zoom Out = CTRL + Mouse wheel down
+
+    - Reset zoom = CTRL + Mouse wheel click or double click on the zoom slider
+### Exchange simulator
+  - Improved the exchange simulator to track real market data more closely
+### Bug Fixes
+  - Fixed a UI crash in some cases when attempting to sort the order book by time
+
+## 1.4.5
+  - Fixed entitlement validation for ICS spreads
+  - Tuned performance logging
+
+## 1.4.4
+
+### Performance Improvements
+  - Reduced performance logging activity
+
+## 1.4.3
+
+### Ladder Usability Improvements
+  - Controls found in Left Pane of the ladder can now be hidden via the checkboxes found under Ladder Form -> Settings button -> Left Pane tab
+  - The Left Pane is now scrollable with the mouse wheel when there isn't enough vertical space to accomodate the entire form
+  - The Left Pane can now grown and shrunk horizontally via the splitter between the left pane and the main ladder area
+### Performance Improvements
+  - Requoting and hedging latency has been reduced by up to 30%
+### Software Development Kit (SDK) Improvements
+  - The API now enforces authentication
+### Bug Fixes
+  - The Market View form now honors filters when new securities are added
+
+## 1.4.2
+
+### User Interface Improvements
+  - When creating a synthetic from the Market View window, securities are now added to the Create Synthetic dialog in the same sort order as they appear in grid
+### Bug Fixes
+  - Shell no longer crashes when removing a security from the synthetics window
+  - It is now possible to create a new synthetic using the same name as a previously deleted synthetic
+### SDK Improvements
+  - Improved Client and Server SDK clarity and documentation
+### Performance Improvements
+  - Synthetic calculation speed was improved by 400%.
+### Risk / Entitlement System
+  - Risk Admin UI
+    - Spark users now have separate Trading, RiskAdmin, and/or MDAdmin permissions
+    - Spark users can now only add other users or accounts if permissioned to do so.
+  - Market Data Entitlement
+    - Automated market data reporting
+
+
+## 1.4.1
+
+### Tabs
+  - Tabs now shrink and expand to make better use of horizontal space
+### Bug Fixes
+  - Fixed a bug that could cause excessive CPU usage when a connection broke
+  - Fixed a layout bug in hedging/requoting performance reports
+  - Fixed account name mismatching bug for old fills
+### Absolute Position limits
+  - Positions can now be netted or absolute for purposes of limit enforcing
+
+## 1.4.0
+
+### Speed Enhancements
+  - Improved requoting and hedging performance
+  - Improved order book form performance
+### Server Side SDK
+  - Added a beta version of the server side SDK
+### Front End
+  - Added status bars to all UI grids
+### Reporting
+  - Added reporting of hedging and requoting performance
+
+## 1.3.8
+
+### Speed Enhancements
+  - Further speed enhancements
+### Conditional (new quoting parameter)
+  - Reduces re-quoting updates by only quoting if N ticks away from the opposite inside market.
+  - Quotes are canceled when it gets further away.
+  - "Price Type" dictates if this logic applies to the quoting market or to the synthetic spread market.
+
+## 1.3.7
+
+### Net Change pricing
+  - The ability to price synthetics based off of net change
+
+## 1.3.5
+
+### Speed Enhancements
+  - Further speed enhancements
+### Separation of implied and direct market data
+  - Viewing within a market view will now show 3 columns
+    - Direct
+    - Implied
+    - Merged
+  - Ability to configure the market data size in a ladder as the following
+    - Direct
+    - Implied
+    - Merged
+### Synthetic creation option to choose Merged or Direct
+  - The ability to choose Merged or Direct per leg when creating a synthetic
+### Volume at price
+  - Within a ladder, you can now see a volume at price graph
+  - This is configurable from a ladder's settings as well
+### Customized settings in all windows
+ - All windows now contain a settings page where grid settings (font color, size, etc) are applied
+ - These settings can be saved on a per-window basis or as a Default for newly-launched windows
+
+
+## 1.3.1 Through 1.3.4
+
+### Speed Enhancements
+  - Further speed enhancements
+
+## 1.2
+
+### Market View
+  - Synthetic future leg working orders are now shown
+
+### Order Book
+  - Many visual facets of the Order Book grid can now be customized
+    - The Order Book settings page can be accessed from the gears icon in the Order Book toolbar or from the right-click menu
+  - The Expand/Collapse All button has been separated into two different buttons on the toolbar
+
+### Sound Alerts
+  - Sounds alerts can now be configured for various order events
+  - Configure sounds under the Sound/Notification Triggers on the Order Book settings page
+
+### Fills
+  - Newly launched fill windows will filter out start-of-day records by default
+
+### PnL
+  - Prices are now formatted with penny precision
+  - Day PnL has been added to the Totals footer at the bottom of the window
+
+### Ladder
+  - Order badges now have been improved to communicate a variety of order state
+    - Background color now indicates order side
+    - Child orders will display a one-character abbreviation to their parent order enclosed in a rectangle; the rectangle will have a fill color matching the parent order
+    - Border color around badge will match the associated algo color
+    - Exchange orders will present with a solid dot in the badge's top-right corner
+    - When multiple orders reside at the same level, a solid triangle will be present in badge's bottom-right corner; all other badge decorations will not be visible in this state
+    - Hedge orders will present with a vertical line inside the badge's left border
+  - Trigger orders, once triggered, will no longer be shown in the ladder
+  - Session High and Low visual indicators have been added to the Price column
+  - Session range is now indicated in the Price column
+  - A button reflecting position has been added above the quantity field
+    - This button reflects the position for the currently selected account
+    - Pressing the button will copy the position size to the quantity field
+  - Order quantity button size values are now customizable
+    - Default size preset values can now be changed, as well as added or removed
+    - This can be configured per ladder instance (i.e. per security) by making use of the Custom tab
+  - Synthetic ladders that were saved as part of a workspace are now restored on startup
+
+### Synthetics
+  - Hedge Ratio parameter has been renamed to Lean Qty Factor
+  - Spreader now includes a Hidden mode parameter
+    - A quoting leg can be hidden, ie an order will not be sent out until the market crosses the computed quoting price. When that happens, the quote is sent out as an IOC order.
+
+### Algos
+  - Trigger orders now can contain a Trigger Exit parameter based on time or price; once the exit condition has been met the Trigger order will cancel
+  - IOC/FAK order type added
+  - Cover order type added
+  - Canceling a synthetic order no longer cancels any working hedge orders
+
+### General
+  - A manual page has been added to the launcher Help menu
+  - Various help buttons and tooltips were added throughout the application
+  - Various cash per point values updated
+  - Time-based grid filters now have tick up/down buttons
+  - Numerous performance improvements and stability enhancements
+  - Automated position and Order Book maintenance during daily recycle
+    - Individual fills will be compacted into the Start of Day position records
+    - Orders from the day before the recycle will be deleted
+
+
+## 1.1
+
+### 1.1.22
+- Order Book: Orders in Error status can now be recreated
+- Order Book: Orders containing child orders will now be collapsed by default.
+- Order Book: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
+- Order Book: Symbol column added to allow for filtering based on group symbol
+- Time filters now have a quick Set Last Hour hyperlink which will set the Period Begin filter to (Now - 1 hour)
+
+### 1.1.20
+- Update CME market data handler for price precision extension.
+  https://www.cmegroup.com/confluence/display/EPICSANDBOX/Price+Precision+Extension
+
+### 1.1.8
+- Enabled wild cards in risk management UI
+- Fixed spreader order updates
+
+### 1.1.7
+- Fixed spread leg price fill
+- Fixed HO cash per point value
+
+### 1.1.6
+- Fixed occasional crash when filtering fills
+- Fixed several cash per point multipliers
+- Colored time of sales row with aggressor side
+- Iceberg orders now cancel (rather than an error) when a child is canceled
+
+### 1.1.0
+- Added per user self match protection, location and self match protection FIX tag settings
+- Implemented a new "Trigger Order" algo
+- Corrected tick formatting for a number of products
+- Added the ability to recreate canceled orders
+- Added leg fills to facilitate futures based PnL
+- PnL report now has PnL columns calculated based on your Start of Day, current day, and total position
+- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
+- Corrected cash per point values, used in pnl calculations, for a number of products
+- PnL report will now remember any customizations you make to filtering and/or sorting criteria

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "2.1.508",
-    "rollForward": "latestMajor"
-  }
-}

spark_client.sln

@@ -23,6 +23,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\sp
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_excel", "src\client\spark_excel\spark_excel.csproj", "{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_fix", "src\client\spark_fix\spark_fix.csproj", "{D8031C13-7BEA-430A-BB60-40927D15CFC3}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_historical", "src\client\spark_historical\spark_historical.csproj", "{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_analytics", "src\client\spark_analytics\spark_analytics.csproj", "{22DBF34A-961F-4620-91B8-21B0C257A8C6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -219,6 +225,74 @@ Global
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.ActiveCfg = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.Build.0 = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x64.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x86.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x64.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x64.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x86.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x86.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -229,4 +303,7 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 EndGlobal

src/client/CMakeLists.txt

@@ -0,0 +1,19 @@
+
+cmake_minimum_required(VERSION 3.5)
+project(spark_client)
+include(${CMAKE_SOURCE_DIR}/tools/safe_dep_glob.cmake)
+
+file(GLOB_RECURSE spark_client_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cs)
+file(GLOB_RECURSE spark_client_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.csproj)
+file(GLOB_RECURSE spark_ui_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.cs)
+file(GLOB_RECURSE spark_ui_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.csproj)
+list(REMOVE_ITEM spark_ui_sources ${spark_client_sources} )
+list(REMOVE_ITEM spark_ui_csproj ${spark_client_csproj} )
+
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/../
+   COMMAND /bin/bash ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh
+   ARGS ${CMAKE_BUILD_TYPE} ${CMAKE_BINARY_DIR}
+   COMMENT "Building sparkcmd in netcore3"
+   DEPENDS libspark ${spark_client_sources} ${spark_client_csproj} ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh)
+
+   add_custom_target(spark_client_dir ALL DEPENDS ${CMAKE_BINARY_DIR}/../)   

src/client/Directory.Build.props

@@ -5,6 +5,7 @@
     <SparkRoot>$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\..))</SparkRoot>
     <SourceRoot>$([System.IO.Path]::GetFullPath($(SparkRoot)\src))</SourceRoot>
     <Externals>$(SparkRoot)\externals\windows</Externals>
+    <Externals Condition=" '$(OS)' != 'Windows_NT' ">$(SparkRoot)/externals/linux</Externals>
     <DepotName>$([System.IO.Path]::GetFileName($([System.IO.Path]::GetFullPath($(SparkRoot)))))</DepotName>
     <ErrorReport>None</ErrorReport>
     <Prefer32Bit>False</Prefer32Bit>
@@ -19,24 +20,25 @@
     <OutputPath>$(BaseOutputPath)\bin</OutputPath>
     <BaseIntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
     <IntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</IntermediateOutputPath>
+    <BaseIntermediateOutputPath Condition=" '$(OS)' != 'Windows_NT' ">$(BaseOutputPath)\linuxobj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <IntermediateOutputPath Condition=" '$(OS)' != 'Windows_NT' ">$(BaseOutputPath)\linuxobj\$(MSBuildProjectName)</IntermediateOutputPath>
     <GenDir>$(BaseOutputPath)\gen\$(MSBuildProjectName)</GenDir>
 </PropertyGroup>
 
-<PropertyGroup Condition="'$(VCTargetsPath)'==''">
-    <VCTargetsPath>C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\Common7\IDE\VC\VCTargets</VCTargetsPath>
-</PropertyGroup>
 
 <Import Project="$(ComputerSpecificConfig)" Condition="Exists($(ComputerSpecificConfig))" />
 
 <ItemGroup>
     <AvailableItemName Include="Proto2">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto2</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto2Linux</Targets>
     </AvailableItemName>
     <AvailableItemName Include="Proto3">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto3Linux</Targets>
     </AvailableItemName>
     <AvailableItemName Include="Proto3Excel">
-        <Targets>CompileProto3Excel</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' ">CompileProto3Excel</Targets>
     </AvailableItemName>    
 </ItemGroup>
 
@@ -47,6 +49,14 @@
     <protogen>$(Externals)\bin\protogen.exe</protogen>
 </PropertyGroup>
 
+<PropertyGroup Label="ProtoBuf" Condition=" '$(OS)' != 'Windows_NT' ">
+    <ProtocPluginBase>$(SparkRoot)/src/lib/api/service_compiler</ProtocPluginBase>
+    <ProtoInclude>$(Externals)/include</ProtoInclude>
+    <protoc>$(Externals)/bin/protoc</protoc>
+    <protoc_rename>$(Externals)/bin/lowercase_files.sh</protoc_rename>
+    <protogen>$(Externals)/bin/protogen</protogen>
+</PropertyGroup>
+
 <ItemDefinitionGroup>
     <Proto2>
     </Proto2>

src/client/Directory.Build.targets

@@ -3,7 +3,7 @@
 
 
 <Target Name="CompileProto2" 
-    Condition="'@(Proto2)'!=''"
+    Condition="'@(Proto2)'!='' AND '$(OS)' == 'Windows_NT' "
     Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
     Outputs="$(GenDir)\proto2\%(Proto2.filename).cs"
     AfterTargets="PreBuildEvent">
@@ -28,7 +28,7 @@
     passes wipes out the additional compile includes that we add.
 -->
 <Target Name="CompileProto3" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3)'!=''"
+    Condition="'@(Proto3)'!=''  AND '$(OS)' == 'Windows_NT' "
     Inputs="$(protoc);
     %(Proto3.Identity);$(ProtocPluginBase)\plugin.py;
     $(ProtocPluginBase)\templates\c#\client.pb.cs;
@@ -58,7 +58,7 @@
 </Target>
 
 <Target Name="CompileProto3Excel" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3Excel)'!=''"
+    Condition="'@(Proto3Excel)'!='' AND '$(OS)' == 'Windows_NT' "
     Inputs="$(protoc);
     %(Proto3Excel.Identity);$(ProtocPluginBase)\plugin.py;
     $(ProtocPluginBase)\templates\excel\excel.pb.cs;"
@@ -82,12 +82,74 @@
     </ItemGroup>
 </Target>
 
+
+<Target Name="CompileProto2Linux" 
+    Condition="'@(Proto2)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
+    Outputs="$(GenDir)/proto2/%(Proto2.filename).cs"
+    AfterTargets="PreBuildEvent">
+    <PropertyGroup>
+        <SourceRelativeFile>$([System.String]::Copy('%(proto2.fullpath)').Replace($(SourceRoot), ''))</SourceRelativeFile>
+        <SourceRelativeDir>$([System.IO.Path]::GetDirectoryName($(SourceRelativeFile)))</SourceRelativeDir>
+        <WorkingDirectory>$(GenDir)/proto2</WorkingDirectory>
+    </PropertyGroup>
+    <MakeDir Directories="$(WorkingDirectory)" />
+    <Copy SourceFiles="%(Proto2.FullPath)" DestinationFolder="$(WorkingDirectory)" SkipUnchangedFiles="True" UseHardLinksIfPossible="True" />
+    <Error Text="protogen.exe does not exist. If you forgot to clone with externals try: git submodule update --init --recursive" Condition="!Exists($(protogen))" />
+    <Message Importance="High" Text="Doing Proto2 %(proto2.filename)" />
+    <Exec Command="PYTHONPATH=$(Externals)/python $(Protogen) %(Proto2.filename)%(proto2.extension) --csharp_out=. -I. -I$(SourceRoot) -I$(Externals)/include"
+          WorkingDirectory="$(WorkingDirectory)" />
+    <ItemGroup>
+        <Compile Include="$(WorkingDirectory)/%(Proto2.filename).cs" />
+    </ItemGroup>
+</Target>
+
+<Target Name="CompileProto3Linux" AfterTargets="GetAssemblyVersion"
+    Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(protoc);
+    %(Proto3.Identity);$(ProtocPluginBase)/plugin.py;
+    $(ProtocPluginBase)/templates/c#/client.pb.cs;
+    $(ProtocPluginBase)/templates/c#/server.pb.cs;
+    $(ProtocPluginBase)/templates/c#/immutable.pb.cs;"
+    Outputs="%(Proto3.OutCS);%(Proto3.OutCSServer);%(Proto3.OutCSClient);%(Proto3.OutCSImmutable)">
+    <MakeDir Directories="%(Proto3.OutBase)" />
+    <MakeDir Directories="$(IntermediateOutputPath)/gen" />
+   
+    <Message Importance="High" Text="Generating service %(Proto3.FileName)" />
+    <Exec Command="
+       PYTHONPATH=$(Externals)/python $(protoc) \
+       %(Proto3.FullPath) \
+       --error_format=gcc \
+       --proto_path=$(protoinc):$(SourceRoot) \
+       --plugin=&apos;protoc-gen-service=$(ProtocPluginBase)/plugin.py&apos; \
+       --service_out=lang=c#,strip_out_path=1:%(Proto3.OutBase) \
+       --csharp_out=%(CompileOptions)%(Proto3.OutBase)"
+      WorkingDirectory="%(relativedir)" />    
+      
+    <!-- <Exec Command="$(protoc_rename) %(Proto3.OutBase)"  />    -->
+            
+    <ItemGroup>
+       <Compile Include="%(Proto3.OutCS)" />
+       <Compile Include="%(Proto3.OutCSClient)" />
+       <Compile Include="%(Proto3.OutCSServer)" />
+       <Compile Include="%(Proto3.OutCSImmutable)" />
+    </ItemGroup>
+</Target>
+
+<!-- <Target Name="RenameProto3Linux" AfterTargets="CompileProto3Linux"
+   Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' ">
+   <Move SourceFiles="$(OutBase)\Apicore.cs" DestinationFiles="$(OutBase)\apicore.cs" />  
+   <Move SourceFiles="$(OutBase)\Apioptions.cs" DestinationFiles="$(OutBase)\apioptions.cs" />  
+   <Move SourceFiles="$(OutBase)\Co2.cs" DestinationFiles="$(OutBase)\co2.cs" />  
+</Target> -->
+
+
 <Target Name="AfterClean" AfterTargets="Clean">
     <Delete Files="%(Proto3.OutCS)" />
     <Delete Files="%(Proto3.OutCSClient)" />
     <Delete Files="%(Proto3.OutCSServer)" />
     <Delete Files="%(Proto3.OutCSImmutable)" />
-    <Delete Files="$(GenDir)\proto2\*" />
+    <Delete Files="$(GenDir)/proto2/*" />
     <Delete Files="%(Proto3Excel.OutCSExcel)" />
 </Target>
 

src/client/build_spark_dotnetcore.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#$1 = Debug / Release (CMAKE_BUILD_TYPE), $2 = CMAKE_BINARY_DIR
+cd "$(dirname "$0")"
+echo $PWD dotnet build -c $1 ./sparkcmd/
+dotnet build -c $1 -o $2/../$1/bin/netcoreapp3.1/linux-x64 ./sparkcmd/ 
+
+if [ "$1" == "Release" ]
+then
+   dotnet publish -c $1 -o $2/../$1/bin/netcoreapp3.1/linux-x64 ./sparkcmd/ 
+   ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.1/linux-x64/publish
+fi
+
+ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.1/linux-x64

src/client/spark/Modules/Spreader/ExecutionStrategyEditor.xaml

@@ -549,48 +549,24 @@
               </u:BindingUpdateGroupCollection>
             </u:Behavior.BindingUpdateGroups>
           </TextBox>
-          <StackPanel
+          <TextBlock
             x:Name="PART_TickFactors_Label"
             Grid.Column="2"
+            VerticalAlignment="Center"
+            Margin="{StaticResource Margin.LHeaderPanelItem}"
+            Text="Ticks"
             IsEnabled="True"
-            Orientation="Horizontal"
             Visibility="{Binding LegTuples.Count, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource IntNotZeroVisibilityConverter}}"
-           >
-            <TextBlock
-              VerticalAlignment="Center"
-              Margin="{StaticResource Margin.LHeaderPanelItem}"
-              Text="Ticks ("
-            />
-            <CheckBox
-              Margin="1,0,0,0"
-              IsChecked="{Binding UseFractionalTicks, RelativeSource={RelativeSource TemplatedParent}}"
-              VerticalAlignment="Center"
-              VerticalContentAlignment="Center"
-              Padding="0"
-            >
-              fractional
-            </CheckBox>
-            <TextBlock VerticalAlignment="Center" Margin="1,0,0,0">)</TextBlock>
-          </StackPanel>
-          <Grid
+          />
+          <c:TickFactorsPicker
             x:Name="PART_TickFactors_Picker"
             Grid.Column="3"
-            IsEnabled="True"
             Margin="{StaticResource Margin.LHeaderPanelItem}"
+            HorizontalAlignment="Stretch"
+            IsEnabled="True"
+            SelectedTickFactors="{Binding TickFactors, RelativeSource={RelativeSource TemplatedParent}, Mode=TwoWay}"
             Visibility="{Binding LegTuples.Count, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource IntNotZeroVisibilityConverter}}"
-          >
-            <c:TickFactorsPicker
-              HorizontalAlignment="Stretch"
-              SelectedTickFactors="{Binding TickFactors, RelativeSource={RelativeSource TemplatedParent}, Mode=TwoWay}"
-              Visibility="{Binding UseFractionalTicks, RelativeSource={RelativeSource TemplatedParent}, Converter={x:Static cv:BooleanToVisibilityConverter.HiddenInstance}}"
-            />
-            <c:TickUpDownInt
-              HorizontalAlignment="Stretch"
-              Minimum="1"
-              Value="{Binding IntegerTickSize, RelativeSource={RelativeSource TemplatedParent}}"
-              Visibility="{Binding UseFractionalTicks, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource FalseVisibilityConverter}}"
-            />
-          </Grid>
+          />
           <TextBlock
             x:Name="PART_Pricing_Label"
             Grid.Column="4"

src/client/spark/Modules/Spreader/ExecutionStrategyEditor.xaml.cs

@@ -52,26 +52,6 @@ namespace mbts.ui.spark {
       public TickFactors TickFactors { get => GetValue(TickFactorsProperty) as TickFactors; set => SetValue(TickFactorsProperty, value); }
       #endregion
 
-      #region UseFractionalTicks
-      public static readonly DependencyProperty UseFractionalTicksProperty = WPFHelper.create_dp<bool, ExecutionStrategyEditor>(
-         nameof(UseFractionalTicks), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, true);
-
-      public bool UseFractionalTicks {
-         get => (bool)GetValue(UseFractionalTicksProperty);
-         set => SetValue(UseFractionalTicksProperty, value);
-      }
-      #endregion
-
-      #region IntegerTickSize
-      public static readonly DependencyProperty IntegerTickSizeProperty = WPFHelper.create_dp<int, ExecutionStrategyEditor>(
-         nameof(IntegerTickSize), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, 1);
-
-      public int IntegerTickSize {
-         get => (int)GetValue(IntegerTickSizeProperty);
-         set => SetValue(IntegerTickSizeProperty, value);
-      }
-      #endregion
-
       #region IsNew
       public static readonly DependencyProperty IsNewProperty =
          DependencyProperty.Register(

src/client/spark/Modules/Spreader/SyntheticEditorView.xaml

@@ -36,8 +36,6 @@
       ContractPickerModel="{Binding contract_picker_model}"
       HedgePricing="{Binding hedge_pricing}"
       HedgePricingItemsSource="{Binding hedge_pricing_options}"
-      UseFractionalTicks="{Binding use_fractional_ticks}"
-      IntegerTickSize="{Binding integer_tick_size}"
     />
     <c:DialogSkirtOKCancel Grid.Row="1" OKCommand="{Binding commit}" CloseOnOK="False"/>
   </Grid>

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -2,6 +2,7 @@
 using bts.collections;
 using bts.shell;
 using bts.spark;
+using bts.utils;
 using mbts.ui;
 using mbts.ui.shell;
 using mbts.ui.spark;
@@ -29,16 +30,8 @@ namespace mbts.spark {
       readonly Property<bool> _has_working_orders, _can_add_leg;
 
       Security _synthetic;
-      bool
-         _set_name_automatically,
-         _set_ticks_automatically,
-         _set_quantities_automatically,
-         _set_price_mults_automatically,
-         _is_setting_quantities,
-         _is_setting_price_mults,
-         _use_fractional_ticks = true;
+      bool _set_name_automatically, _set_ticks_automatically, _set_quantities_automatically, _set_price_mults_automatically, _is_setting_quantities, _is_setting_price_mults;
       TickFactors _tick_factors;
-      int _integer_tick_size = 1;
       ExecutionStrategyImmutable _strategy;
       HedgePricing _hedge_pricing;
 
@@ -51,10 +44,6 @@ namespace mbts.spark {
          : this(synthetic, strategy, false) {
          ExceptionAssert.InvalidOperation.That(synthetic.Id != 0, "This constructor may only be used for already-existing synthetic securities (id must be non-zero).");
          _tick_factors = TickUtils.get_regional_ticks(synthetic.ContractSpec).low_tick;
-         if (TickUtils.is_integer_tick(synthetic.ContractSpec)) {
-            _use_fractional_ticks = false;
-            _integer_tick_size = DoubleHelper.to_optional_int32(synthetic.ContractSpec.LowTick) ?? 0;
-         }
          title = $"Edit '{synthetic.Synthetic.Name}'";
          icon = Icons.ModifySynthetic;
          _has_working_orders = backing(nameof(has_working_orders), security_has_working_orders.OnDispatcher());
@@ -142,14 +131,14 @@ namespace mbts.spark {
                flash_error_msg("Name already taken", SparkUtils.UI.MessageFlashDuration, nameof(name)).ignore();
                valid = false;
             }
-            if (use_fractional_ticks ? (_tick_factors == null || !_tick_factors.is_valid()) : integer_tick_size == 0) {
-               flash_error_msg("Please select ticks", SparkUtils.UI.MessageFlashDuration, use_fractional_ticks ? nameof(tick_factors) : nameof(integer_tick_size)).ignore();
+            if (_tick_factors == null || !_tick_factors.is_valid()) {
+               flash_error_msg("Please select ticks", SparkUtils.UI.MessageFlashDuration, nameof(tick_factors)).ignore();
                valid = false;
             }
             if (valid) {
                _synthetic.Synthetic.Legs.SetItems(leg_tuples.Select(l => l.leg));
                _synthetic.Symbol = _deduce_symbol(_synthetic);
-               _synthetic.ContractSpec = use_fractional_ticks ? _tick_factors.to_spec() : TickUtils.create_integer_tick_spec(integer_tick_size);
+               _synthetic.ContractSpec = _tick_factors.to_spec();
                uint? id = await model.create_synthetic(new SecurityImmutable(_synthetic));
                if (id.HasValue && id != 0) {
                   if (await model.save_strategy(id.Value, get_strategy())) {
@@ -306,20 +295,6 @@ namespace mbts.spark {
       public bool has_working_orders => _has_working_orders.Value;
       public bool is_new { get; }
       public bool is_new_leg_popup_open { get; set; }
-      public bool use_fractional_ticks {
-         get => _use_fractional_ticks;
-         set {
-            if (Set(ref _use_fractional_ticks, value))
-               _set_ticks_automatically = false;
-         }
-      }
-      public int integer_tick_size {
-         get => _integer_tick_size;
-         set {
-            if (Set(ref _integer_tick_size, value))
-               _set_ticks_automatically = false;
-         }
-      }
       public bool can_add_leg => _can_add_leg.Value;
       public SyntheticContract.Types.PriceCalc price_calc { get => _synthetic.Synthetic.PriceCalc; set => _synthetic.Synthetic.PriceCalc = value; }
       public ObservableCollection<LegSpreaderParamsTuple> leg_tuples { get; } = new ObservableCollection<LegSpreaderParamsTuple>();
@@ -348,16 +323,6 @@ namespace mbts.spark {
          _strategy = strategy;
          hedge_pricing = strategy.HedgePricing;
          leg_tuples.SetItems(_get_legs(_synthetic, strategy));
-         if (synthetic.HasContractSpec) {
-            /// assign the fields <see cref="_use_fractional_ticks"/> and <see cref="_integer_tick_size"/>,
-            /// rather than the properties <see cref="use_fractional_ticks"/> and <see cref="integer_tick_size"/>
-            /// because the setters of the former flip <see cref="_set_ticks_automatically"/> to false if the value changes.
-            /// This behavior causes the ticks to be automatically deduced as legs are added/removed,
-            /// until the user explicitly enters a different value (as the properties are bound to tick input controls in the view).
-            Set(ref _use_fractional_ticks, !TickUtils.is_integer_tick(synthetic.ContractSpec), nameof(use_fractional_ticks));
-            if (!use_fractional_ticks)
-               Set(ref _integer_tick_size, DoubleHelper.to_optional_int32(synthetic.ContractSpec.LowTick) ?? 0, nameof(integer_tick_size));
-         }
          RaisePropertyChanged(nameof(name), nameof(tick_factors));
       }
 

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -5,6 +5,7 @@ using bts.collections;
 using bts.shell.fills;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.collections;
 using mbts.persist;
 using mbts.spark.filters;
@@ -23,6 +24,7 @@ using System.Linq;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
 using System.Windows.Data;
+using System.Windows.Input;
 using System.Windows.Media;
 using static bts.api.orderman.Fill.Types;
 using static bts.shell.FillsWindowFactory;
@@ -310,6 +312,7 @@ namespace mbts.spark {
          public FillStatus fill_status => _fill.FillStatus;
          public string fill_key => _fill.FillKey;
          public string order_key => _order.Key.ToString();
+         public string symbol => fill_tuple.security?.Symbol;
          public bool is_selected {
             get => _is_selected;
             set => Set(ref _is_selected, value);
@@ -330,6 +333,28 @@ namespace mbts.spark {
          public override bool Equals(object obj) => Equals(obj as Row);
          public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();
 
+         public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(typeof(Row),
+               new string[] {
+                  nameof(security),
+                  nameof(symbol),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(side),
+                  nameof(fill_time),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(size),
+                  nameof(price),
+                  nameof(fill_kind),
+                  nameof(fill_status),
+                  nameof(fill_key),
+                  nameof(order_key),
+                  nameof(security_key)
+               });
+
          public event Action invalidated;
          void IRowVM.raise_invalidated() => invalidated?.Invoke();
       }

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -534,44 +534,13 @@ namespace mbts.spark {
          _size_from_row(row, _price_list, _asks, _vm.security.ContractSpec);
 
       uint _volume_from_row(int row) {
-         const int max_tries = 3;
-         var errors = new List<string>(0);
-         int num_tries = 0;
-         for (; num_tries < max_tries; num_tries++) {
-            try {
-               var rv = impl();
-               if (num_tries > 0) {
-                  log_errors(true);
-               }
-               return rv;
-            } catch (Exception ex) {
-               errors.Add(ex.ToString());
-            }
-         }
-         log_errors(false);
-         return 0;
-
-         void log_errors(bool succeeded) {
-            // Only log distinct errors to avoid useless noise.
-            // It's likely that the exception will always be the same (IndexOutOfRange),
-            // however, this bug is not fully understood, so assume nothing.
-            var error_msg = string.Join("; ", errors.Distinct());
-            if (succeeded) {
-               Log.Ui.error($"Ladder '{security}' got volume for row at index '{row}' after {num_tries + 1} attempts (price_list array length is {_price_list.Length}).  The following exceptions were observed: {error_msg}");
-            } else {
-               Log.Ui.error($"Ladder '{security}' failed to get volume for row at index '{row}' after {max_tries} attempts (price_list array length is {_price_list.Length}): {error_msg}");
-            }
-         }
-
-         uint impl() {
-            if (_price_list.SafeTryGetAt(row, out var price)) {
-               var vapls = _volume_container.read_new_value();
-               if (vapls != null) {
-                  return vapls.SelectFirstOrDefault(v => MathUtils.epsilon_equals(v.Price, price), v => v.Volume);
-               }
-            }
-            return 0u;
+         if (_price_list.SafeTryGetAt(row, out var price)) {
+            var vapls = _volume_container.read_new_value();
+            if (vapls != null) {
+               return vapls.SelectFirstOrDefault(v => MathUtils.epsilon_equals(v.Price, price), v => v.Volume);
+            } 
          }
+         return 0u;
       }
 
 

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -4,6 +4,7 @@ using bts.api.orderman;
 using bts.collections;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.ui;
 using mbts.ui.controls;
 using System;
@@ -11,6 +12,7 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Runtime.CompilerServices;
+using System.Linq;
 using System.Windows.Input;
 using System.Windows.Media;
 using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
@@ -36,6 +38,7 @@ namespace mbts.spark {
       public OrderBookRowVM(OrderBookVM owner, OrderSecurityRootPair order_tuple, bool is_expanded, Action<bool> on_selection_changed) {
          _owner = owner;
          this.order_tuple = order_tuple;
+         is_visible = !is_child;
          _is_expanded = is_expanded;
          prospective_quantity = this.quantity;
          prospective_price = this.price;
@@ -59,6 +62,35 @@ namespace mbts.spark {
       public event Action<(OrderSecurityRootPair oldval, OrderSecurityRootPair newval)> updated;
       public event PropertyChangedEventHandler PropertyChanged;
 
+      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(
+               typeof(OrderBookRowVM), 
+               new string[] {
+                  nameof(security_text),
+                  nameof(symbol),
+                  nameof(time_created),
+                  nameof(last_change),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(order_sub_type),
+                  nameof(side),
+                  nameof(quantity),
+                  nameof(filled_quantity),
+                  nameof(price),
+                  nameof(avg_price),
+                  nameof(last_price),
+                  nameof(piq),
+                  nameof(status),
+                  // last message (could contains commas, so skipping)
+                  nameof(security_key),
+                  nameof(order_key),
+                  nameof(parent_order_key),
+               });
+
       // Core
       public OrderSecurityRootPair order_tuple { get; private set; }
       public OrderRecordImmutable order => order_tuple.order.order;
@@ -112,6 +144,7 @@ namespace mbts.spark {
       // Tree
       public bool is_parent { get => _is_parent; set => Set(ref _is_parent, value); }
       public bool is_child => order.has_parent();
+      public bool is_visible { get; set; }
       public bool is_expanded {
          get => _is_expanded;
          set {

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -51,7 +51,6 @@ namespace mbts.spark {
       readonly OrderParamsAggregator _selected_params_mon;
       readonly CommonValueMonitor<SecurityImmutable?> _selected_security_mon = new CommonValueMonitor<SecurityImmutable?>();
       readonly RightPanelModel _right_panel = new RightPanelModel();
-      readonly CustomStateSynchronizer _custom_state_sync;
       OrderBookFillsGridSettings _cached_fills_settings = OrderBookStartupData.DefaultFillsGrid;
       double
          _raw_right_panel_width = OrderBookStartupData.Default.right_panel_width,
@@ -125,8 +124,6 @@ namespace mbts.spark {
          _row_collection.CollectionChanged += _row_collection_CollectionChanged;
          _current_row = backing<OrderBookRowVM>(nameof(current_row));
          _columns.filter_changed += _on_filter_changed;
-         _custom_state_sync = new CustomStateSynchronizer(this);
-         // commands
          collapse_all_rows_command = new DelegateCommand(_row_collection.collapse_all);
          expand_all_rows_command = new DelegateCommand(_row_collection.expand_all);
          var modify_selected_orders_async_cmd = new AsyncCommand(_modify_selected_orders_async, () => _selected_params_mon.all_active && order_mod_vm.is_dirty);
@@ -191,7 +188,6 @@ namespace mbts.spark {
                security_plurality = p;
                RaisePropertyChanged(nameof(security_plurality));
             }),
-            _current_row.Subscribe(_ => _custom_state_sync.sync()),
             orders_source.OnDispatcher().Subscribe(t => {
                if (_selected_rows.Count == 0) {
                   _cached_orders = t.state;
@@ -211,8 +207,6 @@ namespace mbts.spark {
                      }
                   }
                }
-               _custom_state_sync.sync(true);
-               orders_changed?.Invoke();
             }),
             model.on_orders_can_attach.OnDispatcher().Subscribe(table => {
                _cached_attachable = new MappedDictionary<uint, KeyValuePair<uint, bool>, bool>(table.state, kv => kv.Value);
@@ -381,7 +375,7 @@ namespace mbts.spark {
       }
 
       public event Action<OrderRecordImmutable?> selected_order_changed;
-      public event Action begin_attaching, end_attaching, orders_changed;
+      public event Action begin_attaching, end_attaching;
       public event Action<(IEnumerable, bool)> request_selection;
 
       public ObservableAppendCollection<DIYGrid.ColumnBase> columns { get; }
@@ -401,7 +395,6 @@ namespace mbts.spark {
          }
       }
       public OrderRecordImmutable? current_order => current_row?.order;
-      public ObservableCollection<NamedValueVM> current_algo_custom_state { get; } = new ObservableCollection<NamedValueVM>();
       public AlgoDefinitionVM algo_vm => order_mod_vm.algo_vm;
       public OrderParamsVM order_mod_vm { get; }
       public FillsVMContainer fills_vm { get; } = new FillsVMContainer();
@@ -481,28 +474,6 @@ namespace mbts.spark {
       public IObservable<bool> subscribe_show_fills() => _show_fills.observe_current(true);
       public IObservable<bool> subscribe_show_ladder() => _show_ladder.observe_current(true);
       public IObservable<bool> subscribe_show_order_params() => _show_order_params.observe_current(true);
-      public IObservable<OrderSecurityAlgo?> subscribe_single_selected_order() =>
-         Observable.Create<OrderSecurityAlgo?>(o => {
-            OrderSecurityAlgo? last_fired = null;
-            orders_changed += notify;
-            _current_row.Changed += on_current_row_changed;
-            notify();
-            return Disposable.Create(dispose);
-            // local methods
-            void notify() {
-               var next = current_row == null ? (OrderSecurityAlgo?)null : current_row.order_tuple.order;
-               if (next != last_fired)
-                  o.OnNext(last_fired = next);
-            }
-
-            void on_current_row_changed(OrderBookRowVM oldval, OrderBookRowVM newval) =>
-               notify();
-
-            void dispose() {
-               orders_changed -= notify;
-               _current_row.Changed -= on_current_row_changed;
-            }
-         });
 
       public void row_selection_changed(SelectionChangedEventArgs e) {
          try {
@@ -532,7 +503,6 @@ namespace mbts.spark {
             _can_attach_selected_orders.refresh();
             _can_detach_selected_orders.refresh();
             selected_order_changed?.Invoke(current_order);
-            _custom_state_sync.sync();
          } catch (Exception ex) {
             // [John]I observed a strange null-reference exception in this callback when the launcher was shutting down (see https://bluetradesys.atlassian.net/browse/SPARK-2089)
             Log.Ui.error($"[{nameof(OrderBookVM)}] Error handling selection change: {ex}");
@@ -753,45 +723,6 @@ namespace mbts.spark {
          }
       }
 
-      class CustomStateSynchronizer {
-         readonly OrderBookVM _vm;
-         OrderBookRowVM _synced_row;
-
-         public CustomStateSynchronizer(OrderBookVM vm) =>
-            _vm = vm;
-
-         public void sync(bool force_refresh = false) {
-            var row = _vm.current_row != null &&
-               _vm.selected_rows.Select(r => r.order_tuple.root.order.Key).Distinct().TryGetSingle(out uint droot) &&
-               _vm.current_row.order_tuple.root.order.Key == droot ?
-                  _vm.current_row : null;
-            if (force_refresh || row != _synced_row) {
-               _synced_row = row;
-               _refresh();
-            }
-         }
-
-         void _refresh() {
-            _vm.current_algo_custom_state.Clear();
-            if (_synced_row == null)
-               return;
-            var dcs = _synced_row.order.dynamic_custom_state();
-            // while dcs is not expected to be null, best to play it safe and double check.
-            if (dcs.is_null_or_empty()) {
-               var descendent_rows = _vm._row_collection.get_descendent_rows(_synced_row.order_key);
-               if (_vm._row_collection.filter != null)
-                  descendent_rows = descendent_rows.Where(child => _vm._row_collection.filter(child.order_tuple));
-               foreach (var dr in descendent_rows) {
-                  dcs = dr.order.dynamic_custom_state();
-                  if (!dcs.is_null_or_empty())
-                     break;
-               }
-            }
-            foreach (var p in dcs.empty_if_null().OrderBy(kv => kv.Key))
-               _vm.current_algo_custom_state.Add(new NamedValueVM(p.Key, DynamicOrderHelper.fmt_param(p.Value)));
-         }
-      }
-
       public class OrderParamsVM : DisposableViewModelBase {
          readonly SelectorProperty<AccountTuple, AccountTuple?> _accounts;
          readonly OrderBookVM _owner;

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -1,17 +1,15 @@
-using bts;
-using bts.api;
+using bts.api;
 using bts.api.orderman;
 using bts.shell;
 using bts.shell.fills;
 using bts.shell.orderbook;
 using bts.spark;
+using bts.utils.csv;
 using bts.view;
 using mbts.persist;
-using mbts.spark.utilities;
 using mbts.ui;
 using mbts.ui.extensions;
 using mbts.ui.spark;
-using mbts.ui.utilities;
 using mbts.ui.utility;
 using mbts.utils;
 using System;
@@ -27,10 +25,7 @@ namespace mbts.spark {
       const string _window_type = "OrderBook", _default_title = "Order Book";
       readonly INotificationService _notification_service;
       readonly IWindowSettingsManager<OrderBookGridSettings> _settings_manager;
-      readonly bool _can_edit_specific_algo_params;
       IDisposable _attachment_exit_sub;
-      // initialized in thread_init
-      List<IDisposable> _thread_init_resources;
       OrderBookStartupData _state;
       OrdersModel _model;
       OrderBookVM _vm;
@@ -40,14 +35,13 @@ namespace mbts.spark {
       SparkCommand[] _commands;
 
       public OrderBookWindow(OrdersModel model, OrderBookWindowFactory factory, IBus bus, INotificationService notification_service,
-         OrderBookStartupData state, ImageSource icon, IWindowSettingsManager<OrderBookGridSettings> settings_manager, bool can_edit_specific_algo_params)
+         OrderBookStartupData state, ImageSource icon, IWindowSettingsManager<OrderBookGridSettings> settings_manager)
          : base(_default_title, bus, factory, SparkCommands.ShowOrders.icon) {
          _model = model;
          _factory = factory;
          _notification_service = notification_service;
          _state = state;
          _settings_manager = settings_manager;
-         _can_edit_specific_algo_params = can_edit_specific_algo_params;
          thread_id = config.Configuration.get<bts.config.MultiThreadedWindows>().order_book ? "OrderBook" : null;
       }
 
@@ -71,14 +65,12 @@ namespace mbts.spark {
       public string thread_id { get; }
 
       public override void Dispose() {
-         Disposable.dispose(_thread_init_resources);
+         _commands.OfType<IDisposable>().DisposeAll();
          Disposable.dispose(ref _attachment_exit_sub);
          Disposable.dispose(ref _vm);
       }
 
       public void thread_init() {
-         Log.assert(_thread_init_resources == null, $"{nameof(_thread_init_resources)} must be initialized in thread_init.");
-         _thread_init_resources = new List<IDisposable>();
          _menu_display_vm = new SparkMenuDisplayVM();
          if (_state != null)
             _menu_display_vm.load_from(_state.menu_display);
@@ -156,15 +148,6 @@ namespace mbts.spark {
             can_execute: () => _vm.selected_orders.Any(x => passes_recreate_search_criteria(x))
          );
          recreate_selected_orders_command.group = CommandGroups.Modify;
-         ICommand edit_algo_params_cmd;
-         if (_can_edit_specific_algo_params) {
-            _thread_init_resources.Add(CommandHelper.bind_param_source(
-               new DialogCommand<OrderSecurityAlgo?, Window>(mk_algo_params_dialog, has_algo_params),
-               _vm.subscribe_single_selected_order(),
-               out edit_algo_params_cmd, true));
-         } else {
-            edit_algo_params_cmd = DelegateCommand.empty;
-         }
 
          var attach_selected_orders_command = new SparkCommandCommand(
             "Attach To Selected",
@@ -212,22 +195,15 @@ namespace mbts.spark {
          show_grid_settings_command.group = CommandGroups.Auxiliary;
 
          _commands = new SparkCommand[] {
-            // modify commands
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(OrderBookRowVM.csv_ordered_properties), "OrderBook")),
             toggle_show_right_panel_command,
             attach_selected_orders_command,
             detach_selected_orders_command,
             recreate_selected_orders_command,
-            new SparkCommandCommand("Algo Params...", Icons.Algo, edit_algo_params_cmd) {
-               group = CommandGroups.Modify,
-               show_in_toolbar = false,
-               show_in_context_menu = _can_edit_specific_algo_params
-            },
-            // cancel commands
             cancel_selected_orders_command,
             cancel_buy_orders_command,
             cancel_sell_orders_command,
             cancel_all_orders_command,
-
             toggle_collapse_all_command,
             toggle_expand_all_command,
             show_grid_settings_command,
@@ -236,12 +212,9 @@ namespace mbts.spark {
             hidden_cmd(new SparkCommandCommand(OrderBookView.CommandDescriptions.RevertOrders, null, _vm.revert_selected_orders_cmd, OrderBookView.KeyboardShortcuts.RevertOrders)),
             _menu_display_vm.spark_cmd
          };
-         _thread_init_resources.AddRange(_commands.OfType<IDisposable>());
-         _thread_init_resources.Add(_vm.as_demuxer()
-            .subscribe<bool>(nameof(OrderBookVM.show_right_panel))
-            .Subscribe(_ => toggle_show_right_panel_command.title = get_toggle_show_right_panel_cmd_title()));
+         _vm.as_demuxer().subscribe<bool>(nameof(OrderBookVM.show_right_panel))
+            .Subscribe(_ => toggle_show_right_panel_command.title = get_toggle_show_right_panel_cmd_title());
          _vm.selected_order_changed += vm_selected_order_changed;
-         _thread_init_resources.Add(Disposable.Create(() => _vm.selected_order_changed -= vm_selected_order_changed));
          // local methods
          string get_toggle_show_right_panel_cmd_title() =>
             _vm.show_right_panel ? OrderBookView.CommandDescriptions.HideRightPanel : OrderBookView.CommandDescriptions.ShowRightPanel;
@@ -263,36 +236,6 @@ namespace mbts.spark {
             attach_selected_orders_command.raiseCanExecuteChanged();
             detach_selected_orders_command.raiseCanExecuteChanged();
          }
-
-         bool has_algo_params(OrderSecurityAlgo? order) =>
-            order.HasValue && !OrderDataHelper.get_metadata(order.Value.algo.value.CustomCase).NoProps;
-
-         Window mk_algo_params_dialog(OrderSecurityAlgo? order) {
-            if (!has_algo_params(order))
-               return null;
-            var algo_def = AlgoDefinitionVM.create(_model, order.Value.algo, editing_permissions: AlgoEditingPermissions.AllowEditValues);
-            var algo_params_vm = new OrderBookAlgoParamsVM(algo_def, order.Value.order.is_non_terminal());
-            var dialog = new OrderBookAlgoParamsDialog {
-               Content = algo_params_vm
-            };
-            ui.WindowStyle.SetTitleContent(dialog, algo_def);
-            dialog.Closed += (s, e) => {
-               Disposable.dispose(ref algo_params_vm);
-               dialog = null;
-            };
-            algo_params_vm.RequestCommit += commit;
-            algo_params_vm.RequestClose += (s, e) => close_dialog();
-            return dialog;
-
-            void close_dialog() {
-               dialog?.Close();
-               dialog = null;
-            }
-            void commit(CustomOrderDefinitionImmutable value) {
-               _model.update_order(order.Value.order, new_algo: value).ignore();
-               close_dialog();
-            }
-         }
       }
 
       protected override void OnParentViewCreated(SparkWindowView parent_view) {

src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs

@@ -1,7 +1,6 @@
 using bts.api;
 using bts.shell.orderbook;
 using bts.spark;
-using mbts.config;
 using mbts.persist;
 using mbts.spark;
 using mbts.ui;
@@ -16,11 +15,8 @@ namespace bts.shell {
       readonly INotificationServiceDataCache _notification_service_data_cache;
       readonly OrdersModel _model;
       readonly UserSettingsHub _settings_hub;
-      readonly OrderBookConfig _config;
 
       public OrderBookWindowFactory(IShellModule owner, IBus bus, INotificationService notification_service, INotificationServiceDataCache notification_service_data_cache, OrdersModel model, UserSettingsHub settings_hub) : base(owner, "SparkOrderBookWindow") {
-         var raw_config = Configuration.get<bts.config.OrderBookOptions>();
-         _config = new OrderBookConfig(raw_config.can_edit_specific_algo_params);
          _bus = bus;
          _notification_service = notification_service;
          _notification_service_data_cache = notification_service_data_cache;
@@ -33,8 +29,7 @@ namespace bts.shell {
          var settings_manager = new WindowSettingsManager<OrderBookGridSettings>(_settings_hub, state.grid_settings_wrapper(),
             us => us.order_book_grid ?? OrderBookStartupData.DefaultGrid,
             (us, s) => us.with(order_book_grid: s));
-         var w = new OrderBookWindow(_model, this, _bus, _notification_service, state,
-            SparkCommands.ShowOrders.icon, settings_manager.as_non_shared(), _config.can_edit_specific_algo_params);
+         var w = new OrderBookWindow(_model, this, _bus, _notification_service, state, SparkCommands.ShowOrders.icon, settings_manager.as_non_shared());
          return Task.FromResult<IShellWindow>(w);
       }
 
@@ -42,12 +37,5 @@ namespace bts.shell {
          var w = await load(startup_data);
          _bus.post(SparkTags.ShowWindow, w);
       }
-
-      readonly struct OrderBookConfig {
-         public readonly bool can_edit_specific_algo_params;
-
-         public OrderBookConfig(bool can_edit_specific_algo_params) =>
-            this.can_edit_specific_algo_params = can_edit_specific_algo_params;
-      }
    }
 }

src/client/spark/Modules/Trading/OrderBook/View/OrderBookAlgoParamsDialog.xaml

@@ -1,40 +0,0 @@
-<Window
-  x:Class="bts.shell.orderbook.OrderBookAlgoParamsDialog"
-  x:ClassModifier="internal"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:sp="clr-namespace:mbts.spark"
-  xmlns:ob="clr-namespace:bts.shell.orderbook"
-  xmlns:usp="clr-namespace:mbts.ui.spark"
-  xmlns:u="clr-namespace:mbts.ui"
-  xmlns:c="clr-namespace:mbts.ui.controls"
-  u:WindowStyle.SizeToContent="WidthAndHeight"
-  Margin="0"
-  Padding="0"
-  Style="{StaticResource BTSWindowTitleTemplate}"
->
-  <u:WindowStyle.TitleContentTemplate>
-    <DataTemplate DataType="{x:Type sp:AlgoDefinitionVM}">
-      <StackPanel HorizontalAlignment="Left" Orientation="Horizontal">
-        <usp:OrderTypeItem Margin="4,0" DisplayName="{Binding name, StringFormat='{}{0} Params'}" BadgeFill="{Binding badge_fill}"/>
-      </StackPanel>
-    </DataTemplate>
-  </u:WindowStyle.TitleContentTemplate>
-  <Window.ContentTemplate>
-    <DataTemplate DataType="{x:Type ob:OrderBookAlgoParamsVM}">
-      <Grid>
-        <Grid.RowDefinitions>
-          <RowDefinition Height="*"/>
-          <RowDefinition Height="Auto"/>
-        </Grid.RowDefinitions>
-        <usp:AlgoEditorGridView
-          Width="390"
-          Margin="5"
-          AlgoParamRows="{Binding param_rows}"
-          CanUserChangeAlgoRoot="False"
-        />
-        <c:DialogSkirt Grid.Row="1" RightItems="{Binding dialog_skirt}"/>
-      </Grid>
-    </DataTemplate>
-  </Window.ContentTemplate>
-</Window>

src/client/spark/Modules/Trading/OrderBook/View/OrderBookAlgoParamsDialog.xaml.cs

@@ -1,49 +0,0 @@
-using bts.api.orderman;
-using bts.collections;
-using mbts.spark;
-using mbts.ui;
-using mbts.ui.controls;
-using mbts.ui.shell;
-using System;
-using System.Collections.Generic;
-using System.Windows;
-
-namespace bts.shell.orderbook {
-   /// <summary>
-   /// Interaction logic for OrderBookAlgoParamsDialog.xaml
-   /// </summary>
-   partial class OrderBookAlgoParamsDialog : Window {
-      public OrderBookAlgoParamsDialog() =>
-         InitializeComponent();
-   }
-
-   class OrderBookAlgoParamsVM : DisposableViewModelBase, IRequestViewClose {
-      readonly ObservableAggregateCollection<AlgoParamRowVM> _param_rows = new ObservableAggregateCollection<AlgoParamRowVM>();
-      readonly DialogSkirtItem _commit, _cancel;
-
-      public OrderBookAlgoParamsVM(AlgoDefinitionVM algo_definition, bool can_user_edit) {
-         this.algo_definition = algo_definition;
-         _dispose = AlgoParamRowsHelper.connect(algo_definition, _param_rows, false);
-         _commit = new DialogSkirtItem("OK", new DelegateCommand(commit, () => can_user_edit)) { is_default = true };
-         _cancel = new DialogSkirtItem("Cancel", new DelegateCommand(close)) { is_cancel = true };
-         // local methods
-         void commit() {
-            RequestCommit?.Invoke(new CustomOrderDefinitionImmutable(algo_definition.algo_root.created_assigned()));
-            close();
-         }
-         void close() => RequestClose?.Invoke(this, EventArgs.Empty);
-      }
-
-      public event EventHandler RequestClose;
-      public event Action<CustomOrderDefinitionImmutable> RequestCommit;
-
-      public AlgoDefinitionVM algo_definition { get; }
-      public IReadOnlyObservableCollection<AlgoParamRowVM> param_rows => _param_rows;
-      public IEnumerable<DialogSkirtItem> dialog_skirt {
-         get {
-            yield return _commit;
-            yield return _cancel;
-         }
-      }
-   }
-}

src/client/spark/Modules/Trading/OrderBook/View/OrderBookView.xaml

@@ -484,7 +484,6 @@
           <Border Padding="12,12,12,6">
             <Grid>
               <Grid.RowDefinitions>
-                <RowDefinition Height="Auto"/>
                 <RowDefinition Height="Auto"/>
                 <RowDefinition Height="*"/>
               </Grid.RowDefinitions>
@@ -496,45 +495,7 @@
                 AlgoParamRows="{Binding param_rows}"
                 IsReadOnly="{Binding DataContext.can_user_modify_selected_orders, ElementName=Root, Converter={StaticResource NotConverter}}"
               />
-              <c:ExpanderToggleButton
-                x:Name="algo_state_toggle"
-                Grid.Row="1"
-                BorderThickness="0"
-                Content="Algo State"
-                HorizontalContentAlignment="Left"
-                Margin="0,4,0,0"
-                VerticalAlignment="Top"
-                Visibility="{Binding current_algo_custom_state.Count, Converter={StaticResource IntNotZeroVisibilityConverter}}"
-              />
-              <c:DIYGrid Grid.Row="1" Rows="{Binding current_algo_custom_state}" Margin="0,28,0,4">
-                <c:DIYGrid.Visibility>
-                  <MultiBinding>
-                    <MultiBinding.Converter>
-                      <cv:MultiValueConverterPipe>
-                        <cv:BooleanAndConverter/>
-                        <cv:BooleanToVisibilityConverter/>
-                      </cv:MultiValueConverterPipe>
-                    </MultiBinding.Converter>
-                    <Binding Path="current_algo_custom_state.Count" Mode="OneWay" Converter="{StaticResource IntNotZeroConverter}"/>
-                    <Binding Path="IsChecked" ElementName="algo_state_toggle"/>
-                  </MultiBinding>
-                </c:DIYGrid.Visibility>
-                <c:DIYGrid.Columns>
-                  <c:DIYGridColumnCollection>
-                    <c:DIYGridColumn id="Name" Header="Param" Width="176"/>
-                    <c:DIYGridColumn id="Value" Header="Value" Width="176"/>
-                  </c:DIYGridColumnCollection>
-                </c:DIYGrid.Columns>
-                <c:DIYGrid.RowTemplate>
-                  <DataTemplate>
-                    <Grid c:DIYGrid.SyncWithColumnGeometry="True">
-                      <c:TextCell c:DIYGrid.ColumnKey="Name" Text="{Binding name}"/>
-                      <c:TextCell c:DIYGrid.ColumnKey="Value" Text="{Binding value}"/>
-                    </Grid>
-                  </DataTemplate>
-                </c:DIYGrid.RowTemplate>
-              </c:DIYGrid>
-              <StackPanel Grid.Row="2" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,8,0,0" Orientation="Horizontal">
+              <StackPanel Grid.Row="1" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,8,0,0" Orientation="Horizontal">
                 <Button x:Name="modify_orders_button" Command="{Binding modify_selected_orders_cmd}">
                   <Button.Margin>
                     <Thickness Right="{StaticResource DialogSkirtButtonPadding.Right}"/>

src/client/spark/Orders/OrderDataHelper.cs

@@ -653,54 +653,25 @@ namespace mbts.spark.utilities {
             return def;
          }
       }
+      
+      static CustomOrderDefinition.CustomOneofCase[] algo_config(
+         Func<bts.config.AlgoPermissions, List<string>> selector)
+      {
+         var conf = mbts.config.Configuration.get<bts.config.AlgoPermissions>();
+         return selector(conf).Select(x => 
+         {
+            return (CustomOrderDefinition.CustomOneofCase)System.Enum.Parse(typeof(CustomOrderDefinition.CustomOneofCase), x, true);
+         }).ToArray();
+      }
 
       public static readonly CustomOrderDefinition.CustomOneofCase[]
 
          // exchange_order_types are the types which can be submitted on an exchange security
-         exchange_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         spreader_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Spreader,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover, 
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         attachable_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         outright_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-         };
+         exchange_order_types = algo_config(x => x.exchange_order_types),
+         spreader_order_types = algo_config(x => x.spreader_order_types),
+         attachable_order_types = algo_config(x => x.attachable_order_types),
+         outright_order_types = algo_config(x => x.outright_order_types);
+         
       public static readonly CustomOrderDefinition.CustomOneofCase[] all_order_types = exchange_order_types.Union(spreader_order_types).ToArray();
 
       public static void copy_values(CustomOrderDefinitionImmutable source, ref CustomOrderDefinition target) =>

src/client/spark/Orders/OrderTypeVMs/DynamicOrderHelper.cs

@@ -23,32 +23,6 @@ namespace mbts.spark {
 
       public IEnumerable<string> dynamic_types => _dynamic_types.DynamicOrderDesc.Select(d => d.DynamicOrderType);
 
-      public static string fmt_param(DynamicOrderParam dyn_param) {
-         if (dyn_param == null)
-            return null;
-         switch (dyn_param.ValueCase) {
-            case DynamicOrderParam.ValueOneofCase.AsArray:
-               return string.Join(", ", dyn_param.AsArray.Element.Select(fmt_param));
-            case DynamicOrderParam.ValueOneofCase.AsBool:
-               return dyn_param.AsBool.ToString();
-            case DynamicOrderParam.ValueOneofCase.AsDouble:
-               return dyn_param.AsDouble.ToString();
-            case DynamicOrderParam.ValueOneofCase.AsEnumerated:
-               return dyn_param.AsEnumerated.EnumValue;
-            case DynamicOrderParam.ValueOneofCase.AsInt32:
-               return dyn_param.AsInt32.ToString();
-            case DynamicOrderParam.ValueOneofCase.AsQuantity:
-               return dyn_param.AsQuantity.ToString();
-            case DynamicOrderParam.ValueOneofCase.AsString:
-               return dyn_param.AsString;
-            default:
-               // implement additional cases as needed
-               // (as of 4/17/20, this method is only used to format
-               // dynamic orders' CustomState params in the orderbook)
-               return $"({dyn_param.ValueCase})";
-         }
-      }
-
       public void add_dynamic_order_defs(List<CustomOrderDefinitionImmutable> defs, Func<CustomOrderDefinition.CustomOneofCase, CustomOrderDefinition> factory) {
          foreach (var d in _dynamic_types.DynamicOrderDesc) {
             var dorder = factory(CustomOrderDefinition.CustomOneofCase.Dynamic);

src/client/spark/Orders/OrderTypeVMs/GenericOrderHelper.cs

@@ -33,7 +33,7 @@ namespace mbts.spark
          {
             return;
          } 
-         var ev = fd.EnumType.Values.FirstOrDefault(x => x.Name == value)?.Number;
+         var ev = fd.EnumType.Values.First(x => x.Name == value)?.Number;
          if(ev.HasValue)
             fd.Accessor.SetValue(message, ev.Value);
       }

src/client/spark/Orders/OrderTypeVMs/GenericOrderParamVM.cs

@@ -28,15 +28,17 @@ namespace mbts.spark
       where TMutable : class, IMessage, new() 
    {
       TMutable _message;
+      readonly protected Func<IMessage, IMessage> _message_override;
 
-      protected GenericOrderParamVM(FieldDescriptor descriptor)
+      protected GenericOrderParamVM(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null)
       {
          _message = new TMutable();
+         _message_override = message_override;
          this.descriptor = descriptor;
       }
 
       public FieldDescriptor descriptor { get; set; }
-      public Google.Protobuf.IMessage message 
+      public virtual Google.Protobuf.IMessage message 
       {
          get => _message;
          set 
@@ -53,11 +55,12 @@ namespace mbts.spark
       where TMutable : class, IMessage, new()
    {
       readonly Func<TModel, TView> _convert;
-      readonly Func<TView, TModel> _convert_back;
+      readonly Func<TView, TModel> _convert_back;      
       TView _value;
 
-      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, Func<TView, TModel> convert_back) : 
-         base(field_descriptor) 
+      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, 
+                                 Func<TView, TModel> convert_back, Func<IMessage, IMessage> message_override = null) : 
+         base(field_descriptor, message_override) 
       {
          _convert = convert;
          _convert_back = convert_back;
@@ -68,7 +71,8 @@ namespace mbts.spark
          get => _value;
          set 
          {
-            this.descriptor.Accessor.SetValue(this.message, _convert_back(_value = value));
+            this.descriptor.Accessor.SetValue( _message_override == null ? this.message : _message_override(this.message), 
+                                              _convert_back(_value = value));
             RaisePropertyChanged();
          }
       }
@@ -84,7 +88,7 @@ namespace mbts.spark
 
       protected override void on_param_changed() 
       {
-         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(this.message)));
+         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(_message_override == null ? this.message : _message_override(this.message))));
          RaisePropertyChanged(nameof(value));
       }
    }
@@ -92,8 +96,8 @@ namespace mbts.spark
    class GenericOrderEnumeratedVM<TMutable> : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<string>
       where TMutable : class, IMessage, new()   
    {
-      public GenericOrderEnumeratedVM(FieldDescriptor descriptor)
-         : base(descriptor)
+      public GenericOrderEnumeratedVM(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null)
+         : base(descriptor, message_override)
       {
          elements = new ObservableCollection<string>(_get_elements(descriptor));
       }
@@ -102,11 +106,13 @@ namespace mbts.spark
 
       public string selected_element 
       {
-         get => GenericOrderHelper.get_enumerated_value(message, descriptor);
+         get => GenericOrderHelper.get_enumerated_value(_message_override == null ? this.message : _message_override(this.message),
+                                                        descriptor);
          set 
          {
             var orig = selected_element;
-            GenericOrderHelper.set_enumerated_value(message, descriptor, value);
+            GenericOrderHelper.set_enumerated_value(_message_override == null ? this.message : _message_override(this.message), 
+                                                    descriptor, value);
             if (selected_element != orig)
                RaisePropertyChanged();
          }
@@ -148,22 +154,23 @@ namespace mbts.spark
    class GenericOrderQuantityVM<TMutable> : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<QuantityMeasureImmutable?>
       where TMutable : class, IMessage, new()
    {
-      public GenericOrderQuantityVM(FieldDescriptor descriptor)
-         : base(descriptor) { }
+      public GenericOrderQuantityVM(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null)
+         : base(descriptor, message_override) { }
 
       QuantityMeasureImmutable? IGenericOrderParamVM<QuantityMeasureImmutable?>.value { get => quantity; set => quantity= value; }
       public QuantityMeasureImmutable? quantity 
       {
-         get => GenericOrderHelper.get_quantity_value(message, descriptor);
+         get => GenericOrderHelper.get_quantity_value(_message_override == null ? this.message : _message_override(this.message), 
+                                                      descriptor);
          set 
          {
             var orig = quantity;
-            GenericOrderHelper.set_quantity_value(message, descriptor, value);
+            GenericOrderHelper.set_quantity_value(_message_override == null ? this.message : _message_override(this.message), 
+                                                  descriptor, value);
             if (quantity != orig)
                RaisePropertyChanged();
          }
       }      
-
       public IObservable<QuantityMeasureImmutable?> subscribe_value() 
       {
          return Observable.Create<QuantityMeasureImmutable?>(o =>
@@ -174,10 +181,9 @@ namespace mbts.spark
                o.OnNext(quantity);
          });
       }
-
       protected override void on_param_changed()
       {
          RaisePropertyChanged(nameof(quantity));
       }
-   }   
+   }
 }

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -2,6 +2,7 @@
 using bts.api;
 using bts.api.orderman;
 using bts.collections;
+using bts.spark;
 using Google.Protobuf;
 using Google.Protobuf.Reflection;
 using mbts.collections;
@@ -14,19 +15,21 @@ using System.Collections.ObjectModel;
 using System.ComponentModel;
 using System.Linq;
 using System.Reactive.Linq;
-using System.Reflection;
 using AlgoType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
 
 namespace mbts.spark {
-   static partial class OrderTypeVMFactory {
-      internal class GenericOrderFactory {
-         static public IAlgoVM create_vm(AlgoKey key, AlgoType algo_type, string display_name, AlgoEditingPermissions editing_permissions) {
+   static partial class OrderTypeVMFactory 
+   {
+      internal class GenericOrderFactory
+      {
+         static public IAlgoVM create_vm(AlgoKey key, AlgoType algo_type, string display_name, AlgoEditingPermissions editing_permissions)
+         {
             var prop = typeof(CustomOrderDefinition).GetProperty(algo_type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
+               System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public);
             var rt = prop.PropertyType;
 
             var propi = typeof(CustomOrderDefinitionImmutable).GetProperty(algo_type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
+               System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public);
             var rti = propi.PropertyType;
 
             var vmt = typeof(GenericOrderVM<,>);
@@ -35,33 +38,38 @@ namespace mbts.spark {
             object vm = Activator.CreateInstance(makevm, key, algo_type, display_name, editing_permissions);
             return (IAlgoVM)vm;
          }
-         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
+         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type)
+         {
             var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
+               System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public);
             var rt = prop.PropertyType;
             var rtt = Activator.CreateInstance(rt);
-
+            
             var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
-            foreach (var cid in ci)
+            foreach(var cid in ci)
                cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
             prop.SetValue(custom, rtt);
          }
-      }
-
+      }      
+      
       class GenericOrderVM<TMutable, TImmutable> : OrderTypeVMBase<TMutable, TImmutable>
-         where TMutable : class, IMessage, new() where TImmutable : struct, IImmutableMessage {
+         where TMutable : class, IMessage, new() where TImmutable : struct, IImmutableMessage
+      {         
          readonly ObservableKeyedCollection<GenericOrderParamVM<TMutable>, string> _parameters =
             new ObservableKeyedCollection<GenericOrderParamVM<TMutable>, string>(vm => vm.descriptor.Name);
          readonly AlgoType _algo_type;
 
          public GenericOrderVM(AlgoKey key, AlgoType algo_type, string display_name, AlgoEditingPermissions editing_permissions)
-            : base(key, display_name, editing_permissions) {
+            : base(key, display_name, editing_permissions) 
+         {
             _algo_type = algo_type;
             _parameters = new ObservableKeyedCollection<GenericOrderParamVM<TMutable>, string>(vm => vm.descriptor.Name);
             _resources.AddRange(
-               ObservableAutoWrapper.connect_item_hooks(_parameters, p => {
+               ObservableAutoWrapper.connect_item_hooks(_parameters, p =>
+               {
                   p.PropertyChanged += on_param_prop_changed;
-                  return Disposable.Create(() => {
+                  return Disposable.Create(() => 
+                  {
                      p.PropertyChanged -= on_param_prop_changed;
                      remove_property(p.descriptor.Name);
                      p.Dispose();
@@ -72,113 +80,128 @@ namespace mbts.spark {
          }
 
          public string algo_type => _algo_type.ToString();
-         public override string abbreviation =>
+         public override string abbreviation => 
             model == null ? display_name : model.factory.abbreviate_algo(new OneOf<CustomOrderDefinition.CustomOneofCase, string>(_algo_type));
          public override AlgoDisplayGroup group => AlgoDisplayGroup.API;
          public ObservableCollection<GenericOrderParamVM<TMutable>> parameters => _parameters;
 
-         protected override void on_model_internal() =>
+         protected override void on_model_internal() 
+         {
             _sync_generic_params();
+         }
 
-         void _sync_generic_params() {
-            if (model != null) {
-               MessageDescriptor desc = new TMutable().Descriptor;
+         void _sync_generic_params()
+         {
+            if (model != null)
+            {               
+               Google.Protobuf.Reflection.MessageDescriptor desc = new TMutable().Descriptor;               
                _parameters.SyncWithDictionary(
-                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), _mk_param_vm);
+                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), x => _mk_param_vm(x));
+               add_message_params(desc);
             }
          }
-
-         static bool _is_value_param_desc(FieldDescriptor desc) =>
-            (desc.FieldType != FieldType.Message ||
+         void add_message_params(Google.Protobuf.Reflection.MessageDescriptor desc)
+         {
+            var msgs = desc.Fields.InDeclarationOrder().Where(x => !_is_value_param_desc(x) && 
+                                                              x.MessageType.ClrType != typeof(CustomOrderDefinition) );
+            foreach(var message_field in msgs)
+            {
+               foreach (var fd in message_field.MessageType.Fields.InDeclarationOrder())
+               {
+                  if(_is_value_param_desc(fd)) //one level deep
+                  {
+                     var mk = _mk_param_vm(fd, 
+                     outside_object => 
+                     {                        
+                        var msgfield = message_field.Accessor.GetValue(outside_object);
+                        if(msgfield == null)
+                        {
+                           message_field.Accessor.SetValue(outside_object, Activator.CreateInstance(message_field.MessageType.ClrType));
+                           msgfield = message_field.Accessor.GetValue(outside_object);
+                        }
+                        return (IMessage)msgfield;
+                     });
+                     if(mk == null) 
+                        continue;
+                        
+                     if(!_parameters.Add(mk))
+                        _parameters.Add(message_field.Name + "_" + mk.descriptor.Name, mk);                     
+                  }
+               }
+            }            
+         }
+         
+         static bool _is_value_param_desc(FieldDescriptor desc)
+         {
+            return (desc.FieldType != Google.Protobuf.Reflection.FieldType.Message || 
                _is_quantity_measure_desc(desc));
-
-         static bool _is_quantity_measure_desc(FieldDescriptor desc) =>
-            desc.FieldType == FieldType.Message &&
-               desc.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName;
-
-         protected override IEnumerable<ChildAlgoData> get_selector_data() =>
-            OrderDataHelper.get_child_algo_data(_algo_type);
-
-         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor) {
-            ExceptionAssert.Argument.NotNull(descriptor, nameof(descriptor));
-            if (descriptor.FieldType == FieldType.Message &&
-                descriptor.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName) {
-               var qm = new GenericOrderQuantityVM<TMutable>(descriptor);
-               return attached(qm, GenericOrderHelper.get_quantity_value, GenericOrderHelper.set_quantity_value);
-            } else if (descriptor.FieldType == FieldType.Enum) {
-               return EnumParamFactory.create(this, descriptor);
-            } else if (descriptor.FieldType == FieldType.Bool) {
-               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault());
-            } else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float) {
-               return mk<double, double>(descriptor, v => v, v => v, double.NaN);
-            } else if (descriptor.FieldType == FieldType.Int32) {
-               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault());
-            } else if (descriptor.FieldType == FieldType.UInt32) {
-               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault());
-            } else if (descriptor.FieldType == FieldType.String) {
-               return mk<string, string>(descriptor, v => v, v => v);
-            } else return null;
          }
 
-         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(
-            FieldDescriptor fdesc,
-            Func<TModel, TView> convert,
-            Func<TView, TModel> convert_back,
-            TView default_value = default) {
-            //
-            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back);
-            return attached(pvm, get_from_param, assign_to_param, default_value);
+         static bool _is_quantity_measure_desc(FieldDescriptor desc)
+         {
+            return desc.FieldType == Google.Protobuf.Reflection.FieldType.Message &&
+               desc.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName;
+         }
+
+         protected override IEnumerable<ChildAlgoData> get_selector_data() 
+         {
+            var v = OrderDataHelper.get_child_algo_data(_algo_type);
+            return v;
+         }
+
+         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null) 
+         {
+            ExceptionAssert.Argument.NotNull(descriptor, nameof(descriptor));
+
+            if (descriptor.FieldType == FieldType.Message && 
+                descriptor.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName)
+            {
+               var qm = new GenericOrderQuantityVM<TMutable>(descriptor, message_override);
+               return attached(qm, GenericOrderHelper.get_quantity_value, GenericOrderHelper.set_quantity_value, message_override);
+            }
+            else if(descriptor.FieldType == FieldType.Enum)
+            {
+               var pvm = new GenericOrderEnumeratedVM<TMutable>(descriptor, message_override);
+               attached(pvm, GenericOrderHelper.get_enumerated_value, GenericOrderHelper.set_enumerated_value, message_override);
+               return pvm;
+            }
+            else if (descriptor.FieldType == FieldType.Bool)
+               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
+            else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float)
+               return mk<double, double>(descriptor, v => v, v => v, message_override, double.NaN);
+            else if (descriptor.FieldType == FieldType.Int32)
+               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
+            else if (descriptor.FieldType == FieldType.UInt32)
+               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
+            else if (descriptor.FieldType == FieldType.String)
+               return mk<string, string>(descriptor, v => v, v => v, message_override);
+            else
+               return null;
+         }
+         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, 
+                                                                        Func<IMessage, IMessage> message_override = null, TView default_value = default)
+         {
+            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back, message_override);
+            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param,
+                                                                                 message_override, default_value);
+
             TView get_from_param(IMessage param, FieldDescriptor fd) => convert(ConvertUtils.try_convert<TModel>(fdesc.Accessor.GetValue(param)));
             void assign_to_param(IMessage param, FieldDescriptor fd, TView value) => fdesc.Accessor.SetValue(param, convert_back(value));
          }
 
-         TParam attached<T, TParam>(
-            TParam pvm,
-            Func<IMessage, FieldDescriptor, T>
-            selector, Action<IMessage, FieldDescriptor, T> assign_to_param,
-            T default_value = default)
-            where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T> {
-            //
-            add_property(
+         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, 
+                                    Func<IMessage, IMessage> message_override = null, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
+         {
+            add_property<T>(
                pvm.descriptor.Name,
-               mdef => mdef != null ? selector(mdef, pvm.descriptor) : default_value,
-               (mdef, v) => assign_to_param(mdef, pvm.descriptor, v),
+               mdef => mdef != null ? selector( message_override == null ?  mdef : message_override(mdef), pvm.descriptor) : default_value,
+               (mdef, v) => assign_to_param(message_override == null ? mdef : message_override(mdef), pvm.descriptor, v),
                () => pvm.value,
                v => pvm.value = v,
                pvm.subscribe_value(),
                default_value);
             return pvm;
-         }
-
-         static class EnumParamFactory {
-            [ThreadStatic]
-            static Dictionary<Type, IFactory> _factories_backing;
-            static Dictionary<Type, IFactory> _factories =>
-               _factories_backing ?? (_factories_backing = new Dictionary<Type, IFactory>());
-
-            public static GenericOrderParamVM<TMutable> create(GenericOrderVM<TMutable, TImmutable> owner, FieldDescriptor descriptor) {
-               if (descriptor.FieldType != FieldType.Enum)
-                  throw ExceptionBuilder.Argument.unexpected_case(descriptor.FieldType, $"{nameof(descriptor)}.{nameof(FieldDescriptor.FieldType)}");
-               if (!_factories.TryGetValue(descriptor.EnumType.ClrType, out var f)) {
-                  // Impl<T> is a nested type, so parent class's generic type args must be included.
-                  _factories[descriptor.EnumType.ClrType] = f = (IFactory)Activator.CreateInstance(typeof(Impl<>)
-                     .MakeGenericType(typeof(TMutable), typeof(TImmutable), descriptor.EnumType.ClrType));
-               }
-               return f.create(owner, descriptor);
-            }
-
-            interface IFactory {
-               GenericOrderParamVM<TMutable> create(GenericOrderVM<TMutable, TImmutable> owner, FieldDescriptor descriptor);
-            }
-
-            class Impl<T> : IFactory where T : struct {
-               static readonly Func<T, T?> _convert = v => v;
-               static readonly Func<T?, T> _convert_back = v => v ?? default;
-
-               public GenericOrderParamVM<TMutable> create(GenericOrderVM<TMutable, TImmutable> owner, FieldDescriptor descriptor) =>
-                  owner.mk(descriptor, _convert, _convert_back, default);
-            }
-         }
+         }         
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/OffsetOrderVM.cs

@@ -4,14 +4,14 @@ using OffsetDefi = bts.api.orderman.OffsetOrderDefinitionImmutable;
 namespace mbts.spark {
    static partial class OrderTypeVMFactory {
       class OffsetOrderVM : OrderTypeVMBase<OffsetDef, OffsetDefi> {
-         readonly Property<uint?> _tick_offset_on_deploy;
+         readonly Property<int?> _tick_offset_on_deploy;
 
          public OffsetOrderVM(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions)
             : base(key, display_name, editing_permissions) {
-            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0u);
+            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0);
          }
 
-         public uint? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
+         public int? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
       }
    }
 }

src/client/spark/Orders/OrderUtils.cs

@@ -63,11 +63,6 @@ namespace mbts.spark {
       public static string last_msg(this OrderRecordImmutable order) =>
          order.HasState ? order.State.LastMsg : string.Empty;
 
-      public static IReadOnlyDictionary<string, DynamicOrderParam> dynamic_custom_state(this OrderRecordImmutable order) =>
-         order.HasState && order.State.HasDynamic ?
-            (order.State.Dynamic.CustomState ?? DictionaryHelper.empty<string, DynamicOrderParam>()) :
-            DictionaryHelper.empty<string, DynamicOrderParam>();
-
       public static DateTime created(this OrderRecordImmutable order) =>
          order.HasState && order.State.HasCreated ? order.State.Created.ToDateTime().ToLocalTime() : DateTime.MinValue;
 

src/client/spark/Spark.csproj

@@ -9,10 +9,8 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Configurations>Debug;Release</Configurations>
     <Platforms>x64</Platforms>
-    <RuntimeIdentifiers>win7-x64</RuntimeIdentifiers>
     <PlatformTarget>AnyCPU</PlatformTarget>
     <LanguageTargets>$(MSBuildExtensionsPath)\$(VisualStudioVersion)\Bin\Microsoft.CSharp.targets</LanguageTargets>
-    <LanguageTargets Condition="'$(VisualStudioVersion)'=='16.0'">$(MSBuildToolsPath)\Microsoft.CSharp.targets</LanguageTargets>
     <ApplicationIcon>logo.ico</ApplicationIcon>
     <OutputType>WinExe</OutputType>
     <StartupObject>mbts.ui.shell.App</StartupObject>
@@ -37,7 +35,7 @@
     <PackageReference Include="Google.Protobuf" Version="3.6.1" />
     <PackageReference Include="IronPython" Version="2.7.7" />
     <PackageReference Include="NAudio" Version="1.8.4" />
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
     <PackageReference Include="NUnit" Version="3.10.1" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.8.0" />

src/client/spark/core/GlobalExtensions/DoubleHelper.cs

@@ -63,20 +63,5 @@ namespace mbts {
 
       public static double? nan_nullable(double value) =>
          double.IsNaN(value) ? (double?)null : value;
-
-      public static int? to_optional_int32(double value) {
-         if (double.IsNaN(value) || double.IsInfinity(value))
-            return null;
-         if (value > 0) {
-            value = Math.Floor(value);
-            if (value <= int.MaxValue)
-               return (int)value;
-         } else if (value < 0) {
-            value = Math.Ceiling(value);
-            if (value >= int.MinValue)
-               return (int)value;
-         }
-         return null;
-      }
    }
 }

src/client/spark/view/Controls/DIYGrid/Columns/DIYGridColumn.cs

@@ -1,9 +0,0 @@
-namespace mbts.ui.controls {
-   class DIYGridColumn : DIYGrid.ColumnBase<string> {
-      public DIYGridColumn()
-         : base() {
-         is_visible = true;
-         Width = 60;
-      }
-   }
-}

src/client/spark/view/Controls/Order/AlgoContentTemplateSelector.xaml

@@ -687,7 +687,7 @@
             <TextBlock VerticalAlignment="Center">Tick Offset</TextBlock>
             <c:HelpButton Margin="{StaticResource Margin.LP1}" ToolTip="How many ticks off the inside market to deploy the order, this guards against crossing the market inadventently"/>
           </StackPanel>
-          <c:TickUpDownUInt x:Name="PART_TickOffsetOnDeploy_TickUpDownUInt" Grid.Column="2" Value="{Binding tick_offset_on_deploy}"/>
+          <c:TickUpDownInt x:Name="PART_TickOffsetOnDeploy_TickUpDownUInt" Grid.Column="2" Value="{Binding tick_offset_on_deploy}"/>
           <uag:AlgoPickerNodeView
             x:Name="PART_Child_AlgoPickerNodeView"
             Grid.Row="2"
@@ -1005,8 +1005,6 @@
             ContractPickerModel="{Binding contract_picker_model}"
             HedgePricing="{Binding hedge_pricing}"
             HedgePricingItemsSource="{Binding hedge_pricing_options}"
-            UseFractionalTicks="{Binding use_fractional_ticks}"
-            IntegerTickSize="{Binding integer_tick_size}"
           />
           <c:DialogSkirt Grid.Row="1" Margin="-4" VerticalAlignment="Top" RightItems="{Binding skirt_items}"/>
         </Grid>

src/client/spark/view/Converters/BooleanToVisibilityConverter.cs

@@ -12,9 +12,6 @@ namespace mbts.ui.converters
       public Visibility VisibilityForFalse { get { return _visfalse; } set { _visfalse = value; } }
 
       public static BooleanToVisibilityConverter Instance => new BooleanToVisibilityConverter();
-      public static BooleanToVisibilityConverter HiddenInstance => new BooleanToVisibilityConverter {
-         VisibilityForFalse = Visibility.Hidden
-      };
 
       public object Convert(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) {
          bool flag = false;

src/client/spark/view/Extensions/Command/FileDialogCommand.cs

@@ -64,6 +64,25 @@ namespace mbts.ui {
             }
          }, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));
 
+      public static IDialogCommand export_csv(
+         Func<string> get_file_content,
+         string source_window_name) =>
+         new AnonymousFileDialogCommand<object>(_ =>
+            _create_save_file_dialog(
+               Environment.GetFolderPath(Environment.SpecialFolder.Desktop),
+               $"BTS_Spark_{source_window_name}_{DateTime.Now.ToString("yyyyMMdd_HHmm")}.csv",
+               new[] { ("CSV (comma delimited)", new string[] { "csv" }) }
+            ),
+            (_, file) => {
+               try {
+                  File.WriteAllText(file, get_file_content.Invoke());
+               } catch (Exception ex) {
+                  BTSMessageBox.Show($"Error exporting data to csv: {ex}.", "CSV Export Error", BTSMessageBoxButtons.OK, System.Windows.MessageBoxImage.Error, true, false, true);
+               }
+            },
+            // we are always able to perform an export, even when no rows (it will just be a header)
+            (_) => true);
+
       static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
          var rv = new TDialog();
          if (!string.IsNullOrEmpty(initial_dir))

src/client/spark/view/Spark/Core/TickUtils.cs

@@ -49,28 +49,6 @@ namespace mbts.spark {
       public static double parse(this ContractSpecificationImmutable? spec, string text, double display_multiplier, string nan_text) =>
          spec.spec_or_default().parse(text, display_multiplier, nan_text);
 
-      public static bool is_integer_tick(ContractSpecificationImmutable spec) =>
-         spec.MainFraction == 0 &&
-         spec.PriceDisplayFormat == 0 &&
-         spec.LowTick > 0 &&
-         MathUtils.is_int(spec.LowTick) &&
-         spec.HighTick > 0 &&
-         MathUtils.is_int(spec.HighTick);
-
-      public static ContractSpecification create_integer_tick_spec(int low_tick) =>
-         create_integer_tick_spec(low_tick, low_tick);
-
-      public static ContractSpecification create_integer_tick_spec(int low_tick, int high_tick) {
-         var rv = SecurityDataHelper.Defaults.ContractSpecification;
-         rv.MainFraction = 0;
-         rv.SubFraction = 0;
-         rv.PriceDisplayFormat = 0;
-         rv.PriceBreak = 0;
-         rv.LowTick = low_tick;
-         rv.HighTick = high_tick;
-         return rv;
-      }
-
       public static double low_tick(NativePriceSize[] bids, NativePriceSize[] asks) {
          double low = bids[bids.Length - 1].price;
          if (double.IsNaN(low))
@@ -348,7 +326,7 @@ namespace mbts.spark {
          }
 
          static bool _check_tick_size(double tick_size) =>
-            tick_size > 0 && (tick_size <= 0.5 || MathUtils.is_int(tick_size));
+            (tick_size > 0 && tick_size <= 0.5) || tick_size == 1;
 
          struct SpecKey : IEquatable<SpecKey> {
             public readonly uint denominator, display_digits;

src/client/spark/view/Utility/CommandHelper.cs

@@ -1,7 +1,5 @@
 using mbts.ui.converters;
-using System;
 using System.Globalization;
-using System.Reactive.Linq;
 using System.Windows;
 using System.Windows.Data;
 using System.Windows.Input;
@@ -14,31 +12,6 @@ namespace mbts.ui.utilities {
       public static string fmt(string title, KeyGesture key_gesture) =>
          key_gesture == null ? title : $"{title} ({key_gesture.GetDisplayStringForCulture(CultureInfo.CurrentCulture)})";
 
-      public static IDisposable bind_param_source<T>(ICommand command, IObservable<T> param_source, out ICommand bound_cmd, bool dispose_command_on_dispose = false) {
-         bool initialized = false;
-         bool can_execute = false;
-         T param = default;
-         var bcmd = new DelegateCommand(() => command.Execute(param), () => can_execute);
-         bound_cmd = bcmd;
-         var sub = param_source.Subscribe(p => {
-            param = p;
-            if (!initialized) {
-               initialized = true;
-               can_execute = command.CanExecute(p);
-               bcmd.RaiseCanExecuteChanged();
-            } else if (can_execute != command.CanExecute(p)) {
-               can_execute = !can_execute;
-               bcmd.RaiseCanExecuteChanged();
-            }
-         });
-         return dispose_command_on_dispose ?
-            Disposable.Create(() => {
-               Disposable.dispose(ref sub);
-               (command as IDisposable)?.Dispose();
-               command = null;
-            }) : sub;
-      }
-
       #region Paraphrased from https://referencesource.microsoft.com/#PresentationFramework/src/Framework/MS/Internal/Commands/CommandHelpers.cs
 
       public static bool CanExecuteCommandSource(ICommandSource commandSource) {

src/client/spark/view/Utility/EnabledValueVM.cs

@@ -30,21 +30,6 @@ namespace mbts.ui {
       public string name { get; }
    }
 
-   class NamedValueVM : ViewModelBase {
-      string _name;
-      object _value;
-
-      public NamedValueVM() { }
-
-      public NamedValueVM(string name, object value) {
-         _name = name;
-         _value = value;
-      }
-
-      public string name { get => _name; set => Set(ref _name, value); }
-      public object value { get => _value; set => Set(ref _value, value); }
-   }
-
    static class EnabledValueVMExtensions {
       public static T? get_nullable<T>(this EnabledValueVM<T> vm) where T : struct =>
          vm.enabled ? vm.value : (T?)null;

src/client/spark_analytics/BlackScholes.cs

@@ -0,0 +1,243 @@
+using System;
+
+namespace bts.analytics
+{
+   public static class BlackScholes
+   {
+      static public double NormCDF(double z)
+      {
+         //return MathNet.Numerics.Distributions.Normal.CDF(0.0, 1.0, z);
+         return Probability.normal_cdf(z);
+      }
+      static public double NormPDF(double z)
+      {
+         //return MathNet.Numerics.Distributions.Normal.PDF(0.0, 1.0, z);
+         return Probability.normal_pdf(z);
+      }
+
+      // expected size of change in underlying
+      // E(|x(t)-x(0)|), x(0)=Spot, E(x(t))=Spot*exp(GrowthRate*t), Spot return normally distributed
+      // This is the undiscounted straddle value
+      static public double expected_move(double spot, double growth, double vol, double t)
+      {
+         double sqrt_t = Math.Sqrt(t);
+         double d1 = (growth / vol + 0.5 * vol) * sqrt_t;
+         double d2 = d1 - vol * sqrt_t;
+         return spot * (Math.Exp(growth* t) * (2.0 * NormCDF(d1) - 1.0) + 1.0 - 2.0 * NormCDF(d2));
+      }      
+      
+      static public double atm_straddle(double spot, double riskless, double growth, double vol, double bizt, double calt)
+      {
+         return 2.0*Math.Exp((growth-riskless)*calt)*spot*(2.0*NormCDF(0.5*vol*Math.Sqrt(bizt))-1.0);
+      }
+
+      static public void calls(double spot, double[] rawk, double growth, double riskless,
+                               double[] vol, double bizt, double calt, out double[] result)
+      {
+         calls(spot * Math.Exp(growth * calt), rawk, riskless, vol, bizt, calt, out result);
+      }
+      static public void calls(double fwd, double[] rawk, double riskless, double[] vol, double bizt, double calt, out double[] result)
+      {
+         result = new double[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; i++)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            double d2 = d1 - vol_root_t;
+            result[i] = disc * (fwd * NormCDF(d1) - rawk[i] * NormCDF(d2));
+         }
+      }
+
+      static public void put(double spot, double[] rawk, double growth, double riskless,
+                             double[] vol, double bizt, double calt, out double[] result)
+      {
+         puts(spot * Math.Exp(growth * calt), rawk, riskless, vol, bizt, calt, out result);
+      }
+      static public void puts(double fwd, double[] rawk, double riskless, double[] vol, double bizt, double calt, out double[] result)
+      {
+         result = new double[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; i++)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            double d2 = d1 - vol_root_t;
+            result[i] = disc * (fwd * (NormCDF(d1) - 1.0) + rawk[i] * (1.0 - NormCDF(d2)));
+         }
+      }
+
+      static public void vega(double fwd, double[] rawk, double riskless, double[] vol, double bizt, double calt, out double[] result)
+      {
+         result = new double[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; ++i)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            result[i] = disc * fwd * NormPDF(d1) * sqrt_t;
+         }
+      }
+
+      static public double atm_straddle(double fwd, double riskless, double vol, double bizt, double calt)
+      {
+         return 2.0 * Math.Exp((/*growth*/-riskless) * calt) * fwd * (2.0 * NormCDF(0.5 * vol * Math.Sqrt(bizt)) - 1.0);
+      }
+
+      public class PriceDeltaGammaVegaThetaRho
+      {
+         public double strike;
+         public double call, put;
+         public double cdelta, pdelta;
+         public double cvega, pvega;
+         public double cgamma, pgamma;
+         public double cdiscrho, pdiscrho;
+         public double cvoltheta, pvoltheta;
+      }
+
+      static public void analytic_theos(double fwd, double[] rawk, double riskless, double[] vol,
+                                        double bizt, double calt, out PriceDeltaGammaVegaThetaRho[] result)
+      {
+         result = new PriceDeltaGammaVegaThetaRho[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; ++i)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            double d2 = d1 - vol_root_t;
+            double nd1 = NormCDF(d1);
+            double nd2 = NormCDF(d2);
+            result[i].strike = rawk[i];
+            result[i].call = disc * (fwd * nd1 - rawk[i] * nd2);
+            result[i].put = disc * (fwd * (nd1 - 1.0) + rawk[i] * (1.0 - nd2));
+            result[i].cdelta = disc * nd1;
+            result[i].pdelta = disc * (nd1 - 1.0);
+            double dnd1 = NormPDF(d1);
+            result[i].cgamma = result[i].pgamma = disc * dnd1 / (fwd * vol_root_t);
+            result[i].cvega = result[i].pvega = disc * dnd1 * fwd * sqrt_t;
+            result[i].cdiscrho = rawk[i] * calt * disc * nd2;
+            result[i].pdiscrho = -rawk[i] * calt * disc * (1.0 - nd2);
+
+            double theta_base = -disc * fwd * dnd1 * vol[i] / (2 * sqrt_t);
+            result[i].cvoltheta = theta_base - riskless * rawk[i] * disc * nd2;
+            result[i].pvoltheta = theta_base + riskless * rawk[i] * disc * (1.0 - nd2);
+         }
+      }
+
+      /// <summary>
+      /// Estimate Vega/Fwd Via 'moneyness' and vol assuming zero interest rate and using t_estimate
+      /// </summary>
+      /// <param name="x"></param>
+      /// <param name="vol"></param>
+      static public double moneyness_vega(double x, double sigma, double vol, double t_estimate, double fwd_estimate, double riskless_estimate = 0.0, double calt_estimate = 0.0)
+      {
+         double disc = Math.Exp(-riskless_estimate * calt_estimate);
+         double sqrt_t = Math.Sqrt(t_estimate);
+         double vol_root_t = sqrt_t * vol;
+         double ln_k_over_f = x * sigma * sqrt_t;
+         double d1 = -ln_k_over_f / vol_root_t + 0.5 * vol_root_t;
+         double vegaish = disc * fwd_estimate * NormPDF(d1) * sqrt_t;
+         return vegaish / 100.0;
+      }
+
+      public static double implied_vol(double fwd, double rawk, double riskless, double initVol, double bizt, double calt,
+                                       bool is_call, double target_price, int max_iterations = 10)
+      {
+         if (max_iterations <= 0)
+            max_iterations = 10;
+         const double ImpVolTol = 1.0e-8;
+         double retinv = Math.Log(fwd / rawk);
+         double temp_factor = retinv * retinv;
+         double sqrthalftpi = Math.Sqrt(0.5 * bizt / Math.PI);
+         double sqrt_t = Math.Sqrt(bizt);
+
+         double disc = Math.Exp(-riskless * calt);
+         double CurrPrice = 0.0, Vol = initVol;
+         for (int i = 0; i < max_iterations; i++)
+         {
+            double VolRootT = Vol * sqrt_t;
+            double d1 = retinv / VolRootT + 0.5 * VolRootT;
+            double d2 = d1 - VolRootT;
+            if (is_call)
+               CurrPrice = disc * (fwd * NormCDF(d1) - rawk * NormCDF(d2));
+            else
+               CurrPrice = disc * (fwd * (NormCDF(d1) - 1.0) + rawk * (1.0 - NormCDF(d2)));
+            if (Math.Abs(CurrPrice - target_price) < ImpVolTol)
+               break;
+            else
+            {
+               double ratio = temp_factor / (Vol * Vol * Vol * bizt) - 0.25 * Vol * bizt;
+               double Vega = disc * fwd * sqrthalftpi * Math.Exp(-0.5 * d1 * d1);
+               if (Vega < 1e-5)
+                  Vega = 1e-5;
+               double dvol, temp_arg;
+               if (Math.Abs(ratio) > 0.0001 && (temp_arg = 1.0 + 2.0 * (target_price - CurrPrice) * ratio / Vega) > 0.0001)
+                  dvol = (Math.Sqrt(temp_arg) - 1.0) / ratio;
+               else
+                  dvol = (target_price - CurrPrice) / Vega;
+               if (dvol < -0.5 * Vol)
+                  dvol = -0.5 * Vol;
+               else if (dvol > 0.2)
+                  dvol = 0.2;
+               Vol += dvol;
+            }
+         }
+         if (Math.Abs(CurrPrice - target_price) < ImpVolTol && Math.Abs(Vol - initVol) > 0.0000001)
+            return Vol;
+         else
+            return -1.0;
+      }
+
+      public static double implied_bizt(double fwd, double rawk, double riskless, double Vol, double tguess_days,
+                                        bool is_call, double target_price, int max_iterations = 10)
+      {
+         if (max_iterations <= 0)
+            max_iterations = 10;
+         const double ImpVolTol = 1.0e-8;
+         double retinv = Math.Log(fwd / rawk);
+         double temp_factor = retinv * retinv;
+
+         double disc = 1.0; double year = 252.0;
+         double CurrPrice = 0.0, bizt = tguess_days / year;
+         for (int i = 0; i < max_iterations; i++)
+         {
+            double sqrthalftpi = Math.Sqrt(0.5 * bizt / Math.PI);
+            double sqrt_t = Math.Sqrt(bizt);
+            double VolRootT = Vol * sqrt_t;
+            double d1 = retinv / VolRootT + 0.5 * VolRootT;
+            double d2 = d1 - VolRootT;
+            if (is_call)
+               CurrPrice = disc * (fwd * NormCDF(d1) - rawk * NormCDF(d2));
+            else
+               CurrPrice = disc * (fwd * (NormCDF(d1) - 1.0) + rawk * (1.0 - NormCDF(d2)));
+            if (Math.Abs(CurrPrice - target_price) < ImpVolTol)
+               break;
+            else
+            {
+               double ratio = temp_factor / (Vol * Vol * Vol * bizt) - 0.25 * Vol * bizt;
+               double theta = -0.5 * fwd * NormPDF(d1) * Vol / Math.Sqrt(bizt) / year; //-r* blah blah, no rates here
+               if (theta > -1e-5)
+                  theta = -1e-5;
+               // double dt, temp_arg;
+               // if (Math.Abs(ratio) > 0.0001 && (temp_arg = 1.0 + 2.0 * (target_price - CurrPrice) * ratio / Vega) > 0.0001)
+               //    dvol = (Math.Sqrt(temp_arg) - 1.0) / ratio;
+               // else
+               double dt = (CurrPrice - target_price) / theta;
+               if (dt/year < -0.5*bizt)
+                  dt= -0.5*bizt*year;
+               else if (dt/year > 0.5*bizt)
+                  dt = 0.5*bizt*year;
+               bizt += dt/year;
+            }
+         }
+         if (Math.Abs(CurrPrice - target_price) < ImpVolTol && Math.Abs(bizt - tguess_days/year) > 0.0000001)
+            return bizt;
+         else
+            return -1.0;
+      }
+   }
+}

src/client/spark_analytics/DistributionEstimator.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Linq;
+
+namespace bts.analytics
+{
+   public class DistributionEstimator
+   {
+      List<double> _vals = new List<double>();
+      NormalEstimator _norm = new NormalEstimator();
+      public DistributionEstimator() 
+      {
+         
+      }
+      public void add(double x)
+      {
+         _vals.Add(x);
+         _norm.add(x);
+      }
+      public void delete(double x)
+      {
+         _vals.Remove(x);
+         _norm.delete(x);
+      }
+      public long num_samples => _vals.Count;
+      public double mean() => _norm.mean();
+      // {
+      //    if (num_samples == 0)
+      //       return 0.0;
+      //    double sum = _vals.Sum();
+      //    return sum / num_samples;
+      // }
+      public double sum_sq()
+      {
+         double m = mean();
+         double sq = 0.0;
+         for (int i = 0; i < num_samples; i++)
+            sq += (_vals[i] - m) * (_vals[i] - m);
+         return sq;
+      }
+      public double variance() => _norm.variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / num_samples;
+      // }
+      public double unbiased_variance() => _norm.unbiased_variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / (num_samples - 1);
+      // }
+      public double std_deviation() => _norm.std_deviation();
+      // {
+      //    return Math.Sqrt(variance());
+      // }
+      public double unbiased_std_deviation() => _norm.unbiased_std_deviation();
+      // {
+      //    return Math.Sqrt(unbiased_variance());
+      // }
+      public double percentile(int p)
+      {
+         _vals.Sort();
+         double percentile = p / 100.0;
+         double n = (_vals.Count - 1) * percentile + 1;
+         if (n <= 1.0)
+            return _vals[0];
+         else if (n >= _vals.Count)
+            return _vals[_vals.Count - 1];
+         else
+         {
+            int k = (int)n;
+            double d = n - k;
+            return _vals[k - 1] + d * (_vals[k] - _vals[k - 1]);
+         }
+      }
+      public override string ToString()
+      {
+         var f = percentile(5);
+         var t = percentile(10);
+         var tw = percentile(25);
+         var h = percentile(50);
+         var s = percentile(75);
+         var nn = percentile(99);
+         var sum50 = f + t + tw + h;
+         return $@"
+   Normal( mean {mean()}, std_dev {std_deviation()} ) 
+   5th { f }, 
+   10th { t } 
+   25th { tw } 
+   50th { h } 
+   75th { s } 
+   99th { nn } 
+   [numSamples={ num_samples } ]
+";
+      }
+   }
+}

src/client/spark_analytics/Extensions.cs

@@ -0,0 +1,7 @@
+
+using System;
+
+namespace bts.analytics
+{
+}
+

src/client/spark_analytics/Interop/AlmgrenChriss.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace bts.analytics
+{
+   public static class AlmgrenChriss
+   {
+      [DllImport("libspark")]
+      //[return: MarshalAs(UnmanagedType.R8)]
+      extern static public double liquidity_cost(double qty, double tick_size, double[] bookprices, 
+                                                 int price_len, int[] booksizes, int size_len, bool is_buy, 
+                                                 bool reverse, ref double residual);
+
+      [DllImport("libspark")]
+      extern static public double ac_alpha(double s_0, double s_t, bool is_buy, double time_factor);
+
+      [DllImport("libspark")]
+      extern static public double ac_sigma(double vol, double dt, double s_t, double time_factor);
+      [DllImport("libspark")]
+      extern static public double ac_sigma_opt(double vol, double dt, double s_t, double time_factor);
+
+      [DllImport("libspark")] 
+      extern static public double ac_xt(double aggressivity, double vol, double dt, double T, double s_0, double s_t, double x0,
+                                        double cost, double adaptivity, int generation, int ngenerations, bool is_buy, double time_factor);
+
+      [DllImport("libspark")]
+      extern static public double ac_xt_sigma_alpha(double lambda, double expmove_sigma, double dt, double T, double alpha, double x0,
+                                                    double cost, double adaptivity, int generation, int ngenerations);
+
+      [DllImport("libspark")]
+      extern static public double ac_optimal_duration(double vol, double dt_seconds, double s_t, double cost_per_share, double time_factor);
+
+   }
+}
+

src/client/spark_analytics/Interpolation.cs

@@ -0,0 +1,107 @@
+using System;
+using System.Linq;
+using MathNet.Numerics;
+
+namespace bts.analytics
+{
+   public static class Interpolation
+   {
+      static public double interp_spline(double t, double[] x, double[] y)
+      {
+         var cs = MathNet.Numerics.Interpolation.CubicSpline.InterpolateBoundariesSorted(x, y,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.Natural, 0.0,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.Natural, 0.0);
+         return cs.Interpolate(t);
+      }
+
+      static public double interp_spline_derivs(double t, double[] x, double[] y, double dleft, double dright)
+      {
+         var cs = MathNet.Numerics.Interpolation.CubicSpline.InterpolateBoundariesSorted(x, y,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.FirstDerivative, dleft,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.FirstDerivative, dright);
+         return cs.Interpolate(t);
+      }
+
+      static public double interp_poly(double xval, double[] poly)
+      {
+         //double v = Evaluate.Polynomial(xval, poly);
+         double v = Polynomial.Evaluate(xval, poly);         
+         return v;
+      }
+
+      static public double interp_linear(double t, double[] x, double[] y)
+      {
+         var ls = MathNet.Numerics.Interpolation.LinearSpline.InterpolateSorted(x, y);
+         return ls.Interpolate(t);
+      }
+
+      static public double interp_quadratic(double t, double[] x, double[] y)
+      {
+         return quad_interp(x, y, t);
+      }
+
+      //utilities.cpp
+      static private double quad_interp(double[] x, double[] y, double x_val)
+      {
+         if (y == null || x == null || x.Length == 0 || y.Length == 0 || x.Length != y.Length) return double.NaN;
+         int npts = x.Length;
+         if (npts < 3) return interp_linear(x_val, x, y);
+         // k is the 'interior' index of the point to which x_val is closest
+         int k;
+         if (x_val < x[0]) k = 1;
+         else if (x_val > x[npts - 1]) k = npts - 2;
+         else
+         {
+            int klo = 1, khi = npts - 2;
+            while (khi - klo > 1)
+            {
+               k = (khi + klo) >> 1;
+               if (x[k] > x_val) khi = k;
+               else klo = k;
+            }
+            if ((x[khi] - x_val) > (x_val - x[klo]) && klo > 0) k = klo;
+            else k = khi;
+            const double SMALLER = 1.0e-7;
+            if ((x[k + 1] - x[k]) * (x[k] - x[k - 1]) < SMALLER) return interp_linear(x_val, x, y);
+         }
+         return y[k - 1] * (x_val - x[k]) * (x_val - x[k + 1]) / ((x[k - 1] - x[k]) * (x[k - 1] - x[k + 1])) + y[k] * (x_val - x[k - 1]) * (x_val - x[k + 1]) / ((x[k] - x[k - 1]) * (x[k] - x[k + 1])) + y[k + 1] * (x_val - x[k - 1]) * (x_val - x[k]) / ((x[k + 1] - x[k - 1]) * (x[k + 1] - x[k]));
+         // this might be faster, should be same answer:
+         //   return -(y[k-1]*(x[k]-x[k+1])*(x_val-x[k])*(x_val-x[k+1])+y[k]*(x[k+1]-x[k-1])*(x_val-x[k-1])*(x_val-x[k+1])+y[k+1]*(x[k-1]-x[k])*(x_val-x[k-1])*(x_val-x[k]))/((x[k+1]-x[k-1])*(x[k-1]-x[k])*(x[k]-x[k+1]));
+      }
+
+      //copied from Theopoint.cs
+
+      public static double weighted_mean_taylor_interp(double x1, double f1, double df1, double d2f1,
+                                                       double x2, double f2, double df2, double d2f2, double x)
+      {
+         double t_f_x1 = taylor_interp(x1, f1, df1, d2f1, x),
+               t_f_x2 = taylor_interp(x2, f2, df2, d2f2, x);
+         return (t_f_x1 * Math.Abs(x2 - x) + t_f_x2 * Math.Abs(x1 - x)) / (Math.Abs(x2 - x1));
+      }
+
+      public static double taylor_interp(double x1, double f1, double df1, double d2f1, double x)
+      {
+         return f1 + df1 * (x - x1) + d2f1 * Math.Pow(x - x1, 2) * 0.5;
+      }
+
+      static public double interp_taylor(double t, double[] x, double[] y, double[] dy, double[] dy2)
+      {
+         int idx = Array.BinarySearch<double>(x, t);
+         if (idx >= 0 && idx < x.Length)
+            return y[idx];
+         else
+         {
+            idx = ~idx;
+            if (idx == x.Length)
+               return double.NaN;
+            else if (idx == 0)
+               return double.NaN;
+            else
+            {
+               return weighted_mean_taylor_interp(x[idx - 1], y[idx - 1], dy[idx - 1], dy2[idx - 1],
+                                                  x[idx], y[idx], dy[idx], dy2[idx], t);
+            }
+         }
+      }
+   }
+}

src/client/spark_analytics/Moments.cs

@@ -0,0 +1,119 @@
+using System;
+
+namespace bts.analytics
+{
+   public class Moments
+   {
+      long _n;
+      double _m1, _m2, _m3, _m4;
+      double _min = double.MaxValue;
+      double _max = double.MinValue;
+      double _total = 0.0;
+
+      public Moments() { }
+      public void clear()
+      {
+         _n = 0;
+         _m1 = _m2 = _m3 = _m4 = 0.0;
+         _min = _max = _total = 0.0;
+      }
+      double _last = double.NaN;
+      public void add_return(double x)
+      {
+         if(!double.IsNaN(_last))
+         {
+            var ret = Math.Log(x / _last);
+            this.add(ret);
+         }
+         _last = x;
+      }
+      public void add(double x)
+      {
+         _total += x;
+         long n1 = _n;
+         _n++;
+         double d = x - _m1;
+         double dn = d / _n;
+         double dn2 = dn * dn;
+         double t = d * dn * n1;
+         _m1 += dn;
+         _m4 += t * dn2 * (_n*_n - 3.0*_n + 3) + 6.0 * dn2 * _m2 - 4.0 * dn * _m3;
+         _m3 += t * dn * (_n - 2.0) - 3.0 * dn * _m2;
+         _m2 += t;
+
+         if (x < _min)
+            _min = x;
+         if (x > _max)
+            _max = x;
+      }
+
+      public double total() => _total;
+      public long num_samples => _n;
+      public double min() => _min;
+      public double max() => _max;
+      public double mean() => _m1;
+      public double variance()
+      {
+         if (_n > 1)
+            return _m2/_n;
+         else
+            return 0.0;
+      }
+      public double unbiased_variance()
+      {
+         if (_n > 2)
+            return _m2 / (_n - 1.0);
+         else
+            return 0.0;
+      }
+      public double std_deviation()
+      {
+         return Math.Sqrt(variance());
+      }
+      public double unbiased_std_deviation()
+      {
+         return Math.Sqrt(unbiased_variance());
+      }
+      public double skewness()
+      {
+         return Math.Sqrt((double)_n) * _m3 / Math.Pow(_m2, 1.5);
+      }
+      public double kurtosis() 
+      {
+         return ((double)_n) * _m4 / (_m2* _m2) - 3.0;
+      }   
+      
+      public static Moments operator +(Moments a, Moments b)
+      {
+         Moments combined = new Moments();
+         combined._n = a._n + b._n;
+
+         double delta = b._m1 - a._m1;
+         double delta2 = delta * delta;
+         double delta3 = delta * delta2;
+         double delta4 = delta2 * delta2;
+
+         combined._m1 = (a._n * a._m1 + b._n * b._m1) / combined._n;
+
+         combined._m2 = a._m2 + b._m2 +
+                       delta2 * a._n * b._n / combined._n;
+
+         combined._m3 = a._m3 + b._m3 +
+                       delta3 * a._n * b._n * (a._n - b._n) / (combined._n * combined._n);
+         combined._m3 += 3.0 * delta * (a._n * b._m2 - b._n * a._m2) / combined._n;
+
+         combined._m4 = a._m4 + b._m4 + delta4 * a._n * b._n * (a._n * a._n - a._n * b._n + b._n * b._n) /
+                       (combined._n * combined._n * combined._n);
+         combined._m4 += 6.0 * delta2 * (a._n * a._n * b._m2 + b._n * b._n * a._m2) / (combined._n * combined._n) +
+                       4.0 * delta * (a._n * b._m3 - b._n * a._m3) / combined._n;
+
+         return combined;
+      }
+         
+      public override string ToString()
+      {
+         return $"Normal( mean {mean()}, std_dev {std_deviation()} ) [min {_min}, max { _max} ] [numSamples={num_samples}]";
+      }
+   }
+
+}

src/client/spark_analytics/Moneyness.cs

@@ -0,0 +1,33 @@
+using System;
+
+namespace bts.analytics
+{
+   public static class Moneyness
+   {
+      const double default_svr = 1.0;
+      public static double rawk_from_moneyness(double x, double fwd, double sigma, double bizt, double svr = default_svr, double reffwd = 0.0)
+      {
+         if (svr != default_svr)
+         {
+            double rawundk = rawk_from_moneyness(x, fwd, sigma, bizt);
+            double refstd = moneyness(rawundk, reffwd, sigma, bizt);
+            double z = x + svr * (refstd - x);
+            return rawk_from_moneyness(z, reffwd, sigma, bizt);
+         }
+         else
+            return fwd * Math.Exp(x * sigma * Math.Sqrt(bizt));
+      }
+
+      public static double moneyness(double rawk, double fwd, double sigma, double bizt, double svr = default_svr, double reffwd = 0.0)
+      {
+         if (svr != default_svr)
+         {
+            double reff = moneyness(rawk, reffwd, sigma, bizt);
+            double fwdf = moneyness(rawk, fwd, sigma, bizt);
+            return reff + svr * (fwdf - reff);
+         }
+         else
+            return Math.Log(rawk / fwd) / (sigma * Math.Sqrt(bizt));
+      }
+   }
+}

src/client/spark_analytics/NormalEstimator.cs

@@ -0,0 +1,74 @@
+using System;
+
+namespace bts.analytics
+{
+   public class NormalEstimator
+   {
+      private long _n;
+      private double _mu;
+      private double _sq;
+      private double _min = double.MaxValue;
+      private double _max = double.MinValue;
+
+      public NormalEstimator() { }
+      public void add(double x)
+      {
+         _n++;
+         double next_mu = _mu + (x - _mu) / _n;
+         _sq += (x - _mu) * (x - next_mu);
+         _mu = next_mu;
+
+         if (x < _min)
+            _min = x;
+         if (x > _max)
+            _max = x;
+      }
+      public void delete(double x)
+      {
+         if (_n == 0)
+            throw new ArgumentOutOfRangeException("No elements to delete");
+         else if (_n == 1)
+         {
+            _n = 0;
+            _mu = 0.0;
+            _sq = 0.0;
+         }
+         else
+         {
+            double mu = (_n * _mu - x) / (_n - 1);
+            _sq -= (x - _mu) * (x - mu);
+            _mu = mu;
+            --_n;
+         }
+      }
+      public long num_samples => _n;
+      public double mean() => _mu;
+      public double variance()
+      {
+         if (_n > 1)
+            return _sq / _n;
+         else
+            return 0.0;
+      }
+      public double unbiased_variance()
+      {
+         if (_n > 1)
+            return _sq / (_n - 1);
+         else
+            return 0.0;
+      }
+      public double std_deviation()
+      {
+         return Math.Sqrt(variance());
+      }
+      public double unbiased_std_deviation()
+      {
+         return Math.Sqrt(unbiased_variance());
+      }
+      public override string ToString()
+      {
+         return $"Normal( mean {mean()}, std_dev {std_deviation()} ) [min {_min}, max { _max} ] [numSamples={num_samples}]";
+      }
+   }
+
+}

src/client/spark_analytics/Probability.cs

@@ -0,0 +1,81 @@
+using System;
+
+namespace bts.analytics
+{
+   public static class Probability
+   {
+      public static double dN(double z)
+      {
+         const double INVSQRT2PI = 0.39894228040143267793994605993438;
+         return Math.Exp(-0.5 * z * z) * INVSQRT2PI;
+      }
+
+      public static double normal_pdf(double z)
+      {
+         const double MAXSTD = 37.61588707;
+         const double mindN = 2.2250738585134157e-308;
+         if (Math.Abs(z) > MAXSTD) return mindN;
+         else return dN(z);
+      }
+
+      public static double inverse_normal(double x, int guess, double zguess)
+      {
+         if (x > 1.0 || x < 0.0) return 0.0;
+         double z;
+         const double err_tol = 1.0e-15;
+         if (guess == 0) z = (x > 0.5) ? 0.67 : -0.67;
+         else z = zguess;
+         double Nz = normal_cdf(z);
+         int counter = 0;
+         for (; Math.Abs(Nz - x) > err_tol && counter < 20; counter++)
+         {
+            double epsilon = z * (x - Nz) / dN(z);
+            z += epsilon * (1.0 + epsilon * (0.5 + epsilon * 5.0 / 16.0)) / z;
+            Nz = normal_cdf(z);
+         }
+         if (double.IsNaN(z) && guess != 0) return inverse_normal(x, 0, 0.0);
+         else return z;
+      }
+
+      private readonly static double MAXSTD = 37.5193793471444;
+      private readonly static double MinNormCDF = 2.2250738585152E-308;
+      public static double normal_cdf(double x)
+      {
+         double XAbs = Math.Abs(x);
+         double Cumnorm = MinNormCDF;
+         if (XAbs < MAXSTD)
+         {
+            double Exponential = Math.Exp(-0.5 * XAbs * XAbs);
+            if (XAbs < 7.07106781186547)
+            {
+               double build = 3.52624965998911E-02 * XAbs + 0.700383064443688;
+               build = build * XAbs + 6.37396220353165;
+               build = build * XAbs + 33.912866078383;
+               build = build * XAbs + 112.079291497871;
+               build = build * XAbs + 221.213596169931;
+               build = build * XAbs + 220.206867912376;
+               Cumnorm = Exponential * build;
+               build = 8.83883476483184E-02 * XAbs + 1.75566716318264;
+               build = build * XAbs + 16.064177579207;
+               build = build * XAbs + 86.7807322029461;
+               build = build * XAbs + 296.564248779674;
+               build = build * XAbs + 637.333633378831;
+               build = build * XAbs + 793.826512519948;
+               build = build * XAbs + 440.413735824752;
+               Cumnorm = Cumnorm / build;
+            }
+            else
+            {
+               double build = XAbs + 0.65;
+               build = XAbs + 4.0 / build;
+               build = XAbs + 3.0 / build;
+               build = XAbs + 2.0 / build;
+               build = XAbs + 1.0 / build;
+               Cumnorm = Exponential / (build * 2.506628274631);
+            }
+         }
+         if (x > 0) Cumnorm = 1.0 - Cumnorm;
+         return Cumnorm;
+      }
+   }
+}

src/client/spark_analytics/Properties/AssemblyAttributes.cs

@@ -0,0 +1,4 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("sparkcmd")]
+

src/client/spark_analytics/VegaFactor.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts.analytics
+{
+   public static class VegaFactor
+   {
+      public static double interpolate_vol_dates(double target_date, double date1, double date2, double vol1, double vol2)
+      {
+         if (date1 == date2 || date1 > target_date)
+            return vol1;
+         else if (date2 < target_date)
+            return vol2;
+         double a = (date2 - target_date) / (date2 - date1);
+         double vol = (-vol1 + (vol1 - vol2) * Math.Exp(a) + Math.E * vol2) / (Math.E - 1.0);
+         return vol;
+      }
+
+      public static double interpolate_vol_array(double[] x, double[] y, double x_val)
+      {
+         if (y == null || x == null || x.Length == 0 || y.Length == 0 || x.Length != y.Length) return double.NaN;
+         int npts = x.Length;
+         if (npts < 3) return Interpolation.interp_linear(x_val, x, y);
+         int klo = 0, khi = npts - 1, k = 0;
+         while (khi - klo > 1)
+         {
+            k = (khi + klo) >> 1;
+            if (x[k] > x_val) 
+               khi = k;
+            else 
+               klo = k;
+         }
+         return interpolate_vol(x[klo], y[klo], x[khi], y[khi], x_val);
+      }
+
+      public static double interpolate_vol(double t1, double vol1, double t2, double vol2, double t)
+      {
+         if (t1 == t2 || t1 > t)
+            return vol1;
+         else if (t2 < t)
+            return vol2;
+         double a = t2 / (t2 - t1);
+         double vol = (-vol1 + (vol1 - vol2) * Math.Exp(a) + Math.E * vol2) / (Math.E - 1.0);
+         return vol;
+      }
+
+      static public double vega_factor_shortcut(double normal_bizt, double normal_vol, double current_bizt, double current_vol)
+      {
+         normal_bizt = Math.Max(normal_bizt, 0.001); current_bizt = Math.Max(current_bizt, 0.001);
+         double[] fwd = new double[] { 1.0 };
+         double[] normal_vega, current_vega;
+         BlackScholes.vega(fwd[0], fwd, 0.0, new double[] { normal_vol }, normal_bizt, 0.0, out normal_vega);
+         BlackScholes.vega(fwd[0], fwd, 0.0, new double[] { current_vol }, current_bizt, 0.0, out current_vega);
+         const double vega_factor_max = 15.0;
+         double vf = Math.Min(normal_vega[0] / current_vega[0], vega_factor_max);
+         return vf;
+      }
+   }
+}

src/client/spark_analytics/VolCommandOptions.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using bts.api.core;
+using bts.md;
+using CommandLine;
+using System.Linq;
+using bts.history;
+using System.IO;
+using System.Threading.Tasks;
+using bts.api;
+using System.Threading;
+
+namespace bts.analytics
+{
+
+[Verb("vol", HelpText = "Vol Service command")]
+public class VolCommandOptions
+{
+   [Option('!', "server", Required = false, Default = false, HelpText = "Run HistoricalData Server")]
+   public bool server { get; set; }
+
+   [Option('u', "user", Required = false, Default = "bts", HelpText = "User to log into market data")]
+   public string user { get; set; }
+
+   [Option('d', "password", Required = false, Default = "", HelpText = "password for market data entitlement")]
+   public string password { get; set; }
+
+   public override string ToString()
+   {
+      return Stringizer.to_string(this);
+   }
+}
+
+}

src/client/spark_analytics/VolServiceServer.cs

@@ -0,0 +1,262 @@
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using bts.api.core;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.md;
+using System.Threading;
+using System.Collections.Concurrent;
+
+namespace bts.analytics
+{
+   class VolServiceServer : VolServiceBase
+   {
+      static bts.md.MarketDataClient _md = new md.MarketDataClient();
+      static bts.config.system_symbols.OptionDriverMap _driver = 
+         mbts.config.Configuration.get<bts.config.system_symbols.OptionDriverMap>();
+
+      readonly Implied_vol_curvePublisher _publisher;
+      readonly StatefulPublisher<StrikeVolCurve, uint> _stateful;
+      readonly Dictionary< DateTime, SortedList<double, StrikeVol> > _vols = new Dictionary<DateTime, SortedList<double, StrikeVol>>();
+      readonly string _option_symbol;
+      readonly ServiceContext _context;
+      readonly IDisposable _disp;
+      readonly Timer _timer;
+
+      bool _dirty;
+      bool _driver_set;
+      DateTime? _driver_expiration;
+      double _driver_price = double.NaN;
+      readonly ConcurrentDictionary<DateTime, double> _shift = new ConcurrentDictionary<DateTime, double>();
+      readonly Dictionary<string, double> _strike_factor = new Dictionary<string, double>();
+      public VolServiceServer(ServiceContext context, string option_symbol) : base(context, option_symbol)
+      {
+         _context = context;
+         _md.set_auth_token(context.token);
+         _option_symbol = option_symbol;
+         _publisher = make_implied_vol_curve_publisher(option_symbol);
+         _stateful = new StatefulPublisher<StrikeVolCurve, uint>(_publisher.publish, x=> future_key(x.Expiration.ToDateTime()));
+         _strike_factor["ES"] = 1.0;
+         
+         string driversym = "";
+         foreach(var d in _driver.option_driver)
+         {
+            if(option_symbol == d.option_sym)
+            {
+               driversym = d.driver_sym;
+               _driver_expiration = d.driver_expiration;
+               _driver_set = _driver_expiration.HasValue;               
+            }
+         }
+         if(string.IsNullOrEmpty(driversym))
+            throw new NotSupportedException("can't find driver for " + _option_symbol);
+         var obs = _md.subscribe_book( sec => 
+            (sec.Symbol == _option_symbol && sec.security_type() == SecurityType.Option));
+
+         var driverobs =  _md.subscribe_book( sec=> 
+            (sec.Symbol == driversym && sec.security_type() == SecurityType.Future));
+         _disp = mbts.Disposable.Create(  
+            obs.Subscribe(book => on_book(book)),
+            driverobs.Subscribe(book => on_driver_book(book))
+            );
+         _timer = new Timer(on_timer, null, 0, 5000);
+      }
+      
+      void on_timer(object _)
+      {
+         publish();
+      }
+
+      void publish()
+      {
+         if(!_dirty)
+            return;
+         lock(_vols)
+         {
+            foreach(var exp in _vols)
+            {
+               StrikeVolCurve curve = new StrikeVolCurve() {Symbol = _option_symbol,Expiration = exp.Key.ToTimestamp(),DateKey = future_key(exp.Key)};
+               compute_curve(curve, exp.Key, exp.Value.Values, _driver_price);               
+               _stateful.add_or_update(curve);
+               Log.Gen.info($"Publishing Vol Curve {_option_symbol} {curve.AtmVol.ToString("0.##")} ");
+            }
+            _stateful.flush();
+         }         
+         _dirty = false;
+      }
+      
+      void compute_curve(StrikeVolCurve curve, DateTime expir, IList<StrikeVol> values, double driver_price)
+      {
+         curve.CalT = compute_calt(expir);
+
+         _shift.TryGetValue(expir, out double shift);         
+         curve.AtmVol = compute_atmvol(values, driver_price) + shift;
+         
+         double[] x = new double[values.Count];
+         double[] v = new double[values.Count];
+         for(int i = 0; i < values.Count; i++)
+         {
+            x[i] = values[i].StdStrike = Moneyness.moneyness(values[i].Strike, driver_price, curve.AtmVol, curve.CalT);
+            v[i] = values[i].ImpliedVol + shift;
+         }
+         //curve.Xcurve.AddRange(values);      
+         
+         curve.P2 = x_p2;
+         curve.Vp2 = Interpolation.interp_quadratic(curve.P2, x, v);
+         curve.P1 = x_p1;
+         curve.Vp1 = Interpolation.interp_quadratic(curve.P1, x, v);
+         curve.C1 = x_c1;
+         curve.Vc1 = Interpolation.interp_quadratic(curve.C1, x, v);
+         curve.C2 = x_c2;
+         curve.Vc2 = Interpolation.interp_quadratic(curve.C2, x, v);
+      }
+
+      static double compute_mid(IMDBook b)
+      {
+         double price = double.NaN;
+         if (b.bids.Count > 0 && b.bids[0].size > 0 && b.asks.Count > 0 && b.asks[0].size > 0)
+            price = 0.5 * (b.bids[0].price + b.asks[0].price);
+         else if (b.bids.Count > 0 && b.bids[0].size > 0)
+            price = b.bids[0].price;
+         else if (b.asks.Count > 0 && b.asks[0].size > 0)
+            price = b.asks[0].price;
+         return price;
+      }
+      
+      private void on_driver_book(IEnumerable<IMDBook> book)
+      {
+         if(!_driver_set)
+         {
+            var min = EnumerableExtensions.Min(book, x => x.security.EarliestExpiration());
+            if(!_driver_expiration.HasValue || min < _driver_expiration.Value)
+               _driver_expiration = min;
+         }
+         foreach(var b in book)
+         {
+            if(_driver_expiration == b.security.EarliestExpiration())
+            {
+               double d = compute_mid(b);
+               if(d != _driver_price)
+               {
+                  _driver_price = d;
+                  _dirty = true;
+               }
+            }
+         }
+      }
+
+      private void on_book(IEnumerable<IMDBook> book)
+      {
+         lock(_vols)
+         {
+            foreach(var b in book)
+            {
+               DateTime expir = DateTime.SpecifyKind(b.security.EarliestExpiration(), DateTimeKind.Utc);
+               if(!_vols.TryGetValue(expir, out SortedList<double, StrikeVol> expvol))
+               {
+                  _vols[expir] = expvol = new SortedList<double, StrikeVol>();
+               }
+               expvol[b.security.Option.Strike] = compute_strike(b);
+            }
+         }
+      }
+
+      const double volguess = 0.15;
+      const double cal_days_per_year = 365.0;
+      const int strike_mult = 100;
+      const int strike_width = 10000 * strike_mult;
+      const double x_p2 = -2.0, x_p1 = -1.0, x_c1 = 0.75, x_c2 = 1.5;
+
+      ulong option_key(DateTime expiration, double strike, PutOrCall cp)
+      {
+         var key = Convert.ToUInt64(expiration.ToOADate()) * strike_width + Convert.ToUInt64(strike * strike_mult) + (ulong)cp;
+         return key;
+      }
+
+      uint future_key(DateTime expiration)
+      {
+         var key = Convert.ToUInt32(expiration.ToOADate());
+         return key;
+      }      
+      
+      public static double compute_calt(in DateTime expir)
+      {
+         double calt = (expir - DateTime.Today).TotalDays / cal_days_per_year;
+         calt = Math.Max(calt, 0.00001);
+         return calt;
+      }
+      
+      double compute_atmvol(IList<StrikeVol> vols, double fwd)
+      {
+         int idx = vols.BinarySearchIndexOf(fwd, x => x.Strike);
+         int comp = ~idx;
+         if(idx >= 0)
+            return vols[idx].ImpliedVol;
+         else if(comp > 0 && comp < vols.Count - 1)
+         {
+            double atm = Interpolation.interp_quadratic(fwd,
+                                                        new[] {vols[comp-1].Strike, vols[comp].Strike, vols[comp+1].Strike },
+                                                        new[] {vols[comp-1].ImpliedVol, vols[comp].ImpliedVol, vols[comp+1].ImpliedVol});
+            return atm;
+         }
+         return double.NaN;
+      }
+
+      StrikeVol compute_strike(IMDBook b)
+      {
+         Log.assert(b.security.security_type() == SecurityType.Option);
+         if (!_strike_factor.TryGetValue(b.security.Symbol, out double kfactor))
+            kfactor = 1.0;
+
+         DateTime expir = DateTime.SpecifyKind(b.security.EarliestExpiration(), DateTimeKind.Utc);
+         var kv = new StrikeVol() 
+         {
+            Symbol = b.security.Symbol,
+            Expiration = expir.ToTimestamp(),
+            Strike = b.security.Option.Strike * kfactor,
+            Cp = b.security.Option.CallPut
+         };
+         kv.Key = option_key(expir, kv.Strike, kv.Cp);
+         double calt = compute_calt(expir);
+         double option_mid = compute_mid(b);
+         kv.ImpliedVol = BlackScholes.implied_vol(_driver_price, kv.Strike, 0.0, volguess, calt, calt, 
+                                                  kv.Cp == PutOrCall.Call, option_mid);
+         if(kv.ImpliedVol <= 0.0)
+         {
+            kv.ImpliedVol = BlackScholes.implied_vol(_driver_price, kv.Strike, 0.0, volguess * 2.0, calt, calt,
+                                                     kv.Cp == PutOrCall.Call, option_mid);
+            if (kv.ImpliedVol <= 0.0)            
+               kv.ImpliedVol = BlackScholes.implied_vol(_driver_price, kv.Strike, 0.0, volguess * 4.0, calt, calt,
+                                                        kv.Cp == PutOrCall.Call, option_mid);
+         }
+         if (kv.ImpliedVol <= 0.0)
+            kv.ImpliedVol = double.NaN;
+            
+         return kv;
+      }
+
+      public override Task<ResultMessage> shift_vol(VolShiftQuery request)
+      {
+         if(request.Symbol == _option_symbol)            
+         {
+            _dirty = true;
+            if(!double.IsNaN(request.Shift))
+            {
+               _shift[request.Expiration.ToDateTime()] = request.Shift;
+               return Task.FromResult(new ResultMessage() { Success = true });
+            }
+            else
+               return Task.FromResult(new ResultMessage() { Success = false });
+         }
+         throw new NotSupportedException($"can only shift vol for {_option_symbol}");
+      }
+      
+      public override void Dispose()
+      {
+         if(_disp != null)
+            _disp.Dispose();
+      }
+   }   
+}

src/client/spark_analytics/VolTermStructureServer.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using bts.api.core;
+using System.Linq;
+using System.Reactive.Disposables;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using System.Threading;
+
+namespace bts.analytics
+{
+   class VolTermStructureServer : VolTermStructureServiceBase
+   {
+      static bts.config.system_symbols.OptionDriverMap _driver =
+         mbts.config.Configuration.get<bts.config.system_symbols.OptionDriverMap>();
+
+      readonly Vol_term_structurePublisher _publisher;
+      readonly StatefulPublisher<VolTermStructure, string> _stateful;
+      readonly SortedDictionary<DateTime, StrikeVolCurve> _vols = new SortedDictionary<DateTime, StrikeVolCurve>();
+      readonly Dictionary<string, VolService> _services = new Dictionary<string, VolService>();
+      readonly ServiceContext _context;
+      readonly CompositeDisposable _disp = new CompositeDisposable();
+      readonly Timer _timer;
+      readonly string _driver_symbol;
+      VolTermStructure _last_term = new VolTermStructure();
+
+      bool _dirty;
+      public VolTermStructureServer(ServiceContext context, string driver_symbol) : base(context, driver_symbol)
+      {
+         _context = context;
+         _driver_symbol = driver_symbol;
+         _publisher = make_vol_term_structure_publisher(driver_symbol);
+         _stateful = new StatefulPublisher<VolTermStructure, string>(_publisher.publish, x => x.DriverSymbol);
+
+         foreach (var d in _driver.option_driver)
+         {
+            if (_driver_symbol == d.driver_sym)
+            {
+               var vs = _services[d.option_sym] = context.locate<VolService>(d.option_sym);
+               _disp.Add( vs.subscribe_implied_vol_curve(d.option_sym).Subscribe( x => 
+               {
+                  lock(_vols)
+                  {
+                     foreach(var val in x.all)
+                        _vols[val.Expiration.ToDateTime()] = val;
+                  }
+                  _dirty = true;
+               }));
+            }
+         }
+         _timer = new Timer(on_timer, null, 0, 5000);
+      }
+
+      void on_timer(object _)
+      {
+         publish();
+      }
+
+      void publish()
+      {
+         if (!_dirty)
+            return;
+         VolTermStructure term = new VolTermStructure() { DriverSymbol = _driver_symbol };
+         lock(_vols)
+         {
+            foreach (var exp in _vols)
+            {
+               VolTermInfo info = new VolTermInfo
+               {
+                  CalT =  exp.Value.CalT,
+                  AtmVol = exp.Value.AtmVol,
+                  Skewness = 0.0,
+               };
+               term.Terms.Add(info);
+            }
+            _stateful.add_or_update(term);      
+            _last_term = term;
+            Log.Gen.info($"Publishing Vol Term Structure {_driver_symbol} ");
+
+            _stateful.flush();
+         }
+         _dirty = false;
+      }
+
+      async public override Task<ResultMessage> global_shift(VolShiftQuery request)
+      {
+         if (request.Symbol != _driver_symbol)
+            throw new NotSupportedException($"can only global shift vol for {_driver_symbol}");
+
+         double[] x = _last_term.Terms.Select( z => z.CalT).ToArray();
+         double[] v = _last_term.Terms.Select( z => z.AtmVol).ToArray();
+         double firstnotnan = v.FirstOrDefault(xx => !double.IsNaN(xx));
+         const double vftime = 1/12.0;
+         double v30day = Interpolation.interp_quadratic(vftime, x, v);
+         if(double.IsNaN(v30day))
+            v30day = Math.Max(firstnotnan, 0.05);
+         foreach(var vs in _services)
+         {
+            List<VolShiftQuery> shifts = new List<VolShiftQuery>();
+            lock(_vols)
+            {
+               foreach(var curve in _vols)
+               {
+                  Timestamp exp = DateTime.SpecifyKind(curve.Key, DateTimeKind.Utc).ToTimestamp();
+                  double vf = VegaFactor.vega_factor_shortcut(vftime, v30day, curve.Value.CalT, curve.Value.AtmVol);
+                  double shift = request.Shift * vf;
+                  shifts.Add(new VolShiftQuery { Symbol = vs.Key, Expiration = exp, Shift = shift});
+               }
+            }
+            //can't await in lock
+            foreach(var s in shifts)
+               if(!double.IsNaN(s.Shift))
+                  await vs.Value.shift_vol_async(s);
+         }
+         _dirty = true;
+         return new ResultMessage() { Success = true };
+      }
+
+      public override void Dispose()
+      {
+         if (_disp != null)
+            _disp.Dispose();
+      }      
+   }
+}

src/client/spark_analytics/spark_analytics.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    
+    <TargetFramework Condition = " '$(OS)' != 'Windows_NT' ">netcoreapp3.0</TargetFramework>
+    <RuntimeIdentifier Condition = " '$(OS)' != 'Windows_NT' ">linux-x64</RuntimeIdentifier>
+    <PublishTrimmed>true</PublishTrimmed>
+    <PublishSingleFile>true</PublishSingleFile>
+    
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Platform>x64</Platform>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <Version>1.0.0</Version>
+    <LangVersion>7.2</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../spark_api/spark_api.csproj" />
+    <ProjectReference Include="../spark_core/spark_core.csproj" />
+    <!--<Proto3 Include="**/*.proto" />-->
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
+    <PackageReference Include="Microsoft.CSharp" Version="4.5.0" />
+    <PackageReference Include="CommandLineParser" Version="2.6.0" />
+    <PackageReference Include="MathNet.Numerics" Version="4.9.0" />
+  </ItemGroup>
+
+</Project>

src/client/spark_api/Co2/Co2NativeMethods.cs

@@ -4,6 +4,8 @@ using System.Runtime.InteropServices;
 using System.Security;
 using Google.Protobuf;
 using security_key = System.UInt32;
+using bts.co2;
+using System.Linq;
 
 namespace mbts.co2
 {
@@ -85,6 +87,7 @@ namespace mbts.co2
 
       [DllImport("libspark")]
       internal unsafe static extern void post_on_scan_security_thread(Action callback);
+      
    }
 
    internal static class Co2NativeTestMethods
@@ -133,5 +136,59 @@ namespace mbts.co2
       [DllImport("libspark")]
       internal static extern void co2_test_get_constants(out ulong max_depth, out ulong max_implied_depth, out sbyte nan_exponent);
 
+      internal unsafe delegate void raw_receive_pb_or_string(byte* buffer, int buffer_size);
+      [DllImport("libspark")]
+      private static unsafe extern void raw_co2_compute_synthetic_book(
+         byte[] security_buffer,
+         int security_length,
+         IntPtr[] leg_books,
+         int[] leg_lengths,
+         int leg_books_length,
+         raw_receive_pb_or_string callback);
+
+      internal static unsafe BookMessageImmutable co2_compute_synthetic_book(SecurityImmutable synthetic, params BookMessageImmutable[] books) {
+         BookMessage result = null;
+         string error = null;
+
+         var security_buffer = synthetic.ToByteArray();
+         // var h = GCHandle.Alloc(null, GCHandleType.Pinned).
+         var book_buffers = 
+            books
+            .Select(b => b.ToByteArray())
+            .ToArray();
+         GCHandle[] book_buffer_handles = 
+            book_buffers
+            .Select(b => GCHandle.Alloc(b,GCHandleType.Pinned))
+            .ToArray();
+         IntPtr[] book_buffer_pointers = 
+            book_buffer_handles
+            .Select(h => h.AddrOfPinnedObject())
+            .ToArray();
+         var lengths = 
+            book_buffers
+            .Select(b => b.Length)
+            .ToArray();
+
+         raw_co2_compute_synthetic_book(
+            security_buffer,
+            security_buffer.Length,
+            book_buffer_pointers,
+            lengths,
+            book_buffers.Length,
+            (byte* buffer, int length) => {
+               if (length < 0) {
+                  error =  new string((char*)buffer,0,-1 * length);
+               }
+               else {
+                  result = Co2NativeMethods.deserialize<BookMessage>(buffer, (ulong)length);
+               }
+            });
+
+         if(error != null) {
+            // C# style multiple return values
+            throw new Exception(error);
+         }
+         return new BookMessageImmutable(result);
+      }
    }
 }

src/client/spark_api/Co2/IntPrice.cs

@@ -52,7 +52,49 @@ namespace mbts.co2 {
          return _mantissa.GetHashCode();
       }
 
+      public static IntPrice from_double(double price, sbyte precision) {
+         if (double.IsNaN(price)) return NaN;
+         if (Math.Abs(price%1) != 0) {
+            double rounding = 0.5 * Math.Pow(10,-precision);
+            price += (price > 0) ? rounding : -rounding;
+            price *= Math.Pow(10,precision);
+            return new IntPrice((long)price, (sbyte)-precision);
+         }
+         return new IntPrice((long)price, 0);
+      }
+
       public static IntPrice NaN = new IntPrice(0,127);
+
+      static double[] powers_10 = {
+  1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10, 1E-9, 1E-8, 1E-7, 1E-6, 1E-5, 1E-4, 1E-3, 1E-2, 1E-1,
+  1E0,
+  1E1, 1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8, 1E9, 1E10, 1E11, 1E12, 1E13, 1E14, 1E15, 1E16, 1E17, 1E18, 1E19, 1E20};
+      static int max_abs_power_10 = powers_10.Length / 2;
+      static double pow10(int power)
+      {
+         int offset = power + max_abs_power_10;
+         int powers_10_sz = powers_10.Length;
+         bts.Log.assert(offset >= 0 && offset < powers_10_sz);
+         if (offset >= 0 && offset < powers_10_sz)
+         {
+            return powers_10[offset];
+         }
+         return double.NaN;
+      }      
+      
+      public static IntPrice from_double(double price, int precision = 9)
+      {
+         if (Double.IsNaN(price)) 
+            return NaN;
+         if (Math.Abs(price - Math.Truncate(price)) != 0.0)
+         {
+            double rounding = 0.5 * pow10(-precision);
+            price += (price > 0.0) ? rounding : -rounding;
+            price *= pow10(precision);
+            return new IntPrice((long)price, Convert.ToSByte(-precision));
+         }
+         return new IntPrice((long)price, 0);
+      }
    }
    
 }

src/client/spark_api/Co2/NativePriceHelpers.cs

@@ -1,4 +1,6 @@
 using bts;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace mbts.co2 {
    public static class NativePriceHelpers {
@@ -38,6 +40,60 @@ namespace mbts.co2 {
       public const uint COCONUT_MAX_DEPTH = 10;
       public const uint COCONUT_MAX_IMPLIED_DEPTH = 2;
       internal const sbyte COCONUT_NAN_EXPONENT = 127;
+
+      public static double side_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if(side == bts.api.Side.Buy)
+            return bid_at(book, level);
+         else
+            return ask_at(book, level);
+      }
+      public static uint sidesize_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if (side == bts.api.Side.Buy)
+            return bidsize_at(book, level);
+         else
+            return asksize_at(book, level);
+      }
+      public static double bid_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static double ask_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static uint bidsize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return lev.Size;
+      }
+      public static uint asksize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return lev.Size;
+      }
+      public static double price(this bts.co2.PriceSizeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }
+      public static double price(this bts.co2.TradeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }      
+
+      public static IList<double> all_bids(this bts.co2.BookMessage book)
+      {
+         var lev = book.Bids.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }
+      public static IList<double> all_asks(this bts.co2.BookMessage book)
+      {
+         var lev = book.Asks.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }      
    }
    
 }

src/client/spark_api/Properties/AssemblyAttributes.cs

@@ -3,4 +3,7 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark_core")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_fix")]
+[assembly: InternalsVisibleTo("spark_historical")]
+[assembly: InternalsVisibleTo("spark_analytics")]
 

src/client/spark_api/api/Apioptions.cs

@@ -0,0 +1,20 @@
+namespace bts.api
+{
+   //keep in sync with apioptions.proto
+   public static partial class ApioptionsReflection
+   {
+      public enum MessageOptions
+      {
+         fix_algostrategy_tagbase = 8506,
+      }
+      public enum FieldOptions
+      {
+         fix_tag = 8507,
+         fix_tag_is_header = 8509,
+      }
+      public enum EnumValueOptions
+      {
+         fix_value = 8508,
+      }
+   }
+}

src/client/spark_api/history/HistoryProtos.cs

@@ -0,0 +1,147 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf;
+using Google.Protobuf.Collections;
+using Google.Protobuf.WellKnownTypes;
+using mbts.co2;
+
+namespace bts.history
+{
+   public interface IHasMsgSeqNum 
+   {
+      uint MsgSeqNum { get; }
+   }
+   
+   public interface IHasDateTimeStamp 
+   {
+      DateTime trade_date { get; set;}
+      DateTime sending_time { get; set;}
+      DateTime transact_time { get; set; }      
+   }
+
+   public interface IHasTimestamp 
+   {
+      Timestamp SendingTime { get; set; }
+      Timestamp TransactTime { get; set; }
+   }
+
+   public interface IHasMdPacket
+   {
+      MarketDataIncrementalRefresh Md{ get; }
+   }
+
+   public interface IHasMdEntry
+   {
+      MDEntry MdEntry { get; }
+      int EntryIndex { get; }
+   }
+   
+   public interface IHasTopBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.TopBookMessage TopBook { get; set; }
+   }
+
+   public interface IHasBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.BookMessage Book { get; set; }
+   }
+
+   public interface IHasTrade : IHasMdEntry, IHasMdPacket
+   {
+      co2.TradeMessage Trade { get; set; }      
+   }   
+
+   public interface IHasMBO : IHasMdPacket
+   {
+      RepeatedField<OrderIDEntry> Orders { get;  }
+   }
+   
+   public sealed partial class HistoryRow : IHasTopBook, IHasMBO, IHasTrade, IHasBook, IHasDateTimeStamp, IHasTimestamp, IHasMsgSeqNum
+   {
+      public DateTime trade_date { get { return DateTime.SpecifyKind(this.TradeDate.ToDateTime(), DateTimeKind.Utc); } 
+                                   set { this.TradeDate = Timestamp.FromDateTime(value); } }
+      public DateTime sending_time { get { return DateTime.SpecifyKind(this.SendingTime.ToDateTime(), DateTimeKind.Utc); } 
+                                     set { this.SendingTime = Timestamp.FromDateTime(value); } }
+      public DateTime transact_time { get { return DateTime.SpecifyKind(this.TransactTime.ToDateTime(), DateTimeKind.Utc); } 
+                                      set { this.TransactTime = Timestamp.FromDateTime(value); } }
+      
+      public HistoryRow add_orders(IEnumerable<OrderIDEntry> orderentries)
+      {
+         this.Orders.AddRange(orderentries);
+         return this;
+      }                                         
+   }
+   
+   public interface IMBOMessage
+   {
+      MDUpdateAction OrderUpdateAction {get;}
+      ulong OrderId {get;}
+      uint SecurityId {get;}
+      string SecurityDesc {get;}
+      ulong MdOrderPriority {get;}
+      double MdEntryPx {get;}
+      uint MdPriceLevel {get;}
+      ulong MdDisplayQty {get;}
+      uint LastQty {get;}
+      MDEntryType MdEntryType {get;}
+      uint ReferenceId {get;}      
+   }
+   
+   public sealed partial class MBOMessage : IMBOMessage
+   {
+      public MDUpdateAction OrderUpdateAction => OrderEntry != null ? OrderEntry.OrderUpdateAction : MdEntry.MdUpdateAction;
+      public ulong OrderId  => OrderEntry != null ? OrderEntry.OrderId : MdEntry.OrderId;
+      public uint SecurityId => MdEntry.SecurityId;
+      public string SecurityDesc => MdEntry.Symbol;
+      public ulong MdOrderPriority => OrderEntry != null ? OrderEntry.MdOrderPriority : MdEntry.MdOrderPriority;
+      public double MdEntryPx => MdEntry.MdEntryPx;
+      public uint MdPriceLevel => MdEntry.MdPriceLevel;
+      public ulong MdDisplayQty => OrderEntry != null ? OrderEntry.MdDisplayQty : MdEntry.MdDisplayQty;
+      public uint LastQty => OrderEntry != null ? OrderEntry.LastQty : MdEntry.LastQty;
+      public MDEntryType MdEntryType => MdEntry.MdEntryType;
+      public uint ReferenceId => OrderEntry != null ? OrderEntry.ReferenceId : MdEntry.ReferenceId;
+      public uint NumberOfOrders => MdEntry.NumberOfOrders;
+   }
+
+   public sealed partial class MarketDataIncrementalRefresh
+   {
+      public MarketDataIncrementalRefresh add_entries(IEnumerable<MDEntry> mdentries, IEnumerable<OrderIDEntry> orderentries)
+      {
+         this.MdEntries.AddRange(mdentries);
+         this.OrderIdEntries.AddRange(orderentries);
+         return this;
+      }
+   }
+   public interface ITradeDetail
+   {
+      ulong OrderId { get; set; }
+      uint LastQty { get; set; }
+      ITradeDetail clone();
+   }
+
+   public sealed partial class OrderIDEntry : ITradeDetail
+   {
+      ITradeDetail ITradeDetail.clone()
+      {
+         return this.Clone();
+      }
+   }
+}
+
+namespace bts.co2
+{
+   public sealed partial class BookMessage
+   {
+      public BookMessage add_bids_and_asks(IEnumerable<PriceSizeMessage> bids, IEnumerable<PriceSizeMessage> asks)
+      {
+         this.Bids.AddRange(bids);
+         this.Asks.AddRange(asks);
+         return this;
+      }
+   }
+   
+   public sealed partial class TradeMessage 
+   {
+      public double price => NativePriceHelpers.to_double(this.Mantissa, (sbyte)this.Exponent);
+   }     
+}

src/client/spark_api/md/MDBook.cs

@@ -38,12 +38,24 @@ namespace bts.md
       {
          this.security = smd.security;
          NativePriceSize[] bids = null, asks = null;
-         if (smd.md.get_depth(ref last_update_time, ref bids, ref asks, NativePriceHelpers.COCONUT_MAX_DEPTH)) 
+         if (smd.md.has_depth && smd.md.get_depth(ref last_update_time, ref bids, ref asks, NativePriceHelpers.COCONUT_MAX_DEPTH)) 
          {            
             this.bids = bids.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
             this.asks = asks.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
             return true;
          }
+         else if(!smd.md.has_depth)
+         {
+            NativePriceSize bid, ask;
+            if(smd.md.get_bbo(ref last_update_time, out bid, out ask))
+            {
+               this.bids = new List<PriceSize>();
+               this.asks = new List<PriceSize>();
+               this.bids.Add(new PriceSize(bid.size, bid.direct_size, bid.implied_size, bid.price));
+               this.bids.Add(new PriceSize(ask.size, ask.direct_size, ask.implied_size, ask.price));
+               return true;
+            }
+         }
          return false;
       }
 

src/client/spark_api/security/SecurityFormatter.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using static bts.api.Security;
 
 namespace bts.api {
-   struct FutureLeg {
+   public struct FutureLeg {
       public readonly Venue exchange;
       public readonly string symbol, exchange_desc;
       public readonly FutureContractImmutable future;

src/client/spark_api/security/SecurityHelper.cs

@@ -4,7 +4,7 @@ using System.Linq;
 using static bts.api.Security;
 
 namespace bts.api {
-   internal static class SecurityHelper {
+   public static class SecurityHelper {
       public static IEnumerable<FutureLeg> get_flat_future_legs(SecurityImmutable sec) {
          switch (sec.ContractCase) {
             case ContractOneofCase.Spread: return _get_flat_future_legs(sec.Spread.Legs, 1);
@@ -34,14 +34,17 @@ namespace bts.api {
       }
 
       #region Extensions
-      public static DateTime expiration_date(this FutureContract future) { return new DateTime((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay); }
+      public static DateTime expiration_date(this FutureContract future) { return new DateTime((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay); }      
       public static DateTime expiration_date(this FutureContractImmutable future) { return new DateTime((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay); }
+      public static DateTime expiration_date(this OptionContract option) { return new DateTime((int)option.ExpirationYear, (int)option.ExpirationMonth, (int)option.ExpirationDay); }
+      public static DateTime expiration_date(this OptionContractImmutable option) { return new DateTime((int)option.ExpirationYear, (int)option.ExpirationMonth, (int)option.ExpirationDay); }
 
       internal static DateTime EarliestExpiration(this SecurityImmutable security) {
          switch (security.ContractCase) {
             case ContractOneofCase.Synthetic: return security.Synthetic.Legs.First().Security.EarliestExpiration();
             case ContractOneofCase.Spread: return security.Spread.Legs.First().Security.EarliestExpiration();
             case ContractOneofCase.Future: return security.Future.expiration_date();
+            case ContractOneofCase.Option: return security.Option.expiration_date();
             default: throw new NotImplementedException();
          }
       }
@@ -63,6 +66,8 @@ namespace bts.api {
             return security.Glbx.SecurityDesc;
          else if(security.HasIce)
             return security.Ice.SecurityDesc;
+         else if(security.HasSynthetic)
+            return security.Synthetic.Name;
          else
          {
             Log.assert(false, "security_desc not implemented for exchange");
@@ -83,24 +88,22 @@ namespace bts.api {
          }
 
          public static int Compare(SecurityImmutable x, SecurityImmutable y) {
-            if (x.ContractCase == ContractOneofCase.Future) {
-               return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
-            } else if (y.ContractCase == ContractOneofCase.Future) {
-               return 1;
-            }
+            if (x.ContractCase == ContractOneofCase.Future) return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
+            else if (y.ContractCase == ContractOneofCase.Future) return 1;
             var xlegs = get_flat_future_legs(x).ToList();
             var ylegs = get_flat_future_legs(y).ToList();
             int max = Math.Min(xlegs.Count, ylegs.Count);
             for (int i = 0; i < max; i++) {
                int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future);
-               if (rv != 0)
-                  return rv;
-               rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
-               if (rv != 0)
-                  return rv;
-               rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
-               if (rv != 0)
-                  return rv;
+               if (rv != 0) return rv;
+            }
+            for (int i = 0; i < max; i++) {
+               int rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
+               if (rv != 0) return rv;
+            }
+            for (int i = 0; i < max; i++) {
+               int rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
+               if (rv != 0) return rv;
             }
             return xlegs.Count.CompareTo(ylegs.Count);
          }

src/client/spark_api/spark_api.csproj

@@ -26,7 +26,7 @@
     <Proto3 Include="**/*.proto" />
     <Proto3 Include="../../lib/api/*.proto" />
     <Proto3 Include="../../lib/api/services/*.proto" />
-    <Proto3 Include="../../lib/md/protos/co2.proto" />
+    <Proto3 Include="../../lib/md/protos/Co2.proto" />
     <Proto3 Include="../../lib/domain_model/Security.proto" />
     <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
     <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">

src/client/spark_api/utils/BTSNativeMethods.cs

@@ -28,5 +28,8 @@ namespace bts.utils
 
       [DllImport("libspark")]
       public static extern bool initialize_nothrow(StringBuilder buffer, int length);
+
+      //public static string bts_data_path()
+      //use mbts.config.Configuration.get<bts.config.System>().data_path;
    }
 }

src/client/spark_api/utils/Locator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Threading.Tasks;
 using bts.api;
 using bts.api.core;
@@ -8,10 +9,14 @@ namespace bts.api
    public static class Locator
    {
       readonly static Lazy<ServiceContext> _context = new Lazy<ServiceContext>(() => new ServiceContext(), true);      
+      public static void init() 
+      {
+         var sc = _context.Value;
+      }
       public static T locate<T>(string qualifier="")  where T : class, IApiClient
       {
          return _context.Value.locate<T>(qualifier);
-      }
+      }            
       static public async Task<LoginResult> login(string username, string password)
       {
          return await _context.Value.login(username, password);
@@ -22,4 +27,20 @@ namespace bts.api
       }      
       static public AuthTokenImmutable? token => _context.Value.token;
    }
+   
+   public class LocatorCache<T> 
+      where T : class, IApiClient
+   {
+      static readonly ConcurrentDictionary<string, T> _cache = new ConcurrentDictionary<string, T>();
+      public static T locate_cached(string qualifier = "") 
+      {
+         qualifier = qualifier ?? "";
+         if(!_cache.TryGetValue(qualifier, out T svc))
+         {
+            _cache[qualifier] = svc = Locator.locate<T>(qualifier);
+         }
+         return svc;
+      }
+   }
+   
 }

src/client/spark_api/utils/Log.cs

@@ -36,6 +36,7 @@ namespace bts {
       Plugin,
       Sdk,
       Http,
+      Fix
       // Please keep this list in sync with logging.h/logging.cpp
    }
    
@@ -102,9 +103,10 @@ namespace bts {
       internal static Logger Assert => new Logger(Subsystem.Assert);
       internal static Logger Ui => new Logger(Subsystem.Ui);
       internal static Logger Testing => new Logger(Subsystem.Testing);
-      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);
+      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);      
       public static Logger Sdk => new Logger(Subsystem.Sdk);
       public static Logger Http => new Logger(Subsystem.Http);
+      internal static Logger FIX => new Logger(Subsystem.Fix);
 
       internal static void log_at(this Logger logger, Level level, string message) {
          switch (level) {

src/client/spark_api/utils/StatefulPublisher.cs

@@ -0,0 +1,74 @@
+
+using System;
+using System.Collections.Generic;
+
+namespace bts.api
+{
+   public class StatefulPublisher<T, TKey>
+   {
+      readonly bool _keep_state;
+      readonly List<T> _added = new List<T>();
+      readonly HashSet<TKey> _erased = new HashSet<TKey>();
+      readonly List<T> _updated = new List<T>();
+
+      readonly HashSet<TKey> _state = new HashSet<TKey>();
+      readonly Dictionary<TKey, T> _state_data = new Dictionary<TKey, T>();
+      readonly Action<List<T>, HashSet<TKey>, List<T>> _publish;
+      readonly Func<T, TKey> _key;
+      public StatefulPublisher(Action< List<T>, HashSet<TKey>, List<T> > pubfn, Func<T, TKey> key, bool keep_state = false) 
+      { 
+         _keep_state = keep_state;
+         _publish = pubfn;
+         _key = key;
+      }
+
+      public void add_or_update(T  m)
+      {
+         var key = _key(m);
+         if (!_has_key(key))
+         {
+            _added.Add(m);
+            if(_keep_state)
+               _state_data[key] = m;
+            else
+               _state.Add(key);
+         }
+         else
+         {
+            // If an item has already been added update to new value
+            Predicate<T> match = (item) => _key(item).Equals( key );
+            var add_idx = _added.FindIndex(match);
+            if (add_idx >= 0)
+            {
+               _added[add_idx] = m;
+            }
+            else
+            {
+               var upd_idx = _updated.FindIndex(match);
+               if (upd_idx < 0 || _updated.Count == 0)
+                  _updated.Add(m);
+               else
+                  _updated[upd_idx] = m;
+            }
+         }
+      }
+      public void erase(TKey m)
+      {
+         if (_has_key(m))
+         {
+            _erased.Add(m);
+         }
+      }
+      public void flush()
+      {
+         _publish(_added, _erased, _updated);
+         _added.Clear();
+         _updated.Clear();
+         _erased.Clear();
+      }
+      bool _has_key(TKey k)
+      {
+         return _keep_state ? _state_data.ContainsKey(k) : _state.Contains(k) ;
+      }
+   }
+}

src/client/spark_api/utils/Stringizer.cs

@@ -0,0 +1,40 @@
+
+using System;
+using System.Collections.Generic;
+
+namespace bts
+{
+   public static class Stringizer
+   {
+      public static string all_to_string(IEnumerable<object> thi, string[] group = null)
+      {
+         string s = string.Empty;
+         foreach (var th in thi)
+            s += Stringizer.to_string(th, group);
+         return s;
+      }
+      public static string to_string(object thi, string[] group = null)
+      {
+         if (thi == null)
+            return string.Empty;
+         string val = thi.GetType().Name + Environment.NewLine;
+         int previ = -1;
+         foreach (var pi in thi.GetType().GetProperties())
+         {
+            var sub = pi.Name.Substring(0, Math.Max(pi.Name.IndexOf('_'), 0));
+            if (group == null)
+               val += pi.Name + ": " + pi.GetValue(thi)?.ToString() + "  " + Environment.NewLine;
+            else
+            {
+               int i = Array.IndexOf(group, sub);
+               if (i >= 0 && (i == previ))
+                  val += pi.Name + ": " + pi.GetValue(thi)?.ToString() + "  ";
+               else
+                  val += Environment.NewLine + pi.Name + ": " + pi.GetValue(thi)?.ToString() + "  ";
+               previ = i;
+            }
+         }
+         return val;
+      }
+   }   
+}

src/client/spark_api/utils/TimestampExtensions.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.api
+{
+   public static class TimestampExtensions
+   {
+      public static Timestamp add_nanos(this Timestamp lhs, int nanos)
+      {
+         var c = new Timestamp(lhs);
+         c.Nanos += nanos;
+         return c;
+      }
+      public static bool less(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) < 0;
+      }
+      public static bool less_or_equal(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) <= 0;
+      }
+      public static bool greater(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) > 0;
+      }
+      public static bool greater_or_equal(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) >= 0;
+      }
+      public static int compare(this Timestamp lhs, Timestamp rhs)
+      {
+         int secdiff = lhs.Seconds.CompareTo(rhs.Seconds);
+         return (secdiff != 0) ? secdiff : (lhs.Nanos.CompareTo(rhs.Nanos));
+      }
+      public static Comparer<Timestamp> timestamp_comparer => Comparer<Timestamp>.Create((x, y) => x.compare(y));
+
+      public static Timestamp parse_timestamp(string data)
+      {
+         DateTime dt = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+         try
+         {
+            if (data.Length <= 8)
+            {
+               dt = DateTime.ParseExact(data, "yyyyMMdd", System.Globalization.CultureInfo.InvariantCulture);
+            }
+            else
+            {
+               int sep = 14;
+               var yyMMddhhmmss = data.Substring(0, sep);
+               dt = DateTime.ParseExact(yyMMddhhmmss, "yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture);
+               var nnanos = data.Substring(sep, data.Length - sep);
+               long nano = long.Parse(nnanos);
+               dt = dt.AddTicks(nano / 100);
+            }
+         }
+         catch (Exception _)
+         {
+            System.Diagnostics.Debug.Assert(_ != null); //UNUSED(_)
+         }
+         dt = DateTime.SpecifyKind(dt, DateTimeKind.Utc);
+         var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(dt);
+         return ts;
+      }
+      public static string to_fix_string(this Timestamp ts)
+      {
+         return ts.ToDateTime().ToString("yyyyMMdd-HH:mm:ss.fff");
+      }
+   }
+}

src/client/spark_api/utils/TypeUtils.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text.RegularExpressions;
+
+namespace bts.api
+{
+   public class TypeUtils
+   {
+      static public List<Type> find_all_types_in_all_assemblies(string type, string assembly_regex = null)
+      {
+         List<Type> res = new List<Type>();
+         var asses = AppDomain.CurrentDomain.GetAssemblies();
+         foreach (Assembly a in asses)
+         {
+            if (!string.IsNullOrEmpty(assembly_regex))
+            {
+               if (!Regex.Match(a.GetName().Name, assembly_regex).Success)
+                  continue;
+            }
+            Type t = a.GetType(type, false, true);
+            if (t != null)
+               res.Add(t);
+         }
+         return res;
+      }
+   }   
+}

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -8,6 +8,34 @@ using System.Reactive.Linq;
 namespace System.Collections.Generic {
    public static class EnumerableExtensions {
       #region Ordered
+
+      public static int BinarySearchIndexOf<T>(this IList<T> list, T value, IComparer<T> comparer = null)
+      {
+         return BinarySearchIndexOf<T, T>(list, value, x => x, comparer);
+      }
+      
+      public static int BinarySearchIndexOf<T, K>(this IList<T> list, K key, Func<T, K> key_func, IComparer<K> comparer = null)
+      {
+         if(list == null)
+            throw new ArgumentNullException(nameof(list));
+         comparer = comparer ?? Comparer<K>.Default;
+
+         int lower = 0;
+         int upper = list.Count - 1;
+         while(lower <= upper)
+         {
+            int middle = lower + (upper - lower) / 2;
+            int result = comparer.Compare(key, key_func(list[middle]));
+            if(result == 0)
+               return middle;
+            else if(result < 0)
+               upper = middle - 1;
+            else
+               lower = middle + 1;
+         }
+         return ~lower;
+      }
+            
       public static IOrderedEnumerable<T> Ordered<T>(this IEnumerable<T> enumerable) =>
          enumerable.OrderBy(t => t);
 

src/client/spark_core/Properties/AssemblyAttributes.cs

@@ -2,4 +2,6 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_historical")]
+[assembly: InternalsVisibleTo("spark_analytics")]
 

src/client/spark_core/TestRunner/TestRunner.cs

@@ -70,10 +70,13 @@ namespace bts.test {
          psi.RedirectStandardOutput = true;
          psi.RedirectStandardError = true;
          psi.UseShellExecute = false;
-
+ 
          try{
             var p = Process.Start(psi);
             p.WaitForExit((int)time_out.TotalMilliseconds);
+            var errorlevel = p.ExitCode;
+            string stdout = p.StandardOutput.ReadToEnd();
+            string stderr = p.StandardError.ReadToEnd();
             if (p.HasExited) {
                return 0;
             }

src/client/spark_core/admin/AdminModelWrapper.cs

@@ -0,0 +1,64 @@
+using bts;
+using bts.api.admin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.api.core
+{
+   public class AdminModelWrapper
+   {
+      readonly AdminDataImmutable? _admin_data;
+      public AdminDataImmutable? admin_data => _admin_data;
+      readonly string _user_name;
+      readonly List<BTSAccountImmutable> _allowed_accounts;
+      readonly Dictionary<string, string> _allowed_account_dict = new Dictionary<string, string>();
+      public AdminModelWrapper(AdminDataImmutable? admin_data = null, string user_name = null)
+      {
+         _admin_data = admin_data;
+         _user_name = user_name ?? string.Empty;
+         _allowed_accounts = search_admin(u => u.Name == _user_name).Where(x =>
+         {
+              return (x.Item2 != null && x.Item2.HasPermissions) || (x.Item3 != null && x.Item3.HasPermissions);
+         }).Select(xx => xx.Item1).ToList();
+         foreach (var val in _allowed_accounts)
+            _allowed_account_dict[val.Name] = val.Name;
+      }
+      public IEnumerable<BTSAccountImmutable> viewable_accounts()
+      {
+         return _allowed_accounts;
+      }
+      public bool can_view_account(string account)
+      {
+         return _allowed_account_dict.ContainsKey(account) || _user_name.ToLower().Equals("bts");
+      }
+
+      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>> 
+         search_admin(Func<BTSUserImmutable, bool> predicate)
+      {
+         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>>();
+         if (_admin_data.HasValue)
+         {
+            var users = _admin_data.Value.Users.Where(predicate);
+            if (users != null && users.Count() == 1)
+            {
+               var user = users.ElementAt(0);
+               if (user.HasUserData && user.UserData.HasAccountPermissions)
+               {
+                  foreach (var kv in user.UserData.AccountPermissions.AccountPermissions)
+                  {
+                     var account_id = kv.Key;
+                     var accounts = _admin_data.Value.Accounts.Where(a => a.Id == account_id);
+                     if (accounts != null && accounts.Count() == 1)
+                     {
+                        var account = accounts.ElementAt(0);
+                        rv.Add(Tuple.Create(account, kv.Value.TradingPermission, kv.Value.AdminPermission));
+                     }
+                  }
+               }
+            }
+         }
+         return rv;
+      }
+   }
+}

src/client/spark_core/admin/PermissionFilters.cs

@@ -0,0 +1,64 @@
+using bts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.position;
+using bts.spark;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Concurrency;
+using System.Reactive.Linq;
+
+namespace bts.api.core
+{
+   public static class PermissionFilters
+   {
+      public static IObservable<Update<PositionUnitImmutable, int>> 
+         PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<PositionUnitImmutable, bool>>(observer =>
+         {
+            return am.Subscribe(x =>
+            {
+               Func<PositionUnitImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+            });
+         });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<OrderRecordImmutable, uint>> 
+         PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<OrderRecordImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.Def.Account ?? string.Empty));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<FillImmutable, string>> 
+         PermissionFilter(this IObservable<Update<FillImmutable, string>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<FillImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<FillImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+   }
+}

src/client/spark_core/spark_core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Platforms>x64</Platforms>
     <PlatformTarget>AnyCPU</PlatformTarget>

src/client/spark_core/utils/CsvHelper.cs

@@ -0,0 +1,52 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reflection;
+
+namespace bts.utils.csv {
+   public static class CsvHelper {
+
+      public static string ToCsv<T>(this IEnumerable<T> items, OrderedDictionary properties = null) where T : class {
+         var output = "";
+         var delimiter = ",";
+         if (properties == null) {
+            properties = new OrderedDictionary();
+            foreach (var pi in typeof(T).GetProperties()) {
+               properties.Add(pi.Name, pi);
+            }
+         }
+
+         using (var sw = new StringWriter()) {
+            var dict_values = properties.Values.OfType<PropertyInfo>();
+            var header = dict_values.Select(n => n.Name).Aggregate((first, second) => first + delimiter + second);
+            sw.WriteLine(header);
+            foreach (var i in items) {
+               var row = dict_values.Select(x => x.GetValue(i, null))
+                  .Select(x => x == null ? "null" : x.ToString())
+                  .Aggregate((first, second) => first + delimiter + second);
+               sw.WriteLine(row);
+            }
+            output = sw.ToString();
+         }
+         return output;
+      }
+
+      public static OrderedDictionary create_ordering_maintainer(Type t, string[] property_names_in_order) {
+         OrderedDictionary od = new OrderedDictionary();
+         var props = t.GetProperties();
+         foreach (var n in property_names_in_order) {
+            var query = props.Where(pi => pi.Name == n);
+            if (query.Any()) {
+               var property_info = query.First();
+               if (!od.Contains(property_info)) {
+                  od.Add(n, property_info);
+               }
+            }
+         }
+         return od;
+      }
+
+   }
+}

src/client/spark_core/utils/FilePathHelper.cs

@@ -1,21 +0,0 @@
-using System;
-using System.IO;
-
-namespace bts.utils {
-   public static class FilePathHelper {
-      public static string GetUniquePathWithDate(string path_root, string ext) =>
-         GetUniquePath($"{path_root}_{DateTime.Now:MM - dd - yyyy_hh - mm}", ext);
-
-      public static string GetUniquePath(string path_root, string ext) {
-         var path = $"{path_root}.{ext}";
-         const int max_copies = 2 ^ 14;
-         int i = 1;
-         while (File.Exists(path)) {
-            path = $"{path_root}({i++}).{ext}";
-            if (i == max_copies)
-               throw new InvalidOperationException("Unable to make path unique by concating a number.");
-         }
-         return path;
-      }
-   }
-}

src/client/spark_core/utils/GlobalExtensions.cs

@@ -66,7 +66,6 @@ namespace mbts {
             }
          }
       }
-
       public static void sync_with_models<T, TModel>(this IList<T> target, IEnumerable<TModel> models, Func<TModel, T> add_factory, Func<T, TModel, T> update_factory) {
          var model_list = models as IList<TModel> ?? models.ToList();
          if (model_list.Count == 0) {

src/client/spark_core/utils/ObservableExtensions.cs

@@ -209,7 +209,7 @@ namespace System.Reactive.Linq {
       /// <returns>An IDisposable object used to unsubscribe from the source sequence, and dispose the latest value subscription.</returns>
       public static IDisposable SubscribeEach<T>(this IObservable<T> source, Func<T, IDisposable> subscribe_value) {
          IDisposable value_sub = null;
-         IDisposable source_sub = source.Subscribe(on_source);
+         IDisposable source_sub = source.Select(v => v).Subscribe(on_source);
          return Disposable.Create(dispose);
          // local methods
          void on_source(T value) {

src/client/spark_core/utils/UpdateExtensions.cs

@@ -1,253 +1,208 @@
-using bts.api.core;
-using bts.collections;
-using bts.utils.proto;
-using Google.Protobuf;
-using mbts;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Reactive.Linq;
-
-namespace bts.utils {
-   public static class UpdateExtensions {
-      public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
-
-      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> with_root,
-         Func<T, TResult> without_root,
-         Func<TResult, TKey> result_key)
-         where TKey : IEquatable<TKey> =>
-         RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
-
-      // TODO: unit tests
-      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent,
-         Func<TResult, TKey> result_key,
-         TKey no_parent_key)
-         where TKey : IEquatable<TKey> =>
-         Observable.Create<TableUpdate<TResult, TKey>>(observer => {
-            var state = DictionaryHelper.empty<TKey, T>();
-            var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
-            var parent_to_children = new SetMap<TKey, T>();
-            return source.Subscribe(update);
-            //
-            void update(TableUpdate<T, TKey> input) {
-               state = input.state;
-               var builder = output.to_builder();
-               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
-               foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
-                  var k = input.key_extractor(v);
-                  parent_to_children.remove(parent_key(v), v);
-               }
-               foreach (var v in input.added.Concat(input.changed)) {
-                  var pk = parent_key(v);
-                  if (!pk.Equals(no_parent_key))
-                     parent_to_children.add(pk, v);
-               }
-               foreach (var v in input.removed) {
-                  var k = input.key_extractor(v);
-                  builder.remove(k);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children) {
-                        var ck = key(child);
-                        builder.add_or_update(ck, get_result_no_parent(child));
-                        foreach (var d in descendents(ck))
-                           builder.add_or_update(key(d), get_result(d, child));
-                     }
-                  }
-               }
-               foreach (var v in input.changed) {
-                  var k = input.key_extractor(v);
-                  var root = get_root(v);
-                  TResult result;
-                  if (key(root).Equals(k)) {
-                     result = get_result_no_parent(v);
-                     foreach (var d in descendents(k))
-                        builder.add_or_update(key(d), get_result(d, v));
-                  } else {
-                     result = get_result(v, root);
-                  }
-                  builder.add_or_update(k, result);
-               }
-               foreach (var v in input.added) {
-                  var k = input.key_extractor(v);
-                  var pk = parent_key(v);
-                  TResult result;
-                  if (pk.Equals(no_parent_key)) {
-                     result = get_result_no_parent(v);
-                     foreach (var d in descendents(k))
-                        builder.add_or_update(key(d), get_result(d, v));
-                  } else {
-                     parent_to_children.add(pk, v);
-                     var root = get_root(v);
-                     if (key(root).Equals(k)) {
-                        result = get_result_no_parent(v);
-                        foreach (var d in descendents(k))
-                           builder.add_or_update(key(d), get_result(d, v));
-                     } else {
-                        result = get_result(v, root);
-                     }
-                  }
-                  builder.add_or_update(k, result);
-               }
-               observer.OnNext(output = builder.to_table());
-            }
-            T get_root(T value) {
-               var pk = parent_key(value);
-               while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
-                  value = parent;
-                  pk = parent_key(value);
-               }
-               return value;
-            }
-            IEnumerable<T> descendents(TKey k) {
-               if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                  foreach (var child in children) {
-                     yield return child;
-                     foreach (var d in descendents(key(child)))
-                        yield return d;
-                  }
-               }
-            }
-         });
-
-      public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
-         source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
-
-      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent)
-         where TResult : IEquatable<TResult>
-         where TKey : IEquatable<TKey> =>
-         ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
-
-      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent,
-         IEqualityComparer<TResult> result_comparer,
-         TKey no_parent_key)
-         where TKey : IEquatable<TKey> =>
-         Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
-            var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
-            var state = DictionaryHelper.empty<TKey, T>();
-            var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
-            var parent_to_children = new SetMap<TKey, T>();
-            return source.Subscribe(update);
-            //
-            void update(TableUpdate<T, TKey> input) {
-               state = input.state;
-               var builder = output.to_builder();
-               foreach (var v in input.removed) {
-                  var k = input.key_extractor(v);
-                  builder.remove(k);
-                  parent_to_children.remove(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
-                  }
-               }
-               foreach (var v in input.changed) {
-                  var k = key(v);
-                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
-                  parent_to_children.set(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
-                  }
-               }
-               foreach (var v in input.added) {
-                  var k = key(v);
-                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
-                  parent_to_children.add(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
-                  }
-               }
-               var result = builder.to_table_checked(value_comparer);
-               if (result.has_changes())
-                  observer.OnNext(output = result);
-            }
-            TResult lookup_result(T child) {
-               var pk = parent_key(child);
-               return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ?  get_result(child, parent) : get_result_no_parent(child);
-            }
-         });
-
-      static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
-         public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
-         public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
-
-         static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
-            TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
-
-         static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
-            TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
-      }
-   }
-}
-namespace bts.utils.diagnostics {
-   public static class DiagnosticUpdateExtensions {
-      // Hard to imagine why this util would be needed outside of diagnosig bugs.
-      // The #if DEBUG directive is an extra precaution to avoid accidentally writing
-      // files like this on a client's machine.
-#if DEBUG
-      static readonly string _proto_capture_ext = "proto_capture";
-
-      public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, Func<T, TProto> to_message)
-         where TKey : IEquatable<TKey>
-         where TProto : class, IMessage<TProto> =>
-         ProtoBufCapture(source,
-            FilePathHelper.GetUniquePathWithDate(Path.Combine(Directory.GetCurrentDirectory(), typeof(TProto).Name), _proto_capture_ext),
-            to_message);
-
-      public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, string path, Func<T, TProto> to_message)
-         where TKey : IEquatable<TKey>
-         where TProto : class, IMessage<TProto> =>
-         Observable.Create<TableUpdate<T, TKey>>(o => {
-            // write the state on dispose, and/or on error
-            IReadOnlyDictionary<TKey, T> state = DictionaryHelper.empty<TKey, T>();
-            var sub = source.Subscribe(table => {
-               state = table.state;
-               o.OnNext(table);
-            }, ex => {
-               save(true);
-               o.OnError(ex);
-            },
-            o.OnCompleted);
-            return Disposable.Create(() => {
-               save(false);
-               Disposable.dispose(ref sub);
-            });
-            void save(bool has_error) {
-               if (state.Count > 0)
-                  FileStreamHelper.WriteProtos(has_error ? $"{path}(error)" : path, state.Values.Select(to_message).ExceptNull());
-            }
-         });
-#endif
-   }
-}
+using bts.api.core;
+using bts.collections;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+
+namespace bts.utils {
+   public static class UpdateExtensions {
+      public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
+
+      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> with_root,
+         Func<T, TResult> without_root,
+         Func<TResult, TKey> result_key)
+         where TKey : IEquatable<TKey> =>
+         RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
+
+      // TODO: unit tests
+      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent,
+         Func<TResult, TKey> result_key,
+         TKey no_parent_key)
+         where TKey : IEquatable<TKey> =>
+         Observable.Create<TableUpdate<TResult, TKey>>(observer => {
+            var state = DictionaryHelper.empty<TKey, T>();
+            var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
+            var parent_to_children = new SetMap<TKey, T>();
+            return source.Subscribe(update);
+            //
+            void update(TableUpdate<T, TKey> input) {
+               state = input.state;
+               var builder = output.to_builder();
+               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
+               foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
+                  var k = input.key_extractor(v);
+                  parent_to_children.remove(parent_key(v), v);
+               }
+               foreach (var v in input.added.Concat(input.changed)) {
+                  var pk = parent_key(v);
+                  if (!pk.Equals(no_parent_key))
+                     parent_to_children.add(pk, v);
+               }
+               foreach (var v in input.removed) {
+                  var k = input.key_extractor(v);
+                  builder.remove(k);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children) {
+                        var ck = key(child);
+                        builder.add_or_update(ck, get_result_no_parent(child));
+                        foreach (var d in descendents(ck))
+                           builder.add_or_update(key(d), get_result(d, child));
+                     }
+                  }
+               }
+               foreach (var v in input.changed) {
+                  var k = input.key_extractor(v);
+                  var root = get_root(v);
+                  TResult result;
+                  if (key(root).Equals(k)) {
+                     result = get_result_no_parent(v);
+                     foreach (var d in descendents(k))
+                        builder.add_or_update(key(d), get_result(d, v));
+                  } else {
+                     result = get_result(v, root);
+                  }
+                  builder.add_or_update(k, result);
+               }
+               foreach (var v in input.added) {
+                  var k = input.key_extractor(v);
+                  var pk = parent_key(v);
+                  TResult result;
+                  if (pk.Equals(no_parent_key)) {
+                     result = get_result_no_parent(v);
+                     foreach (var d in descendents(k))
+                        builder.add_or_update(key(d), get_result(d, v));
+                  } else {
+                     parent_to_children.add(pk, v);
+                     var root = get_root(v);
+                     if (key(root).Equals(k)) {
+                        result = get_result_no_parent(v);
+                        foreach (var d in descendents(k))
+                           builder.add_or_update(key(d), get_result(d, v));
+                     } else {
+                        result = get_result(v, root);
+                     }
+                  }
+                  builder.add_or_update(k, result);
+               }
+               observer.OnNext(output = builder.to_table());
+            }
+            T get_root(T value) {
+               var pk = parent_key(value);
+               while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
+                  value = parent;
+                  pk = parent_key(value);
+               }
+               return value;
+            }
+            IEnumerable<T> descendents(TKey k) {
+               if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                  foreach (var child in children) {
+                     yield return child;
+                     foreach (var d in descendents(key(child)))
+                        yield return d;
+                  }
+               }
+            }
+         });
+
+      public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
+         source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
+
+      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent)
+         where TResult : IEquatable<TResult>
+         where TKey : IEquatable<TKey> =>
+         ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
+
+      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent,
+         IEqualityComparer<TResult> result_comparer,
+         TKey no_parent_key)
+         where TKey : IEquatable<TKey> =>
+         Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
+            var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
+            var state = DictionaryHelper.empty<TKey, T>();
+            var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
+            var parent_to_children = new SetMap<TKey, T>();
+            return source.Subscribe(update);
+            //
+            void update(TableUpdate<T, TKey> input) {
+               state = input.state;
+               var builder = output.to_builder();
+               foreach (var v in input.removed) {
+                  var k = input.key_extractor(v);
+                  builder.remove(k);
+                  parent_to_children.remove(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
+                  }
+               }
+               foreach (var v in input.changed) {
+                  var k = key(v);
+                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
+                  parent_to_children.set(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
+                  }
+               }
+               foreach (var v in input.added) {
+                  var k = key(v);
+                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
+                  parent_to_children.add(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
+                  }
+               }
+               var result = builder.to_table_checked(value_comparer);
+               if (result.has_changes())
+                  observer.OnNext(output = result);
+            }
+            TResult lookup_result(T child) {
+               var pk = parent_key(child);
+               return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ?  get_result(child, parent) : get_result_no_parent(child);
+            }
+         });
+
+      static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
+         public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
+         public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
+
+         static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
+            TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
+
+         static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
+            TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
+      }
+   }
+}

src/client/spark_excel/ExcelAddin.cs

@@ -12,6 +12,7 @@ using System.Diagnostics;
 using ExcelDna.Integration;
 using ExcelDna.IntelliSense;
 using ExcelDna.Logging;
+using ExcelDna.Registration;
 
 namespace bts.excel
 {
@@ -54,9 +55,14 @@ namespace bts.excel
          Trace.Listeners.Add(new LogDisplayTraceListener("bts"));
          IntelliSenseServer.Install();
          ExcelIntegration.RegisterUnhandledExceptionHandler(ex => ex.ToString());
+         
+         //https://github.com/Excel-DNA/ExcelDna/issues/255
+         ExcelRegistration.GetExcelFunctions()
+                          .ProcessAsyncRegistrations(nativeAsyncIfAvailable: false)
+                          .RegisterFunctions();         
 
          load_assemblies();
-         ExcelComAddInHelper.LoadComAddIn(new ContextTeardown(APIDispatch.service_context.Value));         
+         ExcelComAddInHelper.LoadComAddIn(new ContextTeardown(APIDispatch.service_context.Value));
       }
 
       private void load_assemblies()

src/client/spark_excel/analytics/AnalyticsWrappers.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text.RegularExpressions;
+using ExcelDna.Integration;
+using bts.api;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using bts.analytics;
+using System.Linq;
+
+namespace bts.excel
+{
+   public class AnalyticsWrappers
+   {
+      [ExcelFunction]
+      public static double bts_interpolate_vol(double[] t, double[] vol, double x)
+      {
+         return VegaFactor.interpolate_vol_array(t, vol, x);
+      }   
+      [ExcelFunction]
+      static public double bts_expected_move(double spot, double growth, double vol, double t)
+      {
+         return BlackScholes.expected_move(spot, growth, vol, t);
+      }
+      [ExcelFunction]
+      static public double bts_atm_straddle(double spot, double riskless, double growth, double vol, double bizt, double calt)
+      {
+         return BlackScholes.atm_straddle(spot, riskless, growth, vol, bizt, calt);
+      }
+      [ExcelFunction]
+      static public double[] bts_liquidity_cost(double qty, double tick_size,
+                                              double[] bookprices, double[] booksizes, bool is_buy, bool reverse)
+      {
+         double residual = 0.0;
+         var cost = AlmgrenChriss.liquidity_cost(qty, tick_size, bookprices.ToArray(), bookprices.Length, 
+                                                 booksizes.Select(x => Convert.ToInt32(x)).ToArray(), bookprices.Length, is_buy, reverse, ref residual);
+         return new[]{cost, residual};
+      }
+      [ExcelFunction]
+      static public double bts_ac_alpha(double s_0, double s_t, bool is_buy, double time_factor)
+      {
+         return AlmgrenChriss.ac_alpha(s_0, s_t, is_buy, time_factor);
+      }
+      [ExcelFunction]
+      static public double bts_ac_sigma(double vol, double dt, double s_t, double time_factor)
+      {
+         return AlmgrenChriss.ac_sigma(vol, dt, s_t, time_factor);
+      }
+      [ExcelFunction]
+      static public double bts_ac_sigma_opt(double vol, double dt, double s_t, double time_factor)
+      {
+         return AlmgrenChriss.ac_sigma_opt(vol, dt, s_t, time_factor);
+      }
+      [ExcelFunction] 
+      public static double bts_ac_xt(double aggressivity, double vol, double dt, double T, double s0, double st, double x0,
+                                     double cost, double adaptivity, int generation, int ngenerations, bool is_buy, double time_factor)
+      {
+         return AlmgrenChriss.ac_xt(aggressivity, vol, dt, T, s0, st, x0, cost, adaptivity, generation, ngenerations, is_buy, time_factor);
+      }
+      [ExcelFunction]
+      public static double bts_ac_xt_sigma_alpha(double lambda, double expmove_sigma, double dt, double T, double alpha, double x0,
+                                             double cost, double adaptivity, int generation, int ngenerations)
+      {
+         return AlmgrenChriss.ac_xt_sigma_alpha(lambda, expmove_sigma, dt, T, alpha, x0, cost, adaptivity, generation, ngenerations);
+      }
+      [ExcelFunction] 
+      public static double bts_implied_bizt(double fwd, double rawk, double riskless, double Vol, double tguess,
+                                            bool is_call, double target_price, int max_iterations = 10)
+      {
+         return BlackScholes.implied_bizt(fwd, rawk, riskless, Vol, tguess, is_call, target_price, max_iterations);
+      }
+      [ExcelFunction]
+      static public double bts_ac_optimal_duration(double vol, double dt_seconds, double s_t, double cost_per_share, double time_factor)
+      {
+         return AlmgrenChriss.ac_optimal_duration(vol, dt_seconds, s_t, cost_per_share, time_factor);
+      }
+   }
+}

src/client/spark_excel/extensions/OrderService_Excel.cs

@@ -11,7 +11,7 @@ using bts.excel;
 using bts.excel.extensions;
 using bts.excel.rtd;
 using Newtonsoft.Json;
-
+using System.Threading.Tasks;
 
 namespace bts.api.orderman
 {
@@ -21,7 +21,7 @@ namespace bts.api.orderman
       readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => Locator.locate<admin.AdminService>(), true);
 
       [ExcelFunction(Category = "BTS", Description = "Get All Accessible Accounts")]
-      async public static void spark_get_accounts(string login_token, ExcelAsyncHandle handle)
+      async public static Task<object> spark_get_accounts(string login_token)
       {
          try
          {
@@ -34,11 +34,11 @@ namespace bts.api.orderman
                      accts.Add(acc.Name);
             }
             var res = Utilities.make_column_vector(accts.ToArray());
-            handle.SetResult(res);
+            return res;
          }
          catch (Exception ex)
          {
-            handle.SetException(ex);
+            return ex.ToString();
          }
       }
 

src/client/spark_excel/rtd/formatting/AlphanumComparer.cs

@@ -0,0 +1,157 @@
+/*
+ * The Alphanum Algorithm is an improved sorting algorithm for strings
+ * containing numbers.  Instead of sorting numbers in ASCII order like
+ * a standard sort, this algorithm sorts numbers in numeric order.
+ *
+ * The Alphanum Algorithm is discussed at http://www.DaveKoelle.com
+ *
+ * Based on the Java implementation of Dave Koelle's Alphanum algorithm.
+ * Contributed by Jonathan Ruckwood <jonathan.ruckwood@gmail.com>
+ *
+ * Adapted by Dominik Hurnaus <dominik.hurnaus@gmail.com> to
+ *   - correctly sort words where one word starts with another word
+ *   - have slightly better performance
+ *
+ * Released under the MIT License - https://opensource.org/licenses/MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
+/*
+ * Please compare against the latest Java version at http://www.DaveKoelle.com
+ * to see the most recent modifications
+ */
+namespace Alphanum
+{
+   public class Alphanum
+   {
+      static public IComparer<string> comparer()
+      {
+         return new AlphanumComparator<string>();
+      }
+   }
+   
+   class AlphanumComparator<T> : IComparer<T>
+   {
+      private enum ChunkType { Alphanumeric, Numeric };
+      private bool InChunk(char ch, char otherCh)
+      {
+         ChunkType type = ChunkType.Alphanumeric;
+
+         if (char.IsDigit(otherCh))
+         {
+            type = ChunkType.Numeric;
+         }
+
+         if ((type == ChunkType.Alphanumeric && char.IsDigit(ch))
+             || (type == ChunkType.Numeric && !char.IsDigit(ch)))
+         {
+            return false;
+         }
+
+         return true;
+      }
+
+      public int Compare(T x, T y)
+      {
+         String s1 = x as string;
+         String s2 = y as string;
+         if (s1 == null || s2 == null)
+         {
+            return 0;
+         }
+
+         int thisMarker = 0, thisNumericChunk = 0;
+         int thatMarker = 0, thatNumericChunk = 0;
+
+         while ((thisMarker < s1.Length) || (thatMarker < s2.Length))
+         {
+            if (thisMarker >= s1.Length)
+            {
+               return -1;
+            }
+            else if (thatMarker >= s2.Length)
+            {
+               return 1;
+            }
+            char thisCh = s1[thisMarker];
+            char thatCh = s2[thatMarker];
+
+            StringBuilder thisChunk = new StringBuilder();
+            StringBuilder thatChunk = new StringBuilder();
+
+            while ((thisMarker < s1.Length) && (thisChunk.Length == 0 || InChunk(thisCh, thisChunk[0])))
+            {
+               thisChunk.Append(thisCh);
+               thisMarker++;
+
+               if (thisMarker < s1.Length)
+               {
+                  thisCh = s1[thisMarker];
+               }
+            }
+
+            while ((thatMarker < s2.Length) && (thatChunk.Length == 0 || InChunk(thatCh, thatChunk[0])))
+            {
+               thatChunk.Append(thatCh);
+               thatMarker++;
+
+               if (thatMarker < s2.Length)
+               {
+                  thatCh = s2[thatMarker];
+               }
+            }
+
+            int result = 0;
+            // If both chunks contain numeric characters, sort them numerically
+            if (char.IsDigit(thisChunk[0]) && char.IsDigit(thatChunk[0]))
+            {
+               thisNumericChunk = Convert.ToInt32(thisChunk.ToString());
+               thatNumericChunk = Convert.ToInt32(thatChunk.ToString());
+
+               if (thisNumericChunk < thatNumericChunk)
+               {
+                  result = -1;
+               }
+
+               if (thisNumericChunk > thatNumericChunk)
+               {
+                  result = 1;
+               }
+            }
+            else
+            {
+               result = thisChunk.ToString().CompareTo(thatChunk.ToString());
+            }
+
+            if (result != 0)
+            {
+               return result;
+            }
+         }
+
+         return 0;
+      }
+   }
+}

src/client/spark_excel/rtd/formatting/BookMDFormatter.cs

@@ -16,10 +16,11 @@ namespace bts.excel.rtd
          return format(_update, include_headers);
       }
 
-      static public object[,] format(IMDBook update, bool include_headers = false)
+      static public object[,] format(IMDBook update, bool include_headers = false, int rows = 0)
       {
          const int cols = 5;
-         const int rows = 30;
+         if(rows <= 0)
+            rows = 30;
          var ret = new object[rows, cols];
          for (int i = 0; i < rows; i++)
             for (int j = 0; j < cols; j++)
@@ -28,11 +29,11 @@ namespace bts.excel.rtd
          if (update != null)
          {
             int idx = 0;
-            for (int j = update.asks.Count - 1; j >= 0 && idx < rows; j--)
+            for (int j = 0; j < update.asks.Count && idx < rows/2; j++)
             {
                var ask = update.asks[j];
-               ret[idx, 1] = ask.price;
-               ret[idx, 2] = ask.size;
+               ret[rows/2-idx-1, 1] = ask.price;
+               ret[rows/2-idx-1, 2] = ask.size;
                idx++;
             }
             for (int j = 0; j < update.bids.Count && idx < rows; j++)

src/client/spark_excel/rtd/formatting/JsonFormatter.cs

@@ -88,7 +88,7 @@ namespace bts.excel.rtd
             json = JsonConvert.SerializeObject(data);
 
          JObject jo = JObject.Parse(json);
-         SortedDictionary<string, object> results = new SortedDictionary<string, object>();
+         SortedDictionary<string, object> results = new SortedDictionary<string, object>(Alphanum.Alphanum.comparer());
          parse_object(jo, results);
          return results;
       }
@@ -112,7 +112,7 @@ namespace bts.excel.rtd
       object[,] fill_object_array(IEnumerable<SortedDictionary<string, object>> data, bool include_headers = false)
       {
          int count = 0;
-         SortedDictionary<string, string> keys = new SortedDictionary<string, string>();
+         SortedDictionary<string, string> keys = new SortedDictionary<string, string>(Alphanum.Alphanum.comparer());
          foreach (var d in data)
          {
             count++;

src/client/spark_excel/rtd/subscriptions/MDSubscribe.cs

@@ -8,30 +8,54 @@ using ExcelDna.Registration.Utils;
 using System.Reactive.Linq;
 using bts.api;
 using bts.md;
-using bts.api.orderman;
+using System.Threading.Tasks;
+using System.Reactive.Disposables;
 
 namespace bts.excel.rtd
 {
    public class MDSubscribe
    {
       [ExcelFunction(Category = "BTS", Description = "Log in to BTS Spark")]
-      async public static void spark_login(string login, string password, ExcelAsyncHandle handle) 
+      public static object spark_login(string login = null, string password = null) 
       {
-         try
+         Locator.init();
+         return ObservableRtdUtil.Observe(nameof(observable_login), new object[]{login, password}, 
+                                          () => observable_login(login, password));
+      }
+
+      static IObservable<object> observable_login(string login, string password)
+      {
+         login = login ?? string.Empty;
+         password = password ?? string.Empty;
+         bool logged_in = false;
+         return Observable.Create<object>( obs => 
          {
-            var res = await Locator.login(login, password);
-            if (res.success && Locator.token.HasValue)
+            return Observable.Timer(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5)).Subscribe( async _ =>
             {
-               _md.set_auth_token(Locator.token);
-               handle.SetResult("Login Successful");
-            }
-            else
-               handle.SetResult("Login Error: Could not log in check user/pass " + res.reason);
-         }
-         catch(Exception ex)
-         {
-            handle.SetException(ex);
-         }
+               if(logged_in)
+               {
+                  //obs.OnCompleted(); //don't call this if you want the rtd to restart on open
+                  return;
+               }
+               var now = DateTime.Now.ToShortTimeString();
+               try
+               {
+                  var res = await Locator.login(login, password);
+                  if (res.success && Locator.token.HasValue)
+                  {
+                     _md.set_auth_token(Locator.token);
+                     obs.OnNext("Login Successful " + now);
+                     logged_in = true;
+                  }
+                  else
+                     obs.OnNext("Login Error: Could not log in check user/pass " + res.reason + " " + now);
+               }
+               catch(Exception ex)
+               {
+                  obs.OnNext("Login Exception: " + ex.ToString() + " " + now);
+               }               
+            });
+         });
       }
 
       static MarketDataClient _md = new MarketDataClient();
@@ -92,16 +116,16 @@ namespace bts.excel.rtd
                                     [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
                                     [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,                                    
                                     [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
-                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
+                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null, int rows = 0)
       {
 
-         var param = new object[] { contract, symbol, security_type, exchange };
+         var param = new object[] { contract, symbol, security_type, exchange, rows };
          return ObservableRtdUtil.Observe(nameof(spark_md_book), param, 
                                           () => md_observable(contract, symbol, security_type, exchange).Select(data =>
                                           {
                                              var x = data.FirstOrDefault();
                                              if (x != null && x is IMDBook upd)
-                                                return BookMDFormatter.format(upd);
+                                                return BookMDFormatter.format(upd, false, rows);
                                              return x;
                                           }));
          // security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;

src/client/spark_excel/spark_excel-addin.dna

@@ -1,6 +1,7 @@
 <DnaLibrary Name="spark_excel" RuntimeVersion="v4.0">
-  <ExternalLibrary Path="spark_excel.dll" ExplicitExports="false" LoadFromBytes="true" Pack="true" />
+   <ExternalLibrary Path="spark_excel.dll" ExplicitExports="false" ExplicitRegistration="true" LoadFromBytes="true" Pack="true" />
   
+   <Reference Path="spark_analytics.dll" Pack="true" ExplicitExports="true" />
    <Reference Path="ExcelDna.IntelliSense.dll" Pack="true" />
    <Reference Path="ExcelDna.Registration.dll" Pack="true" />
    <Reference Path="ExcelDna.Integration.dll" Pack="true" />   

src/client/spark_excel/spark_excel.csproj

@@ -26,13 +26,13 @@
     <PackageReference Include="System.Reactive" Version="3.1.1" />
     <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
 	 <PackageReference Include="NUnit" Version="3.10.1" />  
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="System.Linq.Dynamic" Version="1.0.7" />
     
-    <PackageReference Include="ExcelDna.AddIn" Version="0.34.6" />  
-    <PackageReference Include="ExcelDna.Integration" Version="0.34.6" />      
-    <PackageReference Include="ExcelDna.IntelliSense" Version="1.1.0" />      
-    <PackageReference Include="ExcelDna.Registration" Version="0.34.7" />      
+    <PackageReference Include="ExcelDna.AddIn" Version="1.0.0" />
+    <PackageReference Include="ExcelDna.Integration" Version="1.0.0" />
+    <PackageReference Include="ExcelDna.IntelliSense" Version="1.1.0" />
+    <PackageReference Include="ExcelDna.Registration" Version="1.0.0" />
   </ItemGroup>
   
   <ItemGroup>    
@@ -55,6 +55,7 @@
   
   <ItemGroup>
     <ProjectReference Include="../spark_api/spark_api.csproj"/>
+    <ProjectReference Include="../spark_analytics/spark_analytics.csproj"/>
   </ItemGroup>
   
   <ItemGroup>
@@ -74,10 +75,10 @@
   </Target>  
    -->
   
-  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets" Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets')" /> -->
-  <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets"/>
+  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets" Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets')" /> -->
+  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets"/>
   <Target Name="EnsureExcelDnaTargetsImported" BeforeTargets="BeforeBuild" Condition="'$(ExcelDnaTargetsImported)' == ''">
-    <Error Condition="!Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="You are trying to build with ExcelDna, but the NuGet targets file that ExcelDna depends on is not available on this computer. This is probably because the ExcelDna package has not been committed to source control, or NuGet Package Restore is not enabled. Please enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
-    <Error Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="ExcelDna cannot be run because NuGet packages were restored prior to the build running, and the targets file was unavailable when the build started. Please build the project again to include these packages in the build. You may also need to make sure that your build server does not delete packages prior to each build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
-  </Target>  
+    <Error Condition="!Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="You are trying to build with ExcelDna, but the NuGet targets file that ExcelDna depends on is not available on this computer. This is probably because the ExcelDna package has not been committed to source control, or NuGet Package Restore is not enabled. Please enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
+    <Error Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="ExcelDna cannot be run because NuGet packages were restored prior to the build running, and the targets file was unavailable when the build started. Please build the project again to include these packages in the build. You may also need to make sure that your build server does not delete packages prior to each build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
+  </Target>   -->
 </Project>

src/client/spark_excel/utils/APIDispatch.cs

@@ -90,7 +90,14 @@ namespace bts.excel
             {
                if(string.IsNullOrEmpty(json))
                   json = "{}";
-               request = JsonParser.Default.Parse(json, desc);
+               try
+               {
+                  request = JsonParser.Default.Parse(json, desc);
+               }
+               catch(InvalidProtocolBufferException)
+               {
+                  request = (IMessage) Newtonsoft.Json.JsonConvert.DeserializeObject(json, desc.ClrType);
+               }
             }
          }
          return request;