Merge branch 'master' of bitbucket.org:bluetradesys/spark into vwap

# Conflicts:
#	src/client/spark_historical/Notebooks/roll_cover.ipynb

.gitignore

@@ -34,4 +34,7 @@ trace_data/
 /doc/user/public/*
 .ionide/*
 /.ignore
+CMakeDeps.cmake
 /build
+.fake
+.ionide

.vscode/launch.json

@@ -44,9 +44,10 @@
                 "debuggerPath": "/usr/bin/gdb"
             },
             "sourceFileMap": {
-                "/home/bts/spark/": "${workspaceRoot}"
+                "/home/bts/spark/": "${workspaceRoot}",
+                "/home/bts/spark_host/": "${workspaceRoot}",
             }
-        },
+        },     
         {
             "name": "(gdb) Launch",
             "type": "cppdbg",
@@ -98,19 +99,87 @@
             "externalConsole": true,
             "requireExactSource": false
         },
+            {
+         "name": "sparkcmd Launch",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "vol", 
+            "--server",
+             //"--rstart", "11/22/2019",
+             //"--rend", "11/25/2019",
+             //"--rsymbol", "ES",
+             //"--rsecuritytype", "Synthetic",
+             //"--post_process_file_type", "Trade",
+             //"--upload_to_aws", 
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },  
+      {
+         "name": "sparkcmd simulation",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "historical",
+            "--rstart", "11/26/2019",
+            "--rend", "11/26/2019",
+            "--rsymbol", "ZN",
+            "--rsecuritytype", "Synthetic",
+            "--algo_security", "ZNH0;ZNZ9-ZNH0;ZNZ9",
+            "--algo_type", "RoundTrip",
+            "--algo_params", "size=10;tick_size=0.015625,0.0078125,0.015625;hedge_factors=1,1,-1",
+            "--skip_preanalysis", "true",         
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },       
         {
             "name": "Excel Launch",
             "type": "clr",
             "request": "launch",
-            "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
+            "program": "C:/Program Files/Microsoft Office/Office16/EXCEL.EXE",
             "args": [
                 "${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"
             ],
-            "cwd": "${workspaceRoot}",
+            "cwd": "${workspaceRoot}/build/Debug/bin/net471/",
             "stopAtEntry": false,
             "externalConsole": true,
             "requireExactSource": false
         },
+      {
+         "name": "Win C++ Launch",
+         "type": "cppvsdbg",
+         "request": "launch",
+         "program": "C:/Program Files/Microsoft Office/Office16/EXCEL.EXE",
+         "args": [
+            "${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"
+         ],
+         "cwd": "${workspaceRoot}/build/Debug/bin/net471/",
+         "stopAtEntry": false,
+         "environment": [],
+         "externalConsole": false
+      },        
+        {
+           "name": "Fix Launch",
+           "type": "clr",
+           "request": "launch",
+           "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+           "args": [
+              "fix",
+              "-order_session",
+           ],
+           "cwd": "${workspaceRoot}/build/Debug/bin/net471/",
+           "stopAtEntry": false,
+           "externalConsole": true,
+           "requireExactSource": false
+        },        
         {
             "name": "Client Tests",
             "type": "clr",
@@ -126,11 +195,18 @@
             "requireExactSource": false
         },
         {
-            "name": "Client Attach",
+            "name": ".NET Framework Attach",
             "type": "clr",
             "request": "attach",
             "processId": "${command:pickProcess}"
         },
+         {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickProcess}",
+            "requireExactSource": false
+         },        
         {
             "name": "Spreader Tests",
             "type": "cppdbg",
@@ -337,20 +413,24 @@
                 "quoteArgs": true
             },
             "sourceFileMap": {
-                "/home/bts/spark/": "${workspaceRoot}"
+                "/home/bts/spark_host/": "${workspaceRoot}"
             },
             "justMyCode": true
         },
         {
-            "name": ".NET Core Remote Launch",
+            "name": "sparkcmd vsdbg (c#)",
             "type": "coreclr",
             "request": "launch",
-            "program": "/usr/bin/dotnet",
+            "program": "sparkcmd",
             "args": [
-                "sparkcmd.dll",
-                "test",
-                "test_join_create_and_takeout_market"
+            "historical",
+            "--rstart", "11/20/2019",
+            "--rend", "11/20/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Spread",
+            "--post_process_db", "mbo_history",
             ],
+            "env": {"LD_LIBRARY_PATH" : "."},
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server",
             "pipeTransport": {
@@ -361,11 +441,77 @@
                 "debuggerPath": "/home/bts/vsdbg/vsdbg",
                 "pipeCwd": "${workspaceRoot}",
                 "quoteArgs": true
-            },
+            },           
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             },
             "justMyCode": false
-        },
+        },                  
+         {
+            "name": "sparkcmd gdb (c++)",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/home/bts/spark/build/ninja_debug/server/sparkcmd",
+            "args": [
+            "historical",
+            "--rstart", "10/21/2019",
+            "--rend", "10/21/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Future",
+            "--algo_security", "CLX9",
+            "--algo_type", "TWAP",
+            "--algo_params", "'size=100;profit_ticks=1;tick_size=1,1,1;;hedge_factors=1,1,-1'",
+            "--skip_preanalysis", "true",
+            ],   
+            "stopAtEntry": false,
+            "cwd": "/home/bts/spark/build/ninja_debug/server/",
+            "environment": [],
+            "externalConsole": true,
+            "MIMode": "gdb",
+            "preLaunchTask": "",
+            "logging": {
+               "engineLogging": false
+            },
+            "setupCommands": [
+               {
+                  "description": "Enable pretty-printing for gdb",
+                  "text": "enable pretty-printer",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "Skip STL",
+                  "text": "skip -gfi /usr/include/c++/7/bits/*.h",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "LD LIBRARY",
+                  "text": "set env LD_LIBRARY_PATH=.",
+                  "ignoreFailures": false
+               }
+            ],
+            "pipeTransport": {
+               "pipeCwd": "${workspaceRoot}",
+               "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+               "pipeArgs": [
+                  "bts@sparkvm"
+               ],
+               "debuggerPath": "/usr/bin/gdb"
+            },
+            "sourceFileMap": {
+               "/home/bts/spark/": "${workspaceRoot}"
+            }
+         },
+         {
+            "name": "Jupyter notebook",
+            "type": "coreclr",
+            "request": "launch",
+            "program": "jupyter",
+            "args": ["notebook", "--notebook-dir=c:\\code\\bts\\spark\\src\\client\\spark_historical\\Notebooks"],
+            "cwd": "${workspaceFolder}",
+            "stopAtEntry": false,
+            "console": "internalConsole",
+            
+         },
+         
     ]
 }

CMakeLists.txt

@@ -48,6 +48,7 @@ else ()
   add_subdirectory(doc/user)
   add_subdirectory(src/sdk)
   add_subdirectory(src/data_files)
+  add_subdirectory(src/client)
 
   add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
@@ -59,7 +60,17 @@ else ()
     COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
     COMMENT "Creating server runtime layout"
+    DEPENDS sparkd data_files_dir spark_client_dir
     DEPENDS sparkd data_files_dir)  
 
   add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+  
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.1/linux-x64/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+else()
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.1/linux-x64/publish/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+endif()
+      
 endif()

debug.log

@@ -0,0 +1 @@
+[0108/231015.447:ERROR:file_io.cc(89)] ReadExactly: expected 36, observed 0

doc/user/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
+set(sources /home/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

spark_client.sln

@@ -23,6 +23,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\sp
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_excel", "src\client\spark_excel\spark_excel.csproj", "{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_fix", "src\client\spark_fix\spark_fix.csproj", "{D8031C13-7BEA-430A-BB60-40927D15CFC3}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_historical", "src\client\spark_historical\spark_historical.csproj", "{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_analytics", "src\client\spark_analytics\spark_analytics.csproj", "{22DBF34A-961F-4620-91B8-21B0C257A8C6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -219,6 +225,74 @@ Global
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.ActiveCfg = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.Build.0 = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x64.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Debug|x86.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x64.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x64.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x86.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.Release|x86.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+		{22DBF34A-961F-4620-91B8-21B0C257A8C6}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -229,4 +303,7 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 EndGlobal

src/client/CMakeLists.txt

@@ -0,0 +1,19 @@
+
+cmake_minimum_required(VERSION 3.5)
+project(spark_client)
+include(${CMAKE_SOURCE_DIR}/tools/safe_dep_glob.cmake)
+
+file(GLOB_RECURSE spark_client_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cs)
+file(GLOB_RECURSE spark_client_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.csproj)
+file(GLOB_RECURSE spark_ui_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.cs)
+file(GLOB_RECURSE spark_ui_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.csproj)
+list(REMOVE_ITEM spark_ui_sources ${spark_client_sources} )
+list(REMOVE_ITEM spark_ui_csproj ${spark_client_csproj} )
+
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/../
+   COMMAND /bin/bash ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh
+   ARGS ${CMAKE_BUILD_TYPE} ${CMAKE_BINARY_DIR}
+   COMMENT "Building sparkcmd in netcore3"
+   DEPENDS libspark ${spark_client_sources} ${spark_client_csproj} ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh)
+
+   add_custom_target(spark_client_dir ALL DEPENDS ${CMAKE_BINARY_DIR}/../)   

src/client/Directory.Build.props

@@ -5,6 +5,7 @@
     <SparkRoot>$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\..))</SparkRoot>
     <SourceRoot>$([System.IO.Path]::GetFullPath($(SparkRoot)\src))</SourceRoot>
     <Externals>$(SparkRoot)\externals\windows</Externals>
+    <Externals Condition=" '$(OS)' != 'Windows_NT' ">$(SparkRoot)/externals/linux</Externals>
     <DepotName>$([System.IO.Path]::GetFileName($([System.IO.Path]::GetFullPath($(SparkRoot)))))</DepotName>
     <ErrorReport>None</ErrorReport>
     <Prefer32Bit>False</Prefer32Bit>
@@ -19,20 +20,25 @@
     <OutputPath>$(BaseOutputPath)\bin</OutputPath>
     <BaseIntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
     <IntermediateOutputPath>$(BaseOutputPath)\obj\$(MSBuildProjectName)</IntermediateOutputPath>
+    <BaseIntermediateOutputPath Condition=" '$(OS)' != 'Windows_NT' ">$(BaseOutputPath)\linuxobj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <IntermediateOutputPath Condition=" '$(OS)' != 'Windows_NT' ">$(BaseOutputPath)\linuxobj\$(MSBuildProjectName)</IntermediateOutputPath>
     <GenDir>$(BaseOutputPath)\gen\$(MSBuildProjectName)</GenDir>
 </PropertyGroup>
 
+
 <Import Project="$(ComputerSpecificConfig)" Condition="Exists($(ComputerSpecificConfig))" />
 
 <ItemGroup>
     <AvailableItemName Include="Proto2">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto2</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto2Linux</Targets>
     </AvailableItemName>
     <AvailableItemName Include="Proto3">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto3Linux</Targets>
     </AvailableItemName>
     <AvailableItemName Include="Proto3Excel">
-        <Targets>CompileProto3Excel</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' ">CompileProto3Excel</Targets>
     </AvailableItemName>    
 </ItemGroup>
 
@@ -43,6 +49,14 @@
     <protogen>$(Externals)\bin\protogen.exe</protogen>
 </PropertyGroup>
 
+<PropertyGroup Label="ProtoBuf" Condition=" '$(OS)' != 'Windows_NT' ">
+    <ProtocPluginBase>$(SparkRoot)/src/lib/api/service_compiler</ProtocPluginBase>
+    <ProtoInclude>$(Externals)/include</ProtoInclude>
+    <protoc>$(Externals)/bin/protoc</protoc>
+    <protoc_rename>$(Externals)/bin/lowercase_files.sh</protoc_rename>
+    <protogen>$(Externals)/bin/protogen</protogen>
+</PropertyGroup>
+
 <ItemDefinitionGroup>
     <Proto2>
     </Proto2>

src/client/Directory.Build.targets

@@ -3,7 +3,7 @@
 
 
 <Target Name="CompileProto2" 
-    Condition="'@(Proto2)'!=''"
+    Condition="'@(Proto2)'!='' AND '$(OS)' == 'Windows_NT' "
     Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
     Outputs="$(GenDir)\proto2\%(Proto2.filename).cs"
     AfterTargets="PreBuildEvent">
@@ -28,7 +28,7 @@
     passes wipes out the additional compile includes that we add.
 -->
 <Target Name="CompileProto3" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3)'!=''"
+    Condition="'@(Proto3)'!=''  AND '$(OS)' == 'Windows_NT' "
     Inputs="$(protoc);
     %(Proto3.Identity);$(ProtocPluginBase)\plugin.py;
     $(ProtocPluginBase)\templates\c#\client.pb.cs;
@@ -58,7 +58,7 @@
 </Target>
 
 <Target Name="CompileProto3Excel" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3Excel)'!=''"
+    Condition="'@(Proto3Excel)'!='' AND '$(OS)' == 'Windows_NT' "
     Inputs="$(protoc);
     %(Proto3Excel.Identity);$(ProtocPluginBase)\plugin.py;
     $(ProtocPluginBase)\templates\excel\excel.pb.cs;"
@@ -82,12 +82,74 @@
     </ItemGroup>
 </Target>
 
+
+<Target Name="CompileProto2Linux" 
+    Condition="'@(Proto2)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
+    Outputs="$(GenDir)/proto2/%(Proto2.filename).cs"
+    AfterTargets="PreBuildEvent">
+    <PropertyGroup>
+        <SourceRelativeFile>$([System.String]::Copy('%(proto2.fullpath)').Replace($(SourceRoot), ''))</SourceRelativeFile>
+        <SourceRelativeDir>$([System.IO.Path]::GetDirectoryName($(SourceRelativeFile)))</SourceRelativeDir>
+        <WorkingDirectory>$(GenDir)/proto2</WorkingDirectory>
+    </PropertyGroup>
+    <MakeDir Directories="$(WorkingDirectory)" />
+    <Copy SourceFiles="%(Proto2.FullPath)" DestinationFolder="$(WorkingDirectory)" SkipUnchangedFiles="True" UseHardLinksIfPossible="True" />
+    <Error Text="protogen.exe does not exist. If you forgot to clone with externals try: git submodule update --init --recursive" Condition="!Exists($(protogen))" />
+    <Message Importance="High" Text="Doing Proto2 %(proto2.filename)" />
+    <Exec Command="PYTHONPATH=$(Externals)/python $(Protogen) %(Proto2.filename)%(proto2.extension) --csharp_out=. -I. -I$(SourceRoot) -I$(Externals)/include"
+          WorkingDirectory="$(WorkingDirectory)" />
+    <ItemGroup>
+        <Compile Include="$(WorkingDirectory)/%(Proto2.filename).cs" />
+    </ItemGroup>
+</Target>
+
+<Target Name="CompileProto3Linux" AfterTargets="GetAssemblyVersion"
+    Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(protoc);
+    %(Proto3.Identity);$(ProtocPluginBase)/plugin.py;
+    $(ProtocPluginBase)/templates/c#/client.pb.cs;
+    $(ProtocPluginBase)/templates/c#/server.pb.cs;
+    $(ProtocPluginBase)/templates/c#/immutable.pb.cs;"
+    Outputs="%(Proto3.OutCS);%(Proto3.OutCSServer);%(Proto3.OutCSClient);%(Proto3.OutCSImmutable)">
+    <MakeDir Directories="%(Proto3.OutBase)" />
+    <MakeDir Directories="$(IntermediateOutputPath)/gen" />
+   
+    <Message Importance="High" Text="Generating service %(Proto3.FileName)" />
+    <Exec Command="
+       PYTHONPATH=$(Externals)/python $(protoc) \
+       %(Proto3.FullPath) \
+       --error_format=gcc \
+       --proto_path=$(protoinc):$(SourceRoot) \
+       --plugin=&apos;protoc-gen-service=$(ProtocPluginBase)/plugin.py&apos; \
+       --service_out=lang=c#,strip_out_path=1:%(Proto3.OutBase) \
+       --csharp_out=%(CompileOptions)%(Proto3.OutBase)"
+      WorkingDirectory="%(relativedir)" />    
+      
+    <!-- <Exec Command="$(protoc_rename) %(Proto3.OutBase)"  />    -->
+            
+    <ItemGroup>
+       <Compile Include="%(Proto3.OutCS)" />
+       <Compile Include="%(Proto3.OutCSClient)" />
+       <Compile Include="%(Proto3.OutCSServer)" />
+       <Compile Include="%(Proto3.OutCSImmutable)" />
+    </ItemGroup>
+</Target>
+
+<!-- <Target Name="RenameProto3Linux" AfterTargets="CompileProto3Linux"
+   Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' ">
+   <Move SourceFiles="$(OutBase)\Apicore.cs" DestinationFiles="$(OutBase)\apicore.cs" />  
+   <Move SourceFiles="$(OutBase)\Apioptions.cs" DestinationFiles="$(OutBase)\apioptions.cs" />  
+   <Move SourceFiles="$(OutBase)\Co2.cs" DestinationFiles="$(OutBase)\co2.cs" />  
+</Target> -->
+
+
 <Target Name="AfterClean" AfterTargets="Clean">
     <Delete Files="%(Proto3.OutCS)" />
     <Delete Files="%(Proto3.OutCSClient)" />
     <Delete Files="%(Proto3.OutCSServer)" />
     <Delete Files="%(Proto3.OutCSImmutable)" />
-    <Delete Files="$(GenDir)\proto2\*" />
+    <Delete Files="$(GenDir)/proto2/*" />
     <Delete Files="%(Proto3Excel.OutCSExcel)" />
 </Target>
 

src/client/build_spark_dotnetcore.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#$1 = Debug / Release (CMAKE_BUILD_TYPE), $2 = CMAKE_BINARY_DIR
+cd "$(dirname "$0")"
+echo $PWD dotnet build -c $1 ./sparkcmd/
+dotnet build -c $1 -o $2/../$1/bin/netcoreapp3.1/linux-x64 ./sparkcmd/ 
+
+if [ "$1" == "Release" ]
+then
+   dotnet publish -c $1 -o $2/../$1/bin/netcoreapp3.1/linux-x64 ./sparkcmd/ 
+   ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.1/linux-x64/publish
+fi
+
+ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.1/linux-x64

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -5,6 +5,7 @@ using bts.collections;
 using bts.shell.fills;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.collections;
 using mbts.persist;
 using mbts.spark.filters;
@@ -23,6 +24,7 @@ using System.Linq;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
 using System.Windows.Data;
+using System.Windows.Input;
 using System.Windows.Media;
 using static bts.api.orderman.Fill.Types;
 using static bts.shell.FillsWindowFactory;
@@ -310,6 +312,7 @@ namespace mbts.spark {
          public FillStatus fill_status => _fill.FillStatus;
          public string fill_key => _fill.FillKey;
          public string order_key => _order.Key.ToString();
+         public string symbol => fill_tuple.security?.Symbol;
          public bool is_selected {
             get => _is_selected;
             set => Set(ref _is_selected, value);
@@ -330,6 +333,28 @@ namespace mbts.spark {
          public override bool Equals(object obj) => Equals(obj as Row);
          public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();
 
+         public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(typeof(Row),
+               new string[] {
+                  nameof(security),
+                  nameof(symbol),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(side),
+                  nameof(fill_time),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(size),
+                  nameof(price),
+                  nameof(fill_kind),
+                  nameof(fill_status),
+                  nameof(fill_key),
+                  nameof(order_key),
+                  nameof(security_key)
+               });
+
          public event Action invalidated;
          void IRowVM.raise_invalidated() => invalidated?.Invoke();
       }

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -4,6 +4,7 @@ using bts.api.orderman;
 using bts.collections;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.ui;
 using mbts.ui.controls;
 using System;
@@ -11,6 +12,7 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Runtime.CompilerServices;
+using System.Linq;
 using System.Windows.Input;
 using System.Windows.Media;
 using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
@@ -60,6 +62,35 @@ namespace mbts.spark {
       public event Action<(OrderSecurityRootPair oldval, OrderSecurityRootPair newval)> updated;
       public event PropertyChangedEventHandler PropertyChanged;
 
+      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(
+               typeof(OrderBookRowVM), 
+               new string[] {
+                  nameof(security_text),
+                  nameof(symbol),
+                  nameof(time_created),
+                  nameof(last_change),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(order_sub_type),
+                  nameof(side),
+                  nameof(quantity),
+                  nameof(filled_quantity),
+                  nameof(price),
+                  nameof(avg_price),
+                  nameof(last_price),
+                  nameof(piq),
+                  nameof(status),
+                  // last message (could contains commas, so skipping)
+                  nameof(security_key),
+                  nameof(order_key),
+                  nameof(parent_order_key),
+               });
+
       // Core
       public OrderSecurityRootPair order_tuple { get; private set; }
       public OrderRecordImmutable order => order_tuple.order.order;

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -4,6 +4,7 @@ using bts.shell;
 using bts.shell.fills;
 using bts.shell.orderbook;
 using bts.spark;
+using bts.utils.csv;
 using bts.view;
 using mbts.persist;
 using mbts.ui;
@@ -194,6 +195,7 @@ namespace mbts.spark {
          show_grid_settings_command.group = CommandGroups.Auxiliary;
 
          _commands = new SparkCommand[] {
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(OrderBookRowVM.csv_ordered_properties), "OrderBook")),
             toggle_show_right_panel_command,
             attach_selected_orders_command,
             detach_selected_orders_command,

src/client/spark/Orders/OrderDataHelper.cs

@@ -653,54 +653,25 @@ namespace mbts.spark.utilities {
             return def;
          }
       }
+      
+      static CustomOrderDefinition.CustomOneofCase[] algo_config(
+         Func<bts.config.AlgoPermissions, List<string>> selector)
+      {
+         var conf = mbts.config.Configuration.get<bts.config.AlgoPermissions>();
+         return selector(conf).Select(x => 
+         {
+            return (CustomOrderDefinition.CustomOneofCase)System.Enum.Parse(typeof(CustomOrderDefinition.CustomOneofCase), x, true);
+         }).ToArray();
+      }
 
       public static readonly CustomOrderDefinition.CustomOneofCase[]
 
          // exchange_order_types are the types which can be submitted on an exchange security
-         exchange_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         spreader_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Spreader,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover, 
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         attachable_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         outright_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-         };
+         exchange_order_types = algo_config(x => x.exchange_order_types),
+         spreader_order_types = algo_config(x => x.spreader_order_types),
+         attachable_order_types = algo_config(x => x.attachable_order_types),
+         outright_order_types = algo_config(x => x.outright_order_types);
+         
       public static readonly CustomOrderDefinition.CustomOneofCase[] all_order_types = exchange_order_types.Union(spreader_order_types).ToArray();
 
       public static void copy_values(CustomOrderDefinitionImmutable source, ref CustomOrderDefinition target) =>

src/client/spark/Orders/OrderTypeVMs/GenericOrderParamVM.cs

@@ -28,15 +28,17 @@ namespace mbts.spark
       where TMutable : class, IMessage, new() 
    {
       TMutable _message;
+      readonly protected Func<IMessage, IMessage> _message_override;
 
-      protected GenericOrderParamVM(FieldDescriptor descriptor)
+      protected GenericOrderParamVM(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null)
       {
          _message = new TMutable();
+         _message_override = message_override;
          this.descriptor = descriptor;
       }
 
       public FieldDescriptor descriptor { get; set; }
-      public Google.Protobuf.IMessage message 
+      public virtual Google.Protobuf.IMessage message 
       {
          get => _message;
          set 
@@ -53,11 +55,12 @@ namespace mbts.spark
       where TMutable : class, IMessage, new()
    {
       readonly Func<TModel, TView> _convert;
-      readonly Func<TView, TModel> _convert_back;
+      readonly Func<TView, TModel> _convert_back;      
       TView _value;
 
-      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, Func<TView, TModel> convert_back) : 
-         base(field_descriptor) 
+      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, 
+                                 Func<TView, TModel> convert_back, Func<IMessage, IMessage> message_override = null) : 
+         base(field_descriptor, message_override) 
       {
          _convert = convert;
          _convert_back = convert_back;
@@ -68,7 +71,8 @@ namespace mbts.spark
          get => _value;
          set 
          {
-            this.descriptor.Accessor.SetValue(this.message, _convert_back(_value = value));
+            this.descriptor.Accessor.SetValue( _message_override == null ? this.message : _message_override(this.message), 
+                                              _convert_back(_value = value));
             RaisePropertyChanged();
          }
       }
@@ -84,7 +88,7 @@ namespace mbts.spark
 
       protected override void on_param_changed() 
       {
-         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(this.message)));
+         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(_message_override == null ? this.message : _message_override(this.message))));
          RaisePropertyChanged(nameof(value));
       }
    }
@@ -92,8 +96,8 @@ namespace mbts.spark
    class GenericOrderEnumeratedVM<TMutable> : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<string>
       where TMutable : class, IMessage, new()   
    {
-      public GenericOrderEnumeratedVM(FieldDescriptor descriptor)
-         : base(descriptor)
+      public GenericOrderEnumeratedVM(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null)
+         : base(descriptor, message_override)
       {
          elements = new ObservableCollection<string>(_get_elements(descriptor));
       }
@@ -102,11 +106,13 @@ namespace mbts.spark
 
       public string selected_element 
       {
-         get => GenericOrderHelper.get_enumerated_value(message, descriptor);
+         get => GenericOrderHelper.get_enumerated_value(_message_override == null ? this.message : _message_override(this.message),
+                                                        descriptor);
          set 
          {
             var orig = selected_element;
-            GenericOrderHelper.set_enumerated_value(message, descriptor, value);
+            GenericOrderHelper.set_enumerated_value(_message_override == null ? this.message : _message_override(this.message), 
+                                                    descriptor, value);
             if (selected_element != orig)
                RaisePropertyChanged();
          }
@@ -148,22 +154,23 @@ namespace mbts.spark
    class GenericOrderQuantityVM<TMutable> : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<QuantityMeasureImmutable?>
       where TMutable : class, IMessage, new()
    {
-      public GenericOrderQuantityVM(FieldDescriptor descriptor)
-         : base(descriptor) { }
+      public GenericOrderQuantityVM(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null)
+         : base(descriptor, message_override) { }
 
       QuantityMeasureImmutable? IGenericOrderParamVM<QuantityMeasureImmutable?>.value { get => quantity; set => quantity= value; }
       public QuantityMeasureImmutable? quantity 
       {
-         get => GenericOrderHelper.get_quantity_value(message, descriptor);
+         get => GenericOrderHelper.get_quantity_value(_message_override == null ? this.message : _message_override(this.message), 
+                                                      descriptor);
          set 
          {
             var orig = quantity;
-            GenericOrderHelper.set_quantity_value(message, descriptor, value);
+            GenericOrderHelper.set_quantity_value(_message_override == null ? this.message : _message_override(this.message), 
+                                                  descriptor, value);
             if (quantity != orig)
                RaisePropertyChanged();
          }
       }      
-
       public IObservable<QuantityMeasureImmutable?> subscribe_value() 
       {
          return Observable.Create<QuantityMeasureImmutable?>(o =>
@@ -174,10 +181,9 @@ namespace mbts.spark
                o.OnNext(quantity);
          });
       }
-
       protected override void on_param_changed()
       {
          RaisePropertyChanged(nameof(quantity));
       }
-   }   
+   }
 }

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -96,9 +96,41 @@ namespace mbts.spark {
             {               
                Google.Protobuf.Reflection.MessageDescriptor desc = new TMutable().Descriptor;               
                _parameters.SyncWithDictionary(
-                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), _mk_param_vm);
+                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), x => _mk_param_vm(x));
+               add_message_params(desc);
             }
          }
+         void add_message_params(Google.Protobuf.Reflection.MessageDescriptor desc)
+         {
+            var msgs = desc.Fields.InDeclarationOrder().Where(x => !_is_value_param_desc(x) && 
+                                                              x.MessageType.ClrType != typeof(CustomOrderDefinition) );
+            foreach(var message_field in msgs)
+            {
+               foreach (var fd in message_field.MessageType.Fields.InDeclarationOrder())
+               {
+                  if(_is_value_param_desc(fd)) //one level deep
+                  {
+                     var mk = _mk_param_vm(fd, 
+                     outside_object => 
+                     {                        
+                        var msgfield = message_field.Accessor.GetValue(outside_object);
+                        if(msgfield == null)
+                        {
+                           message_field.Accessor.SetValue(outside_object, Activator.CreateInstance(message_field.MessageType.ClrType));
+                           msgfield = message_field.Accessor.GetValue(outside_object);
+                        }
+                        return (IMessage)msgfield;
+                     });
+                     if(mk == null) 
+                        continue;
+                        
+                     if(!_parameters.Add(mk))
+                        _parameters.Add(message_field.Name + "_" + mk.descriptor.Name, mk);                     
+                  }
+               }
+            }            
+         }
+         
          static bool _is_value_param_desc(FieldDescriptor desc)
          {
             return (desc.FieldType != Google.Protobuf.Reflection.FieldType.Message || 
@@ -117,50 +149,53 @@ namespace mbts.spark {
             return v;
          }
 
-         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor) 
+         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null) 
          {
             ExceptionAssert.Argument.NotNull(descriptor, nameof(descriptor));
 
             if (descriptor.FieldType == FieldType.Message && 
                 descriptor.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName)
             {
-               var qm = new GenericOrderQuantityVM<TMutable>(descriptor);
-               return attached(qm, GenericOrderHelper.get_quantity_value, GenericOrderHelper.set_quantity_value);
+               var qm = new GenericOrderQuantityVM<TMutable>(descriptor, message_override);
+               return attached(qm, GenericOrderHelper.get_quantity_value, GenericOrderHelper.set_quantity_value, message_override);
             }
             else if(descriptor.FieldType == FieldType.Enum)
             {
-               var pvm = new GenericOrderEnumeratedVM<TMutable>(descriptor);
-               attached(pvm, GenericOrderHelper.get_enumerated_value, GenericOrderHelper.set_enumerated_value);
+               var pvm = new GenericOrderEnumeratedVM<TMutable>(descriptor, message_override);
+               attached(pvm, GenericOrderHelper.get_enumerated_value, GenericOrderHelper.set_enumerated_value, message_override);
                return pvm;
             }
             else if (descriptor.FieldType == FieldType.Bool)
-               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
             else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float)
-               return mk<double, double>(descriptor, v => v, v => v, double.NaN);
+               return mk<double, double>(descriptor, v => v, v => v, message_override, double.NaN);
             else if (descriptor.FieldType == FieldType.Int32)
-               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
             else if (descriptor.FieldType == FieldType.UInt32)
-               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
             else if (descriptor.FieldType == FieldType.String)
-               return mk<string, string>(descriptor, v => v, v => v);
+               return mk<string, string>(descriptor, v => v, v => v, message_override);
             else
                return null;
          }
-         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, TView default_value = default)
+         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, 
+                                                                        Func<IMessage, IMessage> message_override = null, TView default_value = default)
          {
-            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back);
-            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param, default_value);
+            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back, message_override);
+            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param,
+                                                                                 message_override, default_value);
 
             TView get_from_param(IMessage param, FieldDescriptor fd) => convert(ConvertUtils.try_convert<TModel>(fdesc.Accessor.GetValue(param)));
             void assign_to_param(IMessage param, FieldDescriptor fd, TView value) => fdesc.Accessor.SetValue(param, convert_back(value));
          }
 
-         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
+         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, 
+                                    Func<IMessage, IMessage> message_override = null, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
          {
             add_property<T>(
                pvm.descriptor.Name,
-               mdef => mdef != null ? selector(mdef, pvm.descriptor) : default_value,
-               (mdef, v) => assign_to_param(mdef, pvm.descriptor, v),
+               mdef => mdef != null ? selector( message_override == null ?  mdef : message_override(mdef), pvm.descriptor) : default_value,
+               (mdef, v) => assign_to_param(message_override == null ? mdef : message_override(mdef), pvm.descriptor, v),
                () => pvm.value,
                v => pvm.value = v,
                pvm.subscribe_value(),

src/client/spark/Orders/OrderTypeVMs/OffsetOrderVM.cs

@@ -4,14 +4,14 @@ using OffsetDefi = bts.api.orderman.OffsetOrderDefinitionImmutable;
 namespace mbts.spark {
    static partial class OrderTypeVMFactory {
       class OffsetOrderVM : OrderTypeVMBase<OffsetDef, OffsetDefi> {
-         readonly Property<uint?> _tick_offset_on_deploy;
+         readonly Property<int?> _tick_offset_on_deploy;
 
          public OffsetOrderVM(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions)
             : base(key, display_name, editing_permissions) {
-            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0u);
+            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0);
          }
 
-         public uint? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
+         public int? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
       }
    }
 }

src/client/spark/Spark.csproj

@@ -9,7 +9,6 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Configurations>Debug;Release</Configurations>
     <Platforms>x64</Platforms>
-    <RuntimeIdentifiers>win7-x64</RuntimeIdentifiers>
     <PlatformTarget>AnyCPU</PlatformTarget>
     <LanguageTargets>$(MSBuildExtensionsPath)\$(VisualStudioVersion)\Bin\Microsoft.CSharp.targets</LanguageTargets>
     <ApplicationIcon>logo.ico</ApplicationIcon>
@@ -36,7 +35,7 @@
     <PackageReference Include="Google.Protobuf" Version="3.6.1" />
     <PackageReference Include="IronPython" Version="2.7.7" />
     <PackageReference Include="NAudio" Version="1.8.4" />
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
     <PackageReference Include="NUnit" Version="3.10.1" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.8.0" />

src/client/spark/view/Controls/Order/AlgoContentTemplateSelector.xaml

@@ -687,7 +687,7 @@
             <TextBlock VerticalAlignment="Center">Tick Offset</TextBlock>
             <c:HelpButton Margin="{StaticResource Margin.LP1}" ToolTip="How many ticks off the inside market to deploy the order, this guards against crossing the market inadventently"/>
           </StackPanel>
-          <c:TickUpDownUInt x:Name="PART_TickOffsetOnDeploy_TickUpDownUInt" Grid.Column="2" Value="{Binding tick_offset_on_deploy}"/>
+          <c:TickUpDownInt x:Name="PART_TickOffsetOnDeploy_TickUpDownUInt" Grid.Column="2" Value="{Binding tick_offset_on_deploy}"/>
           <uag:AlgoPickerNodeView
             x:Name="PART_Child_AlgoPickerNodeView"
             Grid.Row="2"

src/client/spark/view/Extensions/Command/FileDialogCommand.cs

@@ -64,6 +64,25 @@ namespace mbts.ui {
             }
          }, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));
 
+      public static IDialogCommand export_csv(
+         Func<string> get_file_content,
+         string source_window_name) =>
+         new AnonymousFileDialogCommand<object>(_ =>
+            _create_save_file_dialog(
+               Environment.GetFolderPath(Environment.SpecialFolder.Desktop),
+               $"BTS_Spark_{source_window_name}_{DateTime.Now.ToString("yyyyMMdd_HHmm")}.csv",
+               new[] { ("CSV (comma delimited)", new string[] { "csv" }) }
+            ),
+            (_, file) => {
+               try {
+                  File.WriteAllText(file, get_file_content.Invoke());
+               } catch (Exception ex) {
+                  BTSMessageBox.Show($"Error exporting data to csv: {ex}.", "CSV Export Error", BTSMessageBoxButtons.OK, System.Windows.MessageBoxImage.Error, true, false, true);
+               }
+            },
+            // we are always able to perform an export, even when no rows (it will just be a header)
+            (_) => true);
+
       static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
          var rv = new TDialog();
          if (!string.IsNullOrEmpty(initial_dir))

src/client/spark_analytics/BlackScholes.cs

@@ -0,0 +1,243 @@
+using System;
+
+namespace bts.analytics
+{
+   public static class BlackScholes
+   {
+      static public double NormCDF(double z)
+      {
+         //return MathNet.Numerics.Distributions.Normal.CDF(0.0, 1.0, z);
+         return Probability.normal_cdf(z);
+      }
+      static public double NormPDF(double z)
+      {
+         //return MathNet.Numerics.Distributions.Normal.PDF(0.0, 1.0, z);
+         return Probability.normal_pdf(z);
+      }
+
+      // expected size of change in underlying
+      // E(|x(t)-x(0)|), x(0)=Spot, E(x(t))=Spot*exp(GrowthRate*t), Spot return normally distributed
+      // This is the undiscounted straddle value
+      static public double expected_move(double spot, double growth, double vol, double t)
+      {
+         double sqrt_t = Math.Sqrt(t);
+         double d1 = (growth / vol + 0.5 * vol) * sqrt_t;
+         double d2 = d1 - vol * sqrt_t;
+         return spot * (Math.Exp(growth* t) * (2.0 * NormCDF(d1) - 1.0) + 1.0 - 2.0 * NormCDF(d2));
+      }      
+      
+      static public double atm_straddle(double spot, double riskless, double growth, double vol, double bizt, double calt)
+      {
+         return 2.0*Math.Exp((growth-riskless)*calt)*spot*(2.0*NormCDF(0.5*vol*Math.Sqrt(bizt))-1.0);
+      }
+
+      static public void calls(double spot, double[] rawk, double growth, double riskless,
+                               double[] vol, double bizt, double calt, out double[] result)
+      {
+         calls(spot * Math.Exp(growth * calt), rawk, riskless, vol, bizt, calt, out result);
+      }
+      static public void calls(double fwd, double[] rawk, double riskless, double[] vol, double bizt, double calt, out double[] result)
+      {
+         result = new double[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; i++)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            double d2 = d1 - vol_root_t;
+            result[i] = disc * (fwd * NormCDF(d1) - rawk[i] * NormCDF(d2));
+         }
+      }
+
+      static public void put(double spot, double[] rawk, double growth, double riskless,
+                             double[] vol, double bizt, double calt, out double[] result)
+      {
+         puts(spot * Math.Exp(growth * calt), rawk, riskless, vol, bizt, calt, out result);
+      }
+      static public void puts(double fwd, double[] rawk, double riskless, double[] vol, double bizt, double calt, out double[] result)
+      {
+         result = new double[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; i++)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            double d2 = d1 - vol_root_t;
+            result[i] = disc * (fwd * (NormCDF(d1) - 1.0) + rawk[i] * (1.0 - NormCDF(d2)));
+         }
+      }
+
+      static public void vega(double fwd, double[] rawk, double riskless, double[] vol, double bizt, double calt, out double[] result)
+      {
+         result = new double[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; ++i)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            result[i] = disc * fwd * NormPDF(d1) * sqrt_t;
+         }
+      }
+
+      static public double atm_straddle(double fwd, double riskless, double vol, double bizt, double calt)
+      {
+         return 2.0 * Math.Exp((/*growth*/-riskless) * calt) * fwd * (2.0 * NormCDF(0.5 * vol * Math.Sqrt(bizt)) - 1.0);
+      }
+
+      public class PriceDeltaGammaVegaThetaRho
+      {
+         public double strike;
+         public double call, put;
+         public double cdelta, pdelta;
+         public double cvega, pvega;
+         public double cgamma, pgamma;
+         public double cdiscrho, pdiscrho;
+         public double cvoltheta, pvoltheta;
+      }
+
+      static public void analytic_theos(double fwd, double[] rawk, double riskless, double[] vol,
+                                        double bizt, double calt, out PriceDeltaGammaVegaThetaRho[] result)
+      {
+         result = new PriceDeltaGammaVegaThetaRho[rawk.Length];
+         double disc = Math.Exp(-riskless * calt);
+         double sqrt_t = Math.Sqrt(bizt);
+         for (int i = 0; i < rawk.Length; ++i)
+         {
+            double vol_root_t = sqrt_t * vol[i];
+            double d1 = Math.Log(fwd / rawk[i]) / vol_root_t + 0.5 * vol_root_t;
+            double d2 = d1 - vol_root_t;
+            double nd1 = NormCDF(d1);
+            double nd2 = NormCDF(d2);
+            result[i].strike = rawk[i];
+            result[i].call = disc * (fwd * nd1 - rawk[i] * nd2);
+            result[i].put = disc * (fwd * (nd1 - 1.0) + rawk[i] * (1.0 - nd2));
+            result[i].cdelta = disc * nd1;
+            result[i].pdelta = disc * (nd1 - 1.0);
+            double dnd1 = NormPDF(d1);
+            result[i].cgamma = result[i].pgamma = disc * dnd1 / (fwd * vol_root_t);
+            result[i].cvega = result[i].pvega = disc * dnd1 * fwd * sqrt_t;
+            result[i].cdiscrho = rawk[i] * calt * disc * nd2;
+            result[i].pdiscrho = -rawk[i] * calt * disc * (1.0 - nd2);
+
+            double theta_base = -disc * fwd * dnd1 * vol[i] / (2 * sqrt_t);
+            result[i].cvoltheta = theta_base - riskless * rawk[i] * disc * nd2;
+            result[i].pvoltheta = theta_base + riskless * rawk[i] * disc * (1.0 - nd2);
+         }
+      }
+
+      /// <summary>
+      /// Estimate Vega/Fwd Via 'moneyness' and vol assuming zero interest rate and using t_estimate
+      /// </summary>
+      /// <param name="x"></param>
+      /// <param name="vol"></param>
+      static public double moneyness_vega(double x, double sigma, double vol, double t_estimate, double fwd_estimate, double riskless_estimate = 0.0, double calt_estimate = 0.0)
+      {
+         double disc = Math.Exp(-riskless_estimate * calt_estimate);
+         double sqrt_t = Math.Sqrt(t_estimate);
+         double vol_root_t = sqrt_t * vol;
+         double ln_k_over_f = x * sigma * sqrt_t;
+         double d1 = -ln_k_over_f / vol_root_t + 0.5 * vol_root_t;
+         double vegaish = disc * fwd_estimate * NormPDF(d1) * sqrt_t;
+         return vegaish / 100.0;
+      }
+
+      public static double implied_vol(double fwd, double rawk, double riskless, double initVol, double bizt, double calt,
+                                       bool is_call, double target_price, int max_iterations = 10)
+      {
+         if (max_iterations <= 0)
+            max_iterations = 10;
+         const double ImpVolTol = 1.0e-8;
+         double retinv = Math.Log(fwd / rawk);
+         double temp_factor = retinv * retinv;
+         double sqrthalftpi = Math.Sqrt(0.5 * bizt / Math.PI);
+         double sqrt_t = Math.Sqrt(bizt);
+
+         double disc = Math.Exp(-riskless * calt);
+         double CurrPrice = 0.0, Vol = initVol;
+         for (int i = 0; i < max_iterations; i++)
+         {
+            double VolRootT = Vol * sqrt_t;
+            double d1 = retinv / VolRootT + 0.5 * VolRootT;
+            double d2 = d1 - VolRootT;
+            if (is_call)
+               CurrPrice = disc * (fwd * NormCDF(d1) - rawk * NormCDF(d2));
+            else
+               CurrPrice = disc * (fwd * (NormCDF(d1) - 1.0) + rawk * (1.0 - NormCDF(d2)));
+            if (Math.Abs(CurrPrice - target_price) < ImpVolTol)
+               break;
+            else
+            {
+               double ratio = temp_factor / (Vol * Vol * Vol * bizt) - 0.25 * Vol * bizt;
+               double Vega = disc * fwd * sqrthalftpi * Math.Exp(-0.5 * d1 * d1);
+               if (Vega < 1e-5)
+                  Vega = 1e-5;
+               double dvol, temp_arg;
+               if (Math.Abs(ratio) > 0.0001 && (temp_arg = 1.0 + 2.0 * (target_price - CurrPrice) * ratio / Vega) > 0.0001)
+                  dvol = (Math.Sqrt(temp_arg) - 1.0) / ratio;
+               else
+                  dvol = (target_price - CurrPrice) / Vega;
+               if (dvol < -0.5 * Vol)
+                  dvol = -0.5 * Vol;
+               else if (dvol > 0.2)
+                  dvol = 0.2;
+               Vol += dvol;
+            }
+         }
+         if (Math.Abs(CurrPrice - target_price) < ImpVolTol && Math.Abs(Vol - initVol) > 0.0000001)
+            return Vol;
+         else
+            return -1.0;
+      }
+
+      public static double implied_bizt(double fwd, double rawk, double riskless, double Vol, double tguess_days,
+                                        bool is_call, double target_price, int max_iterations = 10)
+      {
+         if (max_iterations <= 0)
+            max_iterations = 10;
+         const double ImpVolTol = 1.0e-8;
+         double retinv = Math.Log(fwd / rawk);
+         double temp_factor = retinv * retinv;
+
+         double disc = 1.0; double year = 252.0;
+         double CurrPrice = 0.0, bizt = tguess_days / year;
+         for (int i = 0; i < max_iterations; i++)
+         {
+            double sqrthalftpi = Math.Sqrt(0.5 * bizt / Math.PI);
+            double sqrt_t = Math.Sqrt(bizt);
+            double VolRootT = Vol * sqrt_t;
+            double d1 = retinv / VolRootT + 0.5 * VolRootT;
+            double d2 = d1 - VolRootT;
+            if (is_call)
+               CurrPrice = disc * (fwd * NormCDF(d1) - rawk * NormCDF(d2));
+            else
+               CurrPrice = disc * (fwd * (NormCDF(d1) - 1.0) + rawk * (1.0 - NormCDF(d2)));
+            if (Math.Abs(CurrPrice - target_price) < ImpVolTol)
+               break;
+            else
+            {
+               double ratio = temp_factor / (Vol * Vol * Vol * bizt) - 0.25 * Vol * bizt;
+               double theta = -0.5 * fwd * NormPDF(d1) * Vol / Math.Sqrt(bizt) / year; //-r* blah blah, no rates here
+               if (theta > -1e-5)
+                  theta = -1e-5;
+               // double dt, temp_arg;
+               // if (Math.Abs(ratio) > 0.0001 && (temp_arg = 1.0 + 2.0 * (target_price - CurrPrice) * ratio / Vega) > 0.0001)
+               //    dvol = (Math.Sqrt(temp_arg) - 1.0) / ratio;
+               // else
+               double dt = (CurrPrice - target_price) / theta;
+               if (dt/year < -0.5*bizt)
+                  dt= -0.5*bizt*year;
+               else if (dt/year > 0.5*bizt)
+                  dt = 0.5*bizt*year;
+               bizt += dt/year;
+            }
+         }
+         if (Math.Abs(CurrPrice - target_price) < ImpVolTol && Math.Abs(bizt - tguess_days/year) > 0.0000001)
+            return bizt;
+         else
+            return -1.0;
+      }
+   }
+}

src/client/spark_analytics/DistributionEstimator.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Linq;
+
+namespace bts.analytics
+{
+   public class DistributionEstimator
+   {
+      List<double> _vals = new List<double>();
+      NormalEstimator _norm = new NormalEstimator();
+      public DistributionEstimator() 
+      {
+         
+      }
+      public void add(double x)
+      {
+         _vals.Add(x);
+         _norm.add(x);
+      }
+      public void delete(double x)
+      {
+         _vals.Remove(x);
+         _norm.delete(x);
+      }
+      public long num_samples => _vals.Count;
+      public double mean() => _norm.mean();
+      // {
+      //    if (num_samples == 0)
+      //       return 0.0;
+      //    double sum = _vals.Sum();
+      //    return sum / num_samples;
+      // }
+      public double sum_sq()
+      {
+         double m = mean();
+         double sq = 0.0;
+         for (int i = 0; i < num_samples; i++)
+            sq += (_vals[i] - m) * (_vals[i] - m);
+         return sq;
+      }
+      public double variance() => _norm.variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / num_samples;
+      // }
+      public double unbiased_variance() => _norm.unbiased_variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / (num_samples - 1);
+      // }
+      public double std_deviation() => _norm.std_deviation();
+      // {
+      //    return Math.Sqrt(variance());
+      // }
+      public double unbiased_std_deviation() => _norm.unbiased_std_deviation();
+      // {
+      //    return Math.Sqrt(unbiased_variance());
+      // }
+      public double percentile(int p)
+      {
+         _vals.Sort();
+         double percentile = p / 100.0;
+         double n = (_vals.Count - 1) * percentile + 1;
+         if (n <= 1.0)
+            return _vals[0];
+         else if (n >= _vals.Count)
+            return _vals[_vals.Count - 1];
+         else
+         {
+            int k = (int)n;
+            double d = n - k;
+            return _vals[k - 1] + d * (_vals[k] - _vals[k - 1]);
+         }
+      }
+      public override string ToString()
+      {
+         var f = percentile(5);
+         var t = percentile(10);
+         var tw = percentile(25);
+         var h = percentile(50);
+         var s = percentile(75);
+         var nn = percentile(99);
+         var sum50 = f + t + tw + h;
+         return $@"
+   Normal( mean {mean()}, std_dev {std_deviation()} ) 
+   5th { f }, 
+   10th { t } 
+   25th { tw } 
+   50th { h } 
+   75th { s } 
+   99th { nn } 
+   [numSamples={ num_samples } ]
+";
+      }
+   }
+}

src/client/spark_analytics/Extensions.cs

@@ -0,0 +1,7 @@
+
+using System;
+
+namespace bts.analytics
+{
+}
+

src/client/spark_analytics/Interop/AlmgrenChriss.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace bts.analytics
+{
+   public static class AlmgrenChriss
+   {
+      [DllImport("libspark")]
+      //[return: MarshalAs(UnmanagedType.R8)]
+      extern static public double liquidity_cost(double qty, double tick_size, double[] bookprices, 
+                                                 int price_len, int[] booksizes, int size_len, bool is_buy, 
+                                                 bool reverse, ref double residual);
+
+      [DllImport("libspark")]
+      extern static public double ac_alpha(double s_0, double s_t, bool is_buy, double time_factor);
+
+      [DllImport("libspark")]
+      extern static public double ac_sigma(double vol, double dt, double s_t, double time_factor);
+      [DllImport("libspark")]
+      extern static public double ac_sigma_opt(double vol, double dt, double s_t, double time_factor);
+
+      [DllImport("libspark")] 
+      extern static public double ac_xt(double aggressivity, double vol, double dt, double T, double s_0, double s_t, double x0,
+                                        double cost, double adaptivity, int generation, int ngenerations, bool is_buy, double time_factor);
+
+      [DllImport("libspark")]
+      extern static public double ac_xt_sigma_alpha(double lambda, double expmove_sigma, double dt, double T, double alpha, double x0,
+                                                    double cost, double adaptivity, int generation, int ngenerations);
+
+      [DllImport("libspark")]
+      extern static public double ac_optimal_duration(double vol, double dt_seconds, double s_t, double cost_per_share, double time_factor);
+
+   }
+}
+

src/client/spark_analytics/Interpolation.cs

@@ -0,0 +1,107 @@
+using System;
+using System.Linq;
+using MathNet.Numerics;
+
+namespace bts.analytics
+{
+   public static class Interpolation
+   {
+      static public double interp_spline(double t, double[] x, double[] y)
+      {
+         var cs = MathNet.Numerics.Interpolation.CubicSpline.InterpolateBoundariesSorted(x, y,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.Natural, 0.0,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.Natural, 0.0);
+         return cs.Interpolate(t);
+      }
+
+      static public double interp_spline_derivs(double t, double[] x, double[] y, double dleft, double dright)
+      {
+         var cs = MathNet.Numerics.Interpolation.CubicSpline.InterpolateBoundariesSorted(x, y,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.FirstDerivative, dleft,
+            MathNet.Numerics.Interpolation.SplineBoundaryCondition.FirstDerivative, dright);
+         return cs.Interpolate(t);
+      }
+
+      static public double interp_poly(double xval, double[] poly)
+      {
+         //double v = Evaluate.Polynomial(xval, poly);
+         double v = Polynomial.Evaluate(xval, poly);         
+         return v;
+      }
+
+      static public double interp_linear(double t, double[] x, double[] y)
+      {
+         var ls = MathNet.Numerics.Interpolation.LinearSpline.InterpolateSorted(x, y);
+         return ls.Interpolate(t);
+      }
+
+      static public double interp_quadratic(double t, double[] x, double[] y)
+      {
+         return quad_interp(x, y, t);
+      }
+
+      //utilities.cpp
+      static private double quad_interp(double[] x, double[] y, double x_val)
+      {
+         if (y == null || x == null || x.Length == 0 || y.Length == 0 || x.Length != y.Length) return double.NaN;
+         int npts = x.Length;
+         if (npts < 3) return interp_linear(x_val, x, y);
+         // k is the 'interior' index of the point to which x_val is closest
+         int k;
+         if (x_val < x[0]) k = 1;
+         else if (x_val > x[npts - 1]) k = npts - 2;
+         else
+         {
+            int klo = 1, khi = npts - 2;
+            while (khi - klo > 1)
+            {
+               k = (khi + klo) >> 1;
+               if (x[k] > x_val) khi = k;
+               else klo = k;
+            }
+            if ((x[khi] - x_val) > (x_val - x[klo]) && klo > 0) k = klo;
+            else k = khi;
+            const double SMALLER = 1.0e-7;
+            if ((x[k + 1] - x[k]) * (x[k] - x[k - 1]) < SMALLER) return interp_linear(x_val, x, y);
+         }
+         return y[k - 1] * (x_val - x[k]) * (x_val - x[k + 1]) / ((x[k - 1] - x[k]) * (x[k - 1] - x[k + 1])) + y[k] * (x_val - x[k - 1]) * (x_val - x[k + 1]) / ((x[k] - x[k - 1]) * (x[k] - x[k + 1])) + y[k + 1] * (x_val - x[k - 1]) * (x_val - x[k]) / ((x[k + 1] - x[k - 1]) * (x[k + 1] - x[k]));
+         // this might be faster, should be same answer:
+         //   return -(y[k-1]*(x[k]-x[k+1])*(x_val-x[k])*(x_val-x[k+1])+y[k]*(x[k+1]-x[k-1])*(x_val-x[k-1])*(x_val-x[k+1])+y[k+1]*(x[k-1]-x[k])*(x_val-x[k-1])*(x_val-x[k]))/((x[k+1]-x[k-1])*(x[k-1]-x[k])*(x[k]-x[k+1]));
+      }
+
+      //copied from Theopoint.cs
+
+      public static double weighted_mean_taylor_interp(double x1, double f1, double df1, double d2f1,
+                                                       double x2, double f2, double df2, double d2f2, double x)
+      {
+         double t_f_x1 = taylor_interp(x1, f1, df1, d2f1, x),
+               t_f_x2 = taylor_interp(x2, f2, df2, d2f2, x);
+         return (t_f_x1 * Math.Abs(x2 - x) + t_f_x2 * Math.Abs(x1 - x)) / (Math.Abs(x2 - x1));
+      }
+
+      public static double taylor_interp(double x1, double f1, double df1, double d2f1, double x)
+      {
+         return f1 + df1 * (x - x1) + d2f1 * Math.Pow(x - x1, 2) * 0.5;
+      }
+
+      static public double interp_taylor(double t, double[] x, double[] y, double[] dy, double[] dy2)
+      {
+         int idx = Array.BinarySearch<double>(x, t);
+         if (idx >= 0 && idx < x.Length)
+            return y[idx];
+         else
+         {
+            idx = ~idx;
+            if (idx == x.Length)
+               return double.NaN;
+            else if (idx == 0)
+               return double.NaN;
+            else
+            {
+               return weighted_mean_taylor_interp(x[idx - 1], y[idx - 1], dy[idx - 1], dy2[idx - 1],
+                                                  x[idx], y[idx], dy[idx], dy2[idx], t);
+            }
+         }
+      }
+   }
+}

src/client/spark_analytics/Moments.cs

@@ -0,0 +1,119 @@
+using System;
+
+namespace bts.analytics
+{
+   public class Moments
+   {
+      long _n;
+      double _m1, _m2, _m3, _m4;
+      double _min = double.MaxValue;
+      double _max = double.MinValue;
+      double _total = 0.0;
+
+      public Moments() { }
+      public void clear()
+      {
+         _n = 0;
+         _m1 = _m2 = _m3 = _m4 = 0.0;
+         _min = _max = _total = 0.0;
+      }
+      double _last = double.NaN;
+      public void add_return(double x)
+      {
+         if(!double.IsNaN(_last))
+         {
+            var ret = Math.Log(x / _last);
+            this.add(ret);
+         }
+         _last = x;
+      }
+      public void add(double x)
+      {
+         _total += x;
+         long n1 = _n;
+         _n++;
+         double d = x - _m1;
+         double dn = d / _n;
+         double dn2 = dn * dn;
+         double t = d * dn * n1;
+         _m1 += dn;
+         _m4 += t * dn2 * (_n*_n - 3.0*_n + 3) + 6.0 * dn2 * _m2 - 4.0 * dn * _m3;
+         _m3 += t * dn * (_n - 2.0) - 3.0 * dn * _m2;
+         _m2 += t;
+
+         if (x < _min)
+            _min = x;
+         if (x > _max)
+            _max = x;
+      }
+
+      public double total() => _total;
+      public long num_samples => _n;
+      public double min() => _min;
+      public double max() => _max;
+      public double mean() => _m1;
+      public double variance()
+      {
+         if (_n > 1)
+            return _m2/_n;
+         else
+            return 0.0;
+      }
+      public double unbiased_variance()
+      {
+         if (_n > 2)
+            return _m2 / (_n - 1.0);
+         else
+            return 0.0;
+      }
+      public double std_deviation()
+      {
+         return Math.Sqrt(variance());
+      }
+      public double unbiased_std_deviation()
+      {
+         return Math.Sqrt(unbiased_variance());
+      }
+      public double skewness()
+      {
+         return Math.Sqrt((double)_n) * _m3 / Math.Pow(_m2, 1.5);
+      }
+      public double kurtosis() 
+      {
+         return ((double)_n) * _m4 / (_m2* _m2) - 3.0;
+      }   
+      
+      public static Moments operator +(Moments a, Moments b)
+      {
+         Moments combined = new Moments();
+         combined._n = a._n + b._n;
+
+         double delta = b._m1 - a._m1;
+         double delta2 = delta * delta;
+         double delta3 = delta * delta2;
+         double delta4 = delta2 * delta2;
+
+         combined._m1 = (a._n * a._m1 + b._n * b._m1) / combined._n;
+
+         combined._m2 = a._m2 + b._m2 +
+                       delta2 * a._n * b._n / combined._n;
+
+         combined._m3 = a._m3 + b._m3 +
+                       delta3 * a._n * b._n * (a._n - b._n) / (combined._n * combined._n);
+         combined._m3 += 3.0 * delta * (a._n * b._m2 - b._n * a._m2) / combined._n;
+
+         combined._m4 = a._m4 + b._m4 + delta4 * a._n * b._n * (a._n * a._n - a._n * b._n + b._n * b._n) /
+                       (combined._n * combined._n * combined._n);
+         combined._m4 += 6.0 * delta2 * (a._n * a._n * b._m2 + b._n * b._n * a._m2) / (combined._n * combined._n) +
+                       4.0 * delta * (a._n * b._m3 - b._n * a._m3) / combined._n;
+
+         return combined;
+      }
+         
+      public override string ToString()
+      {
+         return $"Normal( mean {mean()}, std_dev {std_deviation()} ) [min {_min}, max { _max} ] [numSamples={num_samples}]";
+      }
+   }
+
+}

src/client/spark_analytics/Moneyness.cs

@@ -0,0 +1,33 @@
+using System;
+
+namespace bts.analytics
+{
+   public static class Moneyness
+   {
+      const double default_svr = 1.0;
+      public static double rawk_from_moneyness(double x, double fwd, double sigma, double bizt, double svr = default_svr, double reffwd = 0.0)
+      {
+         if (svr != default_svr)
+         {
+            double rawundk = rawk_from_moneyness(x, fwd, sigma, bizt);
+            double refstd = moneyness(rawundk, reffwd, sigma, bizt);
+            double z = x + svr * (refstd - x);
+            return rawk_from_moneyness(z, reffwd, sigma, bizt);
+         }
+         else
+            return fwd * Math.Exp(x * sigma * Math.Sqrt(bizt));
+      }
+
+      public static double moneyness(double rawk, double fwd, double sigma, double bizt, double svr = default_svr, double reffwd = 0.0)
+      {
+         if (svr != default_svr)
+         {
+            double reff = moneyness(rawk, reffwd, sigma, bizt);
+            double fwdf = moneyness(rawk, fwd, sigma, bizt);
+            return reff + svr * (fwdf - reff);
+         }
+         else
+            return Math.Log(rawk / fwd) / (sigma * Math.Sqrt(bizt));
+      }
+   }
+}

src/client/spark_analytics/NormalEstimator.cs

@@ -0,0 +1,74 @@
+using System;
+
+namespace bts.analytics
+{
+   public class NormalEstimator
+   {
+      private long _n;
+      private double _mu;
+      private double _sq;
+      private double _min = double.MaxValue;
+      private double _max = double.MinValue;
+
+      public NormalEstimator() { }
+      public void add(double x)
+      {
+         _n++;
+         double next_mu = _mu + (x - _mu) / _n;
+         _sq += (x - _mu) * (x - next_mu);
+         _mu = next_mu;
+
+         if (x < _min)
+            _min = x;
+         if (x > _max)
+            _max = x;
+      }
+      public void delete(double x)
+      {
+         if (_n == 0)
+            throw new ArgumentOutOfRangeException("No elements to delete");
+         else if (_n == 1)
+         {
+            _n = 0;
+            _mu = 0.0;
+            _sq = 0.0;
+         }
+         else
+         {
+            double mu = (_n * _mu - x) / (_n - 1);
+            _sq -= (x - _mu) * (x - mu);
+            _mu = mu;
+            --_n;
+         }
+      }
+      public long num_samples => _n;
+      public double mean() => _mu;
+      public double variance()
+      {
+         if (_n > 1)
+            return _sq / _n;
+         else
+            return 0.0;
+      }
+      public double unbiased_variance()
+      {
+         if (_n > 1)
+            return _sq / (_n - 1);
+         else
+            return 0.0;
+      }
+      public double std_deviation()
+      {
+         return Math.Sqrt(variance());
+      }
+      public double unbiased_std_deviation()
+      {
+         return Math.Sqrt(unbiased_variance());
+      }
+      public override string ToString()
+      {
+         return $"Normal( mean {mean()}, std_dev {std_deviation()} ) [min {_min}, max { _max} ] [numSamples={num_samples}]";
+      }
+   }
+
+}

src/client/spark_analytics/Probability.cs

@@ -0,0 +1,81 @@
+using System;
+
+namespace bts.analytics
+{
+   public static class Probability
+   {
+      public static double dN(double z)
+      {
+         const double INVSQRT2PI = 0.39894228040143267793994605993438;
+         return Math.Exp(-0.5 * z * z) * INVSQRT2PI;
+      }
+
+      public static double normal_pdf(double z)
+      {
+         const double MAXSTD = 37.61588707;
+         const double mindN = 2.2250738585134157e-308;
+         if (Math.Abs(z) > MAXSTD) return mindN;
+         else return dN(z);
+      }
+
+      public static double inverse_normal(double x, int guess, double zguess)
+      {
+         if (x > 1.0 || x < 0.0) return 0.0;
+         double z;
+         const double err_tol = 1.0e-15;
+         if (guess == 0) z = (x > 0.5) ? 0.67 : -0.67;
+         else z = zguess;
+         double Nz = normal_cdf(z);
+         int counter = 0;
+         for (; Math.Abs(Nz - x) > err_tol && counter < 20; counter++)
+         {
+            double epsilon = z * (x - Nz) / dN(z);
+            z += epsilon * (1.0 + epsilon * (0.5 + epsilon * 5.0 / 16.0)) / z;
+            Nz = normal_cdf(z);
+         }
+         if (double.IsNaN(z) && guess != 0) return inverse_normal(x, 0, 0.0);
+         else return z;
+      }
+
+      private readonly static double MAXSTD = 37.5193793471444;
+      private readonly static double MinNormCDF = 2.2250738585152E-308;
+      public static double normal_cdf(double x)
+      {
+         double XAbs = Math.Abs(x);
+         double Cumnorm = MinNormCDF;
+         if (XAbs < MAXSTD)
+         {
+            double Exponential = Math.Exp(-0.5 * XAbs * XAbs);
+            if (XAbs < 7.07106781186547)
+            {
+               double build = 3.52624965998911E-02 * XAbs + 0.700383064443688;
+               build = build * XAbs + 6.37396220353165;
+               build = build * XAbs + 33.912866078383;
+               build = build * XAbs + 112.079291497871;
+               build = build * XAbs + 221.213596169931;
+               build = build * XAbs + 220.206867912376;
+               Cumnorm = Exponential * build;
+               build = 8.83883476483184E-02 * XAbs + 1.75566716318264;
+               build = build * XAbs + 16.064177579207;
+               build = build * XAbs + 86.7807322029461;
+               build = build * XAbs + 296.564248779674;
+               build = build * XAbs + 637.333633378831;
+               build = build * XAbs + 793.826512519948;
+               build = build * XAbs + 440.413735824752;
+               Cumnorm = Cumnorm / build;
+            }
+            else
+            {
+               double build = XAbs + 0.65;
+               build = XAbs + 4.0 / build;
+               build = XAbs + 3.0 / build;
+               build = XAbs + 2.0 / build;
+               build = XAbs + 1.0 / build;
+               Cumnorm = Exponential / (build * 2.506628274631);
+            }
+         }
+         if (x > 0) Cumnorm = 1.0 - Cumnorm;
+         return Cumnorm;
+      }
+   }
+}

src/client/spark_analytics/Properties/AssemblyAttributes.cs

@@ -0,0 +1,4 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("sparkcmd")]
+

src/client/spark_analytics/VegaFactor.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts.analytics
+{
+   public static class VegaFactor
+   {
+      public static double interpolate_vol_dates(double target_date, double date1, double date2, double vol1, double vol2)
+      {
+         if (date1 == date2 || date1 > target_date)
+            return vol1;
+         else if (date2 < target_date)
+            return vol2;
+         double a = (date2 - target_date) / (date2 - date1);
+         double vol = (-vol1 + (vol1 - vol2) * Math.Exp(a) + Math.E * vol2) / (Math.E - 1.0);
+         return vol;
+      }
+
+      public static double interpolate_vol_array(double[] x, double[] y, double x_val)
+      {
+         if (y == null || x == null || x.Length == 0 || y.Length == 0 || x.Length != y.Length) return double.NaN;
+         int npts = x.Length;
+         if (npts < 3) return Interpolation.interp_linear(x_val, x, y);
+         int klo = 0, khi = npts - 1, k = 0;
+         while (khi - klo > 1)
+         {
+            k = (khi + klo) >> 1;
+            if (x[k] > x_val) 
+               khi = k;
+            else 
+               klo = k;
+         }
+         return interpolate_vol(x[klo], y[klo], x[khi], y[khi], x_val);
+      }
+
+      public static double interpolate_vol(double t1, double vol1, double t2, double vol2, double t)
+      {
+         if (t1 == t2 || t1 > t)
+            return vol1;
+         else if (t2 < t)
+            return vol2;
+         double a = t2 / (t2 - t1);
+         double vol = (-vol1 + (vol1 - vol2) * Math.Exp(a) + Math.E * vol2) / (Math.E - 1.0);
+         return vol;
+      }
+
+      static public double vega_factor_shortcut(double normal_bizt, double normal_vol, double current_bizt, double current_vol)
+      {
+         normal_bizt = Math.Max(normal_bizt, 0.001); current_bizt = Math.Max(current_bizt, 0.001);
+         double[] fwd = new double[] { 1.0 };
+         double[] normal_vega, current_vega;
+         BlackScholes.vega(fwd[0], fwd, 0.0, new double[] { normal_vol }, normal_bizt, 0.0, out normal_vega);
+         BlackScholes.vega(fwd[0], fwd, 0.0, new double[] { current_vol }, current_bizt, 0.0, out current_vega);
+         const double vega_factor_max = 15.0;
+         double vf = Math.Min(normal_vega[0] / current_vega[0], vega_factor_max);
+         return vf;
+      }
+   }
+}

src/client/spark_analytics/VolCommandOptions.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using bts.api.core;
+using bts.md;
+using CommandLine;
+using System.Linq;
+using bts.history;
+using System.IO;
+using System.Threading.Tasks;
+using bts.api;
+using System.Threading;
+
+namespace bts.analytics
+{
+
+[Verb("vol", HelpText = "Vol Service command")]
+public class VolCommandOptions
+{
+   [Option('!', "server", Required = false, Default = false, HelpText = "Run HistoricalData Server")]
+   public bool server { get; set; }
+
+   [Option('u', "user", Required = false, Default = "bts", HelpText = "User to log into market data")]
+   public string user { get; set; }
+
+   [Option('d', "password", Required = false, Default = "", HelpText = "password for market data entitlement")]
+   public string password { get; set; }
+
+   public override string ToString()
+   {
+      return Stringizer.to_string(this);
+   }
+}
+
+}

src/client/spark_analytics/VolServiceServer.cs

@@ -0,0 +1,262 @@
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using bts.api.core;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.md;
+using System.Threading;
+using System.Collections.Concurrent;
+
+namespace bts.analytics
+{
+   class VolServiceServer : VolServiceBase
+   {
+      static bts.md.MarketDataClient _md = new md.MarketDataClient();
+      static bts.config.system_symbols.OptionDriverMap _driver = 
+         mbts.config.Configuration.get<bts.config.system_symbols.OptionDriverMap>();
+
+      readonly Implied_vol_curvePublisher _publisher;
+      readonly StatefulPublisher<StrikeVolCurve, uint> _stateful;
+      readonly Dictionary< DateTime, SortedList<double, StrikeVol> > _vols = new Dictionary<DateTime, SortedList<double, StrikeVol>>();
+      readonly string _option_symbol;
+      readonly ServiceContext _context;
+      readonly IDisposable _disp;
+      readonly Timer _timer;
+
+      bool _dirty;
+      bool _driver_set;
+      DateTime? _driver_expiration;
+      double _driver_price = double.NaN;
+      readonly ConcurrentDictionary<DateTime, double> _shift = new ConcurrentDictionary<DateTime, double>();
+      readonly Dictionary<string, double> _strike_factor = new Dictionary<string, double>();
+      public VolServiceServer(ServiceContext context, string option_symbol) : base(context, option_symbol)
+      {
+         _context = context;
+         _md.set_auth_token(context.token);
+         _option_symbol = option_symbol;
+         _publisher = make_implied_vol_curve_publisher(option_symbol);
+         _stateful = new StatefulPublisher<StrikeVolCurve, uint>(_publisher.publish, x=> future_key(x.Expiration.ToDateTime()));
+         _strike_factor["ES"] = 1.0;
+         
+         string driversym = "";
+         foreach(var d in _driver.option_driver)
+         {
+            if(option_symbol == d.option_sym)
+            {
+               driversym = d.driver_sym;
+               _driver_expiration = d.driver_expiration;
+               _driver_set = _driver_expiration.HasValue;               
+            }
+         }
+         if(string.IsNullOrEmpty(driversym))
+            throw new NotSupportedException("can't find driver for " + _option_symbol);
+         var obs = _md.subscribe_book( sec => 
+            (sec.Symbol == _option_symbol && sec.security_type() == SecurityType.Option));
+
+         var driverobs =  _md.subscribe_book( sec=> 
+            (sec.Symbol == driversym && sec.security_type() == SecurityType.Future));
+         _disp = mbts.Disposable.Create(  
+            obs.Subscribe(book => on_book(book)),
+            driverobs.Subscribe(book => on_driver_book(book))
+            );
+         _timer = new Timer(on_timer, null, 0, 5000);
+      }
+      
+      void on_timer(object _)
+      {
+         publish();
+      }
+
+      void publish()
+      {
+         if(!_dirty)
+            return;
+         lock(_vols)
+         {
+            foreach(var exp in _vols)
+            {
+               StrikeVolCurve curve = new StrikeVolCurve() {Symbol = _option_symbol,Expiration = exp.Key.ToTimestamp(),DateKey = future_key(exp.Key)};
+               compute_curve(curve, exp.Key, exp.Value.Values, _driver_price);               
+               _stateful.add_or_update(curve);
+               Log.Gen.info($"Publishing Vol Curve {_option_symbol} {curve.AtmVol.ToString("0.##")} ");
+            }
+            _stateful.flush();
+         }         
+         _dirty = false;
+      }
+      
+      void compute_curve(StrikeVolCurve curve, DateTime expir, IList<StrikeVol> values, double driver_price)
+      {
+         curve.CalT = compute_calt(expir);
+
+         _shift.TryGetValue(expir, out double shift);         
+         curve.AtmVol = compute_atmvol(values, driver_price) + shift;
+         
+         double[] x = new double[values.Count];
+         double[] v = new double[values.Count];
+         for(int i = 0; i < values.Count; i++)
+         {
+            x[i] = values[i].StdStrike = Moneyness.moneyness(values[i].Strike, driver_price, curve.AtmVol, curve.CalT);
+            v[i] = values[i].ImpliedVol + shift;
+         }
+         //curve.Xcurve.AddRange(values);      
+         
+         curve.P2 = x_p2;
+         curve.Vp2 = Interpolation.interp_quadratic(curve.P2, x, v);
+         curve.P1 = x_p1;
+         curve.Vp1 = Interpolation.interp_quadratic(curve.P1, x, v);
+         curve.C1 = x_c1;
+         curve.Vc1 = Interpolation.interp_quadratic(curve.C1, x, v);
+         curve.C2 = x_c2;
+         curve.Vc2 = Interpolation.interp_quadratic(curve.C2, x, v);
+      }
+
+      static double compute_mid(IMDBook b)
+      {
+         double price = double.NaN;
+         if (b.bids.Count > 0 && b.bids[0].size > 0 && b.asks.Count > 0 && b.asks[0].size > 0)
+            price = 0.5 * (b.bids[0].price + b.asks[0].price);
+         else if (b.bids.Count > 0 && b.bids[0].size > 0)
+            price = b.bids[0].price;
+         else if (b.asks.Count > 0 && b.asks[0].size > 0)
+            price = b.asks[0].price;
+         return price;
+      }
+      
+      private void on_driver_book(IEnumerable<IMDBook> book)
+      {
+         if(!_driver_set)
+         {
+            var min = EnumerableExtensions.Min(book, x => x.security.EarliestExpiration());
+            if(!_driver_expiration.HasValue || min < _driver_expiration.Value)
+               _driver_expiration = min;
+         }
+         foreach(var b in book)
+         {
+            if(_driver_expiration == b.security.EarliestExpiration())
+            {
+               double d = compute_mid(b);
+               if(d != _driver_price)
+               {
+                  _driver_price = d;
+                  _dirty = true;
+               }
+            }
+         }
+      }
+
+      private void on_book(IEnumerable<IMDBook> book)
+      {
+         lock(_vols)
+         {
+            foreach(var b in book)
+            {
+               DateTime expir = DateTime.SpecifyKind(b.security.EarliestExpiration(), DateTimeKind.Utc);
+               if(!_vols.TryGetValue(expir, out SortedList<double, StrikeVol> expvol))
+               {
+                  _vols[expir] = expvol = new SortedList<double, StrikeVol>();
+               }
+               expvol[b.security.Option.Strike] = compute_strike(b);
+            }
+         }
+      }
+
+      const double volguess = 0.15;
+      const double cal_days_per_year = 365.0;
+      const int strike_mult = 100;
+      const int strike_width = 10000 * strike_mult;
+      const double x_p2 = -2.0, x_p1 = -1.0, x_c1 = 0.75, x_c2 = 1.5;
+
+      ulong option_key(DateTime expiration, double strike, PutOrCall cp)
+      {
+         var key = Convert.ToUInt64(expiration.ToOADate()) * strike_width + Convert.ToUInt64(strike * strike_mult) + (ulong)cp;
+         return key;
+      }
+
+      uint future_key(DateTime expiration)
+      {
+         var key = Convert.ToUInt32(expiration.ToOADate());
+         return key;
+      }      
+      
+      public static double compute_calt(in DateTime expir)
+      {
+         double calt = (expir - DateTime.Today).TotalDays / cal_days_per_year;
+         calt = Math.Max(calt, 0.00001);
+         return calt;
+      }
+      
+      double compute_atmvol(IList<StrikeVol> vols, double fwd)
+      {
+         int idx = vols.BinarySearchIndexOf(fwd, x => x.Strike);
+         int comp = ~idx;
+         if(idx >= 0)
+            return vols[idx].ImpliedVol;
+         else if(comp > 0 && comp < vols.Count - 1)
+         {
+            double atm = Interpolation.interp_quadratic(fwd,
+                                                        new[] {vols[comp-1].Strike, vols[comp].Strike, vols[comp+1].Strike },
+                                                        new[] {vols[comp-1].ImpliedVol, vols[comp].ImpliedVol, vols[comp+1].ImpliedVol});
+            return atm;
+         }
+         return double.NaN;
+      }
+
+      StrikeVol compute_strike(IMDBook b)
+      {
+         Log.assert(b.security.security_type() == SecurityType.Option);
+         if (!_strike_factor.TryGetValue(b.security.Symbol, out double kfactor))
+            kfactor = 1.0;
+
+         DateTime expir = DateTime.SpecifyKind(b.security.EarliestExpiration(), DateTimeKind.Utc);
+         var kv = new StrikeVol() 
+         {
+            Symbol = b.security.Symbol,
+            Expiration = expir.ToTimestamp(),
+            Strike = b.security.Option.Strike * kfactor,
+            Cp = b.security.Option.CallPut
+         };
+         kv.Key = option_key(expir, kv.Strike, kv.Cp);
+         double calt = compute_calt(expir);
+         double option_mid = compute_mid(b);
+         kv.ImpliedVol = BlackScholes.implied_vol(_driver_price, kv.Strike, 0.0, volguess, calt, calt, 
+                                                  kv.Cp == PutOrCall.Call, option_mid);
+         if(kv.ImpliedVol <= 0.0)
+         {
+            kv.ImpliedVol = BlackScholes.implied_vol(_driver_price, kv.Strike, 0.0, volguess * 2.0, calt, calt,
+                                                     kv.Cp == PutOrCall.Call, option_mid);
+            if (kv.ImpliedVol <= 0.0)            
+               kv.ImpliedVol = BlackScholes.implied_vol(_driver_price, kv.Strike, 0.0, volguess * 4.0, calt, calt,
+                                                        kv.Cp == PutOrCall.Call, option_mid);
+         }
+         if (kv.ImpliedVol <= 0.0)
+            kv.ImpliedVol = double.NaN;
+            
+         return kv;
+      }
+
+      public override Task<ResultMessage> shift_vol(VolShiftQuery request)
+      {
+         if(request.Symbol == _option_symbol)            
+         {
+            _dirty = true;
+            if(!double.IsNaN(request.Shift))
+            {
+               _shift[request.Expiration.ToDateTime()] = request.Shift;
+               return Task.FromResult(new ResultMessage() { Success = true });
+            }
+            else
+               return Task.FromResult(new ResultMessage() { Success = false });
+         }
+         throw new NotSupportedException($"can only shift vol for {_option_symbol}");
+      }
+      
+      public override void Dispose()
+      {
+         if(_disp != null)
+            _disp.Dispose();
+      }
+   }   
+}

src/client/spark_analytics/VolTermStructureServer.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using bts.api.core;
+using System.Linq;
+using System.Reactive.Disposables;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using System.Threading;
+
+namespace bts.analytics
+{
+   class VolTermStructureServer : VolTermStructureServiceBase
+   {
+      static bts.config.system_symbols.OptionDriverMap _driver =
+         mbts.config.Configuration.get<bts.config.system_symbols.OptionDriverMap>();
+
+      readonly Vol_term_structurePublisher _publisher;
+      readonly StatefulPublisher<VolTermStructure, string> _stateful;
+      readonly SortedDictionary<DateTime, StrikeVolCurve> _vols = new SortedDictionary<DateTime, StrikeVolCurve>();
+      readonly Dictionary<string, VolService> _services = new Dictionary<string, VolService>();
+      readonly ServiceContext _context;
+      readonly CompositeDisposable _disp = new CompositeDisposable();
+      readonly Timer _timer;
+      readonly string _driver_symbol;
+      VolTermStructure _last_term = new VolTermStructure();
+
+      bool _dirty;
+      public VolTermStructureServer(ServiceContext context, string driver_symbol) : base(context, driver_symbol)
+      {
+         _context = context;
+         _driver_symbol = driver_symbol;
+         _publisher = make_vol_term_structure_publisher(driver_symbol);
+         _stateful = new StatefulPublisher<VolTermStructure, string>(_publisher.publish, x => x.DriverSymbol);
+
+         foreach (var d in _driver.option_driver)
+         {
+            if (_driver_symbol == d.driver_sym)
+            {
+               var vs = _services[d.option_sym] = context.locate<VolService>(d.option_sym);
+               _disp.Add( vs.subscribe_implied_vol_curve(d.option_sym).Subscribe( x => 
+               {
+                  lock(_vols)
+                  {
+                     foreach(var val in x.all)
+                        _vols[val.Expiration.ToDateTime()] = val;
+                  }
+                  _dirty = true;
+               }));
+            }
+         }
+         _timer = new Timer(on_timer, null, 0, 5000);
+      }
+
+      void on_timer(object _)
+      {
+         publish();
+      }
+
+      void publish()
+      {
+         if (!_dirty)
+            return;
+         VolTermStructure term = new VolTermStructure() { DriverSymbol = _driver_symbol };
+         lock(_vols)
+         {
+            foreach (var exp in _vols)
+            {
+               VolTermInfo info = new VolTermInfo
+               {
+                  CalT =  exp.Value.CalT,
+                  AtmVol = exp.Value.AtmVol,
+                  Skewness = 0.0,
+               };
+               term.Terms.Add(info);
+            }
+            _stateful.add_or_update(term);      
+            _last_term = term;
+            Log.Gen.info($"Publishing Vol Term Structure {_driver_symbol} ");
+
+            _stateful.flush();
+         }
+         _dirty = false;
+      }
+
+      async public override Task<ResultMessage> global_shift(VolShiftQuery request)
+      {
+         if (request.Symbol != _driver_symbol)
+            throw new NotSupportedException($"can only global shift vol for {_driver_symbol}");
+
+         double[] x = _last_term.Terms.Select( z => z.CalT).ToArray();
+         double[] v = _last_term.Terms.Select( z => z.AtmVol).ToArray();
+         double firstnotnan = v.FirstOrDefault(xx => !double.IsNaN(xx));
+         const double vftime = 1/12.0;
+         double v30day = Interpolation.interp_quadratic(vftime, x, v);
+         if(double.IsNaN(v30day))
+            v30day = Math.Max(firstnotnan, 0.05);
+         foreach(var vs in _services)
+         {
+            List<VolShiftQuery> shifts = new List<VolShiftQuery>();
+            lock(_vols)
+            {
+               foreach(var curve in _vols)
+               {
+                  Timestamp exp = DateTime.SpecifyKind(curve.Key, DateTimeKind.Utc).ToTimestamp();
+                  double vf = VegaFactor.vega_factor_shortcut(vftime, v30day, curve.Value.CalT, curve.Value.AtmVol);
+                  double shift = request.Shift * vf;
+                  shifts.Add(new VolShiftQuery { Symbol = vs.Key, Expiration = exp, Shift = shift});
+               }
+            }
+            //can't await in lock
+            foreach(var s in shifts)
+               if(!double.IsNaN(s.Shift))
+                  await vs.Value.shift_vol_async(s);
+         }
+         _dirty = true;
+         return new ResultMessage() { Success = true };
+      }
+
+      public override void Dispose()
+      {
+         if (_disp != null)
+            _disp.Dispose();
+      }      
+   }
+}

src/client/spark_analytics/spark_analytics.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    
+    <TargetFramework Condition = " '$(OS)' != 'Windows_NT' ">netcoreapp3.0</TargetFramework>
+    <RuntimeIdentifier Condition = " '$(OS)' != 'Windows_NT' ">linux-x64</RuntimeIdentifier>
+    <PublishTrimmed>true</PublishTrimmed>
+    <PublishSingleFile>true</PublishSingleFile>
+    
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Platform>x64</Platform>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <Version>1.0.0</Version>
+    <LangVersion>7.2</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../spark_api/spark_api.csproj" />
+    <ProjectReference Include="../spark_core/spark_core.csproj" />
+    <!--<Proto3 Include="**/*.proto" />-->
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
+    <PackageReference Include="Microsoft.CSharp" Version="4.5.0" />
+    <PackageReference Include="CommandLineParser" Version="2.6.0" />
+    <PackageReference Include="MathNet.Numerics" Version="4.9.0" />
+  </ItemGroup>
+
+</Project>

src/client/spark_api/Co2/Co2NativeMethods.cs

@@ -4,6 +4,8 @@ using System.Runtime.InteropServices;
 using System.Security;
 using Google.Protobuf;
 using security_key = System.UInt32;
+using bts.co2;
+using System.Linq;
 
 namespace mbts.co2
 {
@@ -85,6 +87,7 @@ namespace mbts.co2
 
       [DllImport("libspark")]
       internal unsafe static extern void post_on_scan_security_thread(Action callback);
+      
    }
 
    internal static class Co2NativeTestMethods
@@ -133,5 +136,59 @@ namespace mbts.co2
       [DllImport("libspark")]
       internal static extern void co2_test_get_constants(out ulong max_depth, out ulong max_implied_depth, out sbyte nan_exponent);
 
+      internal unsafe delegate void raw_receive_pb_or_string(byte* buffer, int buffer_size);
+      [DllImport("libspark")]
+      private static unsafe extern void raw_co2_compute_synthetic_book(
+         byte[] security_buffer,
+         int security_length,
+         IntPtr[] leg_books,
+         int[] leg_lengths,
+         int leg_books_length,
+         raw_receive_pb_or_string callback);
+
+      internal static unsafe BookMessageImmutable co2_compute_synthetic_book(SecurityImmutable synthetic, params BookMessageImmutable[] books) {
+         BookMessage result = null;
+         string error = null;
+
+         var security_buffer = synthetic.ToByteArray();
+         // var h = GCHandle.Alloc(null, GCHandleType.Pinned).
+         var book_buffers = 
+            books
+            .Select(b => b.ToByteArray())
+            .ToArray();
+         GCHandle[] book_buffer_handles = 
+            book_buffers
+            .Select(b => GCHandle.Alloc(b,GCHandleType.Pinned))
+            .ToArray();
+         IntPtr[] book_buffer_pointers = 
+            book_buffer_handles
+            .Select(h => h.AddrOfPinnedObject())
+            .ToArray();
+         var lengths = 
+            book_buffers
+            .Select(b => b.Length)
+            .ToArray();
+
+         raw_co2_compute_synthetic_book(
+            security_buffer,
+            security_buffer.Length,
+            book_buffer_pointers,
+            lengths,
+            book_buffers.Length,
+            (byte* buffer, int length) => {
+               if (length < 0) {
+                  error =  new string((char*)buffer,0,-1 * length);
+               }
+               else {
+                  result = Co2NativeMethods.deserialize<BookMessage>(buffer, (ulong)length);
+               }
+            });
+
+         if(error != null) {
+            // C# style multiple return values
+            throw new Exception(error);
+         }
+         return new BookMessageImmutable(result);
+      }
    }
 }

src/client/spark_api/Co2/IntPrice.cs

@@ -52,7 +52,49 @@ namespace mbts.co2 {
          return _mantissa.GetHashCode();
       }
 
+      public static IntPrice from_double(double price, sbyte precision) {
+         if (double.IsNaN(price)) return NaN;
+         if (Math.Abs(price%1) != 0) {
+            double rounding = 0.5 * Math.Pow(10,-precision);
+            price += (price > 0) ? rounding : -rounding;
+            price *= Math.Pow(10,precision);
+            return new IntPrice((long)price, (sbyte)-precision);
+         }
+         return new IntPrice((long)price, 0);
+      }
+
       public static IntPrice NaN = new IntPrice(0,127);
+
+      static double[] powers_10 = {
+  1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10, 1E-9, 1E-8, 1E-7, 1E-6, 1E-5, 1E-4, 1E-3, 1E-2, 1E-1,
+  1E0,
+  1E1, 1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8, 1E9, 1E10, 1E11, 1E12, 1E13, 1E14, 1E15, 1E16, 1E17, 1E18, 1E19, 1E20};
+      static int max_abs_power_10 = powers_10.Length / 2;
+      static double pow10(int power)
+      {
+         int offset = power + max_abs_power_10;
+         int powers_10_sz = powers_10.Length;
+         bts.Log.assert(offset >= 0 && offset < powers_10_sz);
+         if (offset >= 0 && offset < powers_10_sz)
+         {
+            return powers_10[offset];
+         }
+         return double.NaN;
+      }      
+      
+      public static IntPrice from_double(double price, int precision = 9)
+      {
+         if (Double.IsNaN(price)) 
+            return NaN;
+         if (Math.Abs(price - Math.Truncate(price)) != 0.0)
+         {
+            double rounding = 0.5 * pow10(-precision);
+            price += (price > 0.0) ? rounding : -rounding;
+            price *= pow10(precision);
+            return new IntPrice((long)price, Convert.ToSByte(-precision));
+         }
+         return new IntPrice((long)price, 0);
+      }
    }
    
 }

src/client/spark_api/Co2/NativePriceHelpers.cs

@@ -1,4 +1,6 @@
 using bts;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace mbts.co2 {
    public static class NativePriceHelpers {
@@ -38,6 +40,60 @@ namespace mbts.co2 {
       public const uint COCONUT_MAX_DEPTH = 10;
       public const uint COCONUT_MAX_IMPLIED_DEPTH = 2;
       internal const sbyte COCONUT_NAN_EXPONENT = 127;
+
+      public static double side_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if(side == bts.api.Side.Buy)
+            return bid_at(book, level);
+         else
+            return ask_at(book, level);
+      }
+      public static uint sidesize_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if (side == bts.api.Side.Buy)
+            return bidsize_at(book, level);
+         else
+            return asksize_at(book, level);
+      }
+      public static double bid_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static double ask_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static uint bidsize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return lev.Size;
+      }
+      public static uint asksize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return lev.Size;
+      }
+      public static double price(this bts.co2.PriceSizeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }
+      public static double price(this bts.co2.TradeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }      
+
+      public static IList<double> all_bids(this bts.co2.BookMessage book)
+      {
+         var lev = book.Bids.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }
+      public static IList<double> all_asks(this bts.co2.BookMessage book)
+      {
+         var lev = book.Asks.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }      
    }
    
 }

src/client/spark_api/Properties/AssemblyAttributes.cs

@@ -3,4 +3,7 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark_core")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_fix")]
+[assembly: InternalsVisibleTo("spark_historical")]
+[assembly: InternalsVisibleTo("spark_analytics")]
 

src/client/spark_api/api/Apioptions.cs

@@ -0,0 +1,20 @@
+namespace bts.api
+{
+   //keep in sync with apioptions.proto
+   public static partial class ApioptionsReflection
+   {
+      public enum MessageOptions
+      {
+         fix_algostrategy_tagbase = 8506,
+      }
+      public enum FieldOptions
+      {
+         fix_tag = 8507,
+         fix_tag_is_header = 8509,
+      }
+      public enum EnumValueOptions
+      {
+         fix_value = 8508,
+      }
+   }
+}

src/client/spark_api/history/HistoryProtos.cs

@@ -0,0 +1,147 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf;
+using Google.Protobuf.Collections;
+using Google.Protobuf.WellKnownTypes;
+using mbts.co2;
+
+namespace bts.history
+{
+   public interface IHasMsgSeqNum 
+   {
+      uint MsgSeqNum { get; }
+   }
+   
+   public interface IHasDateTimeStamp 
+   {
+      DateTime trade_date { get; set;}
+      DateTime sending_time { get; set;}
+      DateTime transact_time { get; set; }      
+   }
+
+   public interface IHasTimestamp 
+   {
+      Timestamp SendingTime { get; set; }
+      Timestamp TransactTime { get; set; }
+   }
+
+   public interface IHasMdPacket
+   {
+      MarketDataIncrementalRefresh Md{ get; }
+   }
+
+   public interface IHasMdEntry
+   {
+      MDEntry MdEntry { get; }
+      int EntryIndex { get; }
+   }
+   
+   public interface IHasTopBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.TopBookMessage TopBook { get; set; }
+   }
+
+   public interface IHasBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.BookMessage Book { get; set; }
+   }
+
+   public interface IHasTrade : IHasMdEntry, IHasMdPacket
+   {
+      co2.TradeMessage Trade { get; set; }      
+   }   
+
+   public interface IHasMBO : IHasMdPacket
+   {
+      RepeatedField<OrderIDEntry> Orders { get;  }
+   }
+   
+   public sealed partial class HistoryRow : IHasTopBook, IHasMBO, IHasTrade, IHasBook, IHasDateTimeStamp, IHasTimestamp, IHasMsgSeqNum
+   {
+      public DateTime trade_date { get { return DateTime.SpecifyKind(this.TradeDate.ToDateTime(), DateTimeKind.Utc); } 
+                                   set { this.TradeDate = Timestamp.FromDateTime(value); } }
+      public DateTime sending_time { get { return DateTime.SpecifyKind(this.SendingTime.ToDateTime(), DateTimeKind.Utc); } 
+                                     set { this.SendingTime = Timestamp.FromDateTime(value); } }
+      public DateTime transact_time { get { return DateTime.SpecifyKind(this.TransactTime.ToDateTime(), DateTimeKind.Utc); } 
+                                      set { this.TransactTime = Timestamp.FromDateTime(value); } }
+      
+      public HistoryRow add_orders(IEnumerable<OrderIDEntry> orderentries)
+      {
+         this.Orders.AddRange(orderentries);
+         return this;
+      }                                         
+   }
+   
+   public interface IMBOMessage
+   {
+      MDUpdateAction OrderUpdateAction {get;}
+      ulong OrderId {get;}
+      uint SecurityId {get;}
+      string SecurityDesc {get;}
+      ulong MdOrderPriority {get;}
+      double MdEntryPx {get;}
+      uint MdPriceLevel {get;}
+      ulong MdDisplayQty {get;}
+      uint LastQty {get;}
+      MDEntryType MdEntryType {get;}
+      uint ReferenceId {get;}      
+   }
+   
+   public sealed partial class MBOMessage : IMBOMessage
+   {
+      public MDUpdateAction OrderUpdateAction => OrderEntry != null ? OrderEntry.OrderUpdateAction : MdEntry.MdUpdateAction;
+      public ulong OrderId  => OrderEntry != null ? OrderEntry.OrderId : MdEntry.OrderId;
+      public uint SecurityId => MdEntry.SecurityId;
+      public string SecurityDesc => MdEntry.Symbol;
+      public ulong MdOrderPriority => OrderEntry != null ? OrderEntry.MdOrderPriority : MdEntry.MdOrderPriority;
+      public double MdEntryPx => MdEntry.MdEntryPx;
+      public uint MdPriceLevel => MdEntry.MdPriceLevel;
+      public ulong MdDisplayQty => OrderEntry != null ? OrderEntry.MdDisplayQty : MdEntry.MdDisplayQty;
+      public uint LastQty => OrderEntry != null ? OrderEntry.LastQty : MdEntry.LastQty;
+      public MDEntryType MdEntryType => MdEntry.MdEntryType;
+      public uint ReferenceId => OrderEntry != null ? OrderEntry.ReferenceId : MdEntry.ReferenceId;
+      public uint NumberOfOrders => MdEntry.NumberOfOrders;
+   }
+
+   public sealed partial class MarketDataIncrementalRefresh
+   {
+      public MarketDataIncrementalRefresh add_entries(IEnumerable<MDEntry> mdentries, IEnumerable<OrderIDEntry> orderentries)
+      {
+         this.MdEntries.AddRange(mdentries);
+         this.OrderIdEntries.AddRange(orderentries);
+         return this;
+      }
+   }
+   public interface ITradeDetail
+   {
+      ulong OrderId { get; set; }
+      uint LastQty { get; set; }
+      ITradeDetail clone();
+   }
+
+   public sealed partial class OrderIDEntry : ITradeDetail
+   {
+      ITradeDetail ITradeDetail.clone()
+      {
+         return this.Clone();
+      }
+   }
+}
+
+namespace bts.co2
+{
+   public sealed partial class BookMessage
+   {
+      public BookMessage add_bids_and_asks(IEnumerable<PriceSizeMessage> bids, IEnumerable<PriceSizeMessage> asks)
+      {
+         this.Bids.AddRange(bids);
+         this.Asks.AddRange(asks);
+         return this;
+      }
+   }
+   
+   public sealed partial class TradeMessage 
+   {
+      public double price => NativePriceHelpers.to_double(this.Mantissa, (sbyte)this.Exponent);
+   }     
+}

src/client/spark_api/md/MDBook.cs

@@ -38,12 +38,24 @@ namespace bts.md
       {
          this.security = smd.security;
          NativePriceSize[] bids = null, asks = null;
-         if (smd.md.get_depth(ref last_update_time, ref bids, ref asks, NativePriceHelpers.COCONUT_MAX_DEPTH)) 
+         if (smd.md.has_depth && smd.md.get_depth(ref last_update_time, ref bids, ref asks, NativePriceHelpers.COCONUT_MAX_DEPTH)) 
          {            
             this.bids = bids.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
             this.asks = asks.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
             return true;
          }
+         else if(!smd.md.has_depth)
+         {
+            NativePriceSize bid, ask;
+            if(smd.md.get_bbo(ref last_update_time, out bid, out ask))
+            {
+               this.bids = new List<PriceSize>();
+               this.asks = new List<PriceSize>();
+               this.bids.Add(new PriceSize(bid.size, bid.direct_size, bid.implied_size, bid.price));
+               this.bids.Add(new PriceSize(ask.size, ask.direct_size, ask.implied_size, ask.price));
+               return true;
+            }
+         }
          return false;
       }
 

src/client/spark_api/security/SecurityFormatter.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using static bts.api.Security;
 
 namespace bts.api {
-   struct FutureLeg {
+   public struct FutureLeg {
       public readonly Venue exchange;
       public readonly string symbol, exchange_desc;
       public readonly FutureContractImmutable future;

src/client/spark_api/security/SecurityHelper.cs

@@ -4,7 +4,7 @@ using System.Linq;
 using static bts.api.Security;
 
 namespace bts.api {
-   internal static class SecurityHelper {
+   public static class SecurityHelper {
       public static IEnumerable<FutureLeg> get_flat_future_legs(SecurityImmutable sec) {
          switch (sec.ContractCase) {
             case ContractOneofCase.Spread: return _get_flat_future_legs(sec.Spread.Legs, 1);
@@ -34,14 +34,17 @@ namespace bts.api {
       }
 
       #region Extensions
-      public static DateTime expiration_date(this FutureContract future) { return new DateTime((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay); }
+      public static DateTime expiration_date(this FutureContract future) { return new DateTime((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay); }      
       public static DateTime expiration_date(this FutureContractImmutable future) { return new DateTime((int)future.ExpirationYear, (int)future.ExpirationMonth, (int)future.ExpirationDay); }
+      public static DateTime expiration_date(this OptionContract option) { return new DateTime((int)option.ExpirationYear, (int)option.ExpirationMonth, (int)option.ExpirationDay); }
+      public static DateTime expiration_date(this OptionContractImmutable option) { return new DateTime((int)option.ExpirationYear, (int)option.ExpirationMonth, (int)option.ExpirationDay); }
 
       internal static DateTime EarliestExpiration(this SecurityImmutable security) {
          switch (security.ContractCase) {
             case ContractOneofCase.Synthetic: return security.Synthetic.Legs.First().Security.EarliestExpiration();
             case ContractOneofCase.Spread: return security.Spread.Legs.First().Security.EarliestExpiration();
             case ContractOneofCase.Future: return security.Future.expiration_date();
+            case ContractOneofCase.Option: return security.Option.expiration_date();
             default: throw new NotImplementedException();
          }
       }
@@ -63,6 +66,8 @@ namespace bts.api {
             return security.Glbx.SecurityDesc;
          else if(security.HasIce)
             return security.Ice.SecurityDesc;
+         else if(security.HasSynthetic)
+            return security.Synthetic.Name;
          else
          {
             Log.assert(false, "security_desc not implemented for exchange");

src/client/spark_api/spark_api.csproj

@@ -26,7 +26,7 @@
     <Proto3 Include="**/*.proto" />
     <Proto3 Include="../../lib/api/*.proto" />
     <Proto3 Include="../../lib/api/services/*.proto" />
-    <Proto3 Include="../../lib/md/protos/co2.proto" />
+    <Proto3 Include="../../lib/md/protos/Co2.proto" />
     <Proto3 Include="../../lib/domain_model/Security.proto" />
     <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
     <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">

src/client/spark_api/utils/BTSNativeMethods.cs

@@ -28,5 +28,8 @@ namespace bts.utils
 
       [DllImport("libspark")]
       public static extern bool initialize_nothrow(StringBuilder buffer, int length);
+
+      //public static string bts_data_path()
+      //use mbts.config.Configuration.get<bts.config.System>().data_path;
    }
 }

src/client/spark_api/utils/Locator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Threading.Tasks;
 using bts.api;
 using bts.api.core;
@@ -8,10 +9,14 @@ namespace bts.api
    public static class Locator
    {
       readonly static Lazy<ServiceContext> _context = new Lazy<ServiceContext>(() => new ServiceContext(), true);      
+      public static void init() 
+      {
+         var sc = _context.Value;
+      }
       public static T locate<T>(string qualifier="")  where T : class, IApiClient
       {
          return _context.Value.locate<T>(qualifier);
-      }
+      }            
       static public async Task<LoginResult> login(string username, string password)
       {
          return await _context.Value.login(username, password);
@@ -22,4 +27,20 @@ namespace bts.api
       }      
       static public AuthTokenImmutable? token => _context.Value.token;
    }
+   
+   public class LocatorCache<T> 
+      where T : class, IApiClient
+   {
+      static readonly ConcurrentDictionary<string, T> _cache = new ConcurrentDictionary<string, T>();
+      public static T locate_cached(string qualifier = "") 
+      {
+         qualifier = qualifier ?? "";
+         if(!_cache.TryGetValue(qualifier, out T svc))
+         {
+            _cache[qualifier] = svc = Locator.locate<T>(qualifier);
+         }
+         return svc;
+      }
+   }
+   
 }

src/client/spark_api/utils/Log.cs

@@ -36,6 +36,7 @@ namespace bts {
       Plugin,
       Sdk,
       Http,
+      Fix
       // Please keep this list in sync with logging.h/logging.cpp
    }
    
@@ -102,9 +103,10 @@ namespace bts {
       internal static Logger Assert => new Logger(Subsystem.Assert);
       internal static Logger Ui => new Logger(Subsystem.Ui);
       internal static Logger Testing => new Logger(Subsystem.Testing);
-      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);
+      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);      
       public static Logger Sdk => new Logger(Subsystem.Sdk);
       public static Logger Http => new Logger(Subsystem.Http);
+      internal static Logger FIX => new Logger(Subsystem.Fix);
 
       internal static void log_at(this Logger logger, Level level, string message) {
          switch (level) {

src/client/spark_api/utils/StatefulPublisher.cs

@@ -0,0 +1,74 @@
+
+using System;
+using System.Collections.Generic;
+
+namespace bts.api
+{
+   public class StatefulPublisher<T, TKey>
+   {
+      readonly bool _keep_state;
+      readonly List<T> _added = new List<T>();
+      readonly HashSet<TKey> _erased = new HashSet<TKey>();
+      readonly List<T> _updated = new List<T>();
+
+      readonly HashSet<TKey> _state = new HashSet<TKey>();
+      readonly Dictionary<TKey, T> _state_data = new Dictionary<TKey, T>();
+      readonly Action<List<T>, HashSet<TKey>, List<T>> _publish;
+      readonly Func<T, TKey> _key;
+      public StatefulPublisher(Action< List<T>, HashSet<TKey>, List<T> > pubfn, Func<T, TKey> key, bool keep_state = false) 
+      { 
+         _keep_state = keep_state;
+         _publish = pubfn;
+         _key = key;
+      }
+
+      public void add_or_update(T  m)
+      {
+         var key = _key(m);
+         if (!_has_key(key))
+         {
+            _added.Add(m);
+            if(_keep_state)
+               _state_data[key] = m;
+            else
+               _state.Add(key);
+         }
+         else
+         {
+            // If an item has already been added update to new value
+            Predicate<T> match = (item) => _key(item).Equals( key );
+            var add_idx = _added.FindIndex(match);
+            if (add_idx >= 0)
+            {
+               _added[add_idx] = m;
+            }
+            else
+            {
+               var upd_idx = _updated.FindIndex(match);
+               if (upd_idx < 0 || _updated.Count == 0)
+                  _updated.Add(m);
+               else
+                  _updated[upd_idx] = m;
+            }
+         }
+      }
+      public void erase(TKey m)
+      {
+         if (_has_key(m))
+         {
+            _erased.Add(m);
+         }
+      }
+      public void flush()
+      {
+         _publish(_added, _erased, _updated);
+         _added.Clear();
+         _updated.Clear();
+         _erased.Clear();
+      }
+      bool _has_key(TKey k)
+      {
+         return _keep_state ? _state_data.ContainsKey(k) : _state.Contains(k) ;
+      }
+   }
+}

src/client/spark_api/utils/Stringizer.cs

@@ -0,0 +1,40 @@
+
+using System;
+using System.Collections.Generic;
+
+namespace bts
+{
+   public static class Stringizer
+   {
+      public static string all_to_string(IEnumerable<object> thi, string[] group = null)
+      {
+         string s = string.Empty;
+         foreach (var th in thi)
+            s += Stringizer.to_string(th, group);
+         return s;
+      }
+      public static string to_string(object thi, string[] group = null)
+      {
+         if (thi == null)
+            return string.Empty;
+         string val = thi.GetType().Name + Environment.NewLine;
+         int previ = -1;
+         foreach (var pi in thi.GetType().GetProperties())
+         {
+            var sub = pi.Name.Substring(0, Math.Max(pi.Name.IndexOf('_'), 0));
+            if (group == null)
+               val += pi.Name + ": " + pi.GetValue(thi)?.ToString() + "  " + Environment.NewLine;
+            else
+            {
+               int i = Array.IndexOf(group, sub);
+               if (i >= 0 && (i == previ))
+                  val += pi.Name + ": " + pi.GetValue(thi)?.ToString() + "  ";
+               else
+                  val += Environment.NewLine + pi.Name + ": " + pi.GetValue(thi)?.ToString() + "  ";
+               previ = i;
+            }
+         }
+         return val;
+      }
+   }   
+}

src/client/spark_api/utils/TimestampExtensions.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.api
+{
+   public static class TimestampExtensions
+   {
+      public static Timestamp add_nanos(this Timestamp lhs, int nanos)
+      {
+         var c = new Timestamp(lhs);
+         c.Nanos += nanos;
+         return c;
+      }
+      public static bool less(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) < 0;
+      }
+      public static bool less_or_equal(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) <= 0;
+      }
+      public static bool greater(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) > 0;
+      }
+      public static bool greater_or_equal(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) >= 0;
+      }
+      public static int compare(this Timestamp lhs, Timestamp rhs)
+      {
+         int secdiff = lhs.Seconds.CompareTo(rhs.Seconds);
+         return (secdiff != 0) ? secdiff : (lhs.Nanos.CompareTo(rhs.Nanos));
+      }
+      public static Comparer<Timestamp> timestamp_comparer => Comparer<Timestamp>.Create((x, y) => x.compare(y));
+
+      public static Timestamp parse_timestamp(string data)
+      {
+         DateTime dt = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+         try
+         {
+            if (data.Length <= 8)
+            {
+               dt = DateTime.ParseExact(data, "yyyyMMdd", System.Globalization.CultureInfo.InvariantCulture);
+            }
+            else
+            {
+               int sep = 14;
+               var yyMMddhhmmss = data.Substring(0, sep);
+               dt = DateTime.ParseExact(yyMMddhhmmss, "yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture);
+               var nnanos = data.Substring(sep, data.Length - sep);
+               long nano = long.Parse(nnanos);
+               dt = dt.AddTicks(nano / 100);
+            }
+         }
+         catch (Exception _)
+         {
+            System.Diagnostics.Debug.Assert(_ != null); //UNUSED(_)
+         }
+         dt = DateTime.SpecifyKind(dt, DateTimeKind.Utc);
+         var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(dt);
+         return ts;
+      }
+      public static string to_fix_string(this Timestamp ts)
+      {
+         return ts.ToDateTime().ToString("yyyyMMdd-HH:mm:ss.fff");
+      }
+   }
+}

src/client/spark_api/utils/TypeUtils.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text.RegularExpressions;
+
+namespace bts.api
+{
+   public class TypeUtils
+   {
+      static public List<Type> find_all_types_in_all_assemblies(string type, string assembly_regex = null)
+      {
+         List<Type> res = new List<Type>();
+         var asses = AppDomain.CurrentDomain.GetAssemblies();
+         foreach (Assembly a in asses)
+         {
+            if (!string.IsNullOrEmpty(assembly_regex))
+            {
+               if (!Regex.Match(a.GetName().Name, assembly_regex).Success)
+                  continue;
+            }
+            Type t = a.GetType(type, false, true);
+            if (t != null)
+               res.Add(t);
+         }
+         return res;
+      }
+   }   
+}

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -8,6 +8,34 @@ using System.Reactive.Linq;
 namespace System.Collections.Generic {
    public static class EnumerableExtensions {
       #region Ordered
+
+      public static int BinarySearchIndexOf<T>(this IList<T> list, T value, IComparer<T> comparer = null)
+      {
+         return BinarySearchIndexOf<T, T>(list, value, x => x, comparer);
+      }
+      
+      public static int BinarySearchIndexOf<T, K>(this IList<T> list, K key, Func<T, K> key_func, IComparer<K> comparer = null)
+      {
+         if(list == null)
+            throw new ArgumentNullException(nameof(list));
+         comparer = comparer ?? Comparer<K>.Default;
+
+         int lower = 0;
+         int upper = list.Count - 1;
+         while(lower <= upper)
+         {
+            int middle = lower + (upper - lower) / 2;
+            int result = comparer.Compare(key, key_func(list[middle]));
+            if(result == 0)
+               return middle;
+            else if(result < 0)
+               upper = middle - 1;
+            else
+               lower = middle + 1;
+         }
+         return ~lower;
+      }
+            
       public static IOrderedEnumerable<T> Ordered<T>(this IEnumerable<T> enumerable) =>
          enumerable.OrderBy(t => t);
 

src/client/spark_core/Properties/AssemblyAttributes.cs

@@ -2,4 +2,6 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_historical")]
+[assembly: InternalsVisibleTo("spark_analytics")]
 

src/client/spark_core/TestRunner/TestRunner.cs

@@ -70,10 +70,13 @@ namespace bts.test {
          psi.RedirectStandardOutput = true;
          psi.RedirectStandardError = true;
          psi.UseShellExecute = false;
-
+ 
          try{
             var p = Process.Start(psi);
             p.WaitForExit((int)time_out.TotalMilliseconds);
+            var errorlevel = p.ExitCode;
+            string stdout = p.StandardOutput.ReadToEnd();
+            string stderr = p.StandardError.ReadToEnd();
             if (p.HasExited) {
                return 0;
             }

src/client/spark_core/admin/AdminModelWrapper.cs

@@ -0,0 +1,64 @@
+using bts;
+using bts.api.admin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.api.core
+{
+   public class AdminModelWrapper
+   {
+      readonly AdminDataImmutable? _admin_data;
+      public AdminDataImmutable? admin_data => _admin_data;
+      readonly string _user_name;
+      readonly List<BTSAccountImmutable> _allowed_accounts;
+      readonly Dictionary<string, string> _allowed_account_dict = new Dictionary<string, string>();
+      public AdminModelWrapper(AdminDataImmutable? admin_data = null, string user_name = null)
+      {
+         _admin_data = admin_data;
+         _user_name = user_name ?? string.Empty;
+         _allowed_accounts = search_admin(u => u.Name == _user_name).Where(x =>
+         {
+              return (x.Item2 != null && x.Item2.HasPermissions) || (x.Item3 != null && x.Item3.HasPermissions);
+         }).Select(xx => xx.Item1).ToList();
+         foreach (var val in _allowed_accounts)
+            _allowed_account_dict[val.Name] = val.Name;
+      }
+      public IEnumerable<BTSAccountImmutable> viewable_accounts()
+      {
+         return _allowed_accounts;
+      }
+      public bool can_view_account(string account)
+      {
+         return _allowed_account_dict.ContainsKey(account) || _user_name.ToLower().Equals("bts");
+      }
+
+      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>> 
+         search_admin(Func<BTSUserImmutable, bool> predicate)
+      {
+         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>>();
+         if (_admin_data.HasValue)
+         {
+            var users = _admin_data.Value.Users.Where(predicate);
+            if (users != null && users.Count() == 1)
+            {
+               var user = users.ElementAt(0);
+               if (user.HasUserData && user.UserData.HasAccountPermissions)
+               {
+                  foreach (var kv in user.UserData.AccountPermissions.AccountPermissions)
+                  {
+                     var account_id = kv.Key;
+                     var accounts = _admin_data.Value.Accounts.Where(a => a.Id == account_id);
+                     if (accounts != null && accounts.Count() == 1)
+                     {
+                        var account = accounts.ElementAt(0);
+                        rv.Add(Tuple.Create(account, kv.Value.TradingPermission, kv.Value.AdminPermission));
+                     }
+                  }
+               }
+            }
+         }
+         return rv;
+      }
+   }
+}

src/client/spark_core/admin/PermissionFilters.cs

@@ -0,0 +1,64 @@
+using bts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.position;
+using bts.spark;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Concurrency;
+using System.Reactive.Linq;
+
+namespace bts.api.core
+{
+   public static class PermissionFilters
+   {
+      public static IObservable<Update<PositionUnitImmutable, int>> 
+         PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<PositionUnitImmutable, bool>>(observer =>
+         {
+            return am.Subscribe(x =>
+            {
+               Func<PositionUnitImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+            });
+         });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<OrderRecordImmutable, uint>> 
+         PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<OrderRecordImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.Def.Account ?? string.Empty));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<FillImmutable, string>> 
+         PermissionFilter(this IObservable<Update<FillImmutable, string>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<FillImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<FillImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+   }
+}

src/client/spark_core/spark_core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Platforms>x64</Platforms>
     <PlatformTarget>AnyCPU</PlatformTarget>

src/client/spark_core/utils/CsvHelper.cs

@@ -0,0 +1,52 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reflection;
+
+namespace bts.utils.csv {
+   public static class CsvHelper {
+
+      public static string ToCsv<T>(this IEnumerable<T> items, OrderedDictionary properties = null) where T : class {
+         var output = "";
+         var delimiter = ",";
+         if (properties == null) {
+            properties = new OrderedDictionary();
+            foreach (var pi in typeof(T).GetProperties()) {
+               properties.Add(pi.Name, pi);
+            }
+         }
+
+         using (var sw = new StringWriter()) {
+            var dict_values = properties.Values.OfType<PropertyInfo>();
+            var header = dict_values.Select(n => n.Name).Aggregate((first, second) => first + delimiter + second);
+            sw.WriteLine(header);
+            foreach (var i in items) {
+               var row = dict_values.Select(x => x.GetValue(i, null))
+                  .Select(x => x == null ? "null" : x.ToString())
+                  .Aggregate((first, second) => first + delimiter + second);
+               sw.WriteLine(row);
+            }
+            output = sw.ToString();
+         }
+         return output;
+      }
+
+      public static OrderedDictionary create_ordering_maintainer(Type t, string[] property_names_in_order) {
+         OrderedDictionary od = new OrderedDictionary();
+         var props = t.GetProperties();
+         foreach (var n in property_names_in_order) {
+            var query = props.Where(pi => pi.Name == n);
+            if (query.Any()) {
+               var property_info = query.First();
+               if (!od.Contains(property_info)) {
+                  od.Add(n, property_info);
+               }
+            }
+         }
+         return od;
+      }
+
+   }
+}

src/client/spark_core/utils/GlobalExtensions.cs

@@ -66,7 +66,6 @@ namespace mbts {
             }
          }
       }
-
       public static void sync_with_models<T, TModel>(this IList<T> target, IEnumerable<TModel> models, Func<TModel, T> add_factory, Func<T, TModel, T> update_factory) {
          var model_list = models as IList<TModel> ?? models.ToList();
          if (model_list.Count == 0) {

src/client/spark_excel/ExcelAddin.cs

@@ -12,6 +12,7 @@ using System.Diagnostics;
 using ExcelDna.Integration;
 using ExcelDna.IntelliSense;
 using ExcelDna.Logging;
+using ExcelDna.Registration;
 
 namespace bts.excel
 {
@@ -54,9 +55,14 @@ namespace bts.excel
          Trace.Listeners.Add(new LogDisplayTraceListener("bts"));
          IntelliSenseServer.Install();
          ExcelIntegration.RegisterUnhandledExceptionHandler(ex => ex.ToString());
+         
+         //https://github.com/Excel-DNA/ExcelDna/issues/255
+         ExcelRegistration.GetExcelFunctions()
+                          .ProcessAsyncRegistrations(nativeAsyncIfAvailable: false)
+                          .RegisterFunctions();         
 
          load_assemblies();
-         ExcelComAddInHelper.LoadComAddIn(new ContextTeardown(APIDispatch.service_context.Value));         
+         ExcelComAddInHelper.LoadComAddIn(new ContextTeardown(APIDispatch.service_context.Value));
       }
 
       private void load_assemblies()

src/client/spark_excel/analytics/AnalyticsWrappers.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text.RegularExpressions;
+using ExcelDna.Integration;
+using bts.api;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using bts.analytics;
+using System.Linq;
+
+namespace bts.excel
+{
+   public class AnalyticsWrappers
+   {
+      [ExcelFunction]
+      public static double bts_interpolate_vol(double[] t, double[] vol, double x)
+      {
+         return VegaFactor.interpolate_vol_array(t, vol, x);
+      }   
+      [ExcelFunction]
+      static public double bts_expected_move(double spot, double growth, double vol, double t)
+      {
+         return BlackScholes.expected_move(spot, growth, vol, t);
+      }
+      [ExcelFunction]
+      static public double bts_atm_straddle(double spot, double riskless, double growth, double vol, double bizt, double calt)
+      {
+         return BlackScholes.atm_straddle(spot, riskless, growth, vol, bizt, calt);
+      }
+      [ExcelFunction]
+      static public double[] bts_liquidity_cost(double qty, double tick_size,
+                                              double[] bookprices, double[] booksizes, bool is_buy, bool reverse)
+      {
+         double residual = 0.0;
+         var cost = AlmgrenChriss.liquidity_cost(qty, tick_size, bookprices.ToArray(), bookprices.Length, 
+                                                 booksizes.Select(x => Convert.ToInt32(x)).ToArray(), bookprices.Length, is_buy, reverse, ref residual);
+         return new[]{cost, residual};
+      }
+      [ExcelFunction]
+      static public double bts_ac_alpha(double s_0, double s_t, bool is_buy, double time_factor)
+      {
+         return AlmgrenChriss.ac_alpha(s_0, s_t, is_buy, time_factor);
+      }
+      [ExcelFunction]
+      static public double bts_ac_sigma(double vol, double dt, double s_t, double time_factor)
+      {
+         return AlmgrenChriss.ac_sigma(vol, dt, s_t, time_factor);
+      }
+      [ExcelFunction]
+      static public double bts_ac_sigma_opt(double vol, double dt, double s_t, double time_factor)
+      {
+         return AlmgrenChriss.ac_sigma_opt(vol, dt, s_t, time_factor);
+      }
+      [ExcelFunction] 
+      public static double bts_ac_xt(double aggressivity, double vol, double dt, double T, double s0, double st, double x0,
+                                     double cost, double adaptivity, int generation, int ngenerations, bool is_buy, double time_factor)
+      {
+         return AlmgrenChriss.ac_xt(aggressivity, vol, dt, T, s0, st, x0, cost, adaptivity, generation, ngenerations, is_buy, time_factor);
+      }
+      [ExcelFunction]
+      public static double bts_ac_xt_sigma_alpha(double lambda, double expmove_sigma, double dt, double T, double alpha, double x0,
+                                             double cost, double adaptivity, int generation, int ngenerations)
+      {
+         return AlmgrenChriss.ac_xt_sigma_alpha(lambda, expmove_sigma, dt, T, alpha, x0, cost, adaptivity, generation, ngenerations);
+      }
+      [ExcelFunction] 
+      public static double bts_implied_bizt(double fwd, double rawk, double riskless, double Vol, double tguess,
+                                            bool is_call, double target_price, int max_iterations = 10)
+      {
+         return BlackScholes.implied_bizt(fwd, rawk, riskless, Vol, tguess, is_call, target_price, max_iterations);
+      }
+      [ExcelFunction]
+      static public double bts_ac_optimal_duration(double vol, double dt_seconds, double s_t, double cost_per_share, double time_factor)
+      {
+         return AlmgrenChriss.ac_optimal_duration(vol, dt_seconds, s_t, cost_per_share, time_factor);
+      }
+   }
+}

src/client/spark_excel/extensions/OrderService_Excel.cs

@@ -11,7 +11,7 @@ using bts.excel;
 using bts.excel.extensions;
 using bts.excel.rtd;
 using Newtonsoft.Json;
-
+using System.Threading.Tasks;
 
 namespace bts.api.orderman
 {
@@ -21,7 +21,7 @@ namespace bts.api.orderman
       readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => Locator.locate<admin.AdminService>(), true);
 
       [ExcelFunction(Category = "BTS", Description = "Get All Accessible Accounts")]
-      async public static void spark_get_accounts(string login_token, ExcelAsyncHandle handle)
+      async public static Task<object> spark_get_accounts(string login_token)
       {
          try
          {
@@ -34,11 +34,11 @@ namespace bts.api.orderman
                      accts.Add(acc.Name);
             }
             var res = Utilities.make_column_vector(accts.ToArray());
-            handle.SetResult(res);
+            return res;
          }
          catch (Exception ex)
          {
-            handle.SetException(ex);
+            return ex.ToString();
          }
       }
 

src/client/spark_excel/rtd/formatting/AlphanumComparer.cs

@@ -0,0 +1,157 @@
+/*
+ * The Alphanum Algorithm is an improved sorting algorithm for strings
+ * containing numbers.  Instead of sorting numbers in ASCII order like
+ * a standard sort, this algorithm sorts numbers in numeric order.
+ *
+ * The Alphanum Algorithm is discussed at http://www.DaveKoelle.com
+ *
+ * Based on the Java implementation of Dave Koelle's Alphanum algorithm.
+ * Contributed by Jonathan Ruckwood <jonathan.ruckwood@gmail.com>
+ *
+ * Adapted by Dominik Hurnaus <dominik.hurnaus@gmail.com> to
+ *   - correctly sort words where one word starts with another word
+ *   - have slightly better performance
+ *
+ * Released under the MIT License - https://opensource.org/licenses/MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
+/*
+ * Please compare against the latest Java version at http://www.DaveKoelle.com
+ * to see the most recent modifications
+ */
+namespace Alphanum
+{
+   public class Alphanum
+   {
+      static public IComparer<string> comparer()
+      {
+         return new AlphanumComparator<string>();
+      }
+   }
+   
+   class AlphanumComparator<T> : IComparer<T>
+   {
+      private enum ChunkType { Alphanumeric, Numeric };
+      private bool InChunk(char ch, char otherCh)
+      {
+         ChunkType type = ChunkType.Alphanumeric;
+
+         if (char.IsDigit(otherCh))
+         {
+            type = ChunkType.Numeric;
+         }
+
+         if ((type == ChunkType.Alphanumeric && char.IsDigit(ch))
+             || (type == ChunkType.Numeric && !char.IsDigit(ch)))
+         {
+            return false;
+         }
+
+         return true;
+      }
+
+      public int Compare(T x, T y)
+      {
+         String s1 = x as string;
+         String s2 = y as string;
+         if (s1 == null || s2 == null)
+         {
+            return 0;
+         }
+
+         int thisMarker = 0, thisNumericChunk = 0;
+         int thatMarker = 0, thatNumericChunk = 0;
+
+         while ((thisMarker < s1.Length) || (thatMarker < s2.Length))
+         {
+            if (thisMarker >= s1.Length)
+            {
+               return -1;
+            }
+            else if (thatMarker >= s2.Length)
+            {
+               return 1;
+            }
+            char thisCh = s1[thisMarker];
+            char thatCh = s2[thatMarker];
+
+            StringBuilder thisChunk = new StringBuilder();
+            StringBuilder thatChunk = new StringBuilder();
+
+            while ((thisMarker < s1.Length) && (thisChunk.Length == 0 || InChunk(thisCh, thisChunk[0])))
+            {
+               thisChunk.Append(thisCh);
+               thisMarker++;
+
+               if (thisMarker < s1.Length)
+               {
+                  thisCh = s1[thisMarker];
+               }
+            }
+
+            while ((thatMarker < s2.Length) && (thatChunk.Length == 0 || InChunk(thatCh, thatChunk[0])))
+            {
+               thatChunk.Append(thatCh);
+               thatMarker++;
+
+               if (thatMarker < s2.Length)
+               {
+                  thatCh = s2[thatMarker];
+               }
+            }
+
+            int result = 0;
+            // If both chunks contain numeric characters, sort them numerically
+            if (char.IsDigit(thisChunk[0]) && char.IsDigit(thatChunk[0]))
+            {
+               thisNumericChunk = Convert.ToInt32(thisChunk.ToString());
+               thatNumericChunk = Convert.ToInt32(thatChunk.ToString());
+
+               if (thisNumericChunk < thatNumericChunk)
+               {
+                  result = -1;
+               }
+
+               if (thisNumericChunk > thatNumericChunk)
+               {
+                  result = 1;
+               }
+            }
+            else
+            {
+               result = thisChunk.ToString().CompareTo(thatChunk.ToString());
+            }
+
+            if (result != 0)
+            {
+               return result;
+            }
+         }
+
+         return 0;
+      }
+   }
+}

src/client/spark_excel/rtd/formatting/BookMDFormatter.cs

@@ -16,10 +16,11 @@ namespace bts.excel.rtd
          return format(_update, include_headers);
       }
 
-      static public object[,] format(IMDBook update, bool include_headers = false)
+      static public object[,] format(IMDBook update, bool include_headers = false, int rows = 0)
       {
          const int cols = 5;
-         const int rows = 30;
+         if(rows <= 0)
+            rows = 30;
          var ret = new object[rows, cols];
          for (int i = 0; i < rows; i++)
             for (int j = 0; j < cols; j++)
@@ -28,11 +29,11 @@ namespace bts.excel.rtd
          if (update != null)
          {
             int idx = 0;
-            for (int j = update.asks.Count - 1; j >= 0 && idx < rows; j--)
+            for (int j = 0; j < update.asks.Count && idx < rows/2; j++)
             {
                var ask = update.asks[j];
-               ret[idx, 1] = ask.price;
-               ret[idx, 2] = ask.size;
+               ret[rows/2-idx-1, 1] = ask.price;
+               ret[rows/2-idx-1, 2] = ask.size;
                idx++;
             }
             for (int j = 0; j < update.bids.Count && idx < rows; j++)

src/client/spark_excel/rtd/formatting/JsonFormatter.cs

@@ -88,7 +88,7 @@ namespace bts.excel.rtd
             json = JsonConvert.SerializeObject(data);
 
          JObject jo = JObject.Parse(json);
-         SortedDictionary<string, object> results = new SortedDictionary<string, object>();
+         SortedDictionary<string, object> results = new SortedDictionary<string, object>(Alphanum.Alphanum.comparer());
          parse_object(jo, results);
          return results;
       }
@@ -112,7 +112,7 @@ namespace bts.excel.rtd
       object[,] fill_object_array(IEnumerable<SortedDictionary<string, object>> data, bool include_headers = false)
       {
          int count = 0;
-         SortedDictionary<string, string> keys = new SortedDictionary<string, string>();
+         SortedDictionary<string, string> keys = new SortedDictionary<string, string>(Alphanum.Alphanum.comparer());
          foreach (var d in data)
          {
             count++;

src/client/spark_excel/rtd/subscriptions/MDSubscribe.cs

@@ -8,30 +8,54 @@ using ExcelDna.Registration.Utils;
 using System.Reactive.Linq;
 using bts.api;
 using bts.md;
-using bts.api.orderman;
+using System.Threading.Tasks;
+using System.Reactive.Disposables;
 
 namespace bts.excel.rtd
 {
    public class MDSubscribe
    {
       [ExcelFunction(Category = "BTS", Description = "Log in to BTS Spark")]
-      async public static void spark_login(string login, string password, ExcelAsyncHandle handle) 
+      public static object spark_login(string login = null, string password = null) 
       {
-         try
+         Locator.init();
+         return ObservableRtdUtil.Observe(nameof(observable_login), new object[]{login, password}, 
+                                          () => observable_login(login, password));
+      }
+
+      static IObservable<object> observable_login(string login, string password)
+      {
+         login = login ?? string.Empty;
+         password = password ?? string.Empty;
+         bool logged_in = false;
+         return Observable.Create<object>( obs => 
          {
-            var res = await Locator.login(login, password);
-            if (res.success && Locator.token.HasValue)
+            return Observable.Timer(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5)).Subscribe( async _ =>
             {
-               _md.set_auth_token(Locator.token);
-               handle.SetResult("Login Successful");
-            }
-            else
-               handle.SetResult("Login Error: Could not log in check user/pass " + res.reason);
-         }
-         catch(Exception ex)
-         {
-            handle.SetException(ex);
-         }
+               if(logged_in)
+               {
+                  //obs.OnCompleted(); //don't call this if you want the rtd to restart on open
+                  return;
+               }
+               var now = DateTime.Now.ToShortTimeString();
+               try
+               {
+                  var res = await Locator.login(login, password);
+                  if (res.success && Locator.token.HasValue)
+                  {
+                     _md.set_auth_token(Locator.token);
+                     obs.OnNext("Login Successful " + now);
+                     logged_in = true;
+                  }
+                  else
+                     obs.OnNext("Login Error: Could not log in check user/pass " + res.reason + " " + now);
+               }
+               catch(Exception ex)
+               {
+                  obs.OnNext("Login Exception: " + ex.ToString() + " " + now);
+               }               
+            });
+         });
       }
 
       static MarketDataClient _md = new MarketDataClient();
@@ -92,16 +116,16 @@ namespace bts.excel.rtd
                                     [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
                                     [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,                                    
                                     [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
-                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
+                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null, int rows = 0)
       {
 
-         var param = new object[] { contract, symbol, security_type, exchange };
+         var param = new object[] { contract, symbol, security_type, exchange, rows };
          return ObservableRtdUtil.Observe(nameof(spark_md_book), param, 
                                           () => md_observable(contract, symbol, security_type, exchange).Select(data =>
                                           {
                                              var x = data.FirstOrDefault();
                                              if (x != null && x is IMDBook upd)
-                                                return BookMDFormatter.format(upd);
+                                                return BookMDFormatter.format(upd, false, rows);
                                              return x;
                                           }));
          // security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;

src/client/spark_excel/spark_excel-addin.dna

@@ -1,6 +1,7 @@
 <DnaLibrary Name="spark_excel" RuntimeVersion="v4.0">
-  <ExternalLibrary Path="spark_excel.dll" ExplicitExports="false" LoadFromBytes="true" Pack="true" />
+   <ExternalLibrary Path="spark_excel.dll" ExplicitExports="false" ExplicitRegistration="true" LoadFromBytes="true" Pack="true" />
   
+   <Reference Path="spark_analytics.dll" Pack="true" ExplicitExports="true" />
    <Reference Path="ExcelDna.IntelliSense.dll" Pack="true" />
    <Reference Path="ExcelDna.Registration.dll" Pack="true" />
    <Reference Path="ExcelDna.Integration.dll" Pack="true" />   

src/client/spark_excel/spark_excel.csproj

@@ -26,13 +26,13 @@
     <PackageReference Include="System.Reactive" Version="3.1.1" />
     <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
 	 <PackageReference Include="NUnit" Version="3.10.1" />  
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="System.Linq.Dynamic" Version="1.0.7" />
     
-    <PackageReference Include="ExcelDna.AddIn" Version="0.34.6" />  
-    <PackageReference Include="ExcelDna.Integration" Version="0.34.6" />      
-    <PackageReference Include="ExcelDna.IntelliSense" Version="1.1.0" />      
-    <PackageReference Include="ExcelDna.Registration" Version="0.34.7" />      
+    <PackageReference Include="ExcelDna.AddIn" Version="1.0.0" />
+    <PackageReference Include="ExcelDna.Integration" Version="1.0.0" />
+    <PackageReference Include="ExcelDna.IntelliSense" Version="1.1.0" />
+    <PackageReference Include="ExcelDna.Registration" Version="1.0.0" />
   </ItemGroup>
   
   <ItemGroup>    
@@ -55,6 +55,7 @@
   
   <ItemGroup>
     <ProjectReference Include="../spark_api/spark_api.csproj"/>
+    <ProjectReference Include="../spark_analytics/spark_analytics.csproj"/>
   </ItemGroup>
   
   <ItemGroup>
@@ -74,10 +75,10 @@
   </Target>  
    -->
   
-  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets" Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets')" /> -->
-  <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets"/>
+  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets" Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets')" /> -->
+  <!-- <Import Project="$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets"/>
   <Target Name="EnsureExcelDnaTargetsImported" BeforeTargets="BeforeBuild" Condition="'$(ExcelDnaTargetsImported)' == ''">
-    <Error Condition="!Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="You are trying to build with ExcelDna, but the NuGet targets file that ExcelDna depends on is not available on this computer. This is probably because the ExcelDna package has not been committed to source control, or NuGet Package Restore is not enabled. Please enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
-    <Error Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\0.34.6\tools\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="ExcelDna cannot be run because NuGet packages were restored prior to the build running, and the targets file was unavailable when the build started. Please build the project again to include these packages in the build. You may also need to make sure that your build server does not delete packages prior to each build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
-  </Target>  
+    <Error Condition="!Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="You are trying to build with ExcelDna, but the NuGet targets file that ExcelDna depends on is not available on this computer. This is probably because the ExcelDna package has not been committed to source control, or NuGet Package Restore is not enabled. Please enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
+    <Error Condition="Exists('$(NuGetPackageRoot)\ExcelDna.AddIn\1.0.0\build\ExcelDna.AddIn.targets') And ('$(RunExcelDnaBuild)' != '' And $(RunExcelDnaBuild))" Text="ExcelDna cannot be run because NuGet packages were restored prior to the build running, and the targets file was unavailable when the build started. Please build the project again to include these packages in the build. You may also need to make sure that your build server does not delete packages prior to each build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
+  </Target>   -->
 </Project>

src/client/spark_excel/utils/APIDispatch.cs

@@ -90,7 +90,14 @@ namespace bts.excel
             {
                if(string.IsNullOrEmpty(json))
                   json = "{}";
-               request = JsonParser.Default.Parse(json, desc);
+               try
+               {
+                  request = JsonParser.Default.Parse(json, desc);
+               }
+               catch(InvalidProtocolBufferException)
+               {
+                  request = (IMessage) Newtonsoft.Json.JsonConvert.DeserializeObject(json, desc.ClrType);
+               }
             }
          }
          return request;

src/client/spark_excel/utils/Utilities.cs

@@ -25,7 +25,68 @@ namespace bts.excel
          else if (!string.IsNullOrEmpty(key1))
             val = jo[key1];
          return val.ToString();
-      }      
+      }
+      
+      static Google.Protobuf.JsonFormatter _formatter = 
+         new Google.Protobuf.JsonFormatter(new Google.Protobuf.JsonFormatter.Settings(true));
+      [ExcelFunction]
+      public static object bts_json_create(string type)
+      {         
+         System.Type t = find_full_type_in_all_assemblies(type);
+         object obj = Activator.CreateInstance(t);
+         foreach(var pr in t.GetProperties())
+         {
+            object val = null;
+            try
+            {
+               val = Activator.CreateInstance(pr.PropertyType);
+            }
+            catch {}
+            try
+            {
+               pr.SetValue(obj, val);
+            }
+            catch {}
+         }
+         string s = "{}";         
+         if(obj is Google.Protobuf.IMessage imsg)
+            s = _formatter.Format(imsg);
+         else
+            s = JsonConvert.SerializeObject(obj, Formatting.Indented);
+         return s;
+      }
+      [ExcelFunction]
+      public static object bts_json_fill(string root, string key, string value)
+      {
+         var jo = JObject.Parse(root);
+         jo[key] = value;
+         var s = jo.ToString();
+         return s;
+      }
+
+      [ExcelFunction]
+      public static object bts_date_to_json(object dt, bool is_duration = false)
+      {
+         string s = "{}";
+         if(is_duration)
+         {
+            var span = TimeSpan.FromDays((double)dt);
+            var ts = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(span);
+            s = JsonConvert.SerializeObject(ts, Formatting.None);
+            s = _formatter.Format(ts);
+            s = s.Replace(@"""", "");
+         }
+         else
+         {
+            var dd = DateTime.FromOADate((double)dt);
+            var utc = DateTime.SpecifyKind(dd, DateTimeKind.Utc);
+            var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(utc);
+            s = JsonConvert.SerializeObject(ts, Formatting.None);
+            s = _formatter.Format(ts);
+            s = s.Replace(@"""", "");
+         }
+         return s;
+      }
 
       public static bool parse_venue(string exch, out Venue ven)
       {

src/client/spark_fix/FixInit.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+namespace bts.fix
+{
+   public class FixInit
+   {
+      static FixInit()
+      {
+         AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
+      }
+
+      static void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)
+      {
+         var exception = e.ExceptionObject as Exception;
+         Log.FIX.error($"Unhandled Exception: {e.ExceptionObject} {exception?.StackTrace}");
+         if (e.IsTerminating)
+         {
+            Log.system_is_crashing("Unhandled exception", e.ExceptionObject);
+         }
+      }
+      
+      public static int start_acceptor(string settings_file, bool orders_allowed, string http_address = null)
+      {
+         SessionSettings settings = new SessionSettings(settings_file);
+         IApplication app = new FixRouter( (th, sess) =>
+         {
+            if(orders_allowed)
+               return new OrderSession(th, sess);
+            else
+               return new DropCopySession(th, sess);
+         });
+         IMessageStoreFactory storeFactory = new FileStoreFactory(settings);
+         ILogFactory logFactory = new FileLogFactory(settings);
+         IAcceptor acceptor = new ThreadedSocketAcceptor(app, storeFactory, settings, logFactory);
+
+         acceptor.Start();
+         HttpServer srv = null;
+         if (!string.IsNullOrEmpty(http_address))
+         {
+            if(!http_address.StartsWith("http"))
+               http_address = "http://127.0.0.1:5080/";
+            srv = new HttpServer(http_address, settings);
+            srv.Start();
+         }
+         Console.WriteLine("press <q> to quit");
+         while (Console.Read() != 'q') { }
+         acceptor.Stop();
+         if(srv != null)
+            srv.Stop();
+         return 0;
+
+      }
+   }
+}
+

src/client/spark_fix/FixRouter.cs

@@ -0,0 +1,252 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.md;
+using bts.api.admin;
+
+using security_key = System.UInt32;
+using QuickFix.FIX42;
+using Message = QuickFix.Message;
+
+namespace bts.fix
+{
+   interface IFixRouter
+   {
+      IEnumerable<SecurityImmutable> get_securities();
+      SecurityImmutable? lookup(security_key key);
+      SecurityImmutable? lookup(string desc);
+
+      void send(Message mess, SessionID sess);
+      void send_session_reject(SessionID sess, string text, Message refmessage = null);
+      void send_biz_reject(SessionID sess, string text, Message refmessage, int reason_code = 0);
+   }
+
+   class FixRouter : QuickFix.MessageCracker, IFIXSession, IFixRouter
+   {
+      readonly ConcurrentDictionary<SessionID, QuickFix.IApplication> _sessions = new ConcurrentDictionary<SessionID, IApplication>();
+      readonly Func<IFixRouter, SessionID, IApplication> _factory;      
+      readonly object _lock = new object();
+      readonly MarketDataService _md;
+      private readonly IDisposable _security_sub;
+      readonly ConcurrentDictionary<security_key, SecurityImmutable> _id_to_security = new ConcurrentDictionary<security_key, SecurityImmutable>();
+      readonly ConcurrentDictionary<string, SecurityImmutable> _desc_to_security = new ConcurrentDictionary<string, SecurityImmutable>();
+
+      public FixRouter(Func<IFixRouter, SessionID, IApplication> factory)
+      {
+         _factory = factory;
+         using(var service_context = new ServiceContext())
+         {
+            _md = service_context.locate<MarketDataService>();
+            _security_sub = _md.subscribe_securities().Subscribe( x => 
+            {
+               foreach( var sec in x.all)
+               {
+                  var imm = sec.immutable();
+                  _id_to_security[sec.Id] = imm;                  
+                  _desc_to_security[imm.security_desc()] = imm;
+               }
+            });         
+         }
+      }
+
+      IApplication session(SessionID sessionID)
+      {
+         IApplication app;
+         if (!_sessions.TryGetValue(sessionID, out app))
+         {
+            app = _factory(this, sessionID);
+            _sessions[sessionID] = app;
+         }
+         return app;
+      }
+
+      public void send(Message mess, SessionID sess)
+      {
+         lock (_lock)
+         {
+            try
+            {
+               Session.SendToTarget(mess, sess);
+            }
+            catch (SessionNotFound)
+            {
+               Log.assert(false, "session not found " + sess);
+            }
+            catch (Exception ex)
+            {
+               Log.FIX.error(sess.ToString() + " " + ex.Message + " " + sess);
+            }
+         }
+      }
+
+      public void send_session_reject(SessionID sess, string text, Message refmessage = null)
+      {
+         Reject rej = null;
+         if (refmessage != null && refmessage.Header.IsSetField(Tags.MsgSeqNum))
+            rej = new Reject(new QuickFix.Fields.RefSeqNum(refmessage.Header.GetInt(Tags.MsgSeqNum)));
+         else
+            rej = new Reject();
+         rej.Text = new Text(text);
+         send(rej, sess);
+      }
+
+      public void send_biz_reject(SessionID sess, string text, Message refmessage, int reason_code = 0)
+      {
+         BusinessMessageReject rej = new BusinessMessageReject();
+
+         rej.RefMsgType = new QuickFix.Fields.RefMsgType(refmessage.Header.GetString(Tags.MsgType));
+         rej.RefSeqNum = new QuickFix.Fields.RefSeqNum(refmessage.Header.GetInt(Tags.MsgSeqNum));
+         rej.BusinessRejectReason = new QuickFix.Fields.BusinessRejectReason(reason_code);
+         rej.Text = new Text(text);
+         send(rej, sess);
+      }      
+
+      void handle_session_exception(Exception ex, SessionID sessionID, Message message = null)
+      {
+         string msg = "==ERROR in session " + sessionID.ToString() + "== ";
+         msg += message?.GetType().Name ?? string.Empty;
+         msg += ex?.ToString() ?? string.Empty;
+         if(!string.IsNullOrEmpty(ex.StackTrace))
+            msg += Environment.NewLine + ex.StackTrace;
+
+         bts.Log.FIX.error(msg);
+         Console.WriteLine(msg);
+         //now shut this fucker down
+         try
+         {
+            var sess = Session.LookupSession(sessionID);
+            if(sess != null)
+               sess.Disconnect(msg);
+         }
+         catch { }
+      }
+
+      void handle_field_not_found(FieldNotFoundException ex, SessionID sess, Message message)
+      {
+         if(ex.Field > 0)
+            send_biz_reject(sess, $"Field {ex.Field} was expected but not found", message, 5 );
+         //QF does not include the tag number....
+         //throw new FieldNotFoundException($"Field {ex.Field} was expected but not found", ex);
+         else
+            throw ex;
+      }
+
+      public void FromAdmin(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).FromAdmin(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void FromApp(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).FromApp(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void OnCreate(SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).OnCreate(sessionID);    
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }      
+      }
+
+      public void OnLogon(SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).OnLogon(sessionID);
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }
+      }
+
+      public void OnLogout(SessionID sessionID)
+      {
+         try
+         {
+            //TODO stackoverflow?
+            session(sessionID).OnLogout(sessionID);
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }
+      }
+
+      public void ToAdmin(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).ToAdmin(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) {  throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void ToApp(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).ToApp(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      IEnumerable<SecurityImmutable> IFixRouter.get_securities()
+      {
+         return _id_to_security.Values;
+      }
+
+      SecurityImmutable? IFixRouter.lookup(uint key)
+      {
+         if(_id_to_security.TryGetValue(key, out SecurityImmutable sec))
+            return sec;
+         else
+            return null;
+      }
+
+      SecurityImmutable? IFixRouter.lookup(string desc)
+      {
+         if (_desc_to_security.TryGetValue(desc, out SecurityImmutable sec))
+            return sec;
+         else
+            return null;
+      }
+   }
+}

src/client/spark_fix/HttpServer.cs

@@ -0,0 +1,585 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Web;
+using QuickFix;
+
+namespace bts.fix
+{
+   public class HttpServer : IDisposable
+   {
+      private readonly HttpListener _httpListener;
+      private Thread _connectionThread;
+      private Boolean _running, _disposed;
+      private readonly SessionSettings _sessionSettings;
+      private StringBuilder _sbHtmlHeader;
+
+
+      public HttpServer(string prefix, SessionSettings settings)
+      {
+         if (!HttpListener.IsSupported)
+         {
+            // Requires at least a Windows XP with Service Pack 2
+            throw new NotSupportedException(
+                "The Http Server cannot run on this operating system.");
+         }
+
+         _httpListener = new HttpListener();
+         _httpListener.Prefixes.Add(prefix);
+         _sessionSettings = settings;
+      }
+
+      public void Start()
+      {
+         if (!_httpListener.IsListening)
+         {
+            _httpListener.Start();
+            _running = true;
+            // Use a thread to listen to the Http requests
+            _connectionThread = new Thread(ConnectionThreadStart);
+            _connectionThread.Start();
+         }
+      }
+
+      public void Stop()
+      {
+         if (_httpListener.IsListening)
+         {
+            _running = false;
+            _httpListener.Stop();
+         }
+      }
+
+      private void ConnectionThreadStart()
+      {
+         try
+         {
+            while (_running)
+            {
+               // Grab the context and pass it to the processor methods to handle it
+               HttpListenerContext context = _httpListener.GetContext();
+               HttpListenerRequest request = context.Request;
+               HttpListenerResponse response = context.Response;
+               string responseString;
+
+               _sbHtmlHeader = new StringBuilder(@"<HEAD><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>");
+               _sbHtmlHeader.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEADER><BODY>", request.Url.OriginalString);
+
+               switch (request.Url.AbsolutePath)
+               {
+                  case "/":
+                     responseString = ProcessRoot(request);
+                     break;
+                  case "/session":
+                     responseString = SessionDetails(request);
+                     break;
+                  case "/resetSession":
+                     responseString = ResetSession(request);
+                     break;
+                  case "/resetSessions":
+                     responseString = ResetSessions(request);
+                     break;
+                  // case "/refreshSession":
+                  //    responseString = RefreshSession(request);
+                  //    break;
+                  // case "/refreshSessions":
+                  //    responseString = RefreshSessions(request);
+                  //    break;
+                  case "/enableSessions":
+                     responseString = EnableSessions(request);
+                     break;
+                  case "/disableSessions":
+                     responseString = DisableSessions(request);
+                     break;
+                  default:
+                     responseString = ProcessRoot(request);
+                     break;
+               }
+
+               byte[] buffer = Encoding.UTF8.GetBytes(responseString);
+               response.ContentLength64 = buffer.Length;
+               Stream output = response.OutputStream;
+               output.Write(buffer, 0, buffer.Length);
+               output.Close();
+            }
+         }
+         catch (HttpListenerException)
+         {
+            Console.WriteLine("HTTP server was shut down.");
+         }
+      }
+
+      private string DisableSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Logout();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been disabled</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to disable all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, disable sessions</A>]&nbsp;[<A HREF='/'>NO, do not disable sessions</A>]</CENTER>", request.Url.OriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string EnableSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Logon();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been enabled</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to enable all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, enable sessions</A>]&nbsp;[<A HREF='/'>NO, do not enable sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string RefreshSession(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+
+         string url = "/session?" + GetParameterList(urlOriginalString);
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               sessionDetails.Refresh();
+               url = RemoveQueryStringByKey(urlOriginalString, "confirm");
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL=/session?{0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", GetParameterList(urlOriginalString));
+            sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEAD><BODY>", urlOriginalString);
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='/session?{0}'>{1}</A> has been refreshed</h2></CENTER>", GetParameterList(url), sessionId);
+         }
+         else
+         {
+            sbHtmlPageBody.AppendFormat("<CENTER><h2>Are you sure you want to refresh session <A HREF='{0}'>{1}</A>?</h2></CENTER>", url, sessionId);
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}&confirm=1'>YES, refresh session</A>]&nbsp;[<A HREF='{0}'>NO, do not refresh session</A>]</CENTER>", url);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string RefreshSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Refresh();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been refreshed</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to refresh all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, refresh sessions</A>]&nbsp;[<A HREF='/'>NO, do not refresh sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string ResetSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Reset("Reset from WebInterface");
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been reset</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to reset all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, reset sessions</A>]&nbsp;[<A HREF='/'>NO, do not reset sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string ResetSession(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+
+         string url = "/session?" + GetParameterList(urlOriginalString);
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               sessionDetails.Reset("Reset from WebInterface");
+               url = RemoveQueryStringByKey(urlOriginalString, "confirm");
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL=/session?{0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", GetParameterList(urlOriginalString));
+            sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEAD><BODY>", urlOriginalString);
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='/session?{0}'>{1}</A> has been reset</h2></CENTER>", GetParameterList(url), sessionId);
+         }
+         else
+         {
+            sbHtmlPageBody.AppendFormat("<CENTER><h2>Are you sure you want to reset session <A HREF='{0}'>{1}</A>?</h2></CENTER>", url, sessionId);
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}&confirm=1'>YES, reset session</A>]&nbsp;[<A HREF='{0}'>NO, do not reset session</A>]</CENTER>", url);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+
+      private string ProcessRoot(HttpListenerRequest request)
+      {
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         //Session count
+         sbHtmlPageBody.AppendFormat("<CENTER><b><i>{0} Sessions managed by QuickFIX</b></i></CENTER><HR/>", _sessionSettings.GetSessions().Count);
+
+         //session management links
+         sbHtmlPageBody.AppendFormat(@"<CENTER><A HREF='/resetSessions{0}'>RESET</A>&nbsp;<A HREF='/refreshSessions'>REFRESH</A>&nbsp;<A HREF='/enableSessions'>ENABLE</A>&nbsp;<A HREF='/disableSessions'>DISABLE</A></CENTER><HR/></HEADER><BODY>", GetParameterList(request.Url.OriginalString));
+
+         //Start the table generation
+         sbHtmlPageBody.Append("<table id=\"sessionlist\" style=\"border-width:1; padding:2; width:100%\">");
+
+         sbHtmlPageBody.Append("<tr>");
+         sbHtmlPageBody.Append(AddCell("Session", true));
+         sbHtmlPageBody.Append(AddCell("Type", true));
+         sbHtmlPageBody.Append(AddCell("Enabled", true));
+         sbHtmlPageBody.Append(AddCell("Session Time", true));
+         sbHtmlPageBody.Append(AddCell("Logged On", true));
+         sbHtmlPageBody.Append(AddCell("Next Incoming", true));
+         sbHtmlPageBody.Append(AddCell("Next Outgoing", true));
+         sbHtmlPageBody.Append("</tr>");
+
+         foreach (SessionID session in _sessionSettings.GetSessions())
+         {
+            Session sessionDetails = Session.LookupSession(session);
+            sbHtmlPageBody.Append("<tr>");
+            sbHtmlPageBody.Append(AddCell(String.Format(
+                "<a href=\"session?BeginString={0}&SenderCompID={1}&TargetCompID={2}\">{0}:{1}->{2}</a>",
+                session.BeginString, session.SenderCompID, session.TargetCompID)));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsInitiator ? "initiator" : "acceptor"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsEnabled ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsSessionTime ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsLoggedOn ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.NextTargetMsgSeqNum.ToString()));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.NextSenderMsgSeqNum.ToString()));
+            sbHtmlPageBody.Append("</tr>");
+         }
+
+         sbHtmlPageBody.Append("</table>");
+         return sbHtmlPageBody.ToString();
+      }
+
+      public string SessionDetails(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         string url = request.Url.OriginalString;
+         string urlOriginalString = request.Url.OriginalString;
+
+         if (request.QueryString["enabled"] != null)
+         {
+            if (!Convert.ToBoolean(request.QueryString["enabled"]))
+               sessionDetails.Logout();
+            else
+               sessionDetails.Logon();
+
+            url = RemoveQueryStringByKey(urlOriginalString, "Enabled");
+         }
+
+         if (request.QueryString["next incoming"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["next incoming"]);
+            sessionDetails.NextTargetMsgSeqNum = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "next incoming");
+         }
+
+         if (request.QueryString["Next Outgoing"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["Next Outgoing"]);
+            sessionDetails.NextSenderMsgSeqNum = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "Next Outgoing");
+         }
+
+         if (request.QueryString["SendRedundantResendRequests"] != null)
+         {
+            sessionDetails.SendRedundantResendRequests = Convert.ToBoolean(request.QueryString["SendRedundantResendRequests"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "SendRedundantResendRequests");
+         }
+
+         if (request.QueryString["CheckCompId"] != null)
+         {
+            sessionDetails.CheckCompID = Convert.ToBoolean(request.QueryString["CheckCompId"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "CheckCompId");
+         }
+
+         if (request.QueryString["CheckLatency"] != null)
+         {
+            sessionDetails.CheckLatency = Convert.ToBoolean(request.QueryString["CheckLatency"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "CheckLatency");
+         }
+
+         if (request.QueryString["MaxLatency"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["MaxLatency"]);
+            sessionDetails.MaxLatency = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "MaxLatency");
+         }
+
+         if (request.QueryString["LogonTimeout"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["LogonTimeout"]);
+            sessionDetails.LogonTimeout = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "LogonTimeout");
+         }
+
+         if (request.QueryString["LogoutTimeout"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["LogoutTimeout"]);
+            sessionDetails.LogoutTimeout = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "LogoutTimeout");
+         }
+
+         if (request.QueryString["ResetOnLogon"] != null)
+         {
+            sessionDetails.ResetOnLogon = Convert.ToBoolean(request.QueryString["ResetOnLogon"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnLogon");
+         }
+
+         if (request.QueryString["ResetOnLogout"] != null)
+         {
+            sessionDetails.ResetOnLogout = Convert.ToBoolean(request.QueryString["ResetOnLogout"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnLogout");
+         }
+
+         if (request.QueryString["ResetOnDisconnect"] != null)
+         {
+            sessionDetails.ResetOnDisconnect = Convert.ToBoolean(request.QueryString["ResetOnDisconnect"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnDisconnect");
+         }
+
+         if (request.QueryString["RefreshOnLogon"] != null)
+         {
+            sessionDetails.RefreshOnLogon = Convert.ToBoolean(request.QueryString["RefreshOnLogon"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "RefreshOnLogon");
+         }
+
+         if (request.QueryString["MillisecondsInTimestamp"] != null)
+         {
+            sessionDetails.MillisecondsInTimeStamp = Convert.ToBoolean(request.QueryString["MillisecondsInTimestamp"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "MillisecondsInTimestamp");
+         }
+
+         if (request.QueryString["PersistMessages"] != null)
+         {
+            sessionDetails.PersistMessages = Convert.ToBoolean(request.QueryString["PersistMessages"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "PersistMessages");
+         }
+
+
+         sbHtmlPageBody.AppendFormat(@"<CENTER>{0}</CENTER><HR/>", sessionId);
+         sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/resetSession?{0}'>RESET</A>]&nbsp;[<A HREF='/refreshSession?{0}'>REFRESH</A>]</CENTER><HR/></HEADER><BODY>", GetParameterList(urlOriginalString));
+
+         sbHtmlPageBody.Append("<table id=\"session_details\" style=\"border-width:1; padding:2; width:100%\">");
+
+         sbHtmlPageBody.Append(AddRow("Enabled", sessionDetails.IsEnabled, url));
+         sbHtmlPageBody.Append(AddRow("ConnectionType", sessionDetails.IsInitiator ? "initiator" : "acceptor"));
+         sbHtmlPageBody.Append(AddRow("SessionTime", sessionDetails.IsSessionTime));
+         sbHtmlPageBody.Append(AddRow("LoggedOn", sessionDetails.IsLoggedOn));
+         sbHtmlPageBody.Append(AddRow("Next Incoming", sessionDetails.NextTargetMsgSeqNum, url));
+         sbHtmlPageBody.Append(AddRow("Next Outgoing", sessionDetails.NextSenderMsgSeqNum, url));
+         sbHtmlPageBody.Append(AddRow("SendRedundantResendRequests", sessionDetails.SendRedundantResendRequests, url));
+         sbHtmlPageBody.Append(AddRow("CheckCompId", sessionDetails.CheckCompID, url));
+         sbHtmlPageBody.Append(AddRow("CheckLatency", sessionDetails.CheckLatency, url));
+         sbHtmlPageBody.Append(AddRow("MaxLatency", sessionDetails.MaxLatency, url));
+         sbHtmlPageBody.Append(AddRow("LogonTimeout", sessionDetails.LogonTimeout, url));
+         sbHtmlPageBody.Append(AddRow("LogoutTimeout", sessionDetails.LogoutTimeout, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnLogon", sessionDetails.ResetOnLogon, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnLogout", sessionDetails.ResetOnLogout, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnDisconnect", sessionDetails.ResetOnDisconnect, url));
+         sbHtmlPageBody.Append(AddRow("RefreshOnLogon", sessionDetails.RefreshOnLogon, url));
+         sbHtmlPageBody.Append(AddRow("MillisecondsInTimestamp", sessionDetails.MillisecondsInTimeStamp, url));
+         sbHtmlPageBody.Append(AddRow("PersistMessages", sessionDetails.PersistMessages, url));
+
+         sbHtmlPageBody.Append("</table>");
+         return sbHtmlPageBody.ToString();
+      }
+
+      public static string RemoveQueryStringByKey(string url, string key)
+      {
+         var uri = new Uri(url);
+
+         // this gets all the query string key value pairs as a collection
+         var newQueryString = HttpUtility.ParseQueryString(uri.Query);
+
+         // this removes the key if exists
+         newQueryString.Remove(key);
+
+         // this gets the page path from root without QueryString
+         string pagePathWithoutQueryString = uri.GetLeftPart(UriPartial.Path);
+
+         return newQueryString.Count > 0
+             ? $"{pagePathWithoutQueryString}?{newQueryString}"
+             : pagePathWithoutQueryString;
+      }
+
+      public static string GetParameterList(string url)
+      {
+         return HttpUtility.ParseQueryString((new Uri(url).Query)).ToString();
+      }
+
+      private static string AddRow(string colName, bool value, string url = "")
+      {
+         string valueAsStr = value ? "yes" : "no";
+         string innerHtml = url.Length > 0
+             ? $"<a href=\" {url}&{colName}={!value} \">toggle</a>"
+             : "";
+         return AddRow(colName, valueAsStr, innerHtml);
+      }
+
+      private static string AddRow(string colName, int value, string url = "")
+      {
+         string innerHtml = $"<a href=\" {url}&{colName}={value - 10} \"> << </a>" +
+                            $"<a href=\" {url}&{colName}={value - 1} \"> < </a>" +
+                            " | " +
+                            $"<a href=\" {url}&{colName}={value + 1} \"> > </a>" +
+                            $"<a href=\" {url}&{colName}={value + 10} \"> >> </a>";
+         return AddRow(colName, value.ToString(), innerHtml);
+      }
+      private static string AddRow(string colName, string value, string innerHtml = "")
+      {
+         StringBuilder row = new StringBuilder();
+         row.Append("<tr>");
+         row.Append(AddCell(colName));
+         row.Append(AddCell(value));
+         row.Append(AddCell(innerHtml));
+         row.Append("</tr>");
+         return row.ToString();
+      }
+
+      private static string AddCell(string cellContent, bool header = false)
+      {
+         string entryType = header ? "th" : "td";
+         return "<" + entryType + " align=\"left\">" + cellContent + "</" + entryType + ">";
+      }
+
+      public virtual void Dispose()
+      {
+         Dispose(true);
+         GC.SuppressFinalize(this);
+      }
+
+      private void Dispose(bool disposing)
+      {
+         if (_disposed)
+         {
+            return;
+         }
+         if (disposing)
+         {
+            if (_running)
+            {
+               Stop();
+            }
+            if (_connectionThread != null)
+            {
+               _connectionThread.Abort();
+               _connectionThread = null;
+            }
+         }
+         _disposed = true;
+      }
+   }
+}

src/client/spark_fix/Properties/AssemblyAttributes.cs

@@ -0,0 +1,2 @@
+using System.Runtime.CompilerServices;
+

src/client/spark_fix/ProtoFix.cs

@@ -0,0 +1,293 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using Google.Protobuf;
+using Google.Protobuf.Reflection;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using bts.api;
+
+
+namespace bts.fix
+{
+   public static class ProtoFix
+   {
+      static readonly QuickFix.IMessageFactory _factory = new QuickFix.DefaultMessageFactory();
+      public static T parse_fix<T>(string fixstring) where T : IMessage, new()
+      {
+         return (T) parse_fix(new T().Descriptor, fixstring);
+      }
+
+      public static IMessage parse_fix(MessageDescriptor desc, string fixstring)
+      {
+         QuickFix.Message fix = new QuickFix.Message(fixstring, false);
+         var msg = create_message(desc, fix);
+         return msg;
+      }
+
+      static string get_between(string source, string start, string end)
+      {
+         int spos = source.IndexOf(start, 0);
+         if(spos < 0)
+            return string.Empty;
+         spos += start.Length;
+         int epos = source.IndexOf(end, spos);
+         if(epos < 0)
+            return string.Empty;
+         return source.Substring(spos, epos - spos);
+      }      
+
+      public static Timestamp peek_timestamp(string fixstring, int field = 60) 
+      {
+         string ts = get_between(fixstring, field.ToString() + "=", "\u0001");
+         if(!string.IsNullOrEmpty(ts))
+         {
+            var stamp = TimestampExtensions.parse_timestamp(ts);
+            return stamp;
+         }
+         return null;
+      }
+
+      public static string peek_field(string fixstring, int field)
+      {
+         string ts = get_between(fixstring, field.ToString() + "=", "\u0001");
+         return ts;
+      }
+
+      public static string peek_symbol(string fixstring)
+      {
+         return peek_field(fixstring, 55);
+      }
+
+      public static IMessage create_message(MessageDescriptor desc, QuickFix.FieldMap fix)
+      {
+         object obj = Activator.CreateInstance(desc.ClrType);
+         IMessage msg = (IMessage)obj;
+         return setup_message(msg, fix);
+      }
+
+      public static IMessage setup_message(IMessage msg, QuickFix.FieldMap fix)
+      {
+         foreach (var fd in msg.Descriptor.Fields.InDeclarationOrder())
+         {
+            try
+            {
+               set_field(msg, fd, fix);
+            }
+            catch(QuickFix.FieldNotFoundException) {}
+         }
+         return msg;
+      }
+
+      static int get_tag(IMessage msg, FieldDescriptor fd, out bool is_header)
+      {
+         is_header = false;
+         int fieldnum = 0;
+         if(fd.CustomOptions?.TryGetInt32((int)ApioptionsReflection.FieldOptions.fix_tag, out fieldnum) ?? false)
+         {
+            fd.CustomOptions.TryGetBool((int)ApioptionsReflection.FieldOptions.fix_tag_is_header, out is_header);
+            return fieldnum;
+         }
+         else
+         {
+            int tagbase = 0;
+            if(msg.Descriptor.CustomOptions?.TryGetInt32((int)ApioptionsReflection.MessageOptions.fix_algostrategy_tagbase, out tagbase) ?? false)
+               return fd.FieldNumber + tagbase;
+         }
+         throw new QuickFix.FieldNotFoundException(fd.FieldNumber);
+      }
+
+      static void set_field(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {         
+         switch (fd.FieldType)
+         {
+            case FieldType.Bool:
+               {
+                  int tag = get_tag(msg, fd, out bool is_header);
+                  var data = is_header ? ((QuickFix.Message)fix).Header.GetBoolean(tag) : fix.GetBoolean(tag);
+                  fd.Accessor.SetValue(msg, data);
+               }
+               break;
+            case FieldType.Float:
+               {
+                  int tag = get_tag(msg, fd, out bool is_header);
+                  var data = is_header ? ((QuickFix.Message)fix).Header.GetDecimal(tag) : fix.GetDecimal(tag);
+                  object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                  fd.Accessor.SetValue(msg, conv);
+               }
+               break;
+            case FieldType.Double:
+               {
+                  int tag = get_tag(msg, fd, out bool is_header);
+                  var data = is_header ? ((QuickFix.Message)fix).Header.GetDecimal(tag) : fix.GetDecimal(tag);
+                  double d = decimal.ToDouble(data);
+                  fd.Accessor.SetValue(msg, d);
+               }
+               break;
+            case FieldType.Int32:
+            case FieldType.SInt32:
+            case FieldType.UInt32:
+               {
+                  int tag = get_tag(msg, fd, out bool is_header);
+                  var data = is_header ? ((QuickFix.Message)fix).Header.GetInt(tag) : fix.GetInt(tag);
+                  object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                  fd.Accessor.SetValue(msg, conv);
+               }
+               break;
+            case FieldType.Int64:
+            case FieldType.SInt64:
+            case FieldType.UInt64:
+               {
+                  int tag = get_tag(msg, fd, out bool is_header);
+                  try
+                  {                     
+                     var data = is_header ? ((QuickFix.Message)fix).Header.GetDecimal(tag) : fix.GetDecimal(tag);
+                     object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                     fd.Accessor.SetValue(msg, conv);
+                  }
+                  catch(Exception ex) when (ex is QuickFix.FieldConvertError || ex is FormatException)
+                  {
+                     try
+                     {
+                        var snot = fix.GetString(tag); //scientific notation apparently appears in the files
+                        var data = Double.Parse(snot, System.Globalization.NumberStyles.Float);
+                        object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                        fd.Accessor.SetValue(msg, conv);
+                     }
+                     catch {}
+                  }
+               }
+               break;
+            case FieldType.Enum:
+               {                                    
+                  EnumValueDescriptor val = find_enum_by_attribute(msg, fd, fix);                  
+                  if(val == null) 
+                  {
+                     int tag = get_tag(msg, fd, out bool is_header);
+                     char data = is_header ? ((QuickFix.Message)fix).Header.GetChar(tag) : fix.GetChar(tag);
+                     val = fd.EnumType.FindValueByNumber((int)data);
+                     val = val ?? fd.EnumType.FindValueByNumber(0);
+                  }
+                  fd.Accessor.SetValue(msg, val.Number);
+               }
+               break;
+            case FieldType.String:
+               {
+                  int tag = get_tag(msg, fd, out bool is_header);
+                  var data = is_header ? ((QuickFix.Message)fix).Header.GetString(tag) : fix.GetString(tag);
+                  fd.Accessor.SetValue(msg, data);
+               }
+               break;
+            case FieldType.Message:
+               {
+                  handle_message(msg, fd, fix);
+               }
+               break;
+            default:
+               throw new NotSupportedException("unsupported field type " + fd.FieldType.ToString());
+         }
+      }
+
+      private static EnumValueDescriptor find_enum_by_attribute(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         int tag = get_tag(msg, fd, out bool is_header);
+         var fval = is_header ? ((QuickFix.Message)fix).Header.GetString(tag) : fix.GetString(tag);
+         foreach(var val in fd.EnumType.Values)
+         {
+            string eval = null;
+            if(val.CustomOptions?.TryGetString((int)ApioptionsReflection.EnumValueOptions.fix_value, out eval) ?? false)
+            {
+               if(eval != null && eval == fval)
+                  return val;
+            }
+         }
+         return null;
+      }
+
+      static void handle_timestamp(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         int tag = get_tag(msg, fd, out bool is_header);
+         var data = is_header ? ((QuickFix.Message)fix).Header.GetString(tag) : fix.GetString(tag);
+         Timestamp ts = TimestampExtensions.parse_timestamp(data);         
+         fd.Accessor.SetValue(msg, ts);         
+      }
+      static void handle_quantitymeasure(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         int tag = get_tag(msg, fd, out bool is_header);
+         var data = is_header ? ((QuickFix.Message)fix).Header.GetInt(tag) : fix.GetInt(tag);
+         var qm = new bts.spark.QuantityMeasure { Absolute = (uint) Math.Abs(data) };
+         fd.Accessor.SetValue(msg, qm);
+      }
+
+      static void handle_message(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         if (fd.MessageType.ClrType == typeof(Google.Protobuf.WellKnownTypes.Timestamp))
+         {
+            handle_timestamp(msg, fd, fix);
+         }
+         else if(fd.MessageType.ClrType == typeof(bts.spark.QuantityMeasure))
+         {
+            handle_quantitymeasure(msg, fd, fix);
+         }
+         else if(fd.MessageType.ClrType == typeof(bts.api.orderman.CustomOrderDefinition))
+         {
+            return;
+         }
+         else
+         {
+            if (!fd.IsRepeated)
+            {
+               var submsg = create_message(fd.MessageType, fix);
+               fd.Accessor.SetValue(msg, submsg);
+            }
+            else //for repeating groups we assume that the fd.FieldNumber corresponds to the "no" (number) tag in the fix
+            {
+               int tag = get_tag(msg, fd, out bool is_header);
+               int group = tag;//fd.FieldNumber;
+               int nvals = fix.GetInt(group);
+               IList list = (IList)fd.Accessor.GetValue(msg);
+               if (nvals == 1)
+               {
+                  var submsg = create_message(fd.MessageType, fix);
+                  list.Add(submsg);
+               }
+               else
+               {
+                  var delimproto = fd.MessageType.Fields.InDeclarationOrder()[0];
+                  var delim = delimproto.FieldNumber;
+
+                  List<QuickFix.Fields.IField> repeated = new List<QuickFix.Fields.IField>();
+                  var forder = fix.RepeatedTags.Select(x => x.Tag).Distinct();
+                  var dict = fix.ToDictionary();
+                  foreach(int fo in forder)
+                  {
+                     if(dict.TryGetValue(fo, out QuickFix.Fields.IField field))
+                        repeated.Add(field);
+                  }
+                  repeated.AddRange(fix.RepeatedTags);
+                  QuickFix.FieldMap fm = null;
+                  foreach (var rep in repeated)
+                  {
+                     if (rep.Tag == delim)
+                     {
+                        if (fm != null)
+                        {
+                           var submsg = create_message(fd.MessageType, fm);
+                           list.Add(submsg);
+                        }
+                        fm = new QuickFix.FieldMap();
+                     }
+                     if (fm != null)
+                        fm.SetField(rep);
+                  } //the last one
+                  if (fm != null && fm.Count() > 0)
+                  {
+                     var submsg = create_message(fd.MessageType, fm);
+                     list.Add(submsg);
+                  }
+               }
+            }
+         }         
+      }
+   }
+}

src/client/spark_fix/README.md

@@ -0,0 +1,2 @@
+# BTS Spark FIX Adapter
+

src/client/spark_fix/session/ATDL.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using bts.api.orderman;
+
+namespace bts.fix
+{
+   public class ATDL
+   {
+      readonly Dictionary<string, FIXAlgoDefinition> _atdl = new Dictionary<string, FIXAlgoDefinition>();
+      public IReadOnlyDictionary<string, FIXAlgoDefinition> algo_definitions => _atdl;
+      public ATDL(string path = "./spark_algos.json")
+      {
+         using(var fs = new StreamReader(path))
+         {
+            var algos = Google.Protobuf.JsonParser.Default.Parse<FIXAlgoDefinitions>(fs);
+            foreach(var algo in algos.AlgoStrategy)
+            {
+               _atdl[algo.Name] = algo;
+            }
+         }
+      }
+      
+      public static void write_json(FIXAlgoDefinitions defs, string path)
+      {
+         using (var fs = new StreamWriter(path))
+         {
+            var s = Google.Protobuf.JsonFormatter.Settings.Default;
+
+            var raw = Google.Protobuf.JsonFormatter.Default.Format(defs);
+            object parsed = Newtonsoft.Json.JsonConvert.DeserializeObject(raw);
+
+            string indented = Newtonsoft.Json.JsonConvert.SerializeObject(parsed, Newtonsoft.Json.Formatting.Indented);
+            fs.Write(indented);
+         }         
+      }
+      
+      public static void write_sample_json(string path = "./spark_algos.json")
+      {
+         FIXAlgoDefinition tc = new FIXAlgoDefinition() 
+         {
+            Name = "TickAndCover", 
+            Definition = new CustomOrderDefinition
+            {
+               Cover = new CoverOrderDefinition
+               {
+                  Child = new CustomOrderDefinition
+                  {
+                     Top = new TopOrderDefinition { Child = new CustomOrderDefinition { Limit = new LimitOrderDefinition() } }
+                  },
+                  Hedge = new CustomOrderDefinition
+                  {
+                     Lean = new LeanOrderDefinition { Child = new CustomOrderDefinition { Limit = new LimitOrderDefinition() } }
+                  }                  
+               }
+            }
+         };
+
+         FIXAlgoDefinition sc = new FIXAlgoDefinition()
+         {
+            Name = "SmashAndCover",
+            Definition = new CustomOrderDefinition
+            {
+               Cover = new CoverOrderDefinition
+               {
+                  Child = new CustomOrderDefinition
+                  {
+                     Lean = new LeanOrderDefinition{ Child = new CustomOrderDefinition { Limit = new LimitOrderDefinition() } }
+                  },
+                  Hedge = new CustomOrderDefinition
+                  {
+                     Lean = new LeanOrderDefinition { Child = new CustomOrderDefinition { Limit = new LimitOrderDefinition() } }
+                  }
+               }
+            }
+         };         
+         
+         FIXAlgoDefinitions defs = new FIXAlgoDefinitions();
+         defs.AlgoStrategy.Add(tc);
+         defs.AlgoStrategy.Add(sc);
+         write_json(defs, path);
+      }
+   }
+}

src/client/spark_fix/session/DropCopySession.cs

@@ -0,0 +1,288 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.api.admin;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+using QuickFix.FIX42;
+using Message = QuickFix.Message;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace bts.fix
+{
+   class DropCopySession : SessionBase
+   {
+      protected IDisposable _fillsdisp;
+      protected IDisposable _ordersdisp;
+      protected static Lazy<OrderService> _orderservice = new Lazy<OrderService>(() => bts.api.Locator.locate<OrderService>(), true);
+
+      protected readonly ConcurrentDictionary<uint, OrderRecordImmutable> _order_cache = new ConcurrentDictionary<uint, OrderRecordImmutable>();
+      protected readonly ConcurrentDictionary<uint, string> _fix_clorderids = new ConcurrentDictionary<uint, string>();
+      protected readonly ConcurrentDictionary<string, uint> _external_id_vs_bts_key = new ConcurrentDictionary<string, uint>();
+      public DropCopySession(IFixRouter router, SessionID sess) : base(router, sess)
+      {
+      }
+
+      override public void OnLogon(SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         var msg = sessionID.ToString() + " Logged on";
+         log_info(msg);
+
+         subscribe_fills_and_orders(sessionID);
+      }
+      override public void OnLogout(SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         var msg = sessionID.ToString() + " Logged out";
+         log_info(msg);
+
+         if (_ordersdisp != null)
+            _ordersdisp.Dispose();
+         if (_fillsdisp != null)
+            _fillsdisp.Dispose();
+      }      
+
+      override public void FromApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.ORDER_STATUS_REQUEST:
+            case MsgType.SECURITYDEFINITIONREQUEST:
+               log_info("IN: " + message);
+               Crack(message, sessionID);
+               break;
+            default:
+               log_info("UNSUPPORTED MESSAGE IN: " + type.getValue());
+               throw new UnsupportedMessageType();
+         }
+      }
+
+      virtual public void OnMessage(OrderStatusRequest req, SessionID s)
+      {
+         uint oid = uint.Parse(req.OrderID.getValue());
+         if (_order_cache.TryGetValue(oid, out OrderRecordImmutable order))
+         {
+            var ostat = build_order_status(s, order);
+            //ostat.OrdStatusReqID = new OrdStatusReqID(req.OrdStatusReqID.getValue());
+            //TODO
+            _router.send(ostat, s);
+         }
+      }
+
+      virtual public void OnMessage(SecurityDefinitionRequest req, SessionID s)
+      {
+         if (req.SecurityRequestType.getValue() == SecurityRequestType.REQUEST_LIST_SECURITIES)
+         {
+            var secs = _router.get_securities();
+            if (req.IsSetSymbol())
+               secs = secs.Where(x => x.Symbol.ToLower() == req.Symbol.getValue().ToLower());
+            if (req.IsSetSecurityType())
+               secs = secs.Where(x => security_types_equal(x.security_type(), req.SecurityType));
+            if (req.IsSetSecurityExchange())
+               secs = secs.Where(x => exchanges_equal(x.Exchange, req.SecurityExchange));
+
+            var def = new SecurityDefinition(req.SecurityReqID,
+                                                            new SecurityResponseID(Guid.NewGuid().ToString()),
+                                                            new TotalNumSecurities(secs.Count()));
+            _router.send(def, s);
+         }
+         else
+            throw new IncorrectTagValue(QuickFix.Fields.Tags.SecurityRequestType);
+      }
+
+      void _add_order_info_to_caches(OrderRecordImmutable o) {
+         _order_cache[o.Key] = o;
+         if (o.HasDef && o.ParentKey==0 && !string.IsNullOrEmpty(o.Def.ExternalOrderId)) {
+            _external_id_vs_bts_key[o.Def.ExternalOrderId] = o.Key;
+         }
+      }
+
+      virtual async protected void subscribe_fills_and_orders(SessionID sessionID)
+      {
+         var scheduler = TaskPoolScheduler.Default;
+         var interval = TimeSpan.FromMilliseconds(500);
+         var on_orders = _orderservice.Value.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(_adminobservable).Table().Cache();
+         var on_fills = _orderservice.Value.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(_adminobservable).Table().Cache();
+
+         var odone = new TaskCompletionSource<int>();
+         _ordersdisp = on_orders.Subscribe(o =>
+         {
+            foreach (var order in o.added)
+            {
+               var ostat = build_order_status(sessionID, order);
+               _add_order_info_to_caches(order);
+               _router.send(ostat, sessionID);
+            }
+            foreach (var order in o.changed)
+            {
+               var ostat = build_order_status(sessionID, order, true);
+               _router.send(ostat, sessionID);
+            }
+            odone.TrySetResult(0);
+         });
+         
+         await odone.Task;
+         _fillsdisp = on_fills.Subscribe(f =>
+         {
+            foreach (var fill in f.added)
+            {
+               var fi = build_fill(sessionID, fill);
+               _router.send(fi, sessionID);
+            }
+            foreach (var fill in f.changed)
+            {
+               var fi = build_fill(sessionID, fill, true);
+               _router.send(fi, sessionID);
+            }
+         });
+      }
+
+
+      virtual protected void process_order_state(in OrderStateImmutable state, ref uint filled, ref uint leaves, ref double avgprx,
+                                                 ref QuickFix.Fields.OrdStatus fstatus, ref NativeOrderStateImmutable? native, FillImmutable? fill = null)
+      {
+         filled = state.FilledQuantity;
+         leaves = state.Quantity - state.FilledQuantity;
+         avgprx = state.AvgPrice;
+         fstatus = translate_status(state, ref native);         
+      }
+
+      protected bts.api.SecurityImmutable? resolve_execution_report_security(bts.api.SecurityImmutable? sec, ExecutionReport report)
+      {
+         report.SecurityExchange = convert_exchange(sec.Value.Exchange);
+         var sectype = sec.Value.security_type();
+         report.SecurityType = convert_security_type(sectype);
+         if (sec.Value.HasGlbx)
+            report.SecurityDesc = new SecurityDesc(sec.Value.Glbx.SecurityDesc);
+
+         switch (sectype)
+         {
+            case bts.api.SecurityType.Future:
+               report.MaturityMonthYear = new MaturityMonthYear(sec.Value.EarliestExpiration().ToString("yyyyMMdd"));
+               break;
+            case bts.api.SecurityType.Spread: //TODO this is not quite right for spreads of spreads            
+               var leg = sec.Value.Spread.Legs.OrderBy(x => x.Security.EarliestExpiration()).First();
+               report.MaturityMonthYear = new MaturityMonthYear(leg.Security.EarliestExpiration().ToString("yyyyMMdd"));
+               break;
+            default:
+               throw new UnsupportedMessageType();               
+         }
+         return sec;
+      }
+
+      virtual protected ExecutionReport build_fill(SessionID sess, in FillImmutable fill, bool change = false)
+      {
+         var exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.PARTIAL_FILL);
+         QuickFix.Fields.OrdStatus status = new QuickFix.Fields.OrdStatus();
+         uint filled = 0; uint leaves = 0; double prx = 0.0; uint orig_qty = 0;
+         OrderRecordImmutable order;  NativeOrderStateImmutable? native = null;
+         if (_order_cache.TryGetValue(fill.OrderKey, out order))
+         {
+            orig_qty = order.State.Quantity;
+            process_order_state(order.State, ref filled, ref leaves, ref prx, ref status, ref native);
+            if(Convert.ToDouble(order.State.Quantity) == fill.LastShares)
+               exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.FILL);
+         }
+         if (change)
+            exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.TRADE_CORRECT);
+
+         var sec = _router.lookup(fill.SecurityKey);
+         ExecutionReport report = new ExecutionReport(
+             new OrderID(fill.OrderKey.ToString()),
+             new ExecID(fill.FillKey.ToString()),
+             new QuickFix.Fields.ExecTransType( ExecTransType.NEW ),
+             exec,
+             status,
+             new Symbol(sec?.Symbol ?? string.Empty),
+             new QuickFix.Fields.Side(fill.Side == bts.api.Side.Buy ? QuickFix.Fields.Side.BUY : QuickFix.Fields.Side.SELL),
+             new LeavesQty(leaves),
+             new CumQty(filled),
+             new AvgPx(Convert.ToDecimal(prx)));
+             
+         report.Account = new Account(fill.Account);
+         report.OrderQty = new OrderQty(orig_qty);
+         report.LastShares = new LastShares(Convert.ToDecimal(fill.LastShares));
+         report.LastPx = new LastPx(Convert.ToDecimal(fill.LastPrice));
+         var ts = fill.HasTimestamp ? fill.Timestamp.ToDateTime() : DateTime.MinValue;
+         report.TransactTime = new TransactTime(ts, true);
+         report.TradeDate = new TradeDate(ts.ToString("yyyyMMdd"));
+         sec = resolve_execution_report_security(sec, report);
+         if (native.HasValue && !string.IsNullOrEmpty(native.Value.ExchangeOrderId))
+            report.SecondaryOrderID = new SecondaryOrderID(native.Value.ExchangeOrderId);
+
+         if (sec.HasValue && sec.Value.security_type() == bts.api.SecurityType.Spread || 
+             sec.Value.security_type() == bts.api.SecurityType.Synthetic)
+         {
+            if (fill.FillKind == bts.api.orderman.Fill.Types.FillKind.Leg)
+               report.MultiLegReportingType = new MultiLegReportingType('2');
+            else
+               report.MultiLegReportingType = new MultiLegReportingType('3');
+         }
+         return report;
+      }
+
+      virtual protected ExecutionReport build_order_status(SessionID sess, in OrderRecordImmutable order, bool change = false)
+      {
+         var exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.NEW);
+         if (_order_cache.TryGetValue(order.Key, out OrderRecordImmutable _))
+            exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.ORDER_STATUS);
+            
+         QuickFix.Fields.OrdStatus status = new QuickFix.Fields.OrdStatus();         
+         uint filled = 0; uint leaves = 0; double prx = 0.0;  NativeOrderStateImmutable? native = null;
+         process_order_state(order.State, ref filled, ref leaves, ref prx, ref status, ref native);
+
+         var sec = _router.lookup(order.Def.SecurityKey);
+         ExecutionReport report = new ExecutionReport(
+             new OrderID(order.Key.ToString()),
+             new ExecID("0"),
+             new QuickFix.Fields.ExecTransType(ExecTransType.STATUS),
+             exec,
+             status,
+             new Symbol(sec?.Symbol ?? string.Empty),
+             new QuickFix.Fields.Side(order.Def.Side == bts.api.Side.Buy ? QuickFix.Fields.Side.BUY : QuickFix.Fields.Side.SELL),
+             new LeavesQty(leaves),
+             new CumQty(filled),
+             new AvgPx(Convert.ToDecimal(prx)));
+
+         report.Account = new Account(order.Def.Account);
+         report.OrderQty = new OrderQty(order.State.Quantity);
+         report.Price = new Price(Convert.ToDecimal(order.State.Price));
+         report.TimeInForce = new QuickFix.Fields.TimeInForce((char)order.Def.TimeInForce);
+         var ts = order.State.LastChange.ToDateTime();
+         report.TransactTime = new TransactTime(ts, true);
+         if(!string.IsNullOrEmpty(order.State.LastMsg))
+            report.Text = new Text(order.State.LastMsg);
+
+         sec = resolve_execution_report_security(sec, report);
+         if (native.HasValue)
+         {
+            report.OrdType = new OrdType((char)native.Value.NativeType);        
+            if(!string.IsNullOrEmpty(native.Value.ExchangeOrderId))
+               report.SecondaryOrderID = new SecondaryOrderID(native.Value.ExchangeOrderId);            
+         }
+         //TODO Big race here, we need a correlation id of some sort
+         if (_fix_clorderids.TryGetValue(order.Key, out string clorderid))
+         {
+            report.ClOrdID = new ClOrdID(clorderid);
+         }
+         return report;         
+      }      
+
+   }
+}

src/client/spark_fix/session/IFIXSession.cs

@@ -0,0 +1,8 @@
+using System;
+
+namespace bts.fix
+{
+   interface IFIXSession : QuickFix.IApplication
+   {
+   }
+}

src/client/spark_fix/session/OrderSession.cs

@@ -0,0 +1,204 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.orderman;
+
+using QuickFix.FIX42;
+using Message = QuickFix.Message;
+
+namespace bts.fix
+{
+   /*
+create limit
+8=FIX.4.2
9=156
35=D
49=CQG
56=SPARK
34=1
50=30737
97=N
52=20200109-20:20:50
11=90001008
1=ACC A
21=2
55=SIM:ESH0
207=SIM
54=1
60=20200809-18:20:32
38=10
40=2
44=3200
59=0
10=129

+
+update limit
+8=FIX.4.2
9=110
35=G
49=CQG
56=SPARK
37=13
41=90001008
11=90002008
21=1
55= ESH0
54=1
60=20200113-19:45:29
38=10
40=2
44=3201
10=017

+
+delete limit
+8=FIX.4.2
9=93
35=F
49=CQG
56=SPARK
41= 90001008
37=13
11=90003008
55=ESH0
54=1
60=20200113-19:45:27
38=10
10=247

+
+create algo
+8=FIX.4.2
9=156
35=D
49=CQG
56=SPARK
34=1
50=30737
97=N
52=20200109-20:20:50
11=90001008
1=ACC A
21=2
55=SIM:ESH0
207=SIM
54=1
60=20200809-18:20:32
38=10
40=2
44=3200
59=0
6401=Lean
8401=11
8402=2
8405=0
10=128

+
+      jake:
+      new
+      8=FIX.4.2
9=156
35=D
49=CQG
56=SPARK
34=1
50=30737
97=N
52=20200109-20:20:50
11=8675309
1=ACC A
21=2
55=SIM:ESH0
207=SIM
54=1
60=20200809-18:20:32
38=10
40=2
44=3200
59=0
10=128

+      cancel-replace
+      8=FIX.4.2
9=110
35=G
49=CQG
56=SPARK
37=13
41=8675309
11=8675310
21=1
55=ESH0
54=1
60=20200113-19:45:29
38=10
40=2
44=3201
10=017

+      cancel
+      8=FIX.4.2
9=89
35=F
49=CQG
56=SPARK
41=8675309
37=13
11=8675311
55=ESH0
54=1
60=20200113-19:45:27
38=10
10=123

+
+   */
+   class OrderSession : DropCopySession
+   {
+      readonly ATDL _atdl = new ATDL();
+      public OrderSession(IFixRouter router, SessionID sess) : base(router, sess)
+      {
+      }
+
+      override public void FromApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.ORDER_STATUS_REQUEST: //handled in DropCopySession
+            case MsgType.SECURITYDEFINITIONREQUEST: //handled in DropCopySession
+
+            case MsgType.NEWORDERSINGLE:
+            case MsgType.ORDERCANCELREPLACEREQUEST:
+            case MsgType.ORDERCANCELREQUEST:
+               log_info("IN: " + message);
+               Crack(message, sessionID);
+               break;
+            default:
+               log_info("UNSUPPORTED MESSAGE IN: " + type.getValue());
+               throw new UnsupportedMessageType();
+         }
+      }
+
+      virtual public void OnMessage(OrderCancelRequest cancel, SessionID s)
+      {
+         var original_client_id = cancel.OrigClOrdID.getValue();
+         if (!base._external_id_vs_bts_key.ContainsKey(original_client_id))
+            throw new IncorrectTagValue(QuickFix.Fields.Tags.OrigClOrdID);
+         var key = base._external_id_vs_bts_key[original_client_id];
+         var resp = _orderservice.Value.cancel_order(new Cancel { OrderKey = key });
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            send_cancel_reject(s, resp.ErrorCode, "Cancel Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, cancel);
+         }
+      }
+
+      virtual public void OnMessage(OrderCancelReplaceRequest replace, SessionID s)
+      {
+         var original_client_id = replace.OrigClOrdID.getValue();
+         if (!base._external_id_vs_bts_key.ContainsKey(original_client_id))
+            throw new IncorrectTagValue(QuickFix.Fields.Tags.OrigClOrdID);
+         var key = base._external_id_vs_bts_key[original_client_id];
+         double price = Convert.ToDouble(replace.Price.getValue());
+         uint sz = Convert.ToUInt32(Math.Abs(replace.OrderQty.getValue()));
+         var upd = new Update
+         {
+            OrderKey = key,
+            Price = price,
+            Quantity = sz,
+            Custom = get_custom(replace),
+         };
+         var resp = _orderservice.Value.update_order(upd);
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            send_cancel_reject(s, resp.ErrorCode, "Cancel Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, replace);
+         }
+      }
+
+      virtual public void OnMessage(NewOrderSingle create, SessionID s)
+      {
+         Venue ven = Venue.Glbx;
+         if(create.IsSetSecurityExchange())
+            ven = convert_exchange(create.SecurityExchange);
+         if(create.IsSetExDestination())
+            ven = convert_exchange(create.ExDestination);
+         if (ven == bts.api.Venue.Unset)
+            throw new IncorrectTagValue(QuickFix.Fields.Tags.SecurityExchange);
+
+         var sec = _router.lookup(create.Symbol.getValue());
+         if (!sec.HasValue)
+            sec = _router.lookup(create.SecurityDesc.getValue());
+            
+         if (!sec.HasValue)
+            throw new IncorrectTagValue(QuickFix.Fields.Tags.Symbol);
+
+         OrderDefinition def = (OrderDefinition)ProtoFix.create_message(OrderDefinition.Descriptor, create);
+         def.SecurityKey = sec.Value.Id;
+         def.User = _login_name;
+         def.UserId = lookup_user(_login_name);
+         def.AccountId = lookup_account(def.Account);
+
+         Create cr = new Create();
+         def.CustomDef = get_custom(create);
+         cr.NewOrder = def;         
+         try
+         {
+            if (create.IsSetPrice())
+               cr.Price = Convert.ToDouble(create.Price.getValue());
+         }
+         catch { throw new IncorrectTagValue(QuickFix.Fields.Tags.Price); }
+         try
+         {
+            if(!create.IsSetOrderQty() || create.OrderQty.getValue() < 0)
+               throw new IncorrectTagValue(QuickFix.Fields.Tags.OrderQty);
+            cr.Quantity = Convert.ToUInt32(Math.Abs(create.OrderQty.getValue()));
+         }
+         catch { throw new IncorrectTagValue(QuickFix.Fields.Tags.OrderQty); }
+         if (!create.IsSetClOrdID())
+            throw new IncorrectTagValue(QuickFix.Fields.Tags.ClOrdID);
+         else
+            def.ExternalOrderId = create.ClOrdID.ToString();
+            
+         var resp = _orderservice.Value.create_order(cr);
+         _fix_clorderids[resp.OrderKey] = create.ClOrdID.getValue();         
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            _router.send_biz_reject(s, "Order Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, create);
+         }
+      }
+      
+      CustomOrderDefinition get_custom(Message fix)
+      {
+         FIXAlgoDefinition.Descriptor.FindFieldByNumber(FIXAlgoDefinition.NameFieldNumber).
+            CustomOptions.TryGetInt32((int)ApioptionsReflection.FieldOptions.fix_tag, out int algotag);      
+         if(fix.IsSetField(algotag))
+         {
+            string algotype = fix.GetString(algotag);
+            if(!_atdl.algo_definitions.TryGetValue(algotype, out FIXAlgoDefinition algodef))
+               throw new IncorrectTagValue(algotag);
+            else
+            {
+               var custom = algodef.Definition.Clone();
+               foreach (var fd in CustomOrderDefinition.Descriptor.Fields.InDeclarationOrder())
+               {
+                  var val = fd.Accessor.GetValue(custom);
+                  if(val != null && val is Google.Protobuf.IMessage valmsg)
+                  {
+                     ProtoFix.setup_message(valmsg, fix);
+                     break;
+                  }
+               }
+               return custom;
+            }
+         }
+         else
+         {
+            if(!fix.IsSetField(QuickFix.Fields.Tags.OrdType))
+               throw new IncorrectTagValue(QuickFix.Fields.Tags.OrdType);
+            switch (fix.GetChar(QuickFix.Fields.Tags.OrdType))
+            {
+               case '1':
+                  return new CustomOrderDefinition { Market = new MarketOrderDefinition() };
+               case '2':
+                  return new CustomOrderDefinition { Limit = new LimitOrderDefinition() };
+               case '3':
+                  return new CustomOrderDefinition { Stop = new StopOrderDefinition() 
+                  { 
+                     StopPrice = Convert.ToDouble(fix.GetDecimal(QuickFix.Fields.Tags.StopPx)) } 
+                  };
+               case '4':
+                  return new CustomOrderDefinition { StopLimit = new StopLimitOrderDefinition() 
+                  {
+                     StopPrice = Convert.ToDouble(fix.GetDecimal(QuickFix.Fields.Tags.StopPx)) }
+                  };
+               default:
+                  throw new IncorrectTagValue(QuickFix.Fields.Tags.OrdType);
+            }            
+         }
+      }
+   }
+}

src/client/spark_fix/session/SessionBase.cs

@@ -0,0 +1,383 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using mbts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.api.admin;
+
+using QuickFix.FIX42;
+using Message = QuickFix.Message;
+
+namespace bts.fix
+{
+   abstract class SessionBase : QuickFix.MessageCracker, IFIXSession
+   {      
+      protected static AuthService _authservice = bts.api.Locator.locate<AuthService>();      
+      protected static Lazy<AdminService> _adminservice = new Lazy<AdminService>(() => bts.api.Locator.locate<AdminService>(), true);
+      protected static Lazy<bts.metrics.ShellLatencyService> _latencyservice = 
+         new Lazy<bts.metrics.ShellLatencyService>(() => bts.api.Locator.locate<bts.metrics.ShellLatencyService>(), true);
+
+      protected string _login_name;
+      protected AuthToken _token;
+
+      protected IDisposable _admindisp;
+      protected IObservable<AdminModelWrapper> _adminobservable;
+      protected AdminDataImmutable? _admin_data;      
+
+      protected readonly IFixRouter _router;
+      protected readonly SessionID _sess;
+      
+      public SessionBase(IFixRouter router, SessionID sess)
+      {
+         _router = router;
+         _sess = sess;
+
+         _clockdisp = _latencyservice.Value.subscribe_clock().Subscribe(x =>
+         {
+            var t = x.all.FirstOrDefault();
+            if(t != null)
+               _clock = t.Timestamp;
+         });
+
+         _adminobservable = _adminservice.Value.subscribei_admin_data().Select(x =>
+         {
+            return new AdminModelWrapper(x.all.TryFirst(), _login_name);
+         });
+         _admindisp = _adminobservable.Subscribe(x => _admin_data = x.admin_data);         
+      }
+
+      abstract public void FromApp(Message message, SessionID sessionID);
+      abstract public void OnLogon(SessionID sessionID); //subscribe to your stuff 
+      abstract public void OnLogout(SessionID sessionID); //unsub
+      
+      virtual public void ToApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         log_info("OUT: " + message);
+      }
+
+      readonly IDisposable _clockdisp; ulong _clock = 0;
+      virtual public void FromAdmin(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.LOGON:
+               {
+                  try
+                  {
+                     Logon msg = (Logon)message;
+
+                     var fixlogin = new QuickFix.Fields.SenderCompID();
+                     string login = msg.Header.GetString(fixlogin.Tag).ToLower();
+                     // var fixuser = new QuickFix.Fields.TargetSubID();
+                     // string user = msg.Header.GetString(fixuser.Tag);
+                     _login_name = login;
+                     string pass = string.Empty;                     
+                     if(msg.IsSetRawData())
+                        pass = msg.RawData.getValue();
+                     var resp = _authservice.authenticate_login(new AuthenticateLoginRequest { 
+                                                                Login = login, Password = pass});
+                     if (resp != null && resp.IsValid)
+                        _token = resp.Token;
+                     else
+                        throw new RejectLogon("Logon rejected " + resp?.Reason ?? string.Empty);
+                  }
+                  catch (RejectLogon)
+                  {
+                     throw;
+                  }
+                  catch (Exception ex)
+                  {
+                     throw new RejectLogon("Logon rejected " + ex.ToString());
+                  }
+               }
+               break;
+            case MsgType.HEARTBEAT:
+               {
+                  Heartbeat hb = (Heartbeat)message;
+                  handle_heartbeat(hb, sessionID);
+               }
+               break;               
+            default:
+               log_info("ADMININ: " + message);
+               break;
+         }
+      }
+
+      async protected void handle_heartbeat(Heartbeat hb, SessionID sessionID)
+      {
+         var resp = await _latencyservice.Value.report_async(new bts.metrics.Report { HostName = "BTSFIX_" + sessionID.ToString(), 
+                                                                                      Timestamp = _clock, User = _login_name });
+         if (resp.success)
+            _router.send(hb, sessionID);         
+      }
+
+      virtual public void ToAdmin(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         log_info("ADMINOUT: " + message);
+      }
+
+      virtual public void OnCreate(SessionID sessionID)
+      {
+         log_info("CREATE");
+      }
+
+      virtual public void OnMessage(BusinessMessageReject n, SessionID s)
+      {
+         //Log.assert(false, "biz reject wtf " + n.EncodedText.getValue());
+         Log.FIX.warn(_sess.ToString() + " Business Reject " + n.BusinessRejectReason.getValue() + " " + n.Text.getValue());
+      }
+      virtual public void OnMessage(News n, SessionID s)
+      {
+         log_info("news " + n.EncodedHeadline.getValue());
+      }      
+
+
+      protected void log_info(string message, [CallerFilePath] string file = "", [CallerLineNumber] int line = 0)
+      {
+         Log.FIX.info(_sess.ToString() + " " + message, file, line);
+      }
+
+      protected void log_error(string message, [CallerFilePath] string file = "", [CallerLineNumber] int line = 0)
+      {
+         Log.FIX.error(_sess.ToString() + " " + message, file, line);
+      }
+
+      protected void send_cancel_reject(SessionID sess, OrderError reject_reason, string text, Message refmessage)
+      {
+         OrderCancelReject rej = new OrderCancelReject();
+         if (reject_reason == OrderError.UnknownOrder) {
+            rej.CxlRejReason = new CxlRejReason(CxlRejReason.UNKNOWN_ORDER);
+            rej.OrdStatus = new OrdStatus(OrdStatus.REJECTED);
+            rej.OrderID = new OrderID("NONE");
+         }
+
+         if (refmessage is QuickFix.FIX42.OrderCancelRequest cancel)
+         {
+            if (cancel.IsSetClOrdID())
+               rej.SetField(cancel.ClOrdID);
+            if (cancel.IsSetOrigClOrdID())
+               rej.SetField(cancel.OrigClOrdID);
+            if (cancel.IsSetOrderID() && reject_reason != OrderError.UnknownOrder)
+               rej.SetField(cancel.OrderID);
+            rej.CxlRejResponseTo = new CxlRejResponseTo(CxlRejResponseTo.ORDER_CANCEL_REQUEST);
+            rej.Text = new Text(text);
+            _router.send(rej, sess);
+         }
+         else if (refmessage is OrderCancelReplaceRequest replace)
+         {
+            if (replace.IsSetClOrdID())
+               rej.SetField(replace.ClOrdID);
+            if (replace.IsSetOrigClOrdID())
+               rej.SetField(replace.OrigClOrdID);
+            if (replace.IsSetOrderID() && reject_reason != OrderError.UnknownOrder)
+               rej.SetField(replace.OrderID);
+            rej.CxlRejResponseTo = new CxlRejResponseTo(CxlRejResponseTo.ORDER_CANCEL_REPLACE_REQUEST);
+            rej.Text = new Text(text);
+            _router.send(rej, sess);
+         }
+         else 
+            throw new UnsupportedMessageType();
+      }      
+
+      protected bool security_types_equal(bts.api.SecurityType btstype, QuickFix.Fields.SecurityType fixtype)
+      {
+         return btstype == convert_security_type(fixtype);
+      }
+
+      protected QuickFix.Fields.SecurityType convert_security_type(bts.api.SecurityType btstype)
+      {
+         switch (btstype)
+         {
+            case bts.api.SecurityType.Future:
+               return new QuickFix.Fields.SecurityType("FUT");
+            case bts.api.SecurityType.Option:
+               return new QuickFix.Fields.SecurityType("OPT");
+            case bts.api.SecurityType.Spread:
+               return new QuickFix.Fields.SecurityType("MLEG");
+            case bts.api.SecurityType.Stock:
+               return new QuickFix.Fields.SecurityType("CS");
+            case bts.api.SecurityType.Synthetic:
+               return new QuickFix.Fields.SecurityType("SYNTH");
+            default:
+               return new QuickFix.Fields.SecurityType();
+         }
+      }
+
+      protected bts.api.SecurityType convert_security_type(QuickFix.Fields.SecurityType fixtype)
+      {
+         switch (fixtype.getValue())
+         {
+            case "FUT":
+               return bts.api.SecurityType.Future;
+            case "OPT":
+               return bts.api.SecurityType.Option;
+            case "MLEG":
+               return bts.api.SecurityType.Spread;
+            case "CS":
+               return bts.api.SecurityType.Stock;
+            case "SYNTH":
+               return bts.api.SecurityType.Synthetic;
+            default:
+               return bts.api.SecurityType.Unset;
+         }
+      }
+
+      protected bool exchanges_equal(bts.api.Venue btstype, QuickFix.Fields.SecurityExchange fixtype)
+      {
+         return btstype == convert_exchange(fixtype);
+      }
+
+      protected QuickFix.Fields.SecurityExchange convert_exchange(bts.api.Venue btstype)
+      {
+         switch (btstype)
+         {
+            case bts.api.Venue.Bts:
+               return new QuickFix.Fields.SecurityExchange("BTS");
+            case bts.api.Venue.Cboe:
+               return new QuickFix.Fields.SecurityExchange("XCBO");
+            case bts.api.Venue.Cboecfe:
+               return new QuickFix.Fields.SecurityExchange("XCBF");
+            case bts.api.Venue.Glbx:
+               return new QuickFix.Fields.SecurityExchange("XCME");
+            case bts.api.Venue.Ice:
+               return new QuickFix.Fields.SecurityExchange("IEPA");
+            case bts.api.Venue.Sim:
+               return new QuickFix.Fields.SecurityExchange("SIM");
+            default:
+               return new QuickFix.Fields.SecurityExchange();
+         }
+      }
+
+      protected bts.api.Venue convert_exchange(QuickFix.Fields.SecurityExchange fixtype)
+      {
+         return convert_exchange(fixtype.getValue());
+      }
+
+      protected bts.api.Venue convert_exchange(QuickFix.Fields.ExDestination fixtype)
+      {
+         return convert_exchange(fixtype.getValue());
+      }      
+
+      protected bts.api.Venue convert_exchange(string venue)
+      {
+         switch (venue)
+         {
+            case "BTS":
+               return bts.api.Venue.Bts;
+            case "XCBO":
+               return bts.api.Venue.Cboe;
+            case "XCBF":
+               return bts.api.Venue.Cboecfe;
+            case "GLBX":
+            case "XCME":
+            case "XCBT":
+            case "XCEC":
+            case "XNYM":
+               return bts.api.Venue.Glbx;
+            case "IEPA":
+            case "IFUS":
+            case "IFEU":
+            case "IFLL":
+               return bts.api.Venue.Ice;
+            case "SIM":
+               return bts.api.Venue.Sim;
+            default:
+               return bts.api.Venue.Unset;
+         }
+      }      
+
+      protected QuickFix.Fields.OrdStatus translate_status(in bts.api.orderman.OrderStateImmutable state, 
+                                                           ref bts.api.orderman.NativeOrderStateImmutable? native)
+      {
+         native = null;
+         if (state.HasMarket && state.Market.HasNativeState && state.Market.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Market.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasLimit && state.Limit.HasNativeState && state.Limit.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Limit.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasStop && state.Stop.HasNativeState && state.Stop.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Stop.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasStopLimit && state.StopLimit.HasNativeState && state.StopLimit.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.StopLimit.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else
+         {
+            switch (state.Status)
+            {
+               case OrderStatus.Canceled:
+                  return new OrdStatus(OrdStatus.CANCELED);
+               case OrderStatus.Completed:
+                  return new OrdStatus(OrdStatus.FILLED);
+               case OrderStatus.Disconnected:
+                  return new OrdStatus(OrdStatus.STOPPED);
+               case OrderStatus.Error:
+                  return new OrdStatus(OrdStatus.REJECTED);
+               case OrderStatus.Inactive:
+                  return new OrdStatus(OrdStatus.PENDING_NEW);
+               case OrderStatus.PendingWorking:
+                  return new OrdStatus(OrdStatus.PENDING_NEW);
+               case OrderStatus.RejectedBySpark:
+                  return new OrdStatus(OrdStatus.REJECTED);
+               case OrderStatus.Working:
+                  {
+                     if (state.FilledQuantity < state.Quantity)
+                        return new OrdStatus(OrdStatus.PARTIALLY_FILLED);
+                     else
+                        return new OrdStatus(OrdStatus.NEW);
+                  }
+               default:
+                  return new OrdStatus();
+            }
+         }
+      }
+
+      protected uint lookup_account(string acct)
+      {
+         if (_admin_data.HasValue)
+         {
+            var ac = _admin_data.Value.Accounts.Where(x => x.Name == acct).TryFirst();
+            if(ac.HasValue)
+               return ac.Value.Id;
+         }
+         throw new IncorrectTagValue(new Account().Tag);
+      }
+      
+      protected uint lookup_user(string user)
+      {
+         if (_admin_data.HasValue)
+         {
+            var ac = _admin_data.Value.Users.Where(x => x.Name == user).TryFirst();
+            if (ac.HasValue)
+               return ac.Value.Id;
+         }
+         throw new IncorrectTagValue(new Username().Tag);
+      }
+   }
+}

src/client/spark_fix/spark_algos.json

@@ -0,0 +1,90 @@
+{
+  "algoStrategy": [
+    {
+      "name": "TickAndCover",
+      "definition": 
+      {
+        "cover": 
+        {
+          "child": 
+          {
+            "top": { "child": { "limit": {} } }
+          },
+          "hedge": 
+          {
+            "lean": { "child": { "limit": {} } }
+          }
+        }
+      }
+    },
+    {
+      "name": "SmashAndCover",
+      "definition": 
+      {
+        "cover": 
+        {
+          "child": 
+          {
+            "lean": { "child": { "limit": {} } }
+          },
+          "hedge": 
+          {
+            "lean": { "child": { "limit": {} } }
+          }
+        }
+      }
+    },
+    {
+      "name": "JoinAndCover",
+      "definition": 
+      {
+        "cover": 
+        {
+          "child": 
+          {
+            "join": { "child": { "limit": {} } }
+          },
+          "hedge": 
+          {
+            "lean": { "child": { "limit": {} } }
+          }
+        }
+      }
+    },    
+    {
+      "name": "Prorata",
+      "definition": 
+      {
+        "prorata": 
+        {
+          "child":  { "limit": {} },
+          "hedge": 
+          {
+            "scratch": { "child": { "limit": {} } }
+          },
+          "overfill_hedge": 
+          {
+            "lean": { "child": { "limit": {} } }
+          }          
+        }
+      }
+    },    
+    { "name": "Top", "definition":  { "top": { "child": { "limit": {} } } } },
+    { "name": "Cover", 
+      "definition":  
+      { 
+         "cover": 
+         { 
+            "child" : { "limit": {} }, 
+            "hedge" : { "limit": {} }
+         }
+      }
+    }, 
+    { "name": "Lean", "definition":  { "lean": { "child": { "limit": {} } } } },
+    { "name": "Join", "definition":  { "join": { "child": { "limit": {} } } } }, 
+    { "name": "Iceberg", "definition":  { "iceberg": { "child": { "limit": {} } } } }, 
+    { "name": "Scratch", "definition":  { "scratch": { "child": { "limit": {} } } } }, 
+    { "name": "Spreader", "definition":  { "spreader": {  } } },
+    { "name": "ISpreader", "definition":  { "iceberg": { "child": { "spreader": {} } } } }
+  ]
+}

src/client/spark_fix/spark_fix.cfg

@@ -0,0 +1,42 @@
+#http://www.quickfixengine.org/quickfix/doc/html/configuration.html
+#https://www.onixs.biz/fix-dictionary.html
+
+[DEFAULT]
+ConnectionType=acceptor
+SocketAcceptPort=5001
+SocketReuseAddress=Y
+StartTime=00:00:00
+EndTime=00:00:00
+FileLogPath=log
+UseDataDictionary=N
+ResetOnLogon=Y
+ResetOnLogout=Y
+ResetOnDisconnect=Y
+
+HeartBtInt=30
+LogonTimeout=30
+
+SSLEnable=N
+# SSLEnable=Y
+# # It is recommended to install the certificate and refer to it by name instead of using filename + password
+# SSLCertificate=QuickFixn-TestServer.pfx
+# SSLCertificatePassword=QuickFixn-TestServer
+# # For production refer to certificate by name instead: SSLCertificate=CN=QuickFixn-TestServer
+# SSLCACertificate=QuickFixn-TestCA.cer
+# SSLCheckCertificateRevocation=N
+
+
+[SESSION]
+BeginString=FIX.4.2
+SenderCompID=SPARK
+TargetCompID=CQG
+FileStorePath=store
+DataDictionary=./spec/fix/FIX42.xml
+
+[SESSION]
+BeginString=FIX.4.2
+SenderCompID=SPARK
+TargetCompID=CLIENT2
+FileStorePath=store
+DataDictionary=./spec/fix/FIX42.xml
+

src/client/spark_fix/spark_fix.csproj

@@ -0,0 +1,46 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>  
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Description />
+    <Platforms>x64</Platforms>
+    <PlatformTarget>x64</PlatformTarget>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <LangVersion>7.2</LangVersion>
+    <Version>1.0.0</Version>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
+    <PackageReference Include="System.Reactive" Version="3.1.1" />
+    <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
+	 <PackageReference Include="NUnit" Version="3.10.1" />  
+    
+    <PackageReference Include="QuickFix.Net.NETCore" Version="1.8.1" />     
+    <PackageReference Include="QuickFix.Net.NETCore.FIX42" Version="1.8.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />         
+  </ItemGroup>
+
+  <ItemGroup> 
+    <ProjectReference Include="../spark_api/spark_api.csproj"/>
+    <ProjectReference Include="../spark_core/spark_core.csproj"/>
+  </ItemGroup>
+  
+  <Target Name="copy_fix" AfterTargets="PostBuildEvent"  >
+    <Copy SourceFiles="./spark_fix.cfg" DestinationFolder="$(OutputPath)/" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spark_algos.json" DestinationFolder="$(OutputPath)/" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX42.xml" DestinationFolder="$(OutputPath)/spec/fix/" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX44.xml" DestinationFolder="$(OutputPath)/spec/fix/" SkipUnchangedFiles="true" />
+  </Target>  
+  
+  <Target Name="copy_fix471" AfterTargets="PostBuildEvent"  >
+    <Copy SourceFiles="./spark_fix.cfg" DestinationFolder="$(OutputPath)/../net471" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spark_algos.json" DestinationFolder="$(OutputPath)/../net471" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX42.xml" DestinationFolder="$(OutputPath)/../net471/spec/fix/" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX44.xml" DestinationFolder="$(OutputPath)/../net471/spec/fix/" SkipUnchangedFiles="true" />
+  </Target>  
+</Project>

src/client/spark_fix/spec/fix/FIX40.xml

@@ -0,0 +1,828 @@
+<fix major="4" minor="0">
+  <header>
+    <field name="BeginString" required="Y"/>
+    <field name="BodyLength" required="Y"/>
+    <field name="MsgType" required="Y"/>
+    <field name="SenderCompID" required="Y"/>
+    <field name="TargetCompID" required="Y"/>
+    <field name="OnBehalfOfCompID" required="N"/>
+    <field name="DeliverToCompID" required="N"/>
+    <field name="SecureDataLen" required="N"/>
+    <field name="SecureData" required="N"/>
+    <field name="MsgSeqNum" required="Y"/>
+    <field name="SenderSubID" required="N"/>
+    <field name="TargetSubID" required="N"/>
+    <field name="OnBehalfOfSubID" required="N"/>
+    <field name="DeliverToSubID" required="N"/>
+    <field name="PossDupFlag" required="N"/>
+    <field name="PossResend" required="N"/>
+    <field name="SendingTime" required="Y"/>
+    <field name="OrigSendingTime" required="N"/>
+  </header>
+  <trailer>
+    <field name="SignatureLength" required="N"/>
+    <field name="Signature" required="N"/>
+    <field name="CheckSum" required="Y"/>
+  </trailer>
+  <messages>
+    <message name="Heartbeat" msgtype="0" msgcat="admin">
+      <field name="TestReqID" required="N"/>
+    </message>
+    <message name="Logon" msgtype="A" msgcat="admin">
+      <field name="EncryptMethod" required="Y"/>
+      <field name="HeartBtInt" required="Y"/>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="TestRequest" msgtype="1" msgcat="admin">
+      <field name="TestReqID" required="Y"/>
+    </message>
+    <message name="ResendRequest" msgtype="2" msgcat="admin">
+      <field name="BeginSeqNo" required="Y"/>
+      <field name="EndSeqNo" required="Y"/>
+    </message>
+    <message name="Reject" msgtype="3" msgcat="admin">
+      <field name="RefSeqNum" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="SequenceReset" msgtype="4" msgcat="admin">
+      <field name="GapFillFlag" required="N"/>
+      <field name="NewSeqNo" required="Y"/>
+    </message>
+    <message name="Logout" msgtype="5" msgcat="admin">
+      <field name="Text" required="N"/>
+    </message>
+    <message name="Advertisement" msgtype="7" msgcat="app">
+      <field name="AdvId" required="Y"/>
+      <field name="AdvTransType" required="Y"/>
+      <field name="AdvRefID" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="AdvSide" required="Y"/>
+      <field name="Shares" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TransactTime" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="IndicationofInterest" msgtype="6" msgcat="app">
+      <field name="IOIid" required="Y"/>
+      <field name="IOITransType" required="Y"/>
+      <field name="IOIRefID" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="IOIShares" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="ValidUntilTime" required="N"/>
+      <field name="IOIQltyInd" required="N"/>
+      <field name="IOIOthSvc" required="N"/>
+      <field name="IOINaturalFlag" required="N"/>
+      <field name="IOIQualifier" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="News" msgtype="B" msgcat="app">
+      <field name="OrigTime" required="N"/>
+      <field name="Urgency" required="N"/>
+      <field name="RelatdSym" required="N" repeating="Y"/>
+      <group name="LinesOfText" required="Y">
+        <field name="Text" required="Y"/>
+      </group>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="Email" msgtype="C" msgcat="app">
+      <field name="EmailType" required="Y"/>
+      <field name="OrigTime" required="N"/>
+      <field name="RelatdSym" required="N" repeating="Y"/>
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="N"/>
+      <group name="LinesOfText" required="Y">
+        <field name="Text" required="Y"/>
+      </group>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="QuoteRequest" msgtype="R" msgcat="app">
+      <field name="QuoteReqID" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="N"/>
+      <field name="OrderQty" required="N"/>
+    </message>
+    <message name="Quote" msgtype="S" msgcat="app">
+      <field name="QuoteReqID" required="N"/>
+      <field name="QuoteID" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="BidPx" required="Y"/>
+      <field name="OfferPx" required="N"/>
+      <field name="BidSize" required="N"/>
+      <field name="OfferSize" required="N"/>
+      <field name="ValidUntilTime" required="N"/>
+    </message>
+    <message name="NewOrderSingle" msgtype="D" msgcat="app">
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="ProcessCode" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="LocateReqd" required="N"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="IOIid" required="N"/>
+      <field name="QuoteID" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ExecutionReport" msgtype="8" msgcat="app">
+      <field name="OrderID" required="Y"/>
+      <field name="ClOrdID" required="N"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="ListID" required="N"/>
+      <field name="ExecID" required="Y"/>
+      <field name="ExecTransType" required="Y"/>
+      <field name="ExecRefID" required="N"/>
+      <field name="OrdStatus" required="Y"/>
+      <field name="OrdRejReason" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="N"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="ExecInst" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="LastShares" required="Y"/>
+      <field name="LastPx" required="Y"/>
+      <field name="LastMkt" required="N"/>
+      <field name="LastCapacity" required="N"/>
+      <field name="CumQty" required="Y"/>
+      <field name="AvgPx" required="Y"/>
+      <field name="TradeDate" required="N"/>
+      <field name="TransactTime" required="N"/>
+      <field name="ReportToExch" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <group name="NoMiscFees" required="N">
+        <field name="MiscFeeAmt" required="N"/>
+        <field name="MiscFeeCurr" required="N"/>
+        <field name="MiscFeeType" required="N"/>
+      </group>
+      <field name="NetMoney" required="N"/>
+      <field name="SettlCurrAmt" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="DontKnowTrade" msgtype="Q" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ExecID" required="N"/>
+      <field name="DKReason" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="LastShares" required="Y"/>
+      <field name="LastPx" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelReplaceRequest" msgtype="G" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="OrigClOrdID" required="Y"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ListID" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelRequest" msgtype="F" msgcat="app">
+      <field name="OrigClOrdID" required="Y"/>
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ListID" required="N"/>
+      <field name="CxlType" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelReject" msgtype="9" msgcat="app">
+      <field name="OrderID" required="Y"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="ListID" required="N"/>
+      <field name="CxlRejReason" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderStatusRequest" msgtype="H" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+    </message>
+    <message name="Allocation" msgtype="J" msgcat="app">
+      <field name="AllocID" required="Y"/>
+      <field name="AllocTransType" required="Y"/>
+      <field name="RefAllocID" required="N"/>
+      <group name="NoOrders" required="Y">
+        <field name="ClOrdID" required="Y"/>
+        <field name="OrderID" required="N"/>
+        <field name="ListID" required="N"/>
+        <field name="WaveNo" required="N"/>
+      </group>
+      <group name="NoExecs" required="N">
+        <field name="ExecID" required="N"/>
+        <field name="LastShares" required="N"/>
+        <field name="LastPx" required="N"/>
+        <field name="LastMkt" required="N"/>
+      </group>
+      <field name="Side" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Shares" required="Y"/>
+      <field name="AvgPx" required="Y"/>
+      <field name="Currency" required="N"/>
+      <field name="AvgPrxPrecision" required="N"/>
+      <field name="TradeDate" required="Y"/>
+      <field name="TransactTime" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="NetMoney" required="N"/>
+      <group name="NoMiscFees" required="N">
+        <field name="MiscFeeAmt" required="N"/>
+        <field name="MiscFeeCurr" required="N"/>
+        <field name="MiscFeeType" required="N"/>
+      </group>
+      <field name="SettlCurrAmt" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="OpenClose" required="N"/>
+      <field name="Text" required="N"/>
+      <group name="NoAllocs" required="Y">
+        <field name="AllocAccount" required="Y"/>
+        <field name="AllocShares" required="Y"/>
+        <field name="ProcessCode" required="N"/>
+        <field name="ExecBroker" required="N"/>
+        <field name="ClientID" required="N"/>
+        <field name="Commission" required="N"/>
+        <field name="CommType" required="N"/>
+        <group name="NoDlvyInst" required="N">
+          <field name="BrokerOfCredit" required="N"/>
+          <field name="DlvyInst" required="N"/>
+        </group>
+      </group>
+    </message>
+    <message name="AllocationACK" msgtype="P" msgcat="app">
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="AllocID" required="Y"/>
+      <field name="TradeDate" required="Y"/>
+      <field name="TransactTime" required="N"/>
+      <field name="AllocStatus" required="Y"/>
+      <field name="AllocRejCode" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="NewOrderList" msgtype="E" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="ListSeqNo" required="Y"/>
+      <field name="ListNoOrds" required="Y"/>
+      <field name="ListExecInst" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="ProcessCode" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="LocateReqd" required="N"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListStatus" msgtype="N" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="NoRpts" required="Y"/>
+      <field name="RptSeq" required="Y"/>
+      <group name="NoOrders" required="Y">
+        <field name="ClOrdID" required="Y"/>
+        <field name="CumQty" required="Y"/>
+        <field name="CxlQty" required="Y"/>
+        <field name="AvgPx" required="Y"/>
+      </group>
+    </message>
+    <message name="ListExecute" msgtype="L" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListCancelRequest" msgtype="K" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListStatusRequest" msgtype="M" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+  </messages>
+  <fields>
+    <field number="1" name="Account" type="STRING"/>
+    <field number="2" name="AdvId" type="INT"/>
+    <field number="3" name="AdvRefID" type="INT"/>
+    <field number="4" name="AdvSide" type="STRING">
+      <value enum="B" description="BUY"/>
+      <value enum="S" description="SELL"/>
+      <value enum="X" description="CROSS"/>
+      <value enum="T" description="TRADE"/>
+    </field>
+    <field number="5" name="AdvTransType" type="STRING">
+      <value enum="N" description="NEW"/>
+      <value enum="C" description="CANCEL"/>
+      <value enum="R" description="REPLACE"/>
+    </field>
+    <field number="6" name="AvgPx" type="FLOAT"/>
+    <field number="7" name="BeginSeqNo" type="INT"/>
+    <field number="8" name="BeginString" type="STRING"/>
+    <field number="9" name="BodyLength" type="INT"/>
+    <field number="10" name="CheckSum" type="STRING"/>
+    <field number="11" name="ClOrdID" type="STRING"/>
+    <field number="12" name="Commission" type="FLOAT"/>
+    <field number="13" name="CommType" type="STRING">
+      <value enum="1" description="PER_SHARE"/>
+      <value enum="2" description="PERCENTAGE"/>
+      <value enum="3" description="ABSOLUTE"/>
+    </field>
+    <field number="14" name="CumQty" type="INT"/>
+    <field number="15" name="Currency" type="STRING"/>
+    <field number="16" name="EndSeqNo" type="INT"/>
+    <field number="17" name="ExecID" type="INT"/>
+    <field number="18" name="ExecInst" type="MULTIPLEVALUESTRING">
+      <value enum="1" description="NOT_HELD"/>
+      <value enum="2" description="WORK"/>
+      <value enum="3" description="GO_ALONG"/>
+      <value enum="4" description="OVER_THE_DAY"/>
+      <value enum="5" description="HELD"/>
+      <value enum="6" description="PARTICIPATE_DONT_INITIATE"/>
+      <value enum="7" description="STRICT_SCALE"/>
+      <value enum="8" description="TRY_TO_SCALE"/>
+      <value enum="9" description="STAY_ON_BIDSIDE"/>
+      <value enum="0" description="STAY_ON_OFFERSIDE"/>
+      <value enum="A" description="NO_CROSS"/>
+      <value enum="B" description="OK_TO_CROSS"/>
+      <value enum="C" description="CALL_FIRST"/>
+      <value enum="D" description="PERCENT_OF_VOLUME"/>
+      <value enum="E" description="DO_NOT_INCREASE_DNI"/>
+      <value enum="F" description="DO_NOT_REDUCE_DNR"/>
+      <value enum="G" description="ALL_OR_NONE_AON"/>
+      <value enum="I" description="INSTITUTIONS_ONLY"/>
+      <value enum="L" description="LAST_PEG"/>
+      <value enum="M" description="MIDPRICE_PEG"/>
+      <value enum="N" description="NONNEGOTIABLE"/>
+      <value enum="O" description="OPENING_PEG"/>
+      <value enum="P" description="MARKET_PEG"/>
+      <value enum="R" description="PRIMARY_PEG"/>
+      <value enum="S" description="SUSPEND"/>
+    </field>
+    <field number="19" name="ExecRefID" type="INT"/>
+    <field number="20" name="ExecTransType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="CANCEL"/>
+      <value enum="2" description="CORRECT"/>
+      <value enum="3" description="STATUS"/>
+    </field>
+    <field number="21" name="HandlInst" type="STRING">
+      <value enum="1" description="AUTOMATED_EXECUTION_ORDER_PRIVATE_NO_BROKER_INTERVENTION"/>
+      <value enum="2" description="AUTOMATED_EXECUTION_ORDER_PUBLIC_BROKER_INTERVENTION_OK"/>
+      <value enum="3" description="MANUAL_ORDER_BEST_EXECUTION"/>
+    </field>
+    <field number="22" name="IDSource" type="STRING">
+      <value enum="1" description="CUSIP"/>
+      <value enum="2" description="SEDOL"/>
+      <value enum="3" description="QUIK"/>
+      <value enum="4" description="ISIN_NUMBER"/>
+      <value enum="5" description="RIC_CODE"/>
+    </field>
+    <field number="23" name="IOIid" type="INT"/>
+    <field number="24" name="IOIOthSvc" type="STRING"/>
+    <field number="25" name="IOIQltyInd" type="STRING">
+      <value enum="L" description="LOW"/>
+      <value enum="M" description="MEDIUM"/>
+      <value enum="H" description="HIGH"/>
+    </field>
+    <field number="26" name="IOIRefID" type="INT"/>
+    <field number="27" name="IOIShares" type="STRING"/>
+    <field number="28" name="IOITransType" type="STRING">
+      <value enum="N" description="NEW"/>
+      <value enum="C" description="CANCEL"/>
+      <value enum="R" description="REPLACE"/>
+    </field>
+    <field number="29" name="LastCapacity" type="STRING">
+      <value enum="1" description="AGENT"/>
+      <value enum="2" description="CROSS_AS_AGENT"/>
+      <value enum="3" description="CROSS_AS_PRINCIPAL"/>
+      <value enum="4" description="PRINCIPAL"/>
+    </field>
+    <field number="30" name="LastMkt" type="STRING"/>
+    <field number="31" name="LastPx" type="FLOAT"/>
+    <field number="32" name="LastShares" type="INT"/>
+    <field number="33" name="LinesOfText" type="INT"/>
+    <field number="34" name="MsgSeqNum" type="INT"/>
+    <field number="35" name="MsgType" type="STRING"/>
+    <field number="36" name="NewSeqNo" type="INT"/>
+    <field number="37" name="OrderID" type="STRING"/>
+    <field number="38" name="OrderQty" type="INT"/>
+    <field number="39" name="OrdStatus" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="PARTIALLY_FILLED"/>
+      <value enum="2" description="FILLED"/>
+      <value enum="3" description="DONE_FOR_DAY"/>
+      <value enum="4" description="CANCELED"/>
+      <value enum="5" description="REPLACED"/>
+      <value enum="6" description="PENDING_CANCELREPLACE"/>
+      <value enum="7" description="STOPPED"/>
+      <value enum="8" description="REJECTED"/>
+      <value enum="9" description="SUSPENDED"/>
+      <value enum="A" description="PENDING_NEW"/>
+      <value enum="B" description="CALCULATED"/>
+      <value enum="C" description="EXPIRED"/>
+    </field>
+    <field number="40" name="OrdType" type="STRING">
+      <value enum="1" description="MARKET"/>
+      <value enum="2" description="LIMIT"/>
+      <value enum="3" description="STOP"/>
+      <value enum="4" description="STOP_LIMIT"/>
+      <value enum="5" description="MARKET_ON_CLOSE"/>
+      <value enum="6" description="WITH_OR_WITHOUT"/>
+      <value enum="7" description="LIMIT_OR_BETTER"/>
+      <value enum="8" description="LIMIT_WITH_OR_WITHOUT"/>
+      <value enum="9" description="ON_BASIS"/>
+      <value enum="A" description="ON_CLOSE"/>
+      <value enum="B" description="LIMIT_ON_CLOSE"/>
+      <value enum="C" description="FOREX"/>
+      <value enum="D" description="PREVIOUSLY_QUOTED"/>
+      <value enum="E" description="PREVIOUSLY_INDICATED"/>
+      <value enum="P" description="PEGGED"/>
+    </field>
+    <field number="41" name="OrigClOrdID" type="STRING"/>
+    <field number="42" name="OrigTime" type="TIME"/>
+    <field number="43" name="PossDupFlag" type="STRING">
+      <value enum="Y" description="POSSIBLE_DUPLICATE"/>
+      <value enum="N" description="ORIGINAL_TRANSMISSION"/>
+    </field>
+    <field number="44" name="Price" type="FLOAT"/>
+    <field number="45" name="RefSeqNum" type="INT"/>
+    <field number="46" name="RelatdSym" type="STRING"/>
+    <field number="47" name="Rule80A" type="STRING">
+      <value enum="A" description="AGENCY_SINGLE_ORDER"/>
+      <value enum="D" description="PROGRAM_ORDER_INDEX_ARB_FOR_MEMBER_FIRMORG"/>
+      <value enum="C" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_MEMBER_FIRMORG"/>
+      <value enum="J" description="PROGRAM_ORDER_INDEX_ARB_FOR_INDIVIDUAL_CUSTOMER"/>
+      <value enum="K" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_INDIVIDUAL_CUSTOMER"/>
+      <value enum="U" description="PROGRAM_ORDER_INDEX_ARB_FOR_OTHER_AGENCY"/>
+      <value enum="Y" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_OTHER_AGENCY"/>
+      <value enum="M" description="PROGRAM_ORDER_INDEX_ARB_FOR_OTHER_MEMBER"/>
+      <value enum="N" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_OTHER_MEMBER"/>
+      <value enum="W" description="ALL_OTHER_ORDERS_AS_AGENT_FOR_OTHER_MEMBER"/>
+    </field>
+    <field number="48" name="SecurityID" type="STRING"/>
+    <field number="49" name="SenderCompID" type="STRING"/>
+    <field number="50" name="SenderSubID" type="STRING"/>
+    <field number="52" name="SendingTime" type="TIME"/>
+    <field number="53" name="Shares" type="INT"/>
+    <field number="54" name="Side" type="STRING">
+      <value enum="1" description="BUY"/>
+      <value enum="2" description="SELL"/>
+      <value enum="3" description="BUY_MINUS"/>
+      <value enum="4" description="SELL_PLUS"/>
+      <value enum="5" description="SELL_SHORT"/>
+      <value enum="6" description="SELL_SHORT_EXEMPT"/>
+    </field>
+    <field number="55" name="Symbol" type="STRING"/>
+    <field number="56" name="TargetCompID" type="STRING"/>
+    <field number="57" name="TargetSubID" type="STRING"/>
+    <field number="58" name="Text" type="STRING"/>
+    <field number="59" name="TimeInForce" type="STRING">
+      <value enum="0" description="DAY"/>
+      <value enum="1" description="GOOD_TILL_CANCEL"/>
+      <value enum="2" description="AT_THE_OPENING"/>
+      <value enum="3" description="IMMEDIATE_OR_CANCEL"/>
+      <value enum="4" description="FILL_OR_KILL"/>
+      <value enum="5" description="GOOD_TILL_CROSSING"/>
+      <value enum="6" description="GOOD_TILL_DATE"/>
+    </field>
+    <field number="60" name="TransactTime" type="TIME"/>
+    <field number="61" name="Urgency" type="STRING">
+      <value enum="0" description="NORMAL"/>
+      <value enum="1" description="FLASH"/>
+      <value enum="2" description="BACKGROUND"/>
+    </field>
+    <field number="62" name="ValidUntilTime" type="TIME"/>
+    <field number="63" name="SettlmntTyp" type="STRING">
+      <value enum="0" description="REGULAR"/>
+      <value enum="1" description="CASH"/>
+      <value enum="2" description="NEXT_DAY"/>
+      <value enum="3" description="TPLUS2"/>
+      <value enum="4" description="TPLUS3"/>
+      <value enum="5" description="TPLUS4"/>
+      <value enum="6" description="FUTURE"/>
+      <value enum="7" description="WHEN_ISSUED"/>
+      <value enum="8" description="SELLERS_OPTION"/>
+      <value enum="9" description="TPLUS5"/>
+    </field>
+    <field number="64" name="FutSettDate" type="DATE"/>
+    <field number="65" name="SymbolSfx" type="STRING"/>
+    <field number="66" name="ListID" type="STRING"/>
+    <field number="67" name="ListSeqNo" type="INT"/>
+    <field number="68" name="ListNoOrds" type="INT"/>
+    <field number="69" name="ListExecInst" type="STRING"/>
+    <field number="70" name="AllocID" type="INT"/>
+    <field number="71" name="AllocTransType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="REPLACE"/>
+      <value enum="2" description="CANCEL"/>
+    </field>
+    <field number="72" name="RefAllocID" type="INT"/>
+    <field number="73" name="NoOrders" type="INT"/>
+    <field number="74" name="AvgPrxPrecision" type="INT"/>
+    <field number="75" name="TradeDate" type="DATE"/>
+    <field number="76" name="ExecBroker" type="STRING"/>
+    <field number="77" name="OpenClose" type="STRING"/>
+    <field number="78" name="NoAllocs" type="INT"/>
+    <field number="79" name="AllocAccount" type="STRING"/>
+    <field number="80" name="AllocShares" type="INT"/>
+    <field number="81" name="ProcessCode" type="STRING">
+      <value enum="0" description="REGULAR"/>
+      <value enum="1" description="SOFT_DOLLAR"/>
+      <value enum="2" description="STEPIN"/>
+      <value enum="3" description="STEPOUT"/>
+      <value enum="4" description="SOFTDOLLAR_STEPIN"/>
+      <value enum="5" description="SOFTDOLLAR_STEPOUT"/>
+      <value enum="6" description="PLAN_SPONSOR"/>
+    </field>
+    <field number="82" name="NoRpts" type="INT"/>
+    <field number="83" name="RptSeq" type="INT"/>
+    <field number="84" name="CxlQty" type="INT"/>
+    <field number="85" name="NoDlvyInst" type="INT"/>
+    <field number="86" name="DlvyInst" type="STRING"/>
+    <field number="87" name="AllocStatus" type="INT">
+      <value enum="0" description="ACCEPTED"/>
+      <value enum="1" description="REJECTED"/>
+      <value enum="2" description="PARTIAL_ACCEPT"/>
+      <value enum="3" description="RECEIVED"/>
+    </field>
+    <field number="88" name="AllocRejCode" type="INT">
+      <value enum="0" description="UNKNOWN_ACCOUNT"/>
+      <value enum="1" description="INCORRECT_QUANTITY"/>
+      <value enum="2" description="INCORRECT_AVERAGE_PRICE"/>
+      <value enum="3" description="UNKNOWN_EXECUTING_BROKER_MNEMONIC"/>
+      <value enum="4" description="COMMISSION_DIFFERENCE"/>
+      <value enum="5" description="UNKNOWN_ORDERID"/>
+      <value enum="6" description="UNKNOWN_LISTID"/>
+      <value enum="7" description="OTHER"/>
+    </field>
+    <field number="89" name="Signature" type="DATA"/>
+    <field number="90" name="SecureDataLen" type="INT"/>
+    <field number="91" name="SecureData" type="DATA"/>
+    <field number="92" name="BrokerOfCredit" type="STRING"/>
+    <field number="93" name="SignatureLength" type="INT"/>
+    <field number="94" name="EmailType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="REPLY"/>
+      <value enum="2" description="ADMIN_REPLY"/>
+    </field>
+    <field number="95" name="RawDataLength" type="INT"/>
+    <field number="96" name="RawData" type="DATA"/>
+    <field number="97" name="PossResend" type="STRING"/>
+    <field number="98" name="EncryptMethod" type="INT">
+      <value enum="0" description="NONE_OTHER"/>
+      <value enum="1" description="PKCS"/>
+      <value enum="2" description="DES"/>
+      <value enum="3" description="PKCSDES"/>
+      <value enum="4" description="PGPDES"/>
+      <value enum="5" description="PGPDESMD5"/>
+      <value enum="6" description="PEMDESMD5"/>
+    </field>
+    <field number="99" name="StopPx" type="FLOAT"/>
+    <field number="100" name="ExDestination" type="STRING">
+      <value enum="0" description="NONE"/>
+      <value enum="4" description="POSIT"/>
+    </field>
+    <field number="102" name="CxlRejReason" type="INT">
+      <value enum="0" description="TOO_LATE_TO_CANCEL"/>
+      <value enum="1" description="UNKNOWN_ORDER"/>
+    </field>
+    <field number="103" name="OrdRejReason" type="INT">
+      <value enum="0" description="BROKER_OPTION"/>
+      <value enum="1" description="UNKNOWN_SYMBOL"/>
+      <value enum="2" description="EXCHANGE_CLOSED"/>
+      <value enum="3" description="ORDER_EXCEEDS_LIMIT"/>
+      <value enum="4" description="TOO_LATE_TO_ENTER"/>
+    </field>
+    <field number="104" name="IOIQualifier" type="STRING">
+      <value enum="X" description="CROSSING_OPPORTUNITY"/>
+      <value enum="O" description="AT_THE_OPEN"/>
+      <value enum="M" description="MORE_BEHIND"/>
+      <value enum="P" description="TAKING_A_POSITION"/>
+      <value enum="V" description="VERSUS"/>
+      <value enum="Q" description="CURRENT_QUOTE"/>
+      <value enum="C" description="AT_THE_CLOSE"/>
+      <value enum="S" description="PORTFOLIO_SHOWN"/>
+      <value enum="I" description="IN_TOUCH_WITH"/>
+      <value enum="W" description="INDICATION_WORKING_AWAY"/>
+      <value enum="A" description="ALL_OR_NONE"/>
+      <value enum="L" description="LIMIT"/>
+      <value enum="T" description="THROUGH_THE_DAY"/>
+    </field>
+    <field number="105" name="WaveNo" type="STRING"/>
+    <field number="106" name="Issuer" type="STRING"/>
+    <field number="107" name="SecurityDesc" type="STRING"/>
+    <field number="108" name="HeartBtInt" type="INT"/>
+    <field number="109" name="ClientID" type="STRING"/>
+    <field number="110" name="MinQty" type="INT"/>
+    <field number="111" name="MaxFloor" type="INT"/>
+    <field number="112" name="TestReqID" type="STRING"/>
+    <field number="113" name="ReportToExch" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="114" name="LocateReqd" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="115" name="OnBehalfOfCompID" type="STRING"/>
+    <field number="116" name="OnBehalfOfSubID" type="STRING"/>
+    <field number="117" name="QuoteID" type="STRING"/>
+    <field number="118" name="NetMoney" type="FLOAT"/>
+    <field number="119" name="SettlCurrAmt" type="FLOAT"/>
+    <field number="120" name="SettlCurrency" type="STRING"/>
+    <field number="121" name="ForexReq" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="122" name="OrigSendingTime" type="TIME"/>
+    <field number="123" name="GapFillFlag" type="STRING">
+      <value enum="Y" description="GAP_FILL_MESSAGE_MSGSEQNUM_FIELD_VALID"/>
+      <value enum="N" description="SEQUENCE_RESET_IGNORE_MSGSEQNUM"/>
+    </field>
+    <field number="124" name="NoExecs" type="INT"/>
+    <field number="125" name="CxlType" type="STRING">
+      <value enum="P" description="PARTIAL_CANCEL"/>
+      <value enum="F" description="FULL_REMAINING_QUANTITY"/>
+    </field>
+    <field number="126" name="ExpireTime" type="TIME"/>
+    <field number="127" name="DKReason" type="STRING">
+      <value enum="A" description="UNKNOWN_SYMBOL"/>
+      <value enum="B" description="WRONG_SIDE"/>
+      <value enum="C" description="QUANTITY_EXCEEDS_ORDER"/>
+      <value enum="D" description="NO_MATCHING_ORDER"/>
+      <value enum="E" description="PRICE_EXCEEDS_LIMIT"/>
+      <value enum="Z" description="OTHER" />
+    </field>
+    <field number="128" name="DeliverToCompID" type="STRING"/>
+    <field number="129" name="DeliverToSubID" type="STRING"/>
+    <field number="130" name="IOINaturalFlag" type="STRING">
+      <value enum="Y" description="NATURAL"/>
+      <value enum="N" description="NOT_NATURAL"/>
+    </field>
+    <field number="131" name="QuoteReqID" type="STRING"/>
+    <field number="132" name="BidPx" type="FLOAT"/>
+    <field number="133" name="OfferPx" type="FLOAT"/>
+    <field number="134" name="BidSize" type="INT"/>
+    <field number="135" name="OfferSize" type="INT"/>
+    <field number="136" name="NoMiscFees" type="INT"/>
+    <field number="137" name="MiscFeeAmt" type="FLOAT"/>
+    <field number="138" name="MiscFeeCurr" type="STRING"/>
+    <field number="139" name="MiscFeeType" type="STRING">
+      <value enum="1" description="REGULATORY"/>
+      <value enum="2" description="TAX"/>
+      <value enum="3" description="LOCAL_COMMISSION"/>
+      <value enum="4" description="EXCHANGE_FEES"/>
+      <value enum="5" description="STAMP"/>
+      <value enum="6" description="LEVY"/>
+      <value enum="7" description="OTHER"/>
+    </field>
+    <field number="140" name="PrevClosePx" type="FLOAT"/>
+  </fields>
+</fix>

src/client/spark_fix/spec/fix/FIXT11.xml

@@ -0,0 +1,313 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<fix type='FIXT' major='1' minor='1' servicepack='0'>
+  <header>
+    <field name='BeginString' required='Y'/>
+    <field name='BodyLength' required='Y'/>
+    <field name='MsgType' required='Y'/>
+    <field name='SenderCompID' required='Y'/>
+    <field name='TargetCompID' required='Y'/>
+    <field name='OnBehalfOfCompID' required='N'/>
+    <field name='DeliverToCompID' required='N'/>
+    <field name='SecureDataLen' required='N'/>
+    <field name='SecureData' required='N'/>
+    <field name='MsgSeqNum' required='Y'/>
+    <field name='SenderSubID' required='N'/>
+    <field name='SenderLocationID' required='N'/>
+    <field name='TargetSubID' required='N'/>
+    <field name='TargetLocationID' required='N'/>
+    <field name='OnBehalfOfSubID' required='N'/>
+    <field name='OnBehalfOfLocationID' required='N'/>
+    <field name='DeliverToSubID' required='N'/>
+    <field name='DeliverToLocationID' required='N'/>
+    <field name='PossDupFlag' required='N'/>
+    <field name='PossResend' required='N'/>
+    <field name='SendingTime' required='Y'/>
+    <field name='OrigSendingTime' required='N'/>
+    <field name='XmlDataLen' required='N'/>
+    <field name='XmlData' required='N'/>
+    <field name='MessageEncoding' required='N'/>
+    <field name='LastMsgSeqNumProcessed' required='N'/>
+    <component name='HopGrp' required='N'/>
+    <field name='ApplVerID' required='N'/>
+    <field name='CstmApplVerID' required='N'/>
+  </header>
+  <trailer>
+    <field name='SignatureLength' required='N'/>
+    <field name='Signature' required='N'/>
+    <field name='CheckSum' required='Y'/>
+  </trailer>
+  <messages>
+    <message msgcat='admin' msgtype='0' name='Heartbeat'>
+      <field name='TestReqID' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='1' name='TestRequest'>
+      <field name='TestReqID' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='2' name='ResendRequest'>
+      <field name='BeginSeqNo' required='Y'/>
+      <field name='EndSeqNo' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='3' name='Reject'>
+      <field name='RefSeqNum' required='Y'/>
+      <field name='RefTagID' required='N'/>
+      <field name='RefMsgType' required='N'/>
+      <field name='SessionRejectReason' required='N'/>
+      <field name='Text' required='N'/>
+      <field name='EncodedTextLen' required='N'/>
+      <field name='EncodedText' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='4' name='SequenceReset'>
+      <field name='GapFillFlag' required='N'/>
+      <field name='NewSeqNo' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='5' name='Logout'>
+      <field name='Text' required='N'/>
+      <field name='EncodedTextLen' required='N'/>
+      <field name='EncodedText' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='A' name='Logon'>
+      <field name='EncryptMethod' required='Y'/>
+      <field name='HeartBtInt' required='Y'/>
+      <field name='RawDataLength' required='N'/>
+      <field name='RawData' required='N'/>
+      <field name='ResetSeqNumFlag' required='N'/>
+      <field name='NextExpectedMsgSeqNum' required='N'/>
+      <field name='MaxMessageSize' required='N'/>
+      <field name='TestMessageIndicator' required='N'/>
+      <field name='Username' required='N'/>
+      <field name='Password' required='N'/>
+      <field name='DefaultApplVerID' required='Y'/>
+      <component name='MsgTypeGrp' required='N'/>
+    </message>
+  </messages>
+  <components>
+    <component name='HopGrp'>
+      <group name='NoHops' required='N'>
+        <field name='HopCompID' required='N'/>
+        <field name='HopSendingTime' required='N'/>
+        <field name='HopRefID' required='N'/>
+      </group>
+    </component>
+    <component name='MsgTypeGrp'>
+      <group name='NoMsgTypes' required='N'>
+        <field name='RefMsgType' required='N'/>
+        <field name='MsgDirection' required='N'/>
+        <field name='RefApplVerID' required='N'/>
+        <field name='RefCstmApplVerID' required='N'/>
+      </group>
+    </component>
+  </components>
+  <fields>
+    <field name='BeginSeqNo' number='7' type='SEQNUM'/>
+    <field name='BeginString' number='8' type='STRING'/>
+    <field name='BodyLength' number='9' type='LENGTH'/>
+    <field name='CheckSum' number='10' type='STRING'/>
+    <field name='EndSeqNo' number='16' type='SEQNUM'/>
+    <field name='MsgSeqNum' number='34' type='SEQNUM'/>
+    <field number='35' name='MsgType' type='STRING'>
+      <value enum='0' description='HEARTBEAT'/>
+      <value enum='1' description='TEST_REQUEST'/>
+      <value enum='2' description='RESEND_REQUEST'/>
+      <value enum='3' description='REJECT'/>
+      <value enum='4' description='SEQUENCE_RESET'/>
+      <value enum='5' description='LOGOUT'/>
+      <value enum='6' description='INDICATION_OF_INTEREST'/>
+      <value enum='7' description='ADVERTISEMENT'/>
+      <value enum='8' description='EXECUTION_REPORT'/>
+      <value enum='9' description='ORDER_CANCEL_REJECT'/>
+      <value enum='A' description='LOGON'/>
+      <value enum='B' description='NEWS'/>
+      <value enum='C' description='EMAIL'/>
+      <value enum='D' description='ORDER_SINGLE'/>
+      <value enum='E' description='ORDER_LIST'/>
+      <value enum='F' description='ORDER_CANCEL_REQUEST'/>
+      <value enum='G' description='ORDER_CANCEL_REPLACE_REQUEST'/>
+      <value enum='H' description='ORDER_STATUS_REQUEST'/>
+      <value enum='J' description='ALLOCATION_INSTRUCTION'/>
+      <value enum='K' description='LIST_CANCEL_REQUEST'/>
+      <value enum='L' description='LIST_EXECUTE'/>
+      <value enum='M' description='LIST_STATUS_REQUEST'/>
+      <value enum='N' description='LIST_STATUS'/>
+      <value enum='P' description='ALLOCATION_INSTRUCTION_ACK'/>
+      <value enum='Q' description='DONT_KNOW_TRADE'/>
+      <value enum='R' description='QUOTE_REQUEST'/>
+      <value enum='S' description='QUOTE'/>
+      <value enum='T' description='SETTLEMENT_INSTRUCTIONS'/>
+      <value enum='V' description='MARKET_DATA_REQUEST'/>
+      <value enum='W' description='MARKET_DATA_SNAPSHOT_FULL_REFRESH'/>
+      <value enum='X' description='MARKET_DATA_INCREMENTAL_REFRESH'/>
+      <value enum='Y' description='MARKET_DATA_REQUEST_REJECT'/>
+      <value enum='Z' description='QUOTE_CANCEL'/>
+      <value enum='a' description='QUOTE_STATUS_REQUEST'/>
+      <value enum='b' description='MASS_QUOTE_ACKNOWLEDGEMENT'/>
+      <value enum='c' description='SECURITY_DEFINITION_REQUEST'/>
+      <value enum='d' description='SECURITY_DEFINITION'/>
+      <value enum='e' description='SECURITY_STATUS_REQUEST'/>
+      <value enum='f' description='SECURITY_STATUS'/>
+      <value enum='g' description='TRADING_SESSION_STATUS_REQUEST'/>
+      <value enum='h' description='TRADING_SESSION_STATUS'/>
+      <value enum='i' description='MASS_QUOTE'/>
+      <value enum='j' description='BUSINESS_MESSAGE_REJECT'/>
+      <value enum='k' description='BID_REQUEST'/>
+      <value enum='l' description='BID_RESPONSE'/>
+      <value enum='m' description='LIST_STRIKE_PRICE'/>
+      <value enum='n' description='XML_MESSAGE'/>
+      <value enum='o' description='REGISTRATION_INSTRUCTIONS'/>
+      <value enum='p' description='REGISTRATION_INSTRUCTIONS_RESPONSE'/>
+      <value enum='q' description='ORDER_MASS_CANCEL_REQUEST'/>
+      <value enum='r' description='ORDER_MASS_CANCEL_REPORT'/>
+      <value enum='s' description='NEW_ORDER_CROSS'/>
+      <value enum='t' description='CROSS_ORDER_CANCEL_REPLACE_REQUEST'/>
+      <value enum='u' description='CROSS_ORDER_CANCEL_REQUEST'/>
+      <value enum='v' description='SECURITY_TYPE_REQUEST'/>
+      <value enum='w' description='SECURITY_TYPES'/>
+      <value enum='x' description='SECURITY_LIST_REQUEST'/>
+      <value enum='y' description='SECURITY_LIST'/>
+      <value enum='z' description='DERIVATIVE_SECURITY_LIST_REQUEST'/>
+      <value enum='AA' description='DERIVATIVE_SECURITY_LIST'/>
+      <value enum='AB' description='NEW_ORDER_MULTILEG'/>
+      <value enum='AC' description='MULTILEG_ORDER_CANCEL_REPLACE'/>
+      <value enum='AD' description='TRADE_CAPTURE_REPORT_REQUEST'/>
+      <value enum='AE' description='TRADE_CAPTURE_REPORT'/>
+      <value enum='AF' description='ORDER_MASS_STATUS_REQUEST'/>
+      <value enum='AG' description='QUOTE_REQUEST_REJECT'/>
+      <value enum='AH' description='RFQ_REQUEST'/>
+      <value enum='AI' description='QUOTE_STATUS_REPORT'/>
+      <value enum='AJ' description='QUOTE_RESPONSE'/>
+      <value enum='AK' description='CONFIRMATION'/>
+      <value enum='AL' description='POSITION_MAINTENANCE_REQUEST'/>
+      <value enum='AM' description='POSITION_MAINTENANCE_REPORT'/>
+      <value enum='AN' description='REQUEST_FOR_POSITIONS'/>
+      <value enum='AO' description='REQUEST_FOR_POSITIONS_ACK'/>
+      <value enum='AP' description='POSITION_REPORT'/>
+      <value enum='AQ' description='TRADE_CAPTURE_REPORT_REQUEST_ACK'/>
+      <value enum='AR' description='TRADE_CAPTURE_REPORT_ACK'/>
+      <value enum='AS' description='ALLOCATION_REPORT'/>
+      <value enum='AT' description='ALLOCATION_REPORT_ACK'/>
+      <value enum='AU' description='CONFIRMATION_ACK'/>
+      <value enum='AV' description='SETTLEMENT_INSTRUCTION_REQUEST'/>
+      <value enum='AW' description='ASSIGNMENT_REPORT'/>
+      <value enum='AX' description='COLLATERAL_REQUEST'/>
+      <value enum='AY' description='COLLATERAL_ASSIGNMENT'/>
+      <value enum='AZ' description='COLLATERAL_RESPONSE'/>
+      <value enum='BA' description='COLLATERAL_REPORT'/>
+      <value enum='BB' description='COLLATERAL_INQUIRY'/>
+      <value enum='BC' description='NETWORK_STATUS_REQUEST'/>
+      <value enum='BD' description='NETWORK_STATUS_RESPONSE'/>
+      <value enum='BE' description='USER_REQUEST'/>
+      <value enum='BF' description='USER_RESPONSE'/>
+      <value enum='BG' description='COLLATERAL_INQUIRY_ACK'/>
+      <value enum='BH' description='CONFIRMATION_REQUEST'/>
+      <value enum='BI' description='TRADING_SESSION_LIST_REQUEST'/>
+      <value enum='BJ' description='TRADING_SESSION_LIST'/>
+      <value enum='BK' description='SECURITY_LIST_UPDATE_REPORT'/>
+      <value enum='BL' description='ADJUSTED_POSITION_REPORT'/>
+      <value enum='BM' description='ALLOCATION_INSTRUCTION_ALERT'/>
+      <value enum='BN' description='EXECUTION_ACKNOWLEDGEMENT'/>
+      <value enum='BO' description='CONTRARY_INTENTION_REPORT'/>
+      <value enum='BP' description='SECURITY_DEFINITION_UPDATE_REPORT'/>
+    </field>
+    <field name='NewSeqNo' number='36' type='SEQNUM'/>
+    <field name='PossDupFlag' number='43' type='BOOLEAN'/>
+    <field name='RefSeqNum' number='45' type='SEQNUM'/>
+    <field name='SenderCompID' number='49' type='STRING'/>
+    <field name='SenderSubID' number='50' type='STRING'/>
+    <field name='SendingTime' number='52' type='UTCTIMESTAMP'/>
+    <field name='TargetCompID' number='56' type='STRING'/>
+    <field name='TargetSubID' number='57' type='STRING'/>
+    <field name='Text' number='58' type='STRING'/>
+    <field name='Signature' number='89' type='DATA'/>
+    <field name='SecureDataLen' number='90' type='LENGTH'/>
+    <field name='SecureData' number='91' type='DATA'/>
+    <field name='SignatureLength' number='93' type='LENGTH'/>
+    <field name='RawDataLength' number='95' type='LENGTH'/>
+    <field name='RawData' number='96' type='DATA'/>
+    <field name='PossResend' number='97' type='BOOLEAN'/>
+    <field name='EncryptMethod' number='98' type='INT'>
+      <value description='NONE_OTHER' enum='0'/>
+      <value description='PKCS' enum='1'/>
+      <value description='DES' enum='2'/>
+      <value description='PKCS_DES' enum='3'/>
+      <value description='PGP_DES' enum='4'/>
+      <value description='PGP_DES_MD5' enum='5'/>
+      <value description='PEM_DES_MD5' enum='6'/>
+    </field>
+    <field name='HeartBtInt' number='108' type='INT'/>
+    <field name='TestReqID' number='112' type='STRING'/>
+    <field name='OnBehalfOfCompID' number='115' type='STRING'/>
+    <field name='OnBehalfOfSubID' number='116' type='STRING'/>
+    <field name='OrigSendingTime' number='122' type='UTCTIMESTAMP'/>
+    <field name='GapFillFlag' number='123' type='BOOLEAN'/>
+    <field name='DeliverToCompID' number='128' type='STRING'/>
+    <field name='DeliverToSubID' number='129' type='STRING'/>
+    <field name='ResetSeqNumFlag' number='141' type='BOOLEAN'/>
+    <field name='SenderLocationID' number='142' type='STRING'/>
+    <field name='TargetLocationID' number='143' type='STRING'/>
+    <field name='OnBehalfOfLocationID' number='144' type='STRING'/>
+    <field name='DeliverToLocationID' number='145' type='STRING'/>
+    <field name='XmlDataLen' number='212' type='LENGTH'/>
+    <field name='XmlData' number='213' type='DATA'/>
+    <field number='347' name='MessageEncoding' type='STRING'>
+      <value enum='ISO-2022-JP' description='ISO_2022_JP'/>
+      <value enum='EUC-JP' description='EUC_JP'/>
+      <value enum='SHIFT_JIS' description='SHIFT_JIS'/>
+      <value enum='UTF-8' description='UTF_8'/>
+    </field>
+    <field name='EncodedTextLen' number='354' type='LENGTH'/>
+    <field name='EncodedText' number='355' type='DATA'/>
+    <field name='LastMsgSeqNumProcessed' number='369' type='SEQNUM'/>
+    <field name='RefTagID' number='371' type='INT'/>
+    <field name='RefMsgType' number='372' type='STRING'/>
+    <field name='SessionRejectReason' number='373' type='INT'>
+      <value description='INVALID_TAG_NUMBER' enum='0'/>
+      <value description='REQUIRED_TAG_MISSING' enum='1'/>
+      <value description='SENDINGTIME_ACCURACY_PROBLEM' enum='10'/>
+      <value description='INVALID_MSGTYPE' enum='11'/>
+      <value description='XML_VALIDATION_ERROR' enum='12'/>
+      <value description='TAG_APPEARS_MORE_THAN_ONCE' enum='13'/>
+      <value description='TAG_SPECIFIED_OUT_OF_REQUIRED_ORDER' enum='14'/>
+      <value description='REPEATING_GROUP_FIELDS_OUT_OF_ORDER' enum='15'/>
+      <value description='INCORRECT_NUMINGROUP_COUNT_FOR_REPEATING_GROUP' enum='16'/>
+      <value description='NON_DATA_VALUE_INCLUDES_FIELD_DELIMITER' enum='17'/>
+      <value description='TAG_NOT_DEFINED_FOR_THIS_MESSAGE_TYPE' enum='2'/>
+      <value description='UNDEFINED_TAG' enum='3'/>
+      <value description='TAG_SPECIFIED_WITHOUT_A_VALUE' enum='4'/>
+      <value description='VALUE_IS_INCORRECT' enum='5'/>
+      <value description='INCORRECT_DATA_FORMAT_FOR_VALUE' enum='6'/>
+      <value description='DECRYPTION_PROBLEM' enum='7'/>
+      <value description='SIGNATURE_PROBLEM' enum='8'/>
+      <value description='COMPID_PROBLEM' enum='9'/>
+      <value description='OTHER' enum='99'/>
+    </field>
+    <field name='MaxMessageSize' number='383' type='LENGTH'/>
+    <field name='NoMsgTypes' number='384' type='NUMINGROUP'/>
+    <field name='MsgDirection' number='385' type='CHAR'>
+      <value description='RECEIVE' enum='R'/>
+      <value description='SEND' enum='S'/>
+    </field>
+    <field name='TestMessageIndicator' number='464' type='BOOLEAN'/>
+    <field name='Username' number='553' type='STRING'/>
+    <field name='Password' number='554' type='STRING'/>
+    <field name='NoHops' number='627' type='NUMINGROUP'/>
+    <field name='HopCompID' number='628' type='STRING'/>
+    <field name='HopSendingTime' number='629' type='UTCTIMESTAMP'/>
+    <field name='HopRefID' number='630' type='SEQNUM'/>
+    <field name='NextExpectedMsgSeqNum' number='789' type='SEQNUM'/>
+    <field name='ApplVerID' number='1128' type='STRING'>
+      <value description='FIX27' enum='0'/>
+      <value description='FIX30' enum='1'/>
+      <value description='FIX40' enum='2'/>
+      <value description='FIX41' enum='3'/>
+      <value description='FIX42' enum='4'/>
+      <value description='FIX43' enum='5'/>
+      <value description='FIX44' enum='6'/>
+      <value description='FIX50' enum='7'/>
+      <value description='FIX50SP1' enum='8'/>
+      <value description='FIX50SP2' enum='9'/>
+    </field>
+    <field name='CstmApplVerID' number='1129' type='STRING'/>
+    <field name='RefApplVerID' number='1130' type='STRING'/>
+    <field name='RefCstmApplVerID' number='1131' type='STRING'/>
+    <field name='DefaultApplVerID' number='1137' type='STRING'/>
+  </fields>
+</fix>

src/client/spark_historical/Algos/AlgoBase.cs

@@ -0,0 +1,148 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+
+using bts.co2;
+
+namespace bts.history
+{
+   public interface IAlgo 
+   {
+      bool is_completed();
+      void set_param(string name, string val);
+
+      void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage curr_book,
+                        MarketDataIncrementalRefresh packet);
+      void process_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                          MarketDataIncrementalRefresh packet);
+   }
+   
+   abstract public partial class AlgoBase : IAlgo
+   {
+      public readonly Guid guid = Guid.NewGuid();
+      public ulong order_key { get; private set;}
+      
+      public string[] security_list { get; private set; }
+      public Action<FillEntry> on_fill;
+      public Action<OrderStatus> on_status;
+      
+      protected IOrderSink _sink;
+      public IOrderSink sink => _sink;
+
+      readonly protected SortedDictionary< string, Func<MDEntry, OrderIDEntry, bool> > _filter = new SortedDictionary<string, Func<MDEntry, OrderIDEntry, bool>>();      
+
+      protected AlgoBase(IOrderSink sink, string[] security_list) 
+      {
+         on_fill = new Action<FillEntry>(f => {});
+         on_status = new Action<OrderStatus>(s => {});
+         this._sink = sink;
+         this.security_list = security_list;
+         foreach (var sec in security_list)
+            _filter[sec] = (MDEntry md, OrderIDEntry id) => md.Symbol == sec;
+      }            
+                  
+      readonly Dictionary<string, string> _algo_params = new Dictionary<string, string>();
+      public IReadOnlyDictionary<string, string> algo_params => _algo_params;
+      public virtual void set_param(string name, string val)
+      {
+         _algo_params[name] = val;
+         var pi = this.GetType().GetProperty(name);
+         if(pi != null)
+         {
+            if(val.Contains(','))
+            {
+               var elem = pi.PropertyType.GetElementType();
+               var ary = val.Split(',').Select(x => Convert.ChangeType(x, elem)).ToArray();
+               var tary = Array.CreateInstance(elem, ary.Length);
+               Array.Copy(ary, tary, ary.Length);
+               pi.SetValue(this, tary);
+            }
+            else
+               pi.SetValue(this, Convert.ChangeType(val, pi.PropertyType));
+         }
+      }         
+
+      public virtual void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage curr_book,
+                                       MarketDataIncrementalRefresh packet)
+      {
+         if(!_filter.ContainsKey(security_desc))
+            return;
+         on_book(security_desc, security_key, current_book, curr_book, packet);
+      }
+
+      public virtual void process_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                                         MarketDataIncrementalRefresh packet)      
+      {
+         if (!_filter.ContainsKey(security_desc))
+            return;
+         on_trades(security_desc, security_key, trades, packet);
+      }      
+
+      public abstract object report();
+      public abstract bool is_completed();
+
+      protected virtual void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                     MarketDataIncrementalRefresh packet) { }
+      protected virtual void on_trades(string security_desc, uint security_key, IEnumerable<TradeMessage> trades, 
+                                       MarketDataIncrementalRefresh packet) { }
+                                       
+      protected abstract void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key);
+      
+      protected abstract void do_update(Timestamp asof, double price, uint size = uint.MinValue);
+      protected virtual void do_update(Timestamp asof, Dictionary<string, string> algo_params, double price, uint size = uint.MinValue)
+      {
+         foreach (var s in algo_params)
+            this.set_param(s.Key, s.Value);
+         this.do_update(asof, price, size);
+      }
+      protected abstract void do_cancel(Timestamp asof);
+      protected virtual void get_done(Timestamp asof) {}
+
+      public class Handle : IEquatable<Handle>
+      {
+         readonly AlgoBase _algo;
+         public Handle(AlgoBase algo)
+         {
+            _algo = algo;
+         }
+         
+         public AlgoBase algo => _algo;
+         public ulong order_key => _algo.order_key;
+         public void launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key = ulong.MinValue)
+         {
+            if(order_key == ulong.MinValue)
+               order_key = IDGenerator.next_order_id();
+            _algo.do_launch(security_list, side, price, size, asof, order_key);
+         }
+         public void update(Timestamp asof, double price, uint size = uint.MinValue)
+         {
+            _algo.do_update(asof, price, size);
+         }
+         public void update(Timestamp asof, Dictionary<string, string> algo_params, double price, uint size = uint.MinValue)
+         {
+            foreach (var s in algo_params)
+               _algo.set_param(s.Key, s.Value);
+            update(asof, price, size);
+         }
+         public void cancel(Timestamp asof)
+         {
+            _algo.do_cancel(asof);
+         }
+         public void get_done(Timestamp asof)
+         {
+            _algo.get_done(asof);
+         }
+         public override int GetHashCode()
+         {
+            return this.algo.guid.GetHashCode();
+         }
+         public bool Equals(Handle other)
+         {
+            return this.algo.guid == other.algo.guid;
+         }
+      }
+   }
+}

src/client/spark_historical/Algos/AlgoFactory.cs

@@ -0,0 +1,94 @@
+using System;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+using System.Collections.Generic;
+using System.Reflection;
+using bts.api;
+
+namespace bts.history
+{
+   public class AlgoFactory
+   {
+      static public AlgoBase algo_factory<AlgoType>(IOrderSink sink,string[] security_list, string algo_params = null)
+         where AlgoType : AlgoBase
+      {
+         return AlgoBase.algo_factory(typeof(AlgoType), sink, security_list, algo_params);
+      }
+      static public void set_algo_params(IAlgo inst, string algo_params)
+      {
+         AlgoBase.set_algo_params((AlgoBase) inst, algo_params);
+      }
+      static public void set_algo_params(IAlgo inst, IEnumerable<string> param)
+      {
+         AlgoBase.set_algo_params((AlgoBase)inst, param);
+      }
+      public static AlgoBase.Handle launch(AlgoBase algo, api.Side side, double price, uint size, Timestamp asof,
+                                           Action<AlgoBase.Handle, FillEntry> more_on_fill = null,
+                                           Action<AlgoBase.Handle, OrderStatus> more_on_status = null)
+      {
+         var order_key = IDGenerator.next_order_id();
+         //I am activating myself at the top\         
+         var h = new AlgoBase.Handle(algo);
+         h.launch(algo.security_list, side, price, size, asof, order_key);
+         if (more_on_fill != null)
+            algo.on_fill += f => more_on_fill(h, f);
+         if (more_on_status != null)
+            algo.on_status += s => more_on_status(h, s);
+         return h;
+      }      
+   }   
+   abstract partial class AlgoBase
+   {
+      static public AlgoBase algo_factory(System.Type t, IOrderSink sink, 
+                                          string[] security_list, string algo_params = null)
+      {
+         AlgoBase inst = inst = (AlgoBase)Activator.CreateInstance(t, sink, security_list);
+
+         if(algo_params != null)
+            set_algo_params(inst, algo_params);
+         return inst;
+      }
+      public virtual Handle create<C>(Action<Handle, FillEntry> more_on_fill = null,
+                                      Action<Handle, OrderStatus> more_on_status = null)
+         where C : AlgoBase
+      {
+         var child = AlgoBase.algo_factory(typeof(C), _sink, security_list);
+         Handle h = hookup_child(child, more_on_fill, more_on_status);
+         return h;
+      }
+
+      protected Handle hookup_child(AlgoBase child, Action<Handle, FillEntry> more_on_fill, Action<Handle, OrderStatus> more_on_status)
+      {
+         child.order_key = order_key;
+         AlgoBase.set_algo_params(child, this.algo_params);
+         child.on_fill += f => this.on_fill(f);
+         //child.on_status += s => this.on_status(s);
+         var h = new Handle(child);
+         if (more_on_fill != null)
+            child.on_fill += (f => more_on_fill(h, f));
+         if (more_on_status != null)
+            child.on_status += (s => more_on_status(h, s));
+
+         return h;
+      }      
+      static public void set_algo_params(AlgoBase inst, string algo_params)
+      {
+         string[] param = algo_params.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+         set_algo_params(inst, param);
+      }
+      static public void set_algo_params(AlgoBase inst, IEnumerable<string> param)
+      {
+         foreach (var par in param)
+         {
+            string[] tup = par.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
+            Log.assert(tup.Length == 2);
+            inst.set_param(tup[0], tup[1]);
+         }
+      }
+      static public void set_algo_params(AlgoBase inst, IReadOnlyDictionary<string, string> param)
+      {
+         foreach (var par in param)
+            inst.set_param(par.Key, par.Value);
+      }
+   }
+}

src/client/spark_historical/Algos/CompletionCoverAlgo.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Collections.Generic;
+using mbts.co2;
+
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public class CompletionCoverAlgo<Child, Hedge> : CompositeHedgeAlgo<Child, Hedge>
+      where Child : AlgoBase
+      where Hedge : AlgoBase
+   {
+      public CompletionCoverAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      
+      string[] _security_list;
+      Side _myside;
+      double _myprice;
+      uint _mysize;
+
+      //algo parameters
+      public double hedge_ticks { get; set; } = 0.0;
+      public double hedge_factor { get; set; } = 1.0;
+
+      protected override void do_launch(string[] security_list, Side side, double price, uint size, Timestamp asof, ulong order_key) 
+      {
+         _security_list = security_list;
+         _myside = side;
+         _myprice = price;
+         _mysize = size;
+         _child = this.create<Child>();
+         _child.launch(security_list, side, price, size, asof);
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         if(_child?.Equals(child) ?? false)
+         {            
+            if(stat.state.Status == api.orderman.OrderStatus.Completed)
+            {
+               var side = Utils.opposite_side(_myside);
+               uint size = (uint)(_mysize * this.hedge_factor);
+               double price = (side == Side.Buy) ?
+                  _myprice - this.hedge_ticks :
+                  _myprice + this.hedge_ticks;
+
+               send_hedge(_security_list, side, price, size, stat.TransactTime, null,
+                  (status_handle, status_entry) => { this.on_status?.Invoke(status_entry); } 
+               );
+            }
+            this.on_status?.Invoke(stat);
+         }
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue) 
+      {
+         _child.update(asof, price, size);
+      }
+
+      protected override void do_cancel(Timestamp asof) 
+      {
+         _child.cancel(asof);
+      }
+
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+      }
+   }
+}

src/client/spark_historical/Algos/CompositeAlgo.cs

@@ -0,0 +1,261 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public abstract class CompositeBase<Child, Hedge> : AlgoBase
+      where Hedge : AlgoBase
+   {
+      protected List<Handle> _children = new List<Handle>();
+      protected Handle _child 
+      {
+         get 
+         {
+            Log.assert(_children.Count <= 1);
+            return _children.FirstOrDefault(); 
+         } 
+         set 
+         { 
+            Log.assert(_children.Count <= 1);
+            if(value == null)
+            {
+               if(_children.Count == 1)            
+                  _children.RemoveAt(0);
+            }
+            else
+            {
+               if(_children.Count == 0)
+                  _children.Add(value);
+               else
+                  _children[0] = value;
+            }
+         }
+      }
+      public CompositeBase(IOrderSink sink, string[] security_list) : base(sink, security_list) 
+      {
+      }
+
+      public override Handle create<C>(Action<Handle, FillEntry> more_on_fill = null, 
+                                       Action<Handle, OrderStatus> more_on_status = null)     
+      {         
+         if(typeof(C) == this.GetType())
+         {
+            Log.assert(false);
+            return null;
+            //return AlgoBase.activate_self(this, side, price, size, asof, more_on_fill, more_on_status);
+         }         
+         else
+         {
+            Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => on_child_fill(h, f));
+            if (more_on_fill != null)
+               of += more_on_fill;
+            Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) =>
+            {
+               on_child_status(h, s);
+               if (s.is_terminal())
+                  _children.Remove(h);
+            });
+            if (more_on_status != null)
+               os += more_on_status;
+
+            var handle = base_create<C>(of, os);
+            _children.Add(handle);
+            return handle;
+         }
+      }
+
+      protected Handle base_create<C>(Action<Handle, FillEntry> of, Action<Handle, OrderStatus> os)
+         where C : AlgoBase
+      {
+         return base.create<C>(of, os);
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         foreach(var child in _children.ToArray())
+            child?.update(asof, price, size);
+      }
+
+      protected override void get_done(Timestamp asof)
+      {
+         base.get_done(asof);
+         foreach (var child in _children.ToArray())
+            child?.get_done(asof);
+      }            
+
+      protected override void do_cancel(Timestamp asof)
+      {
+         foreach (var child in _children.ToArray())
+            child?.cancel(asof);
+      }      
+      protected virtual void on_child_fill(Handle child, FillEntry fill) { }
+      
+      protected abstract void on_child_status(Handle child, OrderStatus stat);
+   }
+
+   public abstract class CompositeAlgo<Child> : CompositeBase<Child, NullAlgo>
+   {
+      public CompositeAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+
+      public override bool is_completed()
+      {
+         bool comp = true;
+         foreach (var child in _children)
+            comp &= child.algo.is_completed();
+         return comp;
+      }
+      
+      public override object report()
+      {
+         string c = "";
+         foreach (var child in _children)
+            c+= child.algo.report();
+         return $"{c}";
+      }
+
+      public override void set_param(string name, string val)
+      {
+         base.set_param(name, val);
+         foreach (var child in _children)
+            child?.algo.set_param(name, val);
+      }
+
+      public override void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                        MarketDataIncrementalRefresh packet)
+      {
+         base.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var child in _children.ToArray()) //copy because this could remove / add a child
+            child?.algo.process_book(security_desc, security_key, current_book, working_book, packet);
+      }
+   }
+
+   public abstract class CompositeHedgeAlgo<Child, Hedge> : CompositeBase<Child, Hedge>
+      where Hedge : AlgoBase
+   {
+      protected int _payupcount;
+      protected Dictionary<Handle, Hedge> _hedges = new Dictionary<Handle, Hedge>();
+      public CompositeHedgeAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) {}
+
+      public override bool is_completed() 
+      {
+         bool comp = true;
+         foreach (var child in _children)
+            comp &= child.algo.is_completed();
+         foreach (var sub in _hedges.Values)
+         {
+            var hc = sub.is_completed();
+            if(!hc)
+               Log.assert(true);
+            comp &= hc;
+         }
+         return comp;
+      }
+
+      protected override void get_done(Timestamp asof)
+      {
+         base.get_done(asof);
+         foreach (var child in _children.ToArray())
+            child?.get_done(asof);
+         foreach (var sub in _hedges.Keys.ToArray())
+            sub.get_done(asof);
+      }
+      
+      public override object report() 
+      {
+         string c = "";
+         foreach (var child in _children)
+            c+= child.algo.report();
+         
+         int ih = 0;
+         string sh = string.Empty;
+         foreach (var sub in _hedges.Values)
+         {
+            if(sub.report() is string sr)
+               sh += sr;
+            if(sub.report() is int ir)
+               ih += ir;
+         }
+         return $"{c} Hedge Payups {sh} {ih+_payupcount} Incomplete {_hedges.Values.Sum( x => x.is_completed() ? 0u : 1u) } ";
+      }
+
+      public override void set_param(string name, string val)
+      {
+         base.set_param(name, val);
+         foreach (var child in _children)
+            child?.algo.set_param(name, val);
+         foreach (var sub in _hedges.Values)
+            sub.set_param(name, val);
+      }
+
+      public override void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                                MarketDataIncrementalRefresh packet)
+      {
+         base.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var child in _children.ToArray())
+            child?.algo.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var sub in _hedges.Values.ToArray())
+            sub.process_book(security_desc, security_key, current_book, working_book, packet);
+      }
+
+      public override Handle create<C>(Action<Handle, FillEntry> more_on_fill = null, 
+                                       Action<Handle, OrderStatus> more_on_status = null)
+      {
+         if (typeof(Hedge) == typeof(C))
+         {
+            Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => {} );
+            if (more_on_fill != null)
+               of += more_on_fill;
+            Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) => {} );
+            if (more_on_status != null)
+               os += more_on_status;
+
+            var handle = base_create<Hedge>(of, os);                                         
+            _hedges[handle] = (Hedge)handle.algo;
+            return handle;
+         }
+         else
+            return base.create<C>(more_on_fill, more_on_status);
+      }
+
+      protected void send_hedge(string security_desc, api.Side side, double price, uint size, Timestamp asof,
+                                Action<Handle, FillEntry> more_on_fill = null,
+                                Action<Handle, OrderStatus> more_on_status = null)
+      {
+         send_hedge(new[] { security_desc }, side, price, size, asof, more_on_fill, more_on_status);
+      }
+
+      protected void send_hedge(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
+                                Action<Handle, FillEntry> more_on_fill = null,
+                                Action<Handle, OrderStatus> more_on_status = null)                                
+      {
+         Action<Handle, FillEntry> hedge_on_fill = (hand, f) =>
+         {
+            if(more_on_fill != null)
+               more_on_fill(hand, f);
+            if (f.fully_filled)
+            {
+               if (_hedges.TryGetValue(hand, out Hedge h))
+               {
+                  if (h.report() is int pay)
+                     _payupcount += pay;
+                  _hedges.Remove(hand);
+               }
+            }
+         };
+         Action<Handle, OrderStatus> hedge_on_status = (hand,s) =>
+         {
+            if(more_on_status != null)
+               more_on_status(hand, s);
+            if (s.is_terminal())
+               _hedges.Remove(hand);
+         };
+
+         var handle = this.create<Hedge>(hedge_on_fill, hedge_on_status);
+         handle.launch(security_list, side, price, size, asof);
+      }
+   }
+   
+}