changes in algo backtester to facilitate jupyter, will probably remove backwards compatibility in a bit

make sparkcmd build in the right place on windows again

.vscode/launch.json

@@ -46,7 +46,7 @@
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             }
-        },
+        },     
         {
             "name": "(gdb) Launch",
             "type": "cppdbg",
@@ -98,6 +98,46 @@
             "externalConsole": true,
             "requireExactSource": false
         },
+            {
+         "name": "sparkcmd Launch",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "historical",
+            "--rstart", "12/04/2019",
+            "--rend", "12/05/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Spread",
+            "--post_process_db", "oil_12_4_2019",
+            "--post_process_db_host", "bigblue",
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },  
+      {
+         "name": "sparkcmd simulation",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "historical",
+            "--rstart", "11/26/2019",
+            "--rend", "11/26/2019",
+            "--rsymbol", "ZN",
+            "--rsecuritytype", "Synthetic",
+            "--algo_security", "ZNH0;ZNZ9-ZNH0;ZNZ9",
+            "--algo_type", "RoundTrip",
+            "--algo_params", "size=10;tick_size=0.015625,0.0078125,0.015625;hedge_factors=1,1,-1",
+            "--skip_preanalysis", "true",         
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },       
         {
             "name": "Excel Launch",
             "type": "clr",
@@ -126,11 +166,17 @@
             "requireExactSource": false
         },
         {
-            "name": "Client Attach",
+            "name": ".NET Framework Attach",
             "type": "clr",
             "request": "attach",
             "processId": "${command:pickProcess}"
         },
+         {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+         },        
         {
             "name": "Spreader Tests",
             "type": "cppdbg",
@@ -342,15 +388,19 @@
             "justMyCode": true
         },
         {
-            "name": ".NET Core Remote Launch",
+            "name": "sparkcmd vsdbg (c#)",
             "type": "coreclr",
             "request": "launch",
-            "program": "/usr/bin/dotnet",
+            "program": "sparkcmd",
             "args": [
-                "sparkcmd.dll",
-                "test",
-                "test_join_create_and_takeout_market"
+            "historical",
+            "--rstart", "11/20/2019",
+            "--rend", "11/20/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Spread",
+            "--post_process_db", "mbo_history",
             ],
+            "env": {"LD_LIBRARY_PATH" : "."},
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server",
             "pipeTransport": {
@@ -361,11 +411,77 @@
                 "debuggerPath": "/home/bts/vsdbg/vsdbg",
                 "pipeCwd": "${workspaceRoot}",
                 "quoteArgs": true
-            },
+            },           
             "sourceFileMap": {
                 "/home/bts/spark/": "${workspaceRoot}"
             },
             "justMyCode": false
-        },
+        },                  
+         {
+            "name": "sparkcmd gdb (c++)",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/home/bts/spark/build/ninja_debug/server/sparkcmd",
+            "args": [
+            "historical",
+            "--rstart", "10/21/2019",
+            "--rend", "10/21/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Future",
+            "--algo_security", "CLX9",
+            "--algo_type", "TWAP",
+            "--algo_params", "'size=100;profit_ticks=1;tick_size=1,1,1;;hedge_factors=1,1,-1'",
+            "--skip_preanalysis", "true",
+            ],   
+            "stopAtEntry": false,
+            "cwd": "/home/bts/spark/build/ninja_debug/server/",
+            "environment": [],
+            "externalConsole": true,
+            "MIMode": "gdb",
+            "preLaunchTask": "",
+            "logging": {
+               "engineLogging": false
+            },
+            "setupCommands": [
+               {
+                  "description": "Enable pretty-printing for gdb",
+                  "text": "enable pretty-printer",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "Skip STL",
+                  "text": "skip -gfi /usr/include/c++/7/bits/*.h",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "LD LIBRARY",
+                  "text": "set env LD_LIBRARY_PATH=.",
+                  "ignoreFailures": false
+               }
+            ],
+            "pipeTransport": {
+               "pipeCwd": "${workspaceRoot}",
+               "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+               "pipeArgs": [
+                  "bts@sparkvm"
+               ],
+               "debuggerPath": "/usr/bin/gdb"
+            },
+            "sourceFileMap": {
+               "/home/bts/spark/": "${workspaceRoot}"
+            }
+         },
+         {
+            "name": "Jupyter notebook",
+            "type": "coreclr",
+            "request": "launch",
+            "program": "jupyter",
+            "args": ["notebook", "--notebook-dir=c:\\code\\bts\\spark\\src\\client\\spark_historical\\Notebooks"],
+            "cwd": "${workspaceFolder}",
+            "stopAtEntry": false,
+            "console": "internalConsole",
+            
+         },
+         
     ]
 }

CMakeLists.txt

@@ -48,6 +48,7 @@ else ()
   add_subdirectory(doc/user)
   add_subdirectory(src/sdk)
   add_subdirectory(src/data_files)
+  add_subdirectory(src/client)
 
   add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
@@ -59,7 +60,16 @@ else ()
     COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
     COMMENT "Creating server runtime layout"
-    DEPENDS sparkd data_files_dir)  
+    DEPENDS sparkd data_files_dir spark_client_dir)  
 
   add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+  
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.0/linux-x64/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+else()
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.0/linux-x64/publish/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+endif()
+      
 endif()

spark_client.sln

@@ -23,7 +23,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\sp
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_excel", "src\client\spark_excel\spark_excel.csproj", "{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_fix", "src\client\spark_fix\spark_fix.csproj", "{D8031C13-7BEA-430A-BB60-40927D15CFC3}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_historical", "src\client\spark_historical\spark_historical.csproj", "{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}"
+EndProject
 Global
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Debug|x64 = Debug|x64
@@ -219,6 +226,50 @@ Global
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.ActiveCfg = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.Build.0 = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/client/CMakeLists.txt

@@ -0,0 +1,19 @@
+
+cmake_minimum_required(VERSION 3.5)
+project(spark_client)
+include(${CMAKE_SOURCE_DIR}/tools/safe_dep_glob.cmake)
+
+file(GLOB_RECURSE spark_client_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cs)
+file(GLOB_RECURSE spark_client_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.csproj)
+file(GLOB_RECURSE spark_ui_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.cs)
+file(GLOB_RECURSE spark_ui_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.csproj)
+list(REMOVE_ITEM spark_ui_sources ${spark_client_sources} )
+list(REMOVE_ITEM spark_ui_csproj ${spark_client_csproj} )
+
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/../
+   COMMAND /bin/bash ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh
+   ARGS ${CMAKE_BUILD_TYPE} ${CMAKE_BINARY_DIR}
+   COMMENT "Building sparkcmd in netcore3"
+   DEPENDS libspark ${spark_client_sources} ${spark_client_csproj} ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh)
+
+   add_custom_target(spark_client_dir ALL DEPENDS ${CMAKE_BINARY_DIR}/../)   

src/client/Directory.Build.props

@@ -5,6 +5,7 @@
     <SparkRoot>$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\..))</SparkRoot>
     <SourceRoot>$([System.IO.Path]::GetFullPath($(SparkRoot)\src))</SourceRoot>
     <Externals>$(SparkRoot)\externals\windows</Externals>
+    <Externals Condition=" '$(OS)' != 'Windows_NT' ">$(SparkRoot)/externals/linux</Externals>
     <DepotName>$([System.IO.Path]::GetFileName($([System.IO.Path]::GetFullPath($(SparkRoot)))))</DepotName>
     <ErrorReport>None</ErrorReport>
     <Prefer32Bit>False</Prefer32Bit>
@@ -22,17 +23,20 @@
     <GenDir>$(BaseOutputPath)\gen\$(MSBuildProjectName)</GenDir>
 </PropertyGroup>
 
+
 <Import Project="$(ComputerSpecificConfig)" Condition="Exists($(ComputerSpecificConfig))" />
 
 <ItemGroup>
     <AvailableItemName Include="Proto2">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto2</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto2Linux</Targets>
     </AvailableItemName>
     <AvailableItemName Include="Proto3">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto3Linux</Targets>
     </AvailableItemName>
     <AvailableItemName Include="Proto3Excel">
-        <Targets>CompileProto3Excel</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' ">CompileProto3Excel</Targets>
     </AvailableItemName>    
 </ItemGroup>
 
@@ -43,6 +47,14 @@
     <protogen>$(Externals)\bin\protogen.exe</protogen>
 </PropertyGroup>
 
+<PropertyGroup Label="ProtoBuf" Condition=" '$(OS)' != 'Windows_NT' ">
+    <ProtocPluginBase>$(SparkRoot)/src/lib/api/service_compiler</ProtocPluginBase>
+    <ProtoInclude>$(Externals)/include</ProtoInclude>
+    <protoc>$(Externals)/bin/protoc</protoc>
+    <protoc_rename>$(Externals)/bin/lowercase_files.sh</protoc_rename>
+    <protogen>$(Externals)/bin/protogen</protogen>
+</PropertyGroup>
+
 <ItemDefinitionGroup>
     <Proto2>
     </Proto2>

src/client/Directory.Build.targets

@@ -3,7 +3,7 @@
 
 
 <Target Name="CompileProto2" 
-    Condition="'@(Proto2)'!=''"
+    Condition="'@(Proto2)'!='' AND '$(OS)' == 'Windows_NT' "
     Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
     Outputs="$(GenDir)\proto2\%(Proto2.filename).cs"
     AfterTargets="PreBuildEvent">
@@ -28,7 +28,7 @@
     passes wipes out the additional compile includes that we add.
 -->
 <Target Name="CompileProto3" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3)'!=''"
+    Condition="'@(Proto3)'!=''  AND '$(OS)' == 'Windows_NT' "
     Inputs="$(protoc);
     %(Proto3.Identity);$(ProtocPluginBase)\plugin.py;
     $(ProtocPluginBase)\templates\c#\client.pb.cs;
@@ -58,7 +58,7 @@
 </Target>
 
 <Target Name="CompileProto3Excel" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3Excel)'!=''"
+    Condition="'@(Proto3Excel)'!='' AND '$(OS)' == 'Windows_NT' "
     Inputs="$(protoc);
     %(Proto3Excel.Identity);$(ProtocPluginBase)\plugin.py;
     $(ProtocPluginBase)\templates\excel\excel.pb.cs;"
@@ -82,12 +82,74 @@
     </ItemGroup>
 </Target>
 
+
+<Target Name="CompileProto2Linux" 
+    Condition="'@(Proto2)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
+    Outputs="$(GenDir)/proto2/%(Proto2.filename).cs"
+    AfterTargets="PreBuildEvent">
+    <PropertyGroup>
+        <SourceRelativeFile>$([System.String]::Copy('%(proto2.fullpath)').Replace($(SourceRoot), ''))</SourceRelativeFile>
+        <SourceRelativeDir>$([System.IO.Path]::GetDirectoryName($(SourceRelativeFile)))</SourceRelativeDir>
+        <WorkingDirectory>$(GenDir)/proto2</WorkingDirectory>
+    </PropertyGroup>
+    <MakeDir Directories="$(WorkingDirectory)" />
+    <Copy SourceFiles="%(Proto2.FullPath)" DestinationFolder="$(WorkingDirectory)" SkipUnchangedFiles="True" UseHardLinksIfPossible="True" />
+    <Error Text="protogen.exe does not exist. If you forgot to clone with externals try: git submodule update --init --recursive" Condition="!Exists($(protogen))" />
+    <Message Importance="High" Text="Doing Proto2 %(proto2.filename)" />
+    <Exec Command="PYTHONPATH=$(Externals)/python $(Protogen) %(Proto2.filename)%(proto2.extension) --csharp_out=. -I. -I$(SourceRoot) -I$(Externals)/include"
+          WorkingDirectory="$(WorkingDirectory)" />
+    <ItemGroup>
+        <Compile Include="$(WorkingDirectory)/%(Proto2.filename).cs" />
+    </ItemGroup>
+</Target>
+
+<Target Name="CompileProto3Linux" AfterTargets="GetAssemblyVersion"
+    Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(protoc);
+    %(Proto3.Identity);$(ProtocPluginBase)/plugin.py;
+    $(ProtocPluginBase)/templates/c#/client.pb.cs;
+    $(ProtocPluginBase)/templates/c#/server.pb.cs;
+    $(ProtocPluginBase)/templates/c#/immutable.pb.cs;"
+    Outputs="%(Proto3.OutCS);%(Proto3.OutCSServer);%(Proto3.OutCSClient);%(Proto3.OutCSImmutable)">
+    <MakeDir Directories="%(Proto3.OutBase)" />
+    <MakeDir Directories="$(IntermediateOutputPath)/gen" />
+   
+    <Message Importance="High" Text="Generating service %(Proto3.FileName)" />
+    <Exec Command="
+       PYTHONPATH=$(Externals)/python $(protoc) \
+       %(Proto3.FullPath) \
+       --error_format=gcc \
+       --proto_path=$(protoinc):$(SourceRoot) \
+       --plugin=&apos;protoc-gen-service=$(ProtocPluginBase)/plugin.py&apos; \
+       --service_out=lang=c#,strip_out_path=1:%(Proto3.OutBase) \
+       --csharp_out=%(CompileOptions)%(Proto3.OutBase)"
+      WorkingDirectory="%(relativedir)" />    
+      
+    <!-- <Exec Command="$(protoc_rename) %(Proto3.OutBase)"  />    -->
+            
+    <ItemGroup>
+       <Compile Include="%(Proto3.OutCS)" />
+       <Compile Include="%(Proto3.OutCSClient)" />
+       <Compile Include="%(Proto3.OutCSServer)" />
+       <Compile Include="%(Proto3.OutCSImmutable)" />
+    </ItemGroup>
+</Target>
+
+<!-- <Target Name="RenameProto3Linux" AfterTargets="CompileProto3Linux"
+   Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' ">
+   <Move SourceFiles="$(OutBase)\Apicore.cs" DestinationFiles="$(OutBase)\apicore.cs" />  
+   <Move SourceFiles="$(OutBase)\Apioptions.cs" DestinationFiles="$(OutBase)\apioptions.cs" />  
+   <Move SourceFiles="$(OutBase)\Co2.cs" DestinationFiles="$(OutBase)\co2.cs" />  
+</Target> -->
+
+
 <Target Name="AfterClean" AfterTargets="Clean">
     <Delete Files="%(Proto3.OutCS)" />
     <Delete Files="%(Proto3.OutCSClient)" />
     <Delete Files="%(Proto3.OutCSServer)" />
     <Delete Files="%(Proto3.OutCSImmutable)" />
-    <Delete Files="$(GenDir)\proto2\*" />
+    <Delete Files="$(GenDir)/proto2/*" />
     <Delete Files="%(Proto3Excel.OutCSExcel)" />
 </Target>
 

src/client/build_spark_dotnetcore.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#$1 = Debug / Release (CMAKE_BUILD_TYPE), $2 = CMAKE_BINARY_DIR
+cd "$(dirname "$0")"
+echo $PWD dotnet build -c $1 ./sparkcmd/
+dotnet build -c $1 ./sparkcmd/ 
+
+if [ "$1" == "Release" ]
+then
+   dotnet publish -c $1 ./sparkcmd/ 
+   ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.0/linux-x64/publish
+fi
+
+ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.0/linux-x64

src/client/spark/Orders/OrderDataHelper.cs

@@ -653,54 +653,25 @@ namespace mbts.spark.utilities {
             return def;
          }
       }
+      
+      static CustomOrderDefinition.CustomOneofCase[] algo_config(
+         Func<bts.config.AlgoPermissions, List<string>> selector)
+      {
+         var conf = mbts.config.Configuration.get<bts.config.AlgoPermissions>();
+         return selector(conf).Select(x => 
+         {
+            return (CustomOrderDefinition.CustomOneofCase)System.Enum.Parse(typeof(CustomOrderDefinition.CustomOneofCase), x, true);
+         }).ToArray();
+      }
 
       public static readonly CustomOrderDefinition.CustomOneofCase[]
 
          // exchange_order_types are the types which can be submitted on an exchange security
-         exchange_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         spreader_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Spreader,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover, 
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         attachable_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         outright_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-         };
+         exchange_order_types = algo_config(x => x.exchange_order_types),
+         spreader_order_types = algo_config(x => x.spreader_order_types),
+         attachable_order_types = algo_config(x => x.attachable_order_types),
+         outright_order_types = algo_config(x => x.outright_order_types);
+         
       public static readonly CustomOrderDefinition.CustomOneofCase[] all_order_types = exchange_order_types.Union(spreader_order_types).ToArray();
 
       public static void copy_values(CustomOrderDefinitionImmutable source, ref CustomOrderDefinition target) =>

src/client/spark/Orders/OrderTypeVMs/GenericOrderParamVM.cs

@@ -54,13 +54,16 @@ namespace mbts.spark
    {
       readonly Func<TModel, TView> _convert;
       readonly Func<TView, TModel> _convert_back;
+      readonly Func<IMessage, IMessage> _message_override;
       TView _value;
 
-      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, Func<TView, TModel> convert_back) : 
+      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, 
+                                 Func<TView, TModel> convert_back, Func<IMessage, IMessage> message_override = null) : 
          base(field_descriptor) 
       {
          _convert = convert;
          _convert_back = convert_back;
+         _message_override = message_override;
       }
 
       public TView value 
@@ -68,7 +71,8 @@ namespace mbts.spark
          get => _value;
          set 
          {
-            this.descriptor.Accessor.SetValue(this.message, _convert_back(_value = value));
+            this.descriptor.Accessor.SetValue( _message_override == null ? this.message : _message_override(this.message), 
+                                              _convert_back(_value = value));
             RaisePropertyChanged();
          }
       }
@@ -84,7 +88,7 @@ namespace mbts.spark
 
       protected override void on_param_changed() 
       {
-         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(this.message)));
+         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(_message_override == null ? this.message : _message_override(this.message))));
          RaisePropertyChanged(nameof(value));
       }
    }
@@ -163,7 +167,6 @@ namespace mbts.spark
                RaisePropertyChanged();
          }
       }      
-
       public IObservable<QuantityMeasureImmutable?> subscribe_value() 
       {
          return Observable.Create<QuantityMeasureImmutable?>(o =>
@@ -174,10 +177,9 @@ namespace mbts.spark
                o.OnNext(quantity);
          });
       }
-
       protected override void on_param_changed()
       {
          RaisePropertyChanged(nameof(quantity));
       }
-   }   
+   }
 }

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -96,9 +96,41 @@ namespace mbts.spark {
             {               
                Google.Protobuf.Reflection.MessageDescriptor desc = new TMutable().Descriptor;               
                _parameters.SyncWithDictionary(
-                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), _mk_param_vm);
+                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), x => _mk_param_vm(x));
+               add_message_params(desc);
             }
          }
+         void add_message_params(Google.Protobuf.Reflection.MessageDescriptor desc)
+         {
+            var msgs = desc.Fields.InDeclarationOrder().Where(x => !_is_value_param_desc(x) && 
+                                                              x.MessageType.ClrType != typeof(CustomOrderDefinition) );
+            foreach(var message_field in msgs)
+            {
+               foreach (var fd in message_field.MessageType.Fields.InDeclarationOrder())
+               {
+                  if(_is_value_param_desc(fd)) //one level deep
+                  {
+                     var mk = _mk_param_vm(fd, 
+                     outside_object => 
+                     {                        
+                        var msgfield = message_field.Accessor.GetValue(outside_object);
+                        if(msgfield == null)
+                        {
+                           message_field.Accessor.SetValue(outside_object, Activator.CreateInstance(message_field.MessageType.ClrType));
+                           msgfield = message_field.Accessor.GetValue(outside_object);
+                        }
+                        return (IMessage)msgfield;
+                     });
+                     if(mk == null) 
+                        continue;
+                        
+                     if(!_parameters.Add(mk))
+                        _parameters.Add(message_field.Name + "_" + mk.descriptor.Name, mk);                     
+                  }
+               }
+            }            
+         }
+         
          static bool _is_value_param_desc(FieldDescriptor desc)
          {
             return (desc.FieldType != Google.Protobuf.Reflection.FieldType.Message || 
@@ -117,7 +149,7 @@ namespace mbts.spark {
             return v;
          }
 
-         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor) 
+         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null) 
          {
             ExceptionAssert.Argument.NotNull(descriptor, nameof(descriptor));
 
@@ -134,33 +166,36 @@ namespace mbts.spark {
                return pvm;
             }
             else if (descriptor.FieldType == FieldType.Bool)
-               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
             else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float)
-               return mk<double, double>(descriptor, v => v, v => v, double.NaN);
+               return mk<double, double>(descriptor, v => v, v => v, message_override, double.NaN);
             else if (descriptor.FieldType == FieldType.Int32)
-               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
             else if (descriptor.FieldType == FieldType.UInt32)
-               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
             else if (descriptor.FieldType == FieldType.String)
-               return mk<string, string>(descriptor, v => v, v => v);
+               return mk<string, string>(descriptor, v => v, v => v, message_override);
             else
                return null;
          }
-         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, TView default_value = default)
+         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, 
+                                                                        Func<IMessage, IMessage> message_override = null, TView default_value = default)
          {
-            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back);
-            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param, default_value);
+            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back, message_override);
+            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param,
+                                                                                 message_override, default_value);
 
             TView get_from_param(IMessage param, FieldDescriptor fd) => convert(ConvertUtils.try_convert<TModel>(fdesc.Accessor.GetValue(param)));
             void assign_to_param(IMessage param, FieldDescriptor fd, TView value) => fdesc.Accessor.SetValue(param, convert_back(value));
          }
 
-         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
+         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, 
+                                    Func<IMessage, IMessage> message_override = null, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
          {
             add_property<T>(
                pvm.descriptor.Name,
-               mdef => mdef != null ? selector(mdef, pvm.descriptor) : default_value,
-               (mdef, v) => assign_to_param(mdef, pvm.descriptor, v),
+               mdef => mdef != null ? selector( message_override == null ?  mdef : message_override(mdef), pvm.descriptor) : default_value,
+               (mdef, v) => assign_to_param(message_override == null ? mdef : message_override(mdef), pvm.descriptor, v),
                () => pvm.value,
                v => pvm.value = v,
                pvm.subscribe_value(),

src/client/spark/Orders/OrderTypeVMs/OffsetOrderVM.cs

@@ -4,14 +4,14 @@ using OffsetDefi = bts.api.orderman.OffsetOrderDefinitionImmutable;
 namespace mbts.spark {
    static partial class OrderTypeVMFactory {
       class OffsetOrderVM : OrderTypeVMBase<OffsetDef, OffsetDefi> {
-         readonly Property<uint?> _tick_offset_on_deploy;
+         readonly Property<int?> _tick_offset_on_deploy;
 
          public OffsetOrderVM(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions)
             : base(key, display_name, editing_permissions) {
-            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0u);
+            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0);
          }
 
-         public uint? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
+         public int? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
       }
    }
 }

src/client/spark/Spark.csproj

@@ -9,7 +9,6 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Configurations>Debug;Release</Configurations>
     <Platforms>x64</Platforms>
-    <RuntimeIdentifiers>win7-x64</RuntimeIdentifiers>
     <PlatformTarget>AnyCPU</PlatformTarget>
     <LanguageTargets>$(MSBuildExtensionsPath)\$(VisualStudioVersion)\Bin\Microsoft.CSharp.targets</LanguageTargets>
     <ApplicationIcon>logo.ico</ApplicationIcon>
@@ -35,7 +34,7 @@
     <PackageReference Include="Google.Protobuf" Version="3.6.1" />
     <PackageReference Include="IronPython" Version="2.7.7" />
     <PackageReference Include="NAudio" Version="1.8.4" />
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
     <PackageReference Include="NUnit" Version="3.10.1" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.8.0" />

src/client/spark_api/Co2/Co2NativeMethods.cs

@@ -4,6 +4,8 @@ using System.Runtime.InteropServices;
 using System.Security;
 using Google.Protobuf;
 using security_key = System.UInt32;
+using bts.co2;
+using System.Linq;
 
 namespace mbts.co2
 {
@@ -85,6 +87,7 @@ namespace mbts.co2
 
       [DllImport("libspark")]
       internal unsafe static extern void post_on_scan_security_thread(Action callback);
+      
    }
 
    internal static class Co2NativeTestMethods
@@ -133,5 +136,59 @@ namespace mbts.co2
       [DllImport("libspark")]
       internal static extern void co2_test_get_constants(out ulong max_depth, out ulong max_implied_depth, out sbyte nan_exponent);
 
+      internal unsafe delegate void raw_receive_pb_or_string(byte* buffer, int buffer_size);
+      [DllImport("libspark")]
+      private static unsafe extern void raw_co2_compute_synthetic_book(
+         byte[] security_buffer,
+         int security_length,
+         IntPtr[] leg_books,
+         int[] leg_lengths,
+         int leg_books_length,
+         raw_receive_pb_or_string callback);
+
+      internal static unsafe BookMessageImmutable co2_compute_synthetic_book(SecurityImmutable synthetic, params BookMessageImmutable[] books) {
+         BookMessage result = null;
+         string error = null;
+
+         var security_buffer = synthetic.ToByteArray();
+         // var h = GCHandle.Alloc(null, GCHandleType.Pinned).
+         var book_buffers = 
+            books
+            .Select(b => b.ToByteArray())
+            .ToArray();
+         GCHandle[] book_buffer_handles = 
+            book_buffers
+            .Select(b => GCHandle.Alloc(b,GCHandleType.Pinned))
+            .ToArray();
+         IntPtr[] book_buffer_pointers = 
+            book_buffer_handles
+            .Select(h => h.AddrOfPinnedObject())
+            .ToArray();
+         var lengths = 
+            book_buffers
+            .Select(b => b.Length)
+            .ToArray();
+
+         raw_co2_compute_synthetic_book(
+            security_buffer,
+            security_buffer.Length,
+            book_buffer_pointers,
+            lengths,
+            book_buffers.Length,
+            (byte* buffer, int length) => {
+               if (length < 0) {
+                  error =  new string((char*)buffer,0,-1 * length);
+               }
+               else {
+                  result = Co2NativeMethods.deserialize<BookMessage>(buffer, (ulong)length);
+               }
+            });
+
+         if(error != null) {
+            // C# style multiple return values
+            throw new Exception(error);
+         }
+         return new BookMessageImmutable(result);
+      }
    }
 }

src/client/spark_api/Co2/IntPrice.cs

@@ -52,7 +52,49 @@ namespace mbts.co2 {
          return _mantissa.GetHashCode();
       }
 
+      public static IntPrice from_double(double price, sbyte precision) {
+         if (double.IsNaN(price)) return NaN;
+         if (Math.Abs(price%1) != 0) {
+            double rounding = 0.5 * Math.Pow(10,-precision);
+            price += (price > 0) ? rounding : -rounding;
+            price *= Math.Pow(10,precision);
+            return new IntPrice((long)price, (sbyte)-precision);
+         }
+         return new IntPrice((long)price, 0);
+      }
+
       public static IntPrice NaN = new IntPrice(0,127);
+
+      static double[] powers_10 = {
+  1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10, 1E-9, 1E-8, 1E-7, 1E-6, 1E-5, 1E-4, 1E-3, 1E-2, 1E-1,
+  1E0,
+  1E1, 1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8, 1E9, 1E10, 1E11, 1E12, 1E13, 1E14, 1E15, 1E16, 1E17, 1E18, 1E19, 1E20};
+      static int max_abs_power_10 = powers_10.Length / 2;
+      static double pow10(int power)
+      {
+         int offset = power + max_abs_power_10;
+         int powers_10_sz = powers_10.Length;
+         bts.Log.assert(offset >= 0 && offset < powers_10_sz);
+         if (offset >= 0 && offset < powers_10_sz)
+         {
+            return powers_10[offset];
+         }
+         return double.NaN;
+      }      
+      
+      public static IntPrice from_double(double price, int precision = 9)
+      {
+         if (Double.IsNaN(price)) 
+            return NaN;
+         if (Math.Abs(price - Math.Truncate(price)) != 0.0)
+         {
+            double rounding = 0.5 * pow10(-precision);
+            price += (price > 0.0) ? rounding : -rounding;
+            price *= pow10(precision);
+            return new IntPrice((long)price, Convert.ToSByte(-precision));
+         }
+         return new IntPrice((long)price, 0);
+      }
    }
    
 }

src/client/spark_api/Co2/NativePriceHelpers.cs

@@ -1,4 +1,6 @@
 using bts;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace mbts.co2 {
    public static class NativePriceHelpers {
@@ -38,6 +40,60 @@ namespace mbts.co2 {
       public const uint COCONUT_MAX_DEPTH = 10;
       public const uint COCONUT_MAX_IMPLIED_DEPTH = 2;
       internal const sbyte COCONUT_NAN_EXPONENT = 127;
+
+      public static double side_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if(side == bts.api.Side.Buy)
+            return bid_at(book, level);
+         else
+            return ask_at(book, level);
+      }
+      public static uint sidesize_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if (side == bts.api.Side.Buy)
+            return bidsize_at(book, level);
+         else
+            return asksize_at(book, level);
+      }
+      public static double bid_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static double ask_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static uint bidsize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return lev.Size;
+      }
+      public static uint asksize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return lev.Size;
+      }
+      public static double price(this bts.co2.PriceSizeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }
+      public static double price(this bts.co2.TradeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }      
+
+      public static IList<double> all_bids(this bts.co2.BookMessage book)
+      {
+         var lev = book.Bids.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }
+      public static IList<double> all_asks(this bts.co2.BookMessage book)
+      {
+         var lev = book.Asks.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }      
    }
    
 }

src/client/spark_api/Properties/AssemblyAttributes.cs

@@ -3,4 +3,6 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark_core")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_fix")]
+[assembly: InternalsVisibleTo("spark_historical")]
 

src/client/spark_api/history/HistoryEntry.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public interface IHasTimestamp
+   {
+      DateTime trade_date { get; set;}
+      DateTime sending_time { get; set;}
+      DateTime transact_time { get; set; }
+      DateTime timestamp();
+
+      uint MsgSeqNum { get; }
+   }
+
+   public interface IHasMdPacket
+   {
+      MarketDataIncrementalRefresh Md{ get; }
+   }
+
+   public interface IHasMdEntry
+   {
+      MDEntry MdEntry { get; }
+   }
+   
+   public interface IHasTopBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.TopBookMessage TopBook { get; set; }
+   }
+
+   public interface IHasBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.BookMessage Book { get; set; }
+   }
+
+   public interface IHasTrade : IHasMdEntry, IHasMdPacket
+   {
+      co2.TradeMessage Trade { get; set; }      
+   }   
+
+   public interface IHasMBO : IHasMdPacket
+   {
+      MBOMessage Order { get; set; }
+   }
+   
+   public sealed partial class HistoryRow : IHasTimestamp, IHasTopBook, IHasMBO, IHasTrade, IHasBook
+   {
+      public DateTime trade_date { get { return DateTime.SpecifyKind(this.TradeDate.ToDateTime(), DateTimeKind.Utc); } 
+                                   set { this.TradeDate = Timestamp.FromDateTime(value); } }
+      public DateTime sending_time { get { return DateTime.SpecifyKind(this.SendingTime.ToDateTime(), DateTimeKind.Utc); } 
+                                     set { this.SendingTime = Timestamp.FromDateTime(value); } }
+      public DateTime transact_time { get { return DateTime.SpecifyKind(this.TransactTime.ToDateTime(), DateTimeKind.Utc); } 
+                                      set { this.TransactTime = Timestamp.FromDateTime(value); } }
+                                      
+      public DateTime timestamp()
+      {
+         return this.sending_time;
+      }      
+   }
+   
+   public interface IMBOMessage
+   {
+      MDUpdateAction OrderUpdateAction {get;}
+      ulong OrderId {get;}
+      uint SecurityId {get;}
+      string SecurityDesc {get;}
+      ulong MdOrderPriority {get;}
+      double MdEntryPx {get;}
+      uint MdPriceLevel {get;}
+      ulong MdDisplayQty {get;}
+      MDEntryType MdEntryType {get;}
+      uint ReferenceId {get;}      
+   }
+   
+   public sealed partial class MBOMessage : IMBOMessage
+   {
+      public MDUpdateAction OrderUpdateAction => MdEntry.MdUpdateAction;
+      public ulong OrderId  => OrderEntry != null ? OrderEntry.OrderId : MdEntry.OrderId;
+      public uint SecurityId => MdEntry.SecurityId;
+      public string SecurityDesc => MdEntry.Symbol;
+      public ulong MdOrderPriority => OrderEntry != null ? OrderEntry.MdOrderPriority : MdEntry.MdOrderPriority;
+      public double MdEntryPx => MdEntry.MdEntryPx;
+      public uint MdPriceLevel => MdEntry.MdPriceLevel;
+      public ulong MdDisplayQty => OrderEntry != null ? OrderEntry.MdDisplayQty : MdEntry.MdDisplayQty;
+      public MDEntryType MdEntryType => MdEntry.MdEntryType;
+      public uint ReferenceId => OrderEntry != null ? OrderEntry.ReferenceId : MdEntry.ReferenceId;
+   }
+}
+
+namespace bts.co2
+{
+   public sealed partial class BookMessage
+   {
+      public BookMessage add_bids_and_asks(IEnumerable<PriceSizeMessage> bids, IEnumerable<PriceSizeMessage> asks)
+      {
+         this.Bids.AddRange(bids);
+         this.Asks.AddRange(asks);
+         return this;
+      }
+   }
+}

src/client/spark_api/security/SecurityFormatter.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using static bts.api.Security;
 
 namespace bts.api {
-   struct FutureLeg {
+   public struct FutureLeg {
       public readonly Venue exchange;
       public readonly string symbol, exchange_desc;
       public readonly FutureContractImmutable future;

src/client/spark_api/security/SecurityHelper.cs

@@ -4,7 +4,7 @@ using System.Linq;
 using static bts.api.Security;
 
 namespace bts.api {
-   internal static class SecurityHelper {
+   public static class SecurityHelper {
       public static IEnumerable<FutureLeg> get_flat_future_legs(SecurityImmutable sec) {
          switch (sec.ContractCase) {
             case ContractOneofCase.Spread: return _get_flat_future_legs(sec.Spread.Legs, 1);

src/client/spark_api/spark_api.csproj

@@ -25,7 +25,7 @@
     <Proto3 Include="../../lib/api/*.proto" />
     <Proto3 Include="../../lib/api/services/*.proto" />
     <!--Proto3 Remove="../../lib/api/services/MarketDataRelay.proto" /-->
-    <Proto3 Include="../../lib/md/protos/co2.proto" />
+    <Proto3 Include="../../lib/md/protos/Co2.proto" />
     <Proto3 Include="../../lib/domain_model/Security.proto" />
     <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
     <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">

src/client/spark_api/utils/BTSNativeMethods.cs

@@ -28,5 +28,8 @@ namespace bts.utils
 
       [DllImport("libspark")]
       public static extern bool initialize_nothrow(StringBuilder buffer, int length);
+
+      //public static string bts_data_path()
+      //use mbts.config.Configuration.get<bts.config.System>().data_path;
    }
 }

src/client/spark_api/utils/Log.cs

@@ -36,6 +36,7 @@ namespace bts {
       Plugin,
       Sdk,
       Http,
+      Fix
       // Please keep this list in sync with logging.h/logging.cpp
    }
    
@@ -102,9 +103,10 @@ namespace bts {
       internal static Logger Assert => new Logger(Subsystem.Assert);
       internal static Logger Ui => new Logger(Subsystem.Ui);
       internal static Logger Testing => new Logger(Subsystem.Testing);
-      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);
+      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);      
       public static Logger Sdk => new Logger(Subsystem.Sdk);
       public static Logger Http => new Logger(Subsystem.Http);
+      internal static Logger FIX => new Logger(Subsystem.Fix);
 
       internal static void log_at(this Logger logger, Level level, string message) {
          switch (level) {

src/client/spark_core/Properties/AssemblyAttributes.cs

@@ -2,4 +2,5 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_historical")]
 

src/client/spark_core/TestRunner/TestRunner.cs

@@ -70,10 +70,13 @@ namespace bts.test {
          psi.RedirectStandardOutput = true;
          psi.RedirectStandardError = true;
          psi.UseShellExecute = false;
-
+ 
          try{
             var p = Process.Start(psi);
             p.WaitForExit((int)time_out.TotalMilliseconds);
+            var errorlevel = p.ExitCode;
+            string stdout = p.StandardOutput.ReadToEnd();
+            string stderr = p.StandardError.ReadToEnd();
             if (p.HasExited) {
                return 0;
             }

src/client/spark_core/admin/AdminModelWrapper.cs

@@ -0,0 +1,64 @@
+using bts;
+using bts.api.admin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.api.core
+{
+   public class AdminModelWrapper
+   {
+      readonly AdminDataImmutable? _admin_data;
+      public AdminDataImmutable? admin_data => _admin_data;
+      readonly string _user_name;
+      readonly List<BTSAccountImmutable> _allowed_accounts;
+      readonly Dictionary<string, string> _allowed_account_dict = new Dictionary<string, string>();
+      public AdminModelWrapper(AdminDataImmutable? admin_data = null, string user_name = null)
+      {
+         _admin_data = admin_data;
+         _user_name = user_name ?? string.Empty;
+         _allowed_accounts = search_admin(u => u.Name == _user_name).Where(x =>
+         {
+              return (x.Item2 != null && x.Item2.HasPermissions) || (x.Item3 != null && x.Item3.HasPermissions);
+         }).Select(xx => xx.Item1).ToList();
+         foreach (var val in _allowed_accounts)
+            _allowed_account_dict[val.Name] = val.Name;
+      }
+      public IEnumerable<BTSAccountImmutable> viewable_accounts()
+      {
+         return _allowed_accounts;
+      }
+      public bool can_view_account(string account)
+      {
+         return _allowed_account_dict.ContainsKey(account) || _user_name.ToLower().Equals("bts");
+      }
+
+      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>> 
+         search_admin(Func<BTSUserImmutable, bool> predicate)
+      {
+         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>>();
+         if (_admin_data.HasValue)
+         {
+            var users = _admin_data.Value.Users.Where(predicate);
+            if (users != null && users.Count() == 1)
+            {
+               var user = users.ElementAt(0);
+               if (user.HasUserData && user.UserData.HasAccountPermissions)
+               {
+                  foreach (var kv in user.UserData.AccountPermissions.AccountPermissions)
+                  {
+                     var account_id = kv.Key;
+                     var accounts = _admin_data.Value.Accounts.Where(a => a.Id == account_id);
+                     if (accounts != null && accounts.Count() == 1)
+                     {
+                        var account = accounts.ElementAt(0);
+                        rv.Add(Tuple.Create(account, kv.Value.TradingPermission, kv.Value.AdminPermission));
+                     }
+                  }
+               }
+            }
+         }
+         return rv;
+      }
+   }
+}

src/client/spark_core/admin/PermissionFilters.cs

@@ -0,0 +1,64 @@
+using bts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.position;
+using bts.spark;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Concurrency;
+using System.Reactive.Linq;
+
+namespace bts.api.core
+{
+   public static class PermissionFilters
+   {
+      public static IObservable<Update<PositionUnitImmutable, int>> 
+         PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<PositionUnitImmutable, bool>>(observer =>
+         {
+            return am.Subscribe(x =>
+            {
+               Func<PositionUnitImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+            });
+         });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<OrderRecordImmutable, uint>> 
+         PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<OrderRecordImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.Def.Account ?? string.Empty));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<FillImmutable, string>> 
+         PermissionFilter(this IObservable<Update<FillImmutable, string>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<FillImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<FillImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+   }
+}

src/client/spark_core/spark_core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Platforms>x64</Platforms>
     <PlatformTarget>AnyCPU</PlatformTarget>

src/client/spark_core/utils/GlobalExtensions.cs

@@ -66,7 +66,6 @@ namespace mbts {
             }
          }
       }
-
       public static void sync_with_models<T, TModel>(this IList<T> target, IEnumerable<TModel> models, Func<TModel, T> add_factory, Func<T, TModel, T> update_factory) {
          var model_list = models as IList<TModel> ?? models.ToList();
          if (model_list.Count == 0) {

src/client/spark_excel/spark_excel.csproj

@@ -25,7 +25,7 @@
     <PackageReference Include="System.Reactive" Version="3.1.1" />
     <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
 	 <PackageReference Include="NUnit" Version="3.10.1" />  
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="System.Linq.Dynamic" Version="1.0.7" />
     
     <PackageReference Include="ExcelDna.AddIn" Version="0.34.6" />  

src/client/spark_fix/FixInit.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+namespace bts.fix
+{
+   public class FixInit
+   {
+      static FixInit()
+      {
+         AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
+      }
+
+      static void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)
+      {
+         var exception = e.ExceptionObject as Exception;
+         Log.FIX.error($"Unhandled Exception: {e.ExceptionObject} {exception?.StackTrace}");
+         if (e.IsTerminating)
+         {
+            Log.system_is_crashing("Unhandled exception", e.ExceptionObject);
+         }
+      }
+      
+      public static int start_acceptor(string settings_file, bool orders_allowed, string http_address = null)
+      {
+         SessionSettings settings = new SessionSettings(settings_file);
+         IApplication app = new FixRouter( (th, sess) =>
+         {
+            if(orders_allowed)
+               return new OrderSession(th, sess);
+            else
+               return new DropCopySession(th, sess);
+         });
+         IMessageStoreFactory storeFactory = new FileStoreFactory(settings);
+         ILogFactory logFactory = new FileLogFactory(settings);
+         IAcceptor acceptor = new ThreadedSocketAcceptor(app, storeFactory, settings, logFactory);
+
+         acceptor.Start();
+         HttpServer srv = null;
+         if (!string.IsNullOrEmpty(http_address))
+         {
+            if(!http_address.StartsWith("http"))
+               http_address = "http://127.0.0.1:5080/";
+            srv = new HttpServer(http_address, settings);
+            srv.Start();
+         }
+         Console.WriteLine("press <q> to quit");
+         while (Console.Read() != 'q') { }
+         acceptor.Stop();
+         if(srv != null)
+            srv.Stop();
+         return 0;
+
+      }
+   }
+}
+

src/client/spark_fix/FixRouter.cs

@@ -0,0 +1,231 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.md;
+using bts.api.admin;
+
+using security_key = System.UInt32;
+
+namespace bts.fix
+{
+   interface IFixRouter
+   {
+      IEnumerable<SecurityImmutable> get_securities();
+      SecurityImmutable? lookup(security_key key);
+      SecurityImmutable? lookup(string desc);
+
+      void send(Message mess, SessionID sess);
+      void send_session_reject(SessionID sess, string text, Message refmessage = null);
+      void send_biz_reject(SessionID sess, string text, Message refmessage, int reason_code = 0);
+   }
+
+   class FixRouter : QuickFix.MessageCracker, IFIXSession, IFixRouter
+   {
+      readonly ConcurrentDictionary<SessionID, QuickFix.IApplication> _sessions = new ConcurrentDictionary<SessionID, IApplication>();
+      readonly Func<IFixRouter, SessionID, IApplication> _factory;
+      readonly MarketDataClient _md = new MarketDataClient();
+      readonly object _lock = new object();
+
+      public FixRouter(Func<IFixRouter, SessionID, IApplication> factory)
+      {
+         _factory = factory;
+      }
+
+      IApplication session(SessionID sessionID)
+      {
+         IApplication app;
+         if (!_sessions.TryGetValue(sessionID, out app))
+         {
+            app = _factory(this, sessionID);
+            _sessions[sessionID] = app;
+         }
+         return app;
+      }
+
+      public void send(Message mess, SessionID sess)
+      {
+         lock (_lock)
+         {
+            try
+            {
+               Session.SendToTarget(mess, sess);
+            }
+            catch (SessionNotFound)
+            {
+               Log.assert(false, "session not found " + sess);
+            }
+            catch (Exception ex)
+            {
+               Log.FIX.error(sess.ToString() + " " + ex.Message + " " + sess);
+            }
+         }
+      }
+
+      public void send_session_reject(SessionID sess, string text, Message refmessage = null)
+      {
+         QuickFix.FIX44.Reject rej = null;
+         if (refmessage != null && refmessage.Header.IsSetField(Tags.MsgSeqNum))
+            rej = new QuickFix.FIX44.Reject(new QuickFix.Fields.RefSeqNum(refmessage.Header.GetInt(Tags.MsgSeqNum)));
+         else
+            rej = new QuickFix.FIX44.Reject();
+         rej.Text = new Text(text);
+         send(rej, sess);
+      }
+
+      public void send_biz_reject(SessionID sess, string text, Message refmessage, int reason_code = 0)
+      {
+         QuickFix.FIX44.BusinessMessageReject rej = new QuickFix.FIX44.BusinessMessageReject();
+
+         rej.RefMsgType = new QuickFix.Fields.RefMsgType(refmessage.Header.GetString(Tags.MsgType));
+         rej.RefSeqNum = new QuickFix.Fields.RefSeqNum(refmessage.Header.GetInt(Tags.MsgSeqNum));
+         rej.BusinessRejectReason = new QuickFix.Fields.BusinessRejectReason(reason_code);
+         rej.Text = new Text(text);
+         send(rej, sess);
+      }      
+
+      void handle_session_exception(Exception ex, SessionID sessionID, Message message = null)
+      {
+         string msg = "==ERROR in session " + sessionID.ToString() + "== ";
+         msg += message?.GetType().Name ?? string.Empty;
+         msg += ex?.ToString() ?? string.Empty;
+         if(!string.IsNullOrEmpty(ex.StackTrace))
+            msg += Environment.NewLine + ex.StackTrace;
+
+         bts.Log.FIX.error(msg);
+         Console.WriteLine(msg);
+         //now shut this fucker down
+         try
+         {
+            var sess = Session.LookupSession(sessionID);
+            if(sess != null)
+               sess.Disconnect(msg);
+         }
+         catch { }
+      }
+
+      void handle_field_not_found(FieldNotFoundException ex, SessionID sess, Message message)
+      {
+         if(ex.Field > 0)
+            send_biz_reject(sess, $"Field {ex.Field} was expected but not found", message, 5 );
+         //QF does not include the tag number....
+         //throw new FieldNotFoundException($"Field {ex.Field} was expected but not found", ex);
+         else
+            throw ex;
+      }
+
+      public void FromAdmin(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).FromAdmin(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void FromApp(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).FromApp(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void OnCreate(SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).OnCreate(sessionID);    
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }      
+      }
+
+      public void OnLogon(SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).OnLogon(sessionID);
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }
+      }
+
+      public void OnLogout(SessionID sessionID)
+      {
+         try
+         {
+            //TODO stackoverflow?
+            session(sessionID).OnLogout(sessionID);
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }
+      }
+
+      public void ToAdmin(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).ToAdmin(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) {  throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void ToApp(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).ToApp(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      IEnumerable<SecurityImmutable> IFixRouter.get_securities()
+      {
+         return _md.securities;
+      }
+
+      SecurityImmutable? IFixRouter.lookup(uint key)
+      {
+         return _md.lookup(key);
+      }
+
+      SecurityImmutable? IFixRouter.lookup(string desc)
+      {
+         if (_md.lookup_security_key_from_desc(desc, out uint key))
+            return _md.lookup(key);
+         else
+            return null;
+      }
+   }
+}

src/client/spark_fix/HttpServer.cs

@@ -0,0 +1,585 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Web;
+using QuickFix;
+
+namespace bts.fix
+{
+   public class HttpServer : IDisposable
+   {
+      private readonly HttpListener _httpListener;
+      private Thread _connectionThread;
+      private Boolean _running, _disposed;
+      private readonly SessionSettings _sessionSettings;
+      private StringBuilder _sbHtmlHeader;
+
+
+      public HttpServer(string prefix, SessionSettings settings)
+      {
+         if (!HttpListener.IsSupported)
+         {
+            // Requires at least a Windows XP with Service Pack 2
+            throw new NotSupportedException(
+                "The Http Server cannot run on this operating system.");
+         }
+
+         _httpListener = new HttpListener();
+         _httpListener.Prefixes.Add(prefix);
+         _sessionSettings = settings;
+      }
+
+      public void Start()
+      {
+         if (!_httpListener.IsListening)
+         {
+            _httpListener.Start();
+            _running = true;
+            // Use a thread to listen to the Http requests
+            _connectionThread = new Thread(ConnectionThreadStart);
+            _connectionThread.Start();
+         }
+      }
+
+      public void Stop()
+      {
+         if (_httpListener.IsListening)
+         {
+            _running = false;
+            _httpListener.Stop();
+         }
+      }
+
+      private void ConnectionThreadStart()
+      {
+         try
+         {
+            while (_running)
+            {
+               // Grab the context and pass it to the processor methods to handle it
+               HttpListenerContext context = _httpListener.GetContext();
+               HttpListenerRequest request = context.Request;
+               HttpListenerResponse response = context.Response;
+               string responseString;
+
+               _sbHtmlHeader = new StringBuilder(@"<HEAD><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>");
+               _sbHtmlHeader.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEADER><BODY>", request.Url.OriginalString);
+
+               switch (request.Url.AbsolutePath)
+               {
+                  case "/":
+                     responseString = ProcessRoot(request);
+                     break;
+                  case "/session":
+                     responseString = SessionDetails(request);
+                     break;
+                  case "/resetSession":
+                     responseString = ResetSession(request);
+                     break;
+                  case "/resetSessions":
+                     responseString = ResetSessions(request);
+                     break;
+                  case "/refreshSession":
+                     responseString = RefreshSession(request);
+                     break;
+                  case "/refreshSessions":
+                     responseString = RefreshSessions(request);
+                     break;
+                  case "/enableSessions":
+                     responseString = EnableSessions(request);
+                     break;
+                  case "/disableSessions":
+                     responseString = DisableSessions(request);
+                     break;
+                  default:
+                     responseString = ProcessRoot(request);
+                     break;
+               }
+
+               byte[] buffer = Encoding.UTF8.GetBytes(responseString);
+               response.ContentLength64 = buffer.Length;
+               Stream output = response.OutputStream;
+               output.Write(buffer, 0, buffer.Length);
+               output.Close();
+            }
+         }
+         catch (HttpListenerException)
+         {
+            Console.WriteLine("HTTP server was shut down.");
+         }
+      }
+
+      private string DisableSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Logout();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been disabled</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to disable all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, disable sessions</A>]&nbsp;[<A HREF='/'>NO, do not disable sessions</A>]</CENTER>", request.Url.OriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string EnableSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Logon();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been enabled</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to enable all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, enable sessions</A>]&nbsp;[<A HREF='/'>NO, do not enable sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string RefreshSession(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+
+         string url = "/session?" + GetParameterList(urlOriginalString);
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               sessionDetails.Refresh();
+               url = RemoveQueryStringByKey(urlOriginalString, "confirm");
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL=/session?{0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", GetParameterList(urlOriginalString));
+            sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEAD><BODY>", urlOriginalString);
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='/session?{0}'>{1}</A> has been refreshed</h2></CENTER>", GetParameterList(url), sessionId);
+         }
+         else
+         {
+            sbHtmlPageBody.AppendFormat("<CENTER><h2>Are you sure you want to refresh session <A HREF='{0}'>{1}</A>?</h2></CENTER>", url, sessionId);
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}&confirm=1'>YES, refresh session</A>]&nbsp;[<A HREF='{0}'>NO, do not refresh session</A>]</CENTER>", url);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string RefreshSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Refresh();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been refreshed</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to refresh all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, refresh sessions</A>]&nbsp;[<A HREF='/'>NO, do not refresh sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string ResetSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Reset("Reset from WebInterface");
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been reset</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to reset all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, reset sessions</A>]&nbsp;[<A HREF='/'>NO, do not reset sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string ResetSession(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+
+         string url = "/session?" + GetParameterList(urlOriginalString);
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               sessionDetails.Reset("Reset from WebInterface");
+               url = RemoveQueryStringByKey(urlOriginalString, "confirm");
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL=/session?{0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", GetParameterList(urlOriginalString));
+            sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEAD><BODY>", urlOriginalString);
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='/session?{0}'>{1}</A> has been reset</h2></CENTER>", GetParameterList(url), sessionId);
+         }
+         else
+         {
+            sbHtmlPageBody.AppendFormat("<CENTER><h2>Are you sure you want to reset session <A HREF='{0}'>{1}</A>?</h2></CENTER>", url, sessionId);
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}&confirm=1'>YES, reset session</A>]&nbsp;[<A HREF='{0}'>NO, do not reset session</A>]</CENTER>", url);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+
+      private string ProcessRoot(HttpListenerRequest request)
+      {
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         //Session count
+         sbHtmlPageBody.AppendFormat("<CENTER><b><i>{0} Sessions managed by QuickFIX</b></i></CENTER><HR/>", _sessionSettings.GetSessions().Count);
+
+         //session management links
+         sbHtmlPageBody.AppendFormat(@"<CENTER><A HREF='/resetSessions{0}'>RESET</A>&nbsp;<A HREF='/refreshSessions'>REFRESH</A>&nbsp;<A HREF='/enableSessions'>ENABLE</A>&nbsp;<A HREF='/disableSessions'>DISABLE</A></CENTER><HR/></HEADER><BODY>", GetParameterList(request.Url.OriginalString));
+
+         //Start the table generation
+         sbHtmlPageBody.Append("<table id=\"sessionlist\" style=\"border-width:1; padding:2; width:100%\">");
+
+         sbHtmlPageBody.Append("<tr>");
+         sbHtmlPageBody.Append(AddCell("Session", true));
+         sbHtmlPageBody.Append(AddCell("Type", true));
+         sbHtmlPageBody.Append(AddCell("Enabled", true));
+         sbHtmlPageBody.Append(AddCell("Session Time", true));
+         sbHtmlPageBody.Append(AddCell("Logged On", true));
+         sbHtmlPageBody.Append(AddCell("Next Incoming", true));
+         sbHtmlPageBody.Append(AddCell("Next Outgoing", true));
+         sbHtmlPageBody.Append("</tr>");
+
+         foreach (SessionID session in _sessionSettings.GetSessions())
+         {
+            Session sessionDetails = Session.LookupSession(session);
+            sbHtmlPageBody.Append("<tr>");
+            sbHtmlPageBody.Append(AddCell(String.Format(
+                "<a href=\"session?BeginString={0}&SenderCompID={1}&TargetCompID={2}\">{0}:{1}->{2}</a>",
+                session.BeginString, session.SenderCompID, session.TargetCompID)));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsInitiator ? "initiator" : "acceptor"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsEnabled ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsSessionTime ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsLoggedOn ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.NextTargetMsgSeqNum.ToString()));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.NextSenderMsgSeqNum.ToString()));
+            sbHtmlPageBody.Append("</tr>");
+         }
+
+         sbHtmlPageBody.Append("</table>");
+         return sbHtmlPageBody.ToString();
+      }
+
+      public string SessionDetails(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         string url = request.Url.OriginalString;
+         string urlOriginalString = request.Url.OriginalString;
+
+         if (request.QueryString["enabled"] != null)
+         {
+            if (!Convert.ToBoolean(request.QueryString["enabled"]))
+               sessionDetails.Logout();
+            else
+               sessionDetails.Logon();
+
+            url = RemoveQueryStringByKey(urlOriginalString, "Enabled");
+         }
+
+         if (request.QueryString["next incoming"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["next incoming"]);
+            sessionDetails.NextTargetMsgSeqNum = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "next incoming");
+         }
+
+         if (request.QueryString["Next Outgoing"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["Next Outgoing"]);
+            sessionDetails.NextSenderMsgSeqNum = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "Next Outgoing");
+         }
+
+         if (request.QueryString["SendRedundantResendRequests"] != null)
+         {
+            sessionDetails.SendRedundantResendRequests = Convert.ToBoolean(request.QueryString["SendRedundantResendRequests"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "SendRedundantResendRequests");
+         }
+
+         if (request.QueryString["CheckCompId"] != null)
+         {
+            sessionDetails.CheckCompID = Convert.ToBoolean(request.QueryString["CheckCompId"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "CheckCompId");
+         }
+
+         if (request.QueryString["CheckLatency"] != null)
+         {
+            sessionDetails.CheckLatency = Convert.ToBoolean(request.QueryString["CheckLatency"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "CheckLatency");
+         }
+
+         if (request.QueryString["MaxLatency"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["MaxLatency"]);
+            sessionDetails.MaxLatency = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "MaxLatency");
+         }
+
+         if (request.QueryString["LogonTimeout"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["LogonTimeout"]);
+            sessionDetails.LogonTimeout = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "LogonTimeout");
+         }
+
+         if (request.QueryString["LogoutTimeout"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["LogoutTimeout"]);
+            sessionDetails.LogoutTimeout = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "LogoutTimeout");
+         }
+
+         if (request.QueryString["ResetOnLogon"] != null)
+         {
+            sessionDetails.ResetOnLogon = Convert.ToBoolean(request.QueryString["ResetOnLogon"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnLogon");
+         }
+
+         if (request.QueryString["ResetOnLogout"] != null)
+         {
+            sessionDetails.ResetOnLogout = Convert.ToBoolean(request.QueryString["ResetOnLogout"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnLogout");
+         }
+
+         if (request.QueryString["ResetOnDisconnect"] != null)
+         {
+            sessionDetails.ResetOnDisconnect = Convert.ToBoolean(request.QueryString["ResetOnDisconnect"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnDisconnect");
+         }
+
+         if (request.QueryString["RefreshOnLogon"] != null)
+         {
+            sessionDetails.RefreshOnLogon = Convert.ToBoolean(request.QueryString["RefreshOnLogon"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "RefreshOnLogon");
+         }
+
+         if (request.QueryString["MillisecondsInTimestamp"] != null)
+         {
+            sessionDetails.MillisecondsInTimeStamp = Convert.ToBoolean(request.QueryString["MillisecondsInTimestamp"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "MillisecondsInTimestamp");
+         }
+
+         if (request.QueryString["PersistMessages"] != null)
+         {
+            sessionDetails.PersistMessages = Convert.ToBoolean(request.QueryString["PersistMessages"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "PersistMessages");
+         }
+
+
+         sbHtmlPageBody.AppendFormat(@"<CENTER>{0}</CENTER><HR/>", sessionId);
+         sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/resetSession?{0}'>RESET</A>]&nbsp;[<A HREF='/refreshSession?{0}'>REFRESH</A>]</CENTER><HR/></HEADER><BODY>", GetParameterList(urlOriginalString));
+
+         sbHtmlPageBody.Append("<table id=\"session_details\" style=\"border-width:1; padding:2; width:100%\">");
+
+         sbHtmlPageBody.Append(AddRow("Enabled", sessionDetails.IsEnabled, url));
+         sbHtmlPageBody.Append(AddRow("ConnectionType", sessionDetails.IsInitiator ? "initiator" : "acceptor"));
+         sbHtmlPageBody.Append(AddRow("SessionTime", sessionDetails.IsSessionTime));
+         sbHtmlPageBody.Append(AddRow("LoggedOn", sessionDetails.IsLoggedOn));
+         sbHtmlPageBody.Append(AddRow("Next Incoming", sessionDetails.NextTargetMsgSeqNum, url));
+         sbHtmlPageBody.Append(AddRow("Next Outgoing", sessionDetails.NextSenderMsgSeqNum, url));
+         sbHtmlPageBody.Append(AddRow("SendRedundantResendRequests", sessionDetails.SendRedundantResendRequests, url));
+         sbHtmlPageBody.Append(AddRow("CheckCompId", sessionDetails.CheckCompID, url));
+         sbHtmlPageBody.Append(AddRow("CheckLatency", sessionDetails.CheckLatency, url));
+         sbHtmlPageBody.Append(AddRow("MaxLatency", sessionDetails.MaxLatency, url));
+         sbHtmlPageBody.Append(AddRow("LogonTimeout", sessionDetails.LogonTimeout, url));
+         sbHtmlPageBody.Append(AddRow("LogoutTimeout", sessionDetails.LogoutTimeout, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnLogon", sessionDetails.ResetOnLogon, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnLogout", sessionDetails.ResetOnLogout, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnDisconnect", sessionDetails.ResetOnDisconnect, url));
+         sbHtmlPageBody.Append(AddRow("RefreshOnLogon", sessionDetails.RefreshOnLogon, url));
+         sbHtmlPageBody.Append(AddRow("MillisecondsInTimestamp", sessionDetails.MillisecondsInTimeStamp, url));
+         sbHtmlPageBody.Append(AddRow("PersistMessages", sessionDetails.PersistMessages, url));
+
+         sbHtmlPageBody.Append("</table>");
+         return sbHtmlPageBody.ToString();
+      }
+
+      public static string RemoveQueryStringByKey(string url, string key)
+      {
+         var uri = new Uri(url);
+
+         // this gets all the query string key value pairs as a collection
+         var newQueryString = HttpUtility.ParseQueryString(uri.Query);
+
+         // this removes the key if exists
+         newQueryString.Remove(key);
+
+         // this gets the page path from root without QueryString
+         string pagePathWithoutQueryString = uri.GetLeftPart(UriPartial.Path);
+
+         return newQueryString.Count > 0
+             ? $"{pagePathWithoutQueryString}?{newQueryString}"
+             : pagePathWithoutQueryString;
+      }
+
+      public static string GetParameterList(string url)
+      {
+         return HttpUtility.ParseQueryString((new Uri(url).Query)).ToString();
+      }
+
+      private static string AddRow(string colName, bool value, string url = "")
+      {
+         string valueAsStr = value ? "yes" : "no";
+         string innerHtml = url.Length > 0
+             ? $"<a href=\" {url}&{colName}={!value} \">toggle</a>"
+             : "";
+         return AddRow(colName, valueAsStr, innerHtml);
+      }
+
+      private static string AddRow(string colName, int value, string url = "")
+      {
+         string innerHtml = $"<a href=\" {url}&{colName}={value - 10} \"> << </a>" +
+                            $"<a href=\" {url}&{colName}={value - 1} \"> < </a>" +
+                            " | " +
+                            $"<a href=\" {url}&{colName}={value + 1} \"> > </a>" +
+                            $"<a href=\" {url}&{colName}={value + 10} \"> >> </a>";
+         return AddRow(colName, value.ToString(), innerHtml);
+      }
+      private static string AddRow(string colName, string value, string innerHtml = "")
+      {
+         StringBuilder row = new StringBuilder();
+         row.Append("<tr>");
+         row.Append(AddCell(colName));
+         row.Append(AddCell(value));
+         row.Append(AddCell(innerHtml));
+         row.Append("</tr>");
+         return row.ToString();
+      }
+
+      private static string AddCell(string cellContent, bool header = false)
+      {
+         string entryType = header ? "th" : "td";
+         return "<" + entryType + " align=\"left\">" + cellContent + "</" + entryType + ">";
+      }
+
+      public virtual void Dispose()
+      {
+         Dispose(true);
+         GC.SuppressFinalize(this);
+      }
+
+      private void Dispose(bool disposing)
+      {
+         if (_disposed)
+         {
+            return;
+         }
+         if (disposing)
+         {
+            if (_running)
+            {
+               Stop();
+            }
+            if (_connectionThread != null)
+            {
+               _connectionThread.Abort();
+               _connectionThread = null;
+            }
+         }
+         _disposed = true;
+      }
+   }
+}

src/client/spark_fix/Properties/AssemblyAttributes.cs

@@ -0,0 +1,2 @@
+using System.Runtime.CompilerServices;
+

src/client/spark_fix/README.md

@@ -0,0 +1,2 @@
+# BTS Spark FIX Adapter
+

src/client/spark_fix/session/DropCopySession.cs

@@ -0,0 +1,269 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.api.admin;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+namespace bts.fix
+{
+   class DropCopySession : SessionBase
+   {
+      protected IDisposable _fillsdisp;
+      protected IDisposable _ordersdisp;
+      protected static Lazy<OrderService> _orderservice = new Lazy<OrderService>(() => bts.api.Locator.locate<OrderService>(), true);
+
+      protected readonly ConcurrentDictionary<uint, OrderRecordImmutable> _order_cache = new ConcurrentDictionary<uint, OrderRecordImmutable>();
+      protected readonly ConcurrentDictionary<uint, string> _fix_clorderids = new ConcurrentDictionary<uint, string>();
+      public DropCopySession(IFixRouter router, SessionID sess) : base(router, sess)
+      {
+      }
+
+      override public void OnLogon(SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         var msg = sessionID.ToString() + " Logged on";
+         log_info(msg);
+
+         subscribe_fills_and_orders(sessionID);
+      }
+      override public void OnLogout(SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         var msg = sessionID.ToString() + " Logged out";
+         log_info(msg);
+
+         if (_ordersdisp != null)
+            _ordersdisp.Dispose();
+         if (_fillsdisp != null)
+            _fillsdisp.Dispose();
+      }      
+
+      override public void FromApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.ORDER_STATUS_REQUEST:
+            case MsgType.SECURITYDEFINITIONREQUEST:
+               log_info("IN: " + message);
+               Crack(message, sessionID);
+               break;
+            default:
+               log_info("UNSUPPORTED MESSAGE IN: " + type.getValue());
+               throw new UnsupportedMessageType();
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.OrderStatusRequest req, SessionID s)
+      {
+         uint oid = uint.Parse(req.OrderID.getValue());
+         if (_order_cache.TryGetValue(oid, out OrderRecordImmutable order))
+         {
+            var ostat = build_order_status(s, order);
+            ostat.OrdStatusReqID = new OrdStatusReqID(req.OrdStatusReqID.getValue());
+            _router.send(ostat, s);
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.SecurityDefinitionRequest req, SessionID s)
+      {
+         if (req.SecurityRequestType.getValue() == SecurityRequestType.REQUEST_LIST_SECURITIES)
+         {
+            var secs = _router.get_securities();
+            if (req.IsSetSymbol())
+               secs = secs.Where(x => x.Symbol.ToLower() == req.Symbol.getValue().ToLower());
+            if (req.IsSetSecurityType())
+               secs = secs.Where(x => security_types_equal(x.security_type(), req.SecurityType));
+            if (req.IsSetSecurityExchange())
+               secs = secs.Where(x => exchanges_equal(x.Exchange, req.SecurityExchange));
+
+            var def = new QuickFix.FIX44.SecurityDefinition(req.SecurityReqID,
+                                                            new SecurityResponseID(Guid.NewGuid().ToString()),
+                                                            new SecurityResponseType(SecurityResponseType.LIST_OF_SECURITIES_RETURNED_PER_REQUEST));
+            _router.send(def, s);
+         }
+         else
+            throw new IncorrectTagValue(req.SecurityRequestType.Tag);
+      }
+
+
+
+      virtual protected void subscribe_fills_and_orders(SessionID sessionID)
+      {
+         var scheduler = TaskPoolScheduler.Default;
+         var interval = TimeSpan.FromMilliseconds(500);
+         var on_orders = _orderservice.Value.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(_adminobservable).Table().Cache();
+         var on_fills = _orderservice.Value.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(_adminobservable).Table().Cache();
+
+         _ordersdisp = on_orders.Subscribe(o =>
+         {
+            foreach (var order in o.added)
+            {
+               var ostat = build_order_status(sessionID, order);
+               _order_cache[order.Key] = order;
+               _router.send(ostat, sessionID);
+            }
+            foreach (var order in o.changed)
+            {
+               var ostat = build_order_status(sessionID, order, true);
+               _router.send(ostat, sessionID);
+            }
+         });
+
+         _fillsdisp = on_fills.Subscribe(f =>
+         {
+            foreach (var fill in f.added)
+            {
+               var fi = build_fill(sessionID, fill);
+               _router.send(fi, sessionID);
+            }
+            foreach (var fill in f.changed)
+            {
+               var fi = build_fill(sessionID, fill, true);
+               _router.send(fi, sessionID);
+            }
+         });
+      }
+
+
+      virtual protected void process_order_state(in OrderStateImmutable state, ref uint filled, ref uint leaves, ref double avgprx,
+                                                 ref QuickFix.Fields.OrdStatus fstatus, ref NativeOrderStateImmutable? native, FillImmutable? fill = null)
+      {
+         filled = state.FilledQuantity;
+         leaves = state.Quantity - state.FilledQuantity;
+         avgprx = state.AvgPrice;
+         fstatus = translate_status(state, ref native);         
+      }
+
+      protected bts.api.SecurityImmutable? resolve_execution_report_security(bts.api.SecurityImmutable? sec, QuickFix.FIX44.ExecutionReport report)
+      {
+         report.SecurityExchange = convert_exchange(sec.Value.Exchange);
+         var sectype = sec.Value.security_type();
+         report.SecurityType = convert_security_type(sectype);
+         if (sec.Value.HasGlbx)
+            report.SecurityDesc = new SecurityDesc(sec.Value.Glbx.SecurityDesc);
+
+         switch (sectype)
+         {
+            case bts.api.SecurityType.Future:
+               report.MaturityMonthYear = new MaturityMonthYear(sec.Value.EarliestExpiration().ToString("yyyyMMdd"));
+               break;
+            case bts.api.SecurityType.Spread: //TODO this is not quite right for spreads of spreads            
+               var leg = sec.Value.Spread.Legs.OrderBy(x => x.Security.EarliestExpiration()).First();
+               report.MaturityMonthYear = new MaturityMonthYear(leg.Security.EarliestExpiration().ToString("yyyyMMdd"));
+               break;
+            default:
+               throw new UnsupportedMessageType();               
+         }
+         return sec;
+      }
+
+      virtual protected QuickFix.FIX44.ExecutionReport build_fill(SessionID sess, in FillImmutable fill, bool change = false)
+      {
+         var exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.PARTIAL_FILL);
+         QuickFix.Fields.OrdStatus status = new QuickFix.Fields.OrdStatus();
+         uint filled = 0; uint leaves = 0; double prx = 0.0;
+         OrderRecordImmutable order;  NativeOrderStateImmutable? native = null;
+         if (_order_cache.TryGetValue(fill.OrderKey, out order))
+         {
+            process_order_state(order.State, ref filled, ref leaves, ref prx, ref status, ref native);
+            if(Convert.ToDouble(order.State.Quantity) == fill.LastShares)
+               exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.FILL);
+         }
+         if (change)
+            exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.TRADE_CORRECT);
+
+         var sec = _router.lookup(fill.SecurityKey);
+         QuickFix.FIX44.ExecutionReport report = new QuickFix.FIX44.ExecutionReport(
+             new OrderID(fill.OrderKey.ToString()),
+             new ExecID(fill.FillKey.ToString()), 
+             exec,
+             status,
+             new Symbol(sec?.Symbol ?? string.Empty),
+             new QuickFix.Fields.Side(fill.Side == bts.api.Side.Buy ? QuickFix.Fields.Side.BUY : QuickFix.Fields.Side.SELL),
+             new LeavesQty(leaves),
+             new CumQty(filled),
+             new AvgPx(Convert.ToDecimal(prx)));
+             
+         report.Account = new Account(fill.Account);
+         report.LastQty = new LastQty(Convert.ToDecimal(fill.LastShares));
+         report.LastPx = new LastPx(Convert.ToDecimal(fill.LastPrice));
+         var ts = fill.HasTimestamp ? fill.Timestamp.ToDateTime() : DateTime.MinValue;
+         report.TransactTime = new TransactTime(ts, true);
+         report.TradeDate = new TradeDate(ts.ToString("yyyyMMdd"));
+         sec = resolve_execution_report_security(sec, report);
+         if (native.HasValue && !string.IsNullOrEmpty(native.Value.ExchangeOrderId))
+            report.SecondaryOrderID = new SecondaryOrderID(native.Value.ExchangeOrderId);
+
+         if (sec.HasValue && sec.Value.security_type() == bts.api.SecurityType.Spread || 
+             sec.Value.security_type() == bts.api.SecurityType.Synthetic)
+         {
+            if (fill.FillKind == bts.api.orderman.Fill.Types.FillKind.Leg)
+               report.MultiLegReportingType = new MultiLegReportingType('2');
+            else
+               report.MultiLegReportingType = new MultiLegReportingType('3');
+         }
+         return report;
+      }
+
+      virtual protected QuickFix.FIX44.ExecutionReport build_order_status(SessionID sess, in OrderRecordImmutable order, bool change = false)
+      {
+         var exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.NEW);
+         if (_order_cache.TryGetValue(order.Key, out OrderRecordImmutable _))
+            exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.ORDER_STATUS);
+            
+         QuickFix.Fields.OrdStatus status = new QuickFix.Fields.OrdStatus();         
+         uint filled = 0; uint leaves = 0; double prx = 0.0;  NativeOrderStateImmutable? native = null;
+         process_order_state(order.State, ref filled, ref leaves, ref prx, ref status, ref native);
+
+         var sec = _router.lookup(order.Def.SecurityKey);
+         QuickFix.FIX44.ExecutionReport report = new QuickFix.FIX44.ExecutionReport(
+             new OrderID(order.Key.ToString()),
+             new ExecID("0"),
+             exec,
+             status,
+             new Symbol(sec?.Symbol ?? string.Empty),
+             new QuickFix.Fields.Side(order.Def.Side == bts.api.Side.Buy ? QuickFix.Fields.Side.BUY : QuickFix.Fields.Side.SELL),
+             new LeavesQty(leaves),
+             new CumQty(filled),
+             new AvgPx(Convert.ToDecimal(prx)));
+
+         report.Account = new Account(order.Def.Account);
+         report.OrderQty = new OrderQty(order.State.Quantity);
+         report.Price = new Price(Convert.ToDecimal(order.State.Price));
+         report.TimeInForce = new QuickFix.Fields.TimeInForce((char)order.Def.TimeInForce);
+         var ts = order.State.LastChange.ToDateTime();
+         report.TransactTime = new TransactTime(ts, true);
+         if(!string.IsNullOrEmpty(order.State.LastMsg))
+            report.Text = new Text(order.State.LastMsg);
+
+         sec = resolve_execution_report_security(sec, report);
+         if (native.HasValue)
+         {
+            report.OrdType = new OrdType((char)native.Value.NativeType);        
+            if(!string.IsNullOrEmpty(native.Value.ExchangeOrderId))
+               report.SecondaryOrderID = new SecondaryOrderID(native.Value.ExchangeOrderId);            
+         }
+         //TODO Big race here, we need a correlation id of some sort
+         if (_fix_clorderids.TryGetValue(order.Key, out string clorderid))
+         {
+            report.ClOrdID = new ClOrdID(clorderid);
+         }
+         return report;         
+      }      
+
+   }
+}

src/client/spark_fix/session/IFIXSession.cs

@@ -0,0 +1,8 @@
+using System;
+
+namespace bts.fix
+{
+   interface IFIXSession : QuickFix.IApplication
+   {
+   }
+}

src/client/spark_fix/session/OrderSession.cs

@@ -0,0 +1,167 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.orderman;
+
+namespace bts.fix
+{
+   /* Some test fix
+   order for 4 lot CLZ8 
+   8=FIX.4.4
9=156
35=D
49=SPARK
56=CLIENT1
34=1
50=30737
97=Y
52=20000809-20:20:50
11=90001008
1=ACC A
21=2
55=SIM:CLZ8
54=1
38=4
40=2
59=0
44=74
60=20000809-18:20:32
207=SIM
10=045
   
+   
+   order for 4 lot CLZ8-CLH9
+   8=FIX.4.4
9=156
35=D
49=SPARK
56=CLIENT1
34=1
50=30737
97=Y
52=20000809-20:20:50
11=90001008
1=ACC A
21=2
55=SIM:CLZ8-CLH9
54=1
38=4
40=2
59=0
44=74
60=20000809-18:20:32
207=SIM
10=045

+   
+    */
+
+   class OrderSession : DropCopySession
+   {
+      public OrderSession(IFixRouter router, SessionID sess) : base(router, sess)
+      {
+      }
+
+      override public void FromApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.ORDER_STATUS_REQUEST: //handled in DropCopySession
+            case MsgType.SECURITYDEFINITIONREQUEST: //handled in DropCopySession
+
+            case MsgType.NEWORDERSINGLE:
+            case MsgType.ORDERCANCELREPLACEREQUEST:
+            case MsgType.ORDERCANCELREQUEST:
+               log_info("IN: " + message);
+               Crack(message, sessionID);
+               break;
+            default:
+               log_info("UNSUPPORTED MESSAGE IN: " + type.getValue());
+               throw new UnsupportedMessageType();
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.OrderCancelRequest cancel, SessionID s)
+      {
+         var id = cancel.OrderID.getValue();
+         if(!uint.TryParse(id, out uint key))
+            throw new IncorrectTagValue(cancel.OrderID.Tag);            
+         var resp = _orderservice.Value.cancel_order(new Cancel { OrderKey = key });
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            send_cancel_reject(s, "Cancel Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, cancel);
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.OrderCancelReplaceRequest replace, SessionID s)
+      {
+         var id = replace.OrderID.getValue();
+         if (!uint.TryParse(id, out uint key))
+            throw new IncorrectTagValue(replace.OrderID.Tag);
+         double price = Convert.ToDouble(replace.Price.getValue());
+         uint sz = Convert.ToUInt32(Math.Abs(replace.OrderQty.getValue()));
+         var resp = _orderservice.Value.update_order(new Update
+         {
+            OrderKey = key,
+            Price = price,
+            Quantity = sz,
+         });
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            send_cancel_reject(s, "Cancel Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, replace);
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.NewOrderSingle create, SessionID s)
+      {
+         Venue ven = convert_exchange(create.SecurityExchange);
+         if (ven == bts.api.Venue.Unset)
+            throw new IncorrectTagValue(create.SecurityExchange.Tag);
+
+         var sec = _router.lookup(create.Symbol.getValue());
+         if (!sec.HasValue)
+            sec = _router.lookup(create.SecurityDesc.getValue());
+         if (!sec.HasValue)
+            throw new IncorrectTagValue(create.SecurityDesc.Tag);
+
+         OrderDefinition def = null;
+         try
+         {
+            string manual = "Y";
+            if(create.IsSetField(Tags.ManualOrderIndicator))
+               manual = create.GetField(Tags.ManualOrderIndicator);
+            def = new OrderDefinition()
+            {
+               Account = create.Account.getValue(),
+               AccountId = lookup_account(create.Account.getValue()),
+               SecurityKey = sec.Value.Id,
+               Side = (api.Side)(int)create.Side.getValue(),
+               TimeInForce = (api.orderman.TimeInForce)(int)create.TimeInForce.getValue(),
+               User = _user_name,
+               UserId = lookup_user(_user_name),
+               ManualOrderIndicator = manual == "Y" ?  bts.api.orderman.ManualOrderIndicator.Yes : bts.api.orderman.ManualOrderIndicator.No,
+            };
+         }
+         catch
+         {
+            throw new IncorrectTagValue(create.TimeInForce.Tag);
+         }
+
+         set_ordertype(def, create, out bool price_req);
+         Create cr = new Create();
+         cr.NewOrder = def;
+         try
+         {
+            if (price_req)
+               cr.Price = Convert.ToDouble(create.Price.getValue());
+         }
+         catch
+         {
+            throw new IncorrectTagValue(create.Price.Tag);
+         }
+         try
+         {
+            cr.Quantity = Convert.ToUInt32(Math.Abs(create.OrderQty.getValue()));
+         }
+         catch
+         {
+            throw new IncorrectTagValue(create.OrderQty.Tag);
+         }
+         var resp = _orderservice.Value.create_order(cr);
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            _router.send_biz_reject(s, "Order Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, create);
+         }
+      }
+
+      private void set_ordertype(OrderDefinition def, QuickFix.FIX44.NewOrderSingle req, out bool price_req)
+      {
+         price_req = false;
+         switch (req.OrdType.getValue())
+         {
+            case '1':
+               def.CustomDef = new CustomOrderDefinition { Market = new MarketOrderDefinition() };
+               break;
+            case '2':
+               def.CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition() };
+               price_req = true;
+               break;
+            case '3':
+               def.CustomDef = new CustomOrderDefinition { Stop = new StopOrderDefinition() {StopPrice = Convert.ToDouble(req.StopPx.getValue()) } };
+               break;
+            case '4':
+               def.CustomDef = new CustomOrderDefinition { StopLimit = new StopLimitOrderDefinition() { StopPrice = Convert.ToDouble(req.StopPx.getValue()) } };
+               price_req = true;
+               break;
+            default:
+               throw new IncorrectTagValue(req.OrdType.Tag);
+         }
+      }
+   }
+}

src/client/spark_fix/session/SessionBase.cs

@@ -0,0 +1,361 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using mbts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.api.admin;
+
+
+namespace bts.fix
+{
+   abstract class SessionBase : QuickFix.MessageCracker, IFIXSession
+   {      
+      protected static AuthService _authservice = bts.api.Locator.locate<AuthService>();      
+      protected static Lazy<AdminService> _adminservice = new Lazy<AdminService>(() => bts.api.Locator.locate<AdminService>(), true);
+      protected static Lazy<bts.metrics.ShellLatencyService> _latencyservice = 
+         new Lazy<bts.metrics.ShellLatencyService>(() => bts.api.Locator.locate<bts.metrics.ShellLatencyService>(), true);
+
+      protected string _user_name;
+      protected AuthToken _token;
+
+      protected IDisposable _admindisp;
+      protected IObservable<AdminModelWrapper> _adminobservable;
+      protected AdminDataImmutable? _admin_data;      
+
+      protected readonly IFixRouter _router;
+      protected readonly SessionID _sess;
+      
+      public SessionBase(IFixRouter router, SessionID sess)
+      {
+         _router = router;
+         _sess = sess;
+
+         _clockdisp = _latencyservice.Value.subscribe_clock().Subscribe(x =>
+         {
+            var t = x.all.FirstOrDefault();
+            if(t != null)
+               _clock = t.Timestamp;
+         });
+
+         _adminobservable = _adminservice.Value.subscribei_admin_data().Select(x =>
+         {
+            return new AdminModelWrapper(x.all.TryFirst(), _user_name);
+         });
+         _admindisp = _adminobservable.Subscribe(x => _admin_data = x.admin_data);         
+      }
+
+      abstract public void FromApp(Message message, SessionID sessionID);
+      abstract public void OnLogon(SessionID sessionID); //subscribe to your stuff 
+      abstract public void OnLogout(SessionID sessionID); //unsub
+      
+      virtual public void ToApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         log_info("OUT: " + message);
+      }
+
+      readonly IDisposable _clockdisp; ulong _clock = 0;
+      virtual public void FromAdmin(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.LOGON:
+               {
+                  try
+                  {
+                     QuickFix.FIX44.Logon msg = (QuickFix.FIX44.Logon)message;
+                     _user_name = msg.Username.getValue();
+                     string pass = string.Empty;
+                     if(msg.IsSetPassword())
+                        pass = msg.Password.getValue();
+                     var resp = _authservice.authenticate_login(new AuthenticateLoginRequest { 
+                                                                Login = msg.Username.getValue(), Password = pass});
+                     if (resp != null && resp.IsValid)
+                        _token = resp.Token;
+                     else
+                        throw new RejectLogon("Logon rejected " + resp?.Reason ?? string.Empty);
+                  }
+                  catch (RejectLogon)
+                  {
+                     throw;
+                  }
+                  catch (Exception ex)
+                  {
+                     throw new RejectLogon("Logon rejected " + ex.ToString());
+                  }
+               }
+               break;
+            case MsgType.HEARTBEAT:
+               {
+                  QuickFix.FIX44.Heartbeat hb = (QuickFix.FIX44.Heartbeat)message;
+                  handle_heartbeat(hb, sessionID);
+               }
+               break;               
+            default:
+               log_info("ADMININ: " + message);
+               break;
+         }
+      }
+
+      async protected void handle_heartbeat(QuickFix.FIX44.Heartbeat hb, SessionID sessionID)
+      {
+         var resp = await _latencyservice.Value.report_async(new bts.metrics.Report { HostName = "BTSFIX_" + sessionID.ToString(), 
+                                                                                      Timestamp = _clock, User = _user_name });
+         if (resp.success)
+            _router.send(hb, sessionID);         
+      }
+
+      virtual public void ToAdmin(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         log_info("ADMINOUT: " + message);
+      }
+
+      virtual public void OnCreate(SessionID sessionID)
+      {
+         log_info("CREATE");
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.BusinessMessageReject n, SessionID s)
+      {
+         //Log.assert(false, "biz reject wtf " + n.EncodedText.getValue());
+         Log.FIX.warn(_sess.ToString() + " Business Reject " + n.BusinessRejectReason.getValue() + " " + n.Text.getValue());
+      }
+      virtual public void OnMessage(QuickFix.FIX44.News n, SessionID s)
+      {
+         log_info("news " + n.EncodedHeadline.getValue());
+      }      
+
+
+      protected void log_info(string message, [CallerFilePath] string file = "", [CallerLineNumber] int line = 0)
+      {
+         Log.FIX.info(_sess.ToString() + " " + message, file, line);
+      }
+
+      protected void log_error(string message, [CallerFilePath] string file = "", [CallerLineNumber] int line = 0)
+      {
+         Log.FIX.error(_sess.ToString() + " " + message, file, line);
+      }
+
+      protected void send_cancel_reject(SessionID sess, string text, Message refmessage)
+      {
+         QuickFix.FIX44.OrderCancelReject rej = new QuickFix.FIX44.OrderCancelReject();
+         if (refmessage is QuickFix.FIX44.OrderCancelRequest cancel)
+         {
+            if (cancel.IsSetClOrdID())
+               rej.SetField(cancel.ClOrdID);
+            if (cancel.IsSetOrigClOrdID())
+               rej.SetField(cancel.OrigClOrdID);
+            if (cancel.IsSetOrderID())
+               rej.SetField(cancel.OrderID);
+            rej.CxlRejResponseTo = new CxlRejResponseTo('1');
+            rej.Text = new Text(text);
+            _router.send(rej, sess);
+         }
+         else if (refmessage is QuickFix.FIX44.OrderCancelReplaceRequest replace)
+         {
+            if (replace.IsSetClOrdID())
+               rej.SetField(replace.ClOrdID);
+            if (replace.IsSetOrigClOrdID())
+               rej.SetField(replace.OrigClOrdID);
+            if (replace.IsSetOrderID())
+               rej.SetField(replace.OrderID);
+            rej.CxlRejResponseTo = new CxlRejResponseTo('2');
+            rej.Text = new Text(text);
+            _router.send(rej, sess);
+         }
+         else 
+            throw new UnsupportedMessageType();
+
+      }      
+
+      protected bool security_types_equal(bts.api.SecurityType btstype, QuickFix.Fields.SecurityType fixtype)
+      {
+         return btstype == convert_security_type(fixtype);
+      }
+
+      protected QuickFix.Fields.SecurityType convert_security_type(bts.api.SecurityType btstype)
+      {
+         switch (btstype)
+         {
+            case bts.api.SecurityType.Future:
+               return new QuickFix.Fields.SecurityType("FUT");
+            case bts.api.SecurityType.Option:
+               return new QuickFix.Fields.SecurityType("OPT");
+            case bts.api.SecurityType.Spread:
+               return new QuickFix.Fields.SecurityType("MLEG");
+            case bts.api.SecurityType.Stock:
+               return new QuickFix.Fields.SecurityType("CS");
+            case bts.api.SecurityType.Synthetic:
+               return new QuickFix.Fields.SecurityType("SYNTH");
+            default:
+               return new QuickFix.Fields.SecurityType();
+         }
+      }
+
+      protected bts.api.SecurityType convert_security_type(QuickFix.Fields.SecurityType fixtype)
+      {
+         switch (fixtype.getValue())
+         {
+            case "FUT":
+               return bts.api.SecurityType.Future;
+            case "OPT":
+               return bts.api.SecurityType.Option;
+            case "MLEG":
+               return bts.api.SecurityType.Spread;
+            case "CS":
+               return bts.api.SecurityType.Stock;
+            case "SYNTH":
+               return bts.api.SecurityType.Synthetic;
+            default:
+               return bts.api.SecurityType.Unset;
+         }
+      }
+
+      protected bool exchanges_equal(bts.api.Venue btstype, QuickFix.Fields.SecurityExchange fixtype)
+      {
+         return btstype == convert_exchange(fixtype);
+      }
+
+      protected QuickFix.Fields.SecurityExchange convert_exchange(bts.api.Venue btstype)
+      {
+         switch (btstype)
+         {
+            case bts.api.Venue.Bts:
+               return new QuickFix.Fields.SecurityExchange("BTS");
+            case bts.api.Venue.Cboe:
+               return new QuickFix.Fields.SecurityExchange("XCBO");
+            case bts.api.Venue.Cboecfe:
+               return new QuickFix.Fields.SecurityExchange("XCBF");
+            case bts.api.Venue.Glbx:
+               return new QuickFix.Fields.SecurityExchange("XCME");
+            case bts.api.Venue.Ice:
+               return new QuickFix.Fields.SecurityExchange("IEPA");
+            case bts.api.Venue.Sim:
+               return new QuickFix.Fields.SecurityExchange("SIM");
+            default:
+               return new QuickFix.Fields.SecurityExchange();
+         }
+      }
+
+      protected bts.api.Venue convert_exchange(QuickFix.Fields.SecurityExchange fixtype)
+      {
+         switch (fixtype.getValue())
+         {
+            case "BTS":
+               return bts.api.Venue.Bts;
+            case "XCBO":
+               return bts.api.Venue.Cboe;
+            case "XCBF":
+               return bts.api.Venue.Cboecfe;
+            case "GLBX":
+            case "XCME":
+            case "XCBT":
+            case "XCEC":
+            case "XNYM":
+               return bts.api.Venue.Glbx;
+            case "IEPA":
+            case "IFUS":
+            case "IFEU":
+            case "IFLL":
+               return bts.api.Venue.Ice;
+            case "SIM":
+               return bts.api.Venue.Sim;
+            default:
+               return bts.api.Venue.Unset;
+         }
+      }      
+
+      protected QuickFix.Fields.OrdStatus translate_status(in bts.api.orderman.OrderStateImmutable state, 
+                                                           ref bts.api.orderman.NativeOrderStateImmutable? native)
+      {
+         native = null;
+         if (state.HasMarket && state.Market.HasNativeState && state.Market.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Market.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasLimit && state.Limit.HasNativeState && state.Limit.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Limit.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasStop && state.Stop.HasNativeState && state.Stop.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Stop.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasStopLimit && state.StopLimit.HasNativeState && state.StopLimit.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.StopLimit.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else
+         {
+            switch (state.Status)
+            {
+               case OrderStatus.Canceled:
+                  return new OrdStatus(OrdStatus.CANCELED);
+               case OrderStatus.Completed:
+                  return new OrdStatus(OrdStatus.FILLED);
+               case OrderStatus.Disconnected:
+                  return new OrdStatus(OrdStatus.STOPPED);
+               case OrderStatus.Error:
+                  return new OrdStatus(OrdStatus.REJECTED);
+               case OrderStatus.Inactive:
+                  return new OrdStatus(OrdStatus.PENDING_NEW);
+               case OrderStatus.PendingWorking:
+                  return new OrdStatus(OrdStatus.PENDING_NEW);
+               case OrderStatus.RejectedBySpark:
+                  return new OrdStatus(OrdStatus.REJECTED);
+               case OrderStatus.Working:
+                  {
+                     if (state.FilledQuantity < state.Quantity)
+                        return new OrdStatus(OrdStatus.PARTIALLY_FILLED);
+                     else
+                        return new OrdStatus(OrdStatus.NEW);
+                  }
+               default:
+                  return new OrdStatus();
+            }
+         }
+      }
+
+      protected uint lookup_account(string acct)
+      {
+         if (_admin_data.HasValue)
+         {
+            var ac = _admin_data.Value.Accounts.Where(x => x.Name == acct).TryFirst();
+            if(ac.HasValue)
+               return ac.Value.Id;
+         }
+         throw new IncorrectTagValue(new Account().Tag);
+      }
+      
+      protected uint lookup_user(string user)
+      {
+         if (_admin_data.HasValue)
+         {
+            var ac = _admin_data.Value.Users.Where(x => x.Name == user).TryFirst();
+            if (ac.HasValue)
+               return ac.Value.Id;
+         }
+         throw new IncorrectTagValue(new Username().Tag);
+      }
+   }
+}

src/client/spark_fix/spark_fix.cfg

@@ -0,0 +1,42 @@
+#http://www.quickfixengine.org/quickfix/doc/html/configuration.html
+#https://www.onixs.biz/fix-dictionary.html
+
+[DEFAULT]
+ConnectionType=acceptor
+SocketAcceptPort=5001
+SocketReuseAddress=Y
+StartTime=00:00:00
+EndTime=00:00:00
+FileLogPath=log
+UseDataDictionary=Y
+ResetOnLogon=Y
+ResetOnLogout=Y
+ResetOnDisconnect=Y
+
+HeartBtInt=30
+LogonTimeout=30
+
+SSLEnable=N
+# SSLEnable=Y
+# # It is recommended to install the certificate and refer to it by name instead of using filename + password
+# SSLCertificate=QuickFixn-TestServer.pfx
+# SSLCertificatePassword=QuickFixn-TestServer
+# # For production refer to certificate by name instead: SSLCertificate=CN=QuickFixn-TestServer
+# SSLCACertificate=QuickFixn-TestCA.cer
+# SSLCheckCertificateRevocation=N
+
+
+[SESSION]
+BeginString=FIX.4.4
+SenderCompID=SPARK
+TargetCompID=CLIENT1
+FileStorePath=store
+DataDictionary=./spec/fix/FIX44.xml
+
+[SESSION]
+BeginString=FIX.4.4
+SenderCompID=SPARK
+TargetCompID=CLIENT2
+FileStorePath=store
+DataDictionary=./spec/fix/FIX44.xml
+

src/client/spark_fix/spark_fix.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>  
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Description />
+    <Platforms>x64</Platforms>
+    <PlatformTarget>x64</PlatformTarget>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <LangVersion>7.2</LangVersion>
+    <Version>1.0.0</Version>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
+    <PackageReference Include="System.Reactive" Version="3.1.1" />
+    <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
+	 <PackageReference Include="NUnit" Version="3.10.1" />  
+    
+    <PackageReference Include="QuickFix.Net.NETCore" Version="1.8.1" />     
+    <PackageReference Include="QuickFix.Net.NETCore.FIX44" Version="1.8.1" />         
+  </ItemGroup>
+
+  <ItemGroup> 
+    <ProjectReference Include="../spark_api/spark_api.csproj"/>
+    <ProjectReference Include="../spark_core/spark_core.csproj"/>
+  </ItemGroup>
+  
+  <Target Name="copy_fix" AfterTargets="PostBuildEvent"  >
+    <Copy SourceFiles="./spark_fix.cfg" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX44.xml" DestinationFolder="$(OutputPath)/spec/fix/" SkipUnchangedFiles="true" />
+  </Target>  
+  
+  <Target Name="copy_fix471" AfterTargets="PostBuildEvent"  >
+    <Copy SourceFiles="./spark_fix.cfg" DestinationFolder="$(OutputPath)/../net471" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX44.xml" DestinationFolder="$(OutputPath)/../net471/spec/fix/" SkipUnchangedFiles="true" />
+  </Target>  
+</Project>

src/client/spark_fix/spec/fix/FIX40.xml

@@ -0,0 +1,828 @@
+<fix major="4" minor="0">
+  <header>
+    <field name="BeginString" required="Y"/>
+    <field name="BodyLength" required="Y"/>
+    <field name="MsgType" required="Y"/>
+    <field name="SenderCompID" required="Y"/>
+    <field name="TargetCompID" required="Y"/>
+    <field name="OnBehalfOfCompID" required="N"/>
+    <field name="DeliverToCompID" required="N"/>
+    <field name="SecureDataLen" required="N"/>
+    <field name="SecureData" required="N"/>
+    <field name="MsgSeqNum" required="Y"/>
+    <field name="SenderSubID" required="N"/>
+    <field name="TargetSubID" required="N"/>
+    <field name="OnBehalfOfSubID" required="N"/>
+    <field name="DeliverToSubID" required="N"/>
+    <field name="PossDupFlag" required="N"/>
+    <field name="PossResend" required="N"/>
+    <field name="SendingTime" required="Y"/>
+    <field name="OrigSendingTime" required="N"/>
+  </header>
+  <trailer>
+    <field name="SignatureLength" required="N"/>
+    <field name="Signature" required="N"/>
+    <field name="CheckSum" required="Y"/>
+  </trailer>
+  <messages>
+    <message name="Heartbeat" msgtype="0" msgcat="admin">
+      <field name="TestReqID" required="N"/>
+    </message>
+    <message name="Logon" msgtype="A" msgcat="admin">
+      <field name="EncryptMethod" required="Y"/>
+      <field name="HeartBtInt" required="Y"/>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="TestRequest" msgtype="1" msgcat="admin">
+      <field name="TestReqID" required="Y"/>
+    </message>
+    <message name="ResendRequest" msgtype="2" msgcat="admin">
+      <field name="BeginSeqNo" required="Y"/>
+      <field name="EndSeqNo" required="Y"/>
+    </message>
+    <message name="Reject" msgtype="3" msgcat="admin">
+      <field name="RefSeqNum" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="SequenceReset" msgtype="4" msgcat="admin">
+      <field name="GapFillFlag" required="N"/>
+      <field name="NewSeqNo" required="Y"/>
+    </message>
+    <message name="Logout" msgtype="5" msgcat="admin">
+      <field name="Text" required="N"/>
+    </message>
+    <message name="Advertisement" msgtype="7" msgcat="app">
+      <field name="AdvId" required="Y"/>
+      <field name="AdvTransType" required="Y"/>
+      <field name="AdvRefID" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="AdvSide" required="Y"/>
+      <field name="Shares" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TransactTime" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="IndicationofInterest" msgtype="6" msgcat="app">
+      <field name="IOIid" required="Y"/>
+      <field name="IOITransType" required="Y"/>
+      <field name="IOIRefID" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="IOIShares" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="ValidUntilTime" required="N"/>
+      <field name="IOIQltyInd" required="N"/>
+      <field name="IOIOthSvc" required="N"/>
+      <field name="IOINaturalFlag" required="N"/>
+      <field name="IOIQualifier" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="News" msgtype="B" msgcat="app">
+      <field name="OrigTime" required="N"/>
+      <field name="Urgency" required="N"/>
+      <field name="RelatdSym" required="N" repeating="Y"/>
+      <group name="LinesOfText" required="Y">
+        <field name="Text" required="Y"/>
+      </group>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="Email" msgtype="C" msgcat="app">
+      <field name="EmailType" required="Y"/>
+      <field name="OrigTime" required="N"/>
+      <field name="RelatdSym" required="N" repeating="Y"/>
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="N"/>
+      <group name="LinesOfText" required="Y">
+        <field name="Text" required="Y"/>
+      </group>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="QuoteRequest" msgtype="R" msgcat="app">
+      <field name="QuoteReqID" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="N"/>
+      <field name="OrderQty" required="N"/>
+    </message>
+    <message name="Quote" msgtype="S" msgcat="app">
+      <field name="QuoteReqID" required="N"/>
+      <field name="QuoteID" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="BidPx" required="Y"/>
+      <field name="OfferPx" required="N"/>
+      <field name="BidSize" required="N"/>
+      <field name="OfferSize" required="N"/>
+      <field name="ValidUntilTime" required="N"/>
+    </message>
+    <message name="NewOrderSingle" msgtype="D" msgcat="app">
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="ProcessCode" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="LocateReqd" required="N"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="IOIid" required="N"/>
+      <field name="QuoteID" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ExecutionReport" msgtype="8" msgcat="app">
+      <field name="OrderID" required="Y"/>
+      <field name="ClOrdID" required="N"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="ListID" required="N"/>
+      <field name="ExecID" required="Y"/>
+      <field name="ExecTransType" required="Y"/>
+      <field name="ExecRefID" required="N"/>
+      <field name="OrdStatus" required="Y"/>
+      <field name="OrdRejReason" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="N"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="ExecInst" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="LastShares" required="Y"/>
+      <field name="LastPx" required="Y"/>
+      <field name="LastMkt" required="N"/>
+      <field name="LastCapacity" required="N"/>
+      <field name="CumQty" required="Y"/>
+      <field name="AvgPx" required="Y"/>
+      <field name="TradeDate" required="N"/>
+      <field name="TransactTime" required="N"/>
+      <field name="ReportToExch" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <group name="NoMiscFees" required="N">
+        <field name="MiscFeeAmt" required="N"/>
+        <field name="MiscFeeCurr" required="N"/>
+        <field name="MiscFeeType" required="N"/>
+      </group>
+      <field name="NetMoney" required="N"/>
+      <field name="SettlCurrAmt" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="DontKnowTrade" msgtype="Q" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ExecID" required="N"/>
+      <field name="DKReason" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="LastShares" required="Y"/>
+      <field name="LastPx" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelReplaceRequest" msgtype="G" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="OrigClOrdID" required="Y"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ListID" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelRequest" msgtype="F" msgcat="app">
+      <field name="OrigClOrdID" required="Y"/>
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ListID" required="N"/>
+      <field name="CxlType" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelReject" msgtype="9" msgcat="app">
+      <field name="OrderID" required="Y"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="ListID" required="N"/>
+      <field name="CxlRejReason" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderStatusRequest" msgtype="H" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+    </message>
+    <message name="Allocation" msgtype="J" msgcat="app">
+      <field name="AllocID" required="Y"/>
+      <field name="AllocTransType" required="Y"/>
+      <field name="RefAllocID" required="N"/>
+      <group name="NoOrders" required="Y">
+        <field name="ClOrdID" required="Y"/>
+        <field name="OrderID" required="N"/>
+        <field name="ListID" required="N"/>
+        <field name="WaveNo" required="N"/>
+      </group>
+      <group name="NoExecs" required="N">
+        <field name="ExecID" required="N"/>
+        <field name="LastShares" required="N"/>
+        <field name="LastPx" required="N"/>
+        <field name="LastMkt" required="N"/>
+      </group>
+      <field name="Side" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Shares" required="Y"/>
+      <field name="AvgPx" required="Y"/>
+      <field name="Currency" required="N"/>
+      <field name="AvgPrxPrecision" required="N"/>
+      <field name="TradeDate" required="Y"/>
+      <field name="TransactTime" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="NetMoney" required="N"/>
+      <group name="NoMiscFees" required="N">
+        <field name="MiscFeeAmt" required="N"/>
+        <field name="MiscFeeCurr" required="N"/>
+        <field name="MiscFeeType" required="N"/>
+      </group>
+      <field name="SettlCurrAmt" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="OpenClose" required="N"/>
+      <field name="Text" required="N"/>
+      <group name="NoAllocs" required="Y">
+        <field name="AllocAccount" required="Y"/>
+        <field name="AllocShares" required="Y"/>
+        <field name="ProcessCode" required="N"/>
+        <field name="ExecBroker" required="N"/>
+        <field name="ClientID" required="N"/>
+        <field name="Commission" required="N"/>
+        <field name="CommType" required="N"/>
+        <group name="NoDlvyInst" required="N">
+          <field name="BrokerOfCredit" required="N"/>
+          <field name="DlvyInst" required="N"/>
+        </group>
+      </group>
+    </message>
+    <message name="AllocationACK" msgtype="P" msgcat="app">
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="AllocID" required="Y"/>
+      <field name="TradeDate" required="Y"/>
+      <field name="TransactTime" required="N"/>
+      <field name="AllocStatus" required="Y"/>
+      <field name="AllocRejCode" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="NewOrderList" msgtype="E" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="ListSeqNo" required="Y"/>
+      <field name="ListNoOrds" required="Y"/>
+      <field name="ListExecInst" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="ProcessCode" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="LocateReqd" required="N"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListStatus" msgtype="N" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="NoRpts" required="Y"/>
+      <field name="RptSeq" required="Y"/>
+      <group name="NoOrders" required="Y">
+        <field name="ClOrdID" required="Y"/>
+        <field name="CumQty" required="Y"/>
+        <field name="CxlQty" required="Y"/>
+        <field name="AvgPx" required="Y"/>
+      </group>
+    </message>
+    <message name="ListExecute" msgtype="L" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListCancelRequest" msgtype="K" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListStatusRequest" msgtype="M" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+  </messages>
+  <fields>
+    <field number="1" name="Account" type="STRING"/>
+    <field number="2" name="AdvId" type="INT"/>
+    <field number="3" name="AdvRefID" type="INT"/>
+    <field number="4" name="AdvSide" type="STRING">
+      <value enum="B" description="BUY"/>
+      <value enum="S" description="SELL"/>
+      <value enum="X" description="CROSS"/>
+      <value enum="T" description="TRADE"/>
+    </field>
+    <field number="5" name="AdvTransType" type="STRING">
+      <value enum="N" description="NEW"/>
+      <value enum="C" description="CANCEL"/>
+      <value enum="R" description="REPLACE"/>
+    </field>
+    <field number="6" name="AvgPx" type="FLOAT"/>
+    <field number="7" name="BeginSeqNo" type="INT"/>
+    <field number="8" name="BeginString" type="STRING"/>
+    <field number="9" name="BodyLength" type="INT"/>
+    <field number="10" name="CheckSum" type="STRING"/>
+    <field number="11" name="ClOrdID" type="STRING"/>
+    <field number="12" name="Commission" type="FLOAT"/>
+    <field number="13" name="CommType" type="STRING">
+      <value enum="1" description="PER_SHARE"/>
+      <value enum="2" description="PERCENTAGE"/>
+      <value enum="3" description="ABSOLUTE"/>
+    </field>
+    <field number="14" name="CumQty" type="INT"/>
+    <field number="15" name="Currency" type="STRING"/>
+    <field number="16" name="EndSeqNo" type="INT"/>
+    <field number="17" name="ExecID" type="INT"/>
+    <field number="18" name="ExecInst" type="MULTIPLEVALUESTRING">
+      <value enum="1" description="NOT_HELD"/>
+      <value enum="2" description="WORK"/>
+      <value enum="3" description="GO_ALONG"/>
+      <value enum="4" description="OVER_THE_DAY"/>
+      <value enum="5" description="HELD"/>
+      <value enum="6" description="PARTICIPATE_DONT_INITIATE"/>
+      <value enum="7" description="STRICT_SCALE"/>
+      <value enum="8" description="TRY_TO_SCALE"/>
+      <value enum="9" description="STAY_ON_BIDSIDE"/>
+      <value enum="0" description="STAY_ON_OFFERSIDE"/>
+      <value enum="A" description="NO_CROSS"/>
+      <value enum="B" description="OK_TO_CROSS"/>
+      <value enum="C" description="CALL_FIRST"/>
+      <value enum="D" description="PERCENT_OF_VOLUME"/>
+      <value enum="E" description="DO_NOT_INCREASE_DNI"/>
+      <value enum="F" description="DO_NOT_REDUCE_DNR"/>
+      <value enum="G" description="ALL_OR_NONE_AON"/>
+      <value enum="I" description="INSTITUTIONS_ONLY"/>
+      <value enum="L" description="LAST_PEG"/>
+      <value enum="M" description="MIDPRICE_PEG"/>
+      <value enum="N" description="NONNEGOTIABLE"/>
+      <value enum="O" description="OPENING_PEG"/>
+      <value enum="P" description="MARKET_PEG"/>
+      <value enum="R" description="PRIMARY_PEG"/>
+      <value enum="S" description="SUSPEND"/>
+    </field>
+    <field number="19" name="ExecRefID" type="INT"/>
+    <field number="20" name="ExecTransType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="CANCEL"/>
+      <value enum="2" description="CORRECT"/>
+      <value enum="3" description="STATUS"/>
+    </field>
+    <field number="21" name="HandlInst" type="STRING">
+      <value enum="1" description="AUTOMATED_EXECUTION_ORDER_PRIVATE_NO_BROKER_INTERVENTION"/>
+      <value enum="2" description="AUTOMATED_EXECUTION_ORDER_PUBLIC_BROKER_INTERVENTION_OK"/>
+      <value enum="3" description="MANUAL_ORDER_BEST_EXECUTION"/>
+    </field>
+    <field number="22" name="IDSource" type="STRING">
+      <value enum="1" description="CUSIP"/>
+      <value enum="2" description="SEDOL"/>
+      <value enum="3" description="QUIK"/>
+      <value enum="4" description="ISIN_NUMBER"/>
+      <value enum="5" description="RIC_CODE"/>
+    </field>
+    <field number="23" name="IOIid" type="INT"/>
+    <field number="24" name="IOIOthSvc" type="STRING"/>
+    <field number="25" name="IOIQltyInd" type="STRING">
+      <value enum="L" description="LOW"/>
+      <value enum="M" description="MEDIUM"/>
+      <value enum="H" description="HIGH"/>
+    </field>
+    <field number="26" name="IOIRefID" type="INT"/>
+    <field number="27" name="IOIShares" type="STRING"/>
+    <field number="28" name="IOITransType" type="STRING">
+      <value enum="N" description="NEW"/>
+      <value enum="C" description="CANCEL"/>
+      <value enum="R" description="REPLACE"/>
+    </field>
+    <field number="29" name="LastCapacity" type="STRING">
+      <value enum="1" description="AGENT"/>
+      <value enum="2" description="CROSS_AS_AGENT"/>
+      <value enum="3" description="CROSS_AS_PRINCIPAL"/>
+      <value enum="4" description="PRINCIPAL"/>
+    </field>
+    <field number="30" name="LastMkt" type="STRING"/>
+    <field number="31" name="LastPx" type="FLOAT"/>
+    <field number="32" name="LastShares" type="INT"/>
+    <field number="33" name="LinesOfText" type="INT"/>
+    <field number="34" name="MsgSeqNum" type="INT"/>
+    <field number="35" name="MsgType" type="STRING"/>
+    <field number="36" name="NewSeqNo" type="INT"/>
+    <field number="37" name="OrderID" type="STRING"/>
+    <field number="38" name="OrderQty" type="INT"/>
+    <field number="39" name="OrdStatus" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="PARTIALLY_FILLED"/>
+      <value enum="2" description="FILLED"/>
+      <value enum="3" description="DONE_FOR_DAY"/>
+      <value enum="4" description="CANCELED"/>
+      <value enum="5" description="REPLACED"/>
+      <value enum="6" description="PENDING_CANCELREPLACE"/>
+      <value enum="7" description="STOPPED"/>
+      <value enum="8" description="REJECTED"/>
+      <value enum="9" description="SUSPENDED"/>
+      <value enum="A" description="PENDING_NEW"/>
+      <value enum="B" description="CALCULATED"/>
+      <value enum="C" description="EXPIRED"/>
+    </field>
+    <field number="40" name="OrdType" type="STRING">
+      <value enum="1" description="MARKET"/>
+      <value enum="2" description="LIMIT"/>
+      <value enum="3" description="STOP"/>
+      <value enum="4" description="STOP_LIMIT"/>
+      <value enum="5" description="MARKET_ON_CLOSE"/>
+      <value enum="6" description="WITH_OR_WITHOUT"/>
+      <value enum="7" description="LIMIT_OR_BETTER"/>
+      <value enum="8" description="LIMIT_WITH_OR_WITHOUT"/>
+      <value enum="9" description="ON_BASIS"/>
+      <value enum="A" description="ON_CLOSE"/>
+      <value enum="B" description="LIMIT_ON_CLOSE"/>
+      <value enum="C" description="FOREX"/>
+      <value enum="D" description="PREVIOUSLY_QUOTED"/>
+      <value enum="E" description="PREVIOUSLY_INDICATED"/>
+      <value enum="P" description="PEGGED"/>
+    </field>
+    <field number="41" name="OrigClOrdID" type="STRING"/>
+    <field number="42" name="OrigTime" type="TIME"/>
+    <field number="43" name="PossDupFlag" type="STRING">
+      <value enum="Y" description="POSSIBLE_DUPLICATE"/>
+      <value enum="N" description="ORIGINAL_TRANSMISSION"/>
+    </field>
+    <field number="44" name="Price" type="FLOAT"/>
+    <field number="45" name="RefSeqNum" type="INT"/>
+    <field number="46" name="RelatdSym" type="STRING"/>
+    <field number="47" name="Rule80A" type="STRING">
+      <value enum="A" description="AGENCY_SINGLE_ORDER"/>
+      <value enum="D" description="PROGRAM_ORDER_INDEX_ARB_FOR_MEMBER_FIRMORG"/>
+      <value enum="C" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_MEMBER_FIRMORG"/>
+      <value enum="J" description="PROGRAM_ORDER_INDEX_ARB_FOR_INDIVIDUAL_CUSTOMER"/>
+      <value enum="K" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_INDIVIDUAL_CUSTOMER"/>
+      <value enum="U" description="PROGRAM_ORDER_INDEX_ARB_FOR_OTHER_AGENCY"/>
+      <value enum="Y" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_OTHER_AGENCY"/>
+      <value enum="M" description="PROGRAM_ORDER_INDEX_ARB_FOR_OTHER_MEMBER"/>
+      <value enum="N" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_OTHER_MEMBER"/>
+      <value enum="W" description="ALL_OTHER_ORDERS_AS_AGENT_FOR_OTHER_MEMBER"/>
+    </field>
+    <field number="48" name="SecurityID" type="STRING"/>
+    <field number="49" name="SenderCompID" type="STRING"/>
+    <field number="50" name="SenderSubID" type="STRING"/>
+    <field number="52" name="SendingTime" type="TIME"/>
+    <field number="53" name="Shares" type="INT"/>
+    <field number="54" name="Side" type="STRING">
+      <value enum="1" description="BUY"/>
+      <value enum="2" description="SELL"/>
+      <value enum="3" description="BUY_MINUS"/>
+      <value enum="4" description="SELL_PLUS"/>
+      <value enum="5" description="SELL_SHORT"/>
+      <value enum="6" description="SELL_SHORT_EXEMPT"/>
+    </field>
+    <field number="55" name="Symbol" type="STRING"/>
+    <field number="56" name="TargetCompID" type="STRING"/>
+    <field number="57" name="TargetSubID" type="STRING"/>
+    <field number="58" name="Text" type="STRING"/>
+    <field number="59" name="TimeInForce" type="STRING">
+      <value enum="0" description="DAY"/>
+      <value enum="1" description="GOOD_TILL_CANCEL"/>
+      <value enum="2" description="AT_THE_OPENING"/>
+      <value enum="3" description="IMMEDIATE_OR_CANCEL"/>
+      <value enum="4" description="FILL_OR_KILL"/>
+      <value enum="5" description="GOOD_TILL_CROSSING"/>
+      <value enum="6" description="GOOD_TILL_DATE"/>
+    </field>
+    <field number="60" name="TransactTime" type="TIME"/>
+    <field number="61" name="Urgency" type="STRING">
+      <value enum="0" description="NORMAL"/>
+      <value enum="1" description="FLASH"/>
+      <value enum="2" description="BACKGROUND"/>
+    </field>
+    <field number="62" name="ValidUntilTime" type="TIME"/>
+    <field number="63" name="SettlmntTyp" type="STRING">
+      <value enum="0" description="REGULAR"/>
+      <value enum="1" description="CASH"/>
+      <value enum="2" description="NEXT_DAY"/>
+      <value enum="3" description="TPLUS2"/>
+      <value enum="4" description="TPLUS3"/>
+      <value enum="5" description="TPLUS4"/>
+      <value enum="6" description="FUTURE"/>
+      <value enum="7" description="WHEN_ISSUED"/>
+      <value enum="8" description="SELLERS_OPTION"/>
+      <value enum="9" description="TPLUS5"/>
+    </field>
+    <field number="64" name="FutSettDate" type="DATE"/>
+    <field number="65" name="SymbolSfx" type="STRING"/>
+    <field number="66" name="ListID" type="STRING"/>
+    <field number="67" name="ListSeqNo" type="INT"/>
+    <field number="68" name="ListNoOrds" type="INT"/>
+    <field number="69" name="ListExecInst" type="STRING"/>
+    <field number="70" name="AllocID" type="INT"/>
+    <field number="71" name="AllocTransType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="REPLACE"/>
+      <value enum="2" description="CANCEL"/>
+    </field>
+    <field number="72" name="RefAllocID" type="INT"/>
+    <field number="73" name="NoOrders" type="INT"/>
+    <field number="74" name="AvgPrxPrecision" type="INT"/>
+    <field number="75" name="TradeDate" type="DATE"/>
+    <field number="76" name="ExecBroker" type="STRING"/>
+    <field number="77" name="OpenClose" type="STRING"/>
+    <field number="78" name="NoAllocs" type="INT"/>
+    <field number="79" name="AllocAccount" type="STRING"/>
+    <field number="80" name="AllocShares" type="INT"/>
+    <field number="81" name="ProcessCode" type="STRING">
+      <value enum="0" description="REGULAR"/>
+      <value enum="1" description="SOFT_DOLLAR"/>
+      <value enum="2" description="STEPIN"/>
+      <value enum="3" description="STEPOUT"/>
+      <value enum="4" description="SOFTDOLLAR_STEPIN"/>
+      <value enum="5" description="SOFTDOLLAR_STEPOUT"/>
+      <value enum="6" description="PLAN_SPONSOR"/>
+    </field>
+    <field number="82" name="NoRpts" type="INT"/>
+    <field number="83" name="RptSeq" type="INT"/>
+    <field number="84" name="CxlQty" type="INT"/>
+    <field number="85" name="NoDlvyInst" type="INT"/>
+    <field number="86" name="DlvyInst" type="STRING"/>
+    <field number="87" name="AllocStatus" type="INT">
+      <value enum="0" description="ACCEPTED"/>
+      <value enum="1" description="REJECTED"/>
+      <value enum="2" description="PARTIAL_ACCEPT"/>
+      <value enum="3" description="RECEIVED"/>
+    </field>
+    <field number="88" name="AllocRejCode" type="INT">
+      <value enum="0" description="UNKNOWN_ACCOUNT"/>
+      <value enum="1" description="INCORRECT_QUANTITY"/>
+      <value enum="2" description="INCORRECT_AVERAGE_PRICE"/>
+      <value enum="3" description="UNKNOWN_EXECUTING_BROKER_MNEMONIC"/>
+      <value enum="4" description="COMMISSION_DIFFERENCE"/>
+      <value enum="5" description="UNKNOWN_ORDERID"/>
+      <value enum="6" description="UNKNOWN_LISTID"/>
+      <value enum="7" description="OTHER"/>
+    </field>
+    <field number="89" name="Signature" type="DATA"/>
+    <field number="90" name="SecureDataLen" type="INT"/>
+    <field number="91" name="SecureData" type="DATA"/>
+    <field number="92" name="BrokerOfCredit" type="STRING"/>
+    <field number="93" name="SignatureLength" type="INT"/>
+    <field number="94" name="EmailType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="REPLY"/>
+      <value enum="2" description="ADMIN_REPLY"/>
+    </field>
+    <field number="95" name="RawDataLength" type="INT"/>
+    <field number="96" name="RawData" type="DATA"/>
+    <field number="97" name="PossResend" type="STRING"/>
+    <field number="98" name="EncryptMethod" type="INT">
+      <value enum="0" description="NONE_OTHER"/>
+      <value enum="1" description="PKCS"/>
+      <value enum="2" description="DES"/>
+      <value enum="3" description="PKCSDES"/>
+      <value enum="4" description="PGPDES"/>
+      <value enum="5" description="PGPDESMD5"/>
+      <value enum="6" description="PEMDESMD5"/>
+    </field>
+    <field number="99" name="StopPx" type="FLOAT"/>
+    <field number="100" name="ExDestination" type="STRING">
+      <value enum="0" description="NONE"/>
+      <value enum="4" description="POSIT"/>
+    </field>
+    <field number="102" name="CxlRejReason" type="INT">
+      <value enum="0" description="TOO_LATE_TO_CANCEL"/>
+      <value enum="1" description="UNKNOWN_ORDER"/>
+    </field>
+    <field number="103" name="OrdRejReason" type="INT">
+      <value enum="0" description="BROKER_OPTION"/>
+      <value enum="1" description="UNKNOWN_SYMBOL"/>
+      <value enum="2" description="EXCHANGE_CLOSED"/>
+      <value enum="3" description="ORDER_EXCEEDS_LIMIT"/>
+      <value enum="4" description="TOO_LATE_TO_ENTER"/>
+    </field>
+    <field number="104" name="IOIQualifier" type="STRING">
+      <value enum="X" description="CROSSING_OPPORTUNITY"/>
+      <value enum="O" description="AT_THE_OPEN"/>
+      <value enum="M" description="MORE_BEHIND"/>
+      <value enum="P" description="TAKING_A_POSITION"/>
+      <value enum="V" description="VERSUS"/>
+      <value enum="Q" description="CURRENT_QUOTE"/>
+      <value enum="C" description="AT_THE_CLOSE"/>
+      <value enum="S" description="PORTFOLIO_SHOWN"/>
+      <value enum="I" description="IN_TOUCH_WITH"/>
+      <value enum="W" description="INDICATION_WORKING_AWAY"/>
+      <value enum="A" description="ALL_OR_NONE"/>
+      <value enum="L" description="LIMIT"/>
+      <value enum="T" description="THROUGH_THE_DAY"/>
+    </field>
+    <field number="105" name="WaveNo" type="STRING"/>
+    <field number="106" name="Issuer" type="STRING"/>
+    <field number="107" name="SecurityDesc" type="STRING"/>
+    <field number="108" name="HeartBtInt" type="INT"/>
+    <field number="109" name="ClientID" type="STRING"/>
+    <field number="110" name="MinQty" type="INT"/>
+    <field number="111" name="MaxFloor" type="INT"/>
+    <field number="112" name="TestReqID" type="STRING"/>
+    <field number="113" name="ReportToExch" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="114" name="LocateReqd" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="115" name="OnBehalfOfCompID" type="STRING"/>
+    <field number="116" name="OnBehalfOfSubID" type="STRING"/>
+    <field number="117" name="QuoteID" type="STRING"/>
+    <field number="118" name="NetMoney" type="FLOAT"/>
+    <field number="119" name="SettlCurrAmt" type="FLOAT"/>
+    <field number="120" name="SettlCurrency" type="STRING"/>
+    <field number="121" name="ForexReq" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="122" name="OrigSendingTime" type="TIME"/>
+    <field number="123" name="GapFillFlag" type="STRING">
+      <value enum="Y" description="GAP_FILL_MESSAGE_MSGSEQNUM_FIELD_VALID"/>
+      <value enum="N" description="SEQUENCE_RESET_IGNORE_MSGSEQNUM"/>
+    </field>
+    <field number="124" name="NoExecs" type="INT"/>
+    <field number="125" name="CxlType" type="STRING">
+      <value enum="P" description="PARTIAL_CANCEL"/>
+      <value enum="F" description="FULL_REMAINING_QUANTITY"/>
+    </field>
+    <field number="126" name="ExpireTime" type="TIME"/>
+    <field number="127" name="DKReason" type="STRING">
+      <value enum="A" description="UNKNOWN_SYMBOL"/>
+      <value enum="B" description="WRONG_SIDE"/>
+      <value enum="C" description="QUANTITY_EXCEEDS_ORDER"/>
+      <value enum="D" description="NO_MATCHING_ORDER"/>
+      <value enum="E" description="PRICE_EXCEEDS_LIMIT"/>
+      <value enum="Z" description="OTHER" />
+    </field>
+    <field number="128" name="DeliverToCompID" type="STRING"/>
+    <field number="129" name="DeliverToSubID" type="STRING"/>
+    <field number="130" name="IOINaturalFlag" type="STRING">
+      <value enum="Y" description="NATURAL"/>
+      <value enum="N" description="NOT_NATURAL"/>
+    </field>
+    <field number="131" name="QuoteReqID" type="STRING"/>
+    <field number="132" name="BidPx" type="FLOAT"/>
+    <field number="133" name="OfferPx" type="FLOAT"/>
+    <field number="134" name="BidSize" type="INT"/>
+    <field number="135" name="OfferSize" type="INT"/>
+    <field number="136" name="NoMiscFees" type="INT"/>
+    <field number="137" name="MiscFeeAmt" type="FLOAT"/>
+    <field number="138" name="MiscFeeCurr" type="STRING"/>
+    <field number="139" name="MiscFeeType" type="STRING">
+      <value enum="1" description="REGULATORY"/>
+      <value enum="2" description="TAX"/>
+      <value enum="3" description="LOCAL_COMMISSION"/>
+      <value enum="4" description="EXCHANGE_FEES"/>
+      <value enum="5" description="STAMP"/>
+      <value enum="6" description="LEVY"/>
+      <value enum="7" description="OTHER"/>
+    </field>
+    <field number="140" name="PrevClosePx" type="FLOAT"/>
+  </fields>
+</fix>

src/client/spark_fix/spec/fix/FIXT11.xml

@@ -0,0 +1,313 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<fix type='FIXT' major='1' minor='1' servicepack='0'>
+  <header>
+    <field name='BeginString' required='Y'/>
+    <field name='BodyLength' required='Y'/>
+    <field name='MsgType' required='Y'/>
+    <field name='SenderCompID' required='Y'/>
+    <field name='TargetCompID' required='Y'/>
+    <field name='OnBehalfOfCompID' required='N'/>
+    <field name='DeliverToCompID' required='N'/>
+    <field name='SecureDataLen' required='N'/>
+    <field name='SecureData' required='N'/>
+    <field name='MsgSeqNum' required='Y'/>
+    <field name='SenderSubID' required='N'/>
+    <field name='SenderLocationID' required='N'/>
+    <field name='TargetSubID' required='N'/>
+    <field name='TargetLocationID' required='N'/>
+    <field name='OnBehalfOfSubID' required='N'/>
+    <field name='OnBehalfOfLocationID' required='N'/>
+    <field name='DeliverToSubID' required='N'/>
+    <field name='DeliverToLocationID' required='N'/>
+    <field name='PossDupFlag' required='N'/>
+    <field name='PossResend' required='N'/>
+    <field name='SendingTime' required='Y'/>
+    <field name='OrigSendingTime' required='N'/>
+    <field name='XmlDataLen' required='N'/>
+    <field name='XmlData' required='N'/>
+    <field name='MessageEncoding' required='N'/>
+    <field name='LastMsgSeqNumProcessed' required='N'/>
+    <component name='HopGrp' required='N'/>
+    <field name='ApplVerID' required='N'/>
+    <field name='CstmApplVerID' required='N'/>
+  </header>
+  <trailer>
+    <field name='SignatureLength' required='N'/>
+    <field name='Signature' required='N'/>
+    <field name='CheckSum' required='Y'/>
+  </trailer>
+  <messages>
+    <message msgcat='admin' msgtype='0' name='Heartbeat'>
+      <field name='TestReqID' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='1' name='TestRequest'>
+      <field name='TestReqID' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='2' name='ResendRequest'>
+      <field name='BeginSeqNo' required='Y'/>
+      <field name='EndSeqNo' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='3' name='Reject'>
+      <field name='RefSeqNum' required='Y'/>
+      <field name='RefTagID' required='N'/>
+      <field name='RefMsgType' required='N'/>
+      <field name='SessionRejectReason' required='N'/>
+      <field name='Text' required='N'/>
+      <field name='EncodedTextLen' required='N'/>
+      <field name='EncodedText' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='4' name='SequenceReset'>
+      <field name='GapFillFlag' required='N'/>
+      <field name='NewSeqNo' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='5' name='Logout'>
+      <field name='Text' required='N'/>
+      <field name='EncodedTextLen' required='N'/>
+      <field name='EncodedText' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='A' name='Logon'>
+      <field name='EncryptMethod' required='Y'/>
+      <field name='HeartBtInt' required='Y'/>
+      <field name='RawDataLength' required='N'/>
+      <field name='RawData' required='N'/>
+      <field name='ResetSeqNumFlag' required='N'/>
+      <field name='NextExpectedMsgSeqNum' required='N'/>
+      <field name='MaxMessageSize' required='N'/>
+      <field name='TestMessageIndicator' required='N'/>
+      <field name='Username' required='N'/>
+      <field name='Password' required='N'/>
+      <field name='DefaultApplVerID' required='Y'/>
+      <component name='MsgTypeGrp' required='N'/>
+    </message>
+  </messages>
+  <components>
+    <component name='HopGrp'>
+      <group name='NoHops' required='N'>
+        <field name='HopCompID' required='N'/>
+        <field name='HopSendingTime' required='N'/>
+        <field name='HopRefID' required='N'/>
+      </group>
+    </component>
+    <component name='MsgTypeGrp'>
+      <group name='NoMsgTypes' required='N'>
+        <field name='RefMsgType' required='N'/>
+        <field name='MsgDirection' required='N'/>
+        <field name='RefApplVerID' required='N'/>
+        <field name='RefCstmApplVerID' required='N'/>
+      </group>
+    </component>
+  </components>
+  <fields>
+    <field name='BeginSeqNo' number='7' type='SEQNUM'/>
+    <field name='BeginString' number='8' type='STRING'/>
+    <field name='BodyLength' number='9' type='LENGTH'/>
+    <field name='CheckSum' number='10' type='STRING'/>
+    <field name='EndSeqNo' number='16' type='SEQNUM'/>
+    <field name='MsgSeqNum' number='34' type='SEQNUM'/>
+    <field number='35' name='MsgType' type='STRING'>
+      <value enum='0' description='HEARTBEAT'/>
+      <value enum='1' description='TEST_REQUEST'/>
+      <value enum='2' description='RESEND_REQUEST'/>
+      <value enum='3' description='REJECT'/>
+      <value enum='4' description='SEQUENCE_RESET'/>
+      <value enum='5' description='LOGOUT'/>
+      <value enum='6' description='INDICATION_OF_INTEREST'/>
+      <value enum='7' description='ADVERTISEMENT'/>
+      <value enum='8' description='EXECUTION_REPORT'/>
+      <value enum='9' description='ORDER_CANCEL_REJECT'/>
+      <value enum='A' description='LOGON'/>
+      <value enum='B' description='NEWS'/>
+      <value enum='C' description='EMAIL'/>
+      <value enum='D' description='ORDER_SINGLE'/>
+      <value enum='E' description='ORDER_LIST'/>
+      <value enum='F' description='ORDER_CANCEL_REQUEST'/>
+      <value enum='G' description='ORDER_CANCEL_REPLACE_REQUEST'/>
+      <value enum='H' description='ORDER_STATUS_REQUEST'/>
+      <value enum='J' description='ALLOCATION_INSTRUCTION'/>
+      <value enum='K' description='LIST_CANCEL_REQUEST'/>
+      <value enum='L' description='LIST_EXECUTE'/>
+      <value enum='M' description='LIST_STATUS_REQUEST'/>
+      <value enum='N' description='LIST_STATUS'/>
+      <value enum='P' description='ALLOCATION_INSTRUCTION_ACK'/>
+      <value enum='Q' description='DONT_KNOW_TRADE'/>
+      <value enum='R' description='QUOTE_REQUEST'/>
+      <value enum='S' description='QUOTE'/>
+      <value enum='T' description='SETTLEMENT_INSTRUCTIONS'/>
+      <value enum='V' description='MARKET_DATA_REQUEST'/>
+      <value enum='W' description='MARKET_DATA_SNAPSHOT_FULL_REFRESH'/>
+      <value enum='X' description='MARKET_DATA_INCREMENTAL_REFRESH'/>
+      <value enum='Y' description='MARKET_DATA_REQUEST_REJECT'/>
+      <value enum='Z' description='QUOTE_CANCEL'/>
+      <value enum='a' description='QUOTE_STATUS_REQUEST'/>
+      <value enum='b' description='MASS_QUOTE_ACKNOWLEDGEMENT'/>
+      <value enum='c' description='SECURITY_DEFINITION_REQUEST'/>
+      <value enum='d' description='SECURITY_DEFINITION'/>
+      <value enum='e' description='SECURITY_STATUS_REQUEST'/>
+      <value enum='f' description='SECURITY_STATUS'/>
+      <value enum='g' description='TRADING_SESSION_STATUS_REQUEST'/>
+      <value enum='h' description='TRADING_SESSION_STATUS'/>
+      <value enum='i' description='MASS_QUOTE'/>
+      <value enum='j' description='BUSINESS_MESSAGE_REJECT'/>
+      <value enum='k' description='BID_REQUEST'/>
+      <value enum='l' description='BID_RESPONSE'/>
+      <value enum='m' description='LIST_STRIKE_PRICE'/>
+      <value enum='n' description='XML_MESSAGE'/>
+      <value enum='o' description='REGISTRATION_INSTRUCTIONS'/>
+      <value enum='p' description='REGISTRATION_INSTRUCTIONS_RESPONSE'/>
+      <value enum='q' description='ORDER_MASS_CANCEL_REQUEST'/>
+      <value enum='r' description='ORDER_MASS_CANCEL_REPORT'/>
+      <value enum='s' description='NEW_ORDER_CROSS'/>
+      <value enum='t' description='CROSS_ORDER_CANCEL_REPLACE_REQUEST'/>
+      <value enum='u' description='CROSS_ORDER_CANCEL_REQUEST'/>
+      <value enum='v' description='SECURITY_TYPE_REQUEST'/>
+      <value enum='w' description='SECURITY_TYPES'/>
+      <value enum='x' description='SECURITY_LIST_REQUEST'/>
+      <value enum='y' description='SECURITY_LIST'/>
+      <value enum='z' description='DERIVATIVE_SECURITY_LIST_REQUEST'/>
+      <value enum='AA' description='DERIVATIVE_SECURITY_LIST'/>
+      <value enum='AB' description='NEW_ORDER_MULTILEG'/>
+      <value enum='AC' description='MULTILEG_ORDER_CANCEL_REPLACE'/>
+      <value enum='AD' description='TRADE_CAPTURE_REPORT_REQUEST'/>
+      <value enum='AE' description='TRADE_CAPTURE_REPORT'/>
+      <value enum='AF' description='ORDER_MASS_STATUS_REQUEST'/>
+      <value enum='AG' description='QUOTE_REQUEST_REJECT'/>
+      <value enum='AH' description='RFQ_REQUEST'/>
+      <value enum='AI' description='QUOTE_STATUS_REPORT'/>
+      <value enum='AJ' description='QUOTE_RESPONSE'/>
+      <value enum='AK' description='CONFIRMATION'/>
+      <value enum='AL' description='POSITION_MAINTENANCE_REQUEST'/>
+      <value enum='AM' description='POSITION_MAINTENANCE_REPORT'/>
+      <value enum='AN' description='REQUEST_FOR_POSITIONS'/>
+      <value enum='AO' description='REQUEST_FOR_POSITIONS_ACK'/>
+      <value enum='AP' description='POSITION_REPORT'/>
+      <value enum='AQ' description='TRADE_CAPTURE_REPORT_REQUEST_ACK'/>
+      <value enum='AR' description='TRADE_CAPTURE_REPORT_ACK'/>
+      <value enum='AS' description='ALLOCATION_REPORT'/>
+      <value enum='AT' description='ALLOCATION_REPORT_ACK'/>
+      <value enum='AU' description='CONFIRMATION_ACK'/>
+      <value enum='AV' description='SETTLEMENT_INSTRUCTION_REQUEST'/>
+      <value enum='AW' description='ASSIGNMENT_REPORT'/>
+      <value enum='AX' description='COLLATERAL_REQUEST'/>
+      <value enum='AY' description='COLLATERAL_ASSIGNMENT'/>
+      <value enum='AZ' description='COLLATERAL_RESPONSE'/>
+      <value enum='BA' description='COLLATERAL_REPORT'/>
+      <value enum='BB' description='COLLATERAL_INQUIRY'/>
+      <value enum='BC' description='NETWORK_STATUS_REQUEST'/>
+      <value enum='BD' description='NETWORK_STATUS_RESPONSE'/>
+      <value enum='BE' description='USER_REQUEST'/>
+      <value enum='BF' description='USER_RESPONSE'/>
+      <value enum='BG' description='COLLATERAL_INQUIRY_ACK'/>
+      <value enum='BH' description='CONFIRMATION_REQUEST'/>
+      <value enum='BI' description='TRADING_SESSION_LIST_REQUEST'/>
+      <value enum='BJ' description='TRADING_SESSION_LIST'/>
+      <value enum='BK' description='SECURITY_LIST_UPDATE_REPORT'/>
+      <value enum='BL' description='ADJUSTED_POSITION_REPORT'/>
+      <value enum='BM' description='ALLOCATION_INSTRUCTION_ALERT'/>
+      <value enum='BN' description='EXECUTION_ACKNOWLEDGEMENT'/>
+      <value enum='BO' description='CONTRARY_INTENTION_REPORT'/>
+      <value enum='BP' description='SECURITY_DEFINITION_UPDATE_REPORT'/>
+    </field>
+    <field name='NewSeqNo' number='36' type='SEQNUM'/>
+    <field name='PossDupFlag' number='43' type='BOOLEAN'/>
+    <field name='RefSeqNum' number='45' type='SEQNUM'/>
+    <field name='SenderCompID' number='49' type='STRING'/>
+    <field name='SenderSubID' number='50' type='STRING'/>
+    <field name='SendingTime' number='52' type='UTCTIMESTAMP'/>
+    <field name='TargetCompID' number='56' type='STRING'/>
+    <field name='TargetSubID' number='57' type='STRING'/>
+    <field name='Text' number='58' type='STRING'/>
+    <field name='Signature' number='89' type='DATA'/>
+    <field name='SecureDataLen' number='90' type='LENGTH'/>
+    <field name='SecureData' number='91' type='DATA'/>
+    <field name='SignatureLength' number='93' type='LENGTH'/>
+    <field name='RawDataLength' number='95' type='LENGTH'/>
+    <field name='RawData' number='96' type='DATA'/>
+    <field name='PossResend' number='97' type='BOOLEAN'/>
+    <field name='EncryptMethod' number='98' type='INT'>
+      <value description='NONE_OTHER' enum='0'/>
+      <value description='PKCS' enum='1'/>
+      <value description='DES' enum='2'/>
+      <value description='PKCS_DES' enum='3'/>
+      <value description='PGP_DES' enum='4'/>
+      <value description='PGP_DES_MD5' enum='5'/>
+      <value description='PEM_DES_MD5' enum='6'/>
+    </field>
+    <field name='HeartBtInt' number='108' type='INT'/>
+    <field name='TestReqID' number='112' type='STRING'/>
+    <field name='OnBehalfOfCompID' number='115' type='STRING'/>
+    <field name='OnBehalfOfSubID' number='116' type='STRING'/>
+    <field name='OrigSendingTime' number='122' type='UTCTIMESTAMP'/>
+    <field name='GapFillFlag' number='123' type='BOOLEAN'/>
+    <field name='DeliverToCompID' number='128' type='STRING'/>
+    <field name='DeliverToSubID' number='129' type='STRING'/>
+    <field name='ResetSeqNumFlag' number='141' type='BOOLEAN'/>
+    <field name='SenderLocationID' number='142' type='STRING'/>
+    <field name='TargetLocationID' number='143' type='STRING'/>
+    <field name='OnBehalfOfLocationID' number='144' type='STRING'/>
+    <field name='DeliverToLocationID' number='145' type='STRING'/>
+    <field name='XmlDataLen' number='212' type='LENGTH'/>
+    <field name='XmlData' number='213' type='DATA'/>
+    <field number='347' name='MessageEncoding' type='STRING'>
+      <value enum='ISO-2022-JP' description='ISO_2022_JP'/>
+      <value enum='EUC-JP' description='EUC_JP'/>
+      <value enum='SHIFT_JIS' description='SHIFT_JIS'/>
+      <value enum='UTF-8' description='UTF_8'/>
+    </field>
+    <field name='EncodedTextLen' number='354' type='LENGTH'/>
+    <field name='EncodedText' number='355' type='DATA'/>
+    <field name='LastMsgSeqNumProcessed' number='369' type='SEQNUM'/>
+    <field name='RefTagID' number='371' type='INT'/>
+    <field name='RefMsgType' number='372' type='STRING'/>
+    <field name='SessionRejectReason' number='373' type='INT'>
+      <value description='INVALID_TAG_NUMBER' enum='0'/>
+      <value description='REQUIRED_TAG_MISSING' enum='1'/>
+      <value description='SENDINGTIME_ACCURACY_PROBLEM' enum='10'/>
+      <value description='INVALID_MSGTYPE' enum='11'/>
+      <value description='XML_VALIDATION_ERROR' enum='12'/>
+      <value description='TAG_APPEARS_MORE_THAN_ONCE' enum='13'/>
+      <value description='TAG_SPECIFIED_OUT_OF_REQUIRED_ORDER' enum='14'/>
+      <value description='REPEATING_GROUP_FIELDS_OUT_OF_ORDER' enum='15'/>
+      <value description='INCORRECT_NUMINGROUP_COUNT_FOR_REPEATING_GROUP' enum='16'/>
+      <value description='NON_DATA_VALUE_INCLUDES_FIELD_DELIMITER' enum='17'/>
+      <value description='TAG_NOT_DEFINED_FOR_THIS_MESSAGE_TYPE' enum='2'/>
+      <value description='UNDEFINED_TAG' enum='3'/>
+      <value description='TAG_SPECIFIED_WITHOUT_A_VALUE' enum='4'/>
+      <value description='VALUE_IS_INCORRECT' enum='5'/>
+      <value description='INCORRECT_DATA_FORMAT_FOR_VALUE' enum='6'/>
+      <value description='DECRYPTION_PROBLEM' enum='7'/>
+      <value description='SIGNATURE_PROBLEM' enum='8'/>
+      <value description='COMPID_PROBLEM' enum='9'/>
+      <value description='OTHER' enum='99'/>
+    </field>
+    <field name='MaxMessageSize' number='383' type='LENGTH'/>
+    <field name='NoMsgTypes' number='384' type='NUMINGROUP'/>
+    <field name='MsgDirection' number='385' type='CHAR'>
+      <value description='RECEIVE' enum='R'/>
+      <value description='SEND' enum='S'/>
+    </field>
+    <field name='TestMessageIndicator' number='464' type='BOOLEAN'/>
+    <field name='Username' number='553' type='STRING'/>
+    <field name='Password' number='554' type='STRING'/>
+    <field name='NoHops' number='627' type='NUMINGROUP'/>
+    <field name='HopCompID' number='628' type='STRING'/>
+    <field name='HopSendingTime' number='629' type='UTCTIMESTAMP'/>
+    <field name='HopRefID' number='630' type='SEQNUM'/>
+    <field name='NextExpectedMsgSeqNum' number='789' type='SEQNUM'/>
+    <field name='ApplVerID' number='1128' type='STRING'>
+      <value description='FIX27' enum='0'/>
+      <value description='FIX30' enum='1'/>
+      <value description='FIX40' enum='2'/>
+      <value description='FIX41' enum='3'/>
+      <value description='FIX42' enum='4'/>
+      <value description='FIX43' enum='5'/>
+      <value description='FIX44' enum='6'/>
+      <value description='FIX50' enum='7'/>
+      <value description='FIX50SP1' enum='8'/>
+      <value description='FIX50SP2' enum='9'/>
+    </field>
+    <field name='CstmApplVerID' number='1129' type='STRING'/>
+    <field name='RefApplVerID' number='1130' type='STRING'/>
+    <field name='RefCstmApplVerID' number='1131' type='STRING'/>
+    <field name='DefaultApplVerID' number='1137' type='STRING'/>
+  </fields>
+</fix>

src/client/spark_historical/Algos/AlgoBase.cs

@@ -0,0 +1,233 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+
+using ReactionFunction = System.Action<bts.history.MarketDataIncrementalRefresh /*current_book_packet*/,
+                                       string, /*security_desc*/
+                                       object, /*param*/   
+                                       Google.Protobuf.WellKnownTypes.Timestamp /* asof */
+                                       >;
+using bts.co2;
+
+namespace bts.history
+{
+   class Catalyst
+   {
+      public co2.BookMessage current_book;
+      public co2.BookMessage curr_book;
+      public MarketDataIncrementalRefresh packet;
+      public Timestamp asof;
+   }
+   
+   class Reaction
+   {
+      public ReactionFunction function;
+
+      public MarketDataIncrementalRefresh packet;
+      public string security_desc;
+      public object param;
+      public Timestamp asof;
+   }   
+   
+
+   public interface IAlgo 
+   {
+      void reset();
+
+      void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage curr_book,
+                        MarketDataIncrementalRefresh packet);
+      void process_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                          MarketDataIncrementalRefresh packet);
+      IFillModel fill_model { get; }
+   }
+   
+   abstract partial class AlgoBase : IAlgo
+   {
+      public readonly Guid guid = Guid.NewGuid();
+      public ulong order_key { get; private set;}
+      
+      public string[] security_list { get; private set; }
+      public Bucket bucket { get; private set;}
+      public Action<FillEntry> on_fill;
+      public Action<OrderStatus> on_status;
+      
+      protected IFillModel _fill_model;
+      public IFillModel fill_model => _fill_model;
+      readonly Queue<Reaction> _reactions = new Queue<Reaction>();
+
+      readonly protected SortedDictionary< string, Func<MDEntry, OrderIDEntry, bool> > _filter = new SortedDictionary<string, Func<MDEntry, OrderIDEntry, bool>>();      
+
+      protected AlgoBase(IFillModel fill_model, Bucket bucket, string[] security_list) 
+      {
+         on_fill = new Action<FillEntry>(f => {});
+         on_status = new Action<OrderStatus>(s => {});
+         this._fill_model = fill_model;
+         this.bucket = bucket;
+         this.security_list = security_list;
+      }            
+            
+      //called after is_completed() 
+      public virtual void reset()
+      {
+      }
+      
+      readonly Dictionary<string, string> _algo_params = new Dictionary<string, string>();
+      public IReadOnlyDictionary<string, string> algo_params => _algo_params;
+      public virtual void set_param(string name, string val)
+      {
+         _algo_params[name] = val;
+         var pi = this.GetType().GetProperty(name);
+         if(pi != null)
+         {
+            if(val.Contains(','))
+            {
+               var elem = pi.PropertyType.GetElementType();
+               var ary = val.Split(',').Select(x => Convert.ChangeType(x, elem)).ToArray();
+               var tary = Array.CreateInstance(elem, ary.Length);
+               Array.Copy(ary, tary, ary.Length);
+               pi.SetValue(this, tary);
+            }
+            else
+               pi.SetValue(this, Convert.ChangeType(val, pi.PropertyType));
+         }
+      }         
+
+      public virtual void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage curr_book,
+                                       MarketDataIncrementalRefresh packet)
+      {
+         if(!_filter.ContainsKey(security_desc))
+            return;
+         on_book(security_desc, security_key, current_book, curr_book, packet);
+      }
+
+      public virtual void process_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                                         MarketDataIncrementalRefresh packet)      
+      {
+         if (!_filter.ContainsKey(security_desc))
+            return;
+         on_trades(security_desc, security_key, trades, packet);
+      }      
+
+      public virtual void execute_with_latency(MarketDataIncrementalRefresh packet)
+      {         
+         if(_reactions.Count == 0)
+            return;
+         List<Reaction> rxx = new List<Reaction>();
+         while(_reactions.Count > 0)
+            rxx.Add(_reactions.Dequeue());
+         foreach(var rx in rxx)
+         {
+            //does not have to be a packet from our security to tell time            
+            var delta = (packet.timestamp() - rx.packet.timestamp());
+            double nanos = delta.Seconds * 1e9 + delta.Nanos;
+            if(nanos >= this.bucket.LowerBound)
+            {
+               rx.function(rx.packet, rx.security_desc, rx.param, rx.asof);
+            }
+            else 
+               _reactions.Enqueue(rx);
+         }   
+      }
+
+      protected virtual void queue_reaction(ReactionFunction reaction_function, MarketDataIncrementalRefresh packet, 
+                                            string security_desc, object param = null)
+      {
+         Timestamp asof = packet.timestamp().Clone();
+         asof.Nanos += (int)this.bucket.LowerBound;
+         if(this.bucket.LowerBound == 0)
+         {
+            reaction_function(packet, security_desc, param, asof);
+         }
+         else
+         {
+            _reactions.Enqueue(new Reaction { function = reaction_function, packet = packet.Clone(), 
+                                              security_desc = security_desc, param = param, asof = asof });
+         }
+      }
+                  
+
+      public abstract object report();
+      public abstract bool is_completed();
+
+      public virtual Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
+                                        Action<Handle, FillEntry> more_on_fill = null,
+                                        Action<Handle, OrderStatus> more_on_status = null)
+         where C : AlgoBase
+      {
+         var order_key = IDGenerator.next_order_id();
+         var child = AlgoBase.algo_factory(typeof(C), _fill_model, bucket, security_list);
+         Handle h = hookup_child(child, security_list, side, price, size, asof, more_on_fill, more_on_status, order_key);
+         return h;
+      }
+
+      protected Handle hookup_child(AlgoBase child, string[] security_list, Side side, double price, uint size, Timestamp asof, 
+                                    Action<Handle, FillEntry> more_on_fill, Action<Handle, OrderStatus> more_on_status, uint order_key) 
+      {
+         child.order_key = order_key;
+         AlgoBase.set_algo_params(child, this.algo_params.Values);
+         child.on_fill += f => this.on_fill(f);
+         child.on_status += s => this.on_status(s);
+         var h = new Handle(child);
+         if (more_on_fill != null)
+            child.on_fill += (f => more_on_fill(h, f));
+         if (more_on_status != null)
+            child.on_status += (s => more_on_status(h, s));
+
+         child.do_launch(security_list, side, price, size, asof, order_key);
+         return h;
+      }
+
+      protected virtual void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                     MarketDataIncrementalRefresh packet) { }
+      protected virtual void on_trades(string security_desc, uint security_key, IEnumerable<TradeMessage> trades, 
+                                       MarketDataIncrementalRefresh packet) { }
+                                       
+      protected abstract void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key);
+      
+      protected abstract void do_update(Timestamp asof, double price, uint size = uint.MinValue);
+      protected virtual void do_update(Timestamp asof, Dictionary<string, string> algo_params, double price, uint size = uint.MinValue)
+      {
+         foreach (var s in algo_params)
+            this.set_param(s.Key, s.Value);
+         this.do_update(asof, price, size);
+      }
+      protected abstract void do_cancel(Timestamp asof);
+
+      internal class Handle : IEquatable<Handle>
+      {
+         readonly AlgoBase _algo;
+         public Handle(AlgoBase algo)
+         {
+            _algo = algo;
+         }
+         
+         public AlgoBase algo => _algo;
+         public ulong order_key => _algo.order_key;
+         public void update(Timestamp asof, double price, uint size = uint.MinValue)
+         {
+            _algo.do_update(asof, price, size);
+         }
+         public void update(Timestamp asof, Dictionary<string, string> algo_params, double price, uint size = uint.MinValue)
+         {
+            foreach (var s in algo_params)
+               _algo.set_param(s.Key, s.Value);
+            update(asof, price, size);
+         }
+         public void cancel(Timestamp asof)
+         {
+            _algo.do_cancel(asof);
+         }
+         public override int GetHashCode()
+         {
+            return this.algo.guid.GetHashCode();
+         }
+         public bool Equals(Handle other)
+         {
+            return this.algo.guid == other.algo.guid;
+         }
+      }
+   }
+}

src/client/spark_historical/Algos/AlgoFactory.cs

@@ -0,0 +1,45 @@
+using System;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+using System.Collections.Generic;
+using System.Reflection;
+
+namespace bts.history
+{
+   abstract partial class AlgoBase
+   {
+      static public AlgoBase algo_factory(System.Type t, IFillModel fill_model, Bucket bucket,
+                                          string[] security_list, string algo_params = null)
+      {
+         AlgoBase inst = inst = (AlgoBase)Activator.CreateInstance(t, BindingFlags.NonPublic, null, 
+            new object[] { fill_model, bucket, security_list }, System.Globalization.CultureInfo.CurrentCulture );
+
+         if(algo_params != null)
+            set_algo_params(inst, algo_params);
+         return inst;
+      }      
+      static public void set_algo_params(AlgoBase inst, string algo_params)
+      {
+         string[] param = algo_params.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+         set_algo_params(inst, param);
+      }
+      static public void set_algo_params(AlgoBase inst, IEnumerable<string> param)
+      {
+         foreach (var par in param)
+         {
+            string[] tup = par.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
+            Log.assert(tup.Length == 2);
+            inst.set_param(tup[0], tup[1]);
+         }
+      }      
+      public static Handle activate_self(AlgoBase algo, api.Side side, double price, uint size, Timestamp asof,
+                                         Action<Handle, FillEntry> more_on_fill = null,
+                                         Action<Handle, OrderStatus> more_on_status = null)
+      {
+         var order_key = IDGenerator.next_order_id();
+         //I am activating myself at the top
+         algo.do_launch(algo.security_list, side, price, size, asof, order_key);
+         return new Handle(algo);
+      }
+   }
+}

src/client/spark_historical/Algos/AlgoStatsWrapper.cs

@@ -0,0 +1,171 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using bts.api;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   class AlgoStatsWrapper 
+   {
+      readonly Queue<FillEntry> _tradelog = new Queue<FillEntry>();
+      readonly Queue<FillEntry> _market_tradelog = new Queue<FillEntry>();
+      DateTime _tradedate = DateTime.Now;
+      readonly string _symbol;
+      readonly SecurityType _securitytype;
+      
+      readonly SortedDictionary<string, TradeStats> _stats = new SortedDictionary<string, TradeStats>();
+      public IEnumerable<TradeStats> stats => _stats.Values;
+
+      readonly AlgoBase _algo;
+      public AlgoBase algo => _algo;
+
+      readonly IReadOnlyDictionary<string, IOrderBook> _books;
+      readonly StreamWriter _fs;
+      
+      public AlgoStatsWrapper(AlgoBase algo, IReadOnlyDictionary<string, IOrderBook> shared_book, HistoricalDataOptions options)
+      {
+         _algo = algo;
+         _books = shared_book;
+         foreach (var sec in algo.security_list)
+         {
+            _stats[sec] = new TradeStats();
+            shared_book[sec].on_market_trade += (md) =>
+            {
+               foreach(var mde in md.MdEntries)
+               {
+                  if(mde.MdEntryType == MDEntryType.Trade)
+                  {
+                     //opposite the aggressor
+                     api.Side side = (mde.AggressorSide == 0 ? api.Side.Undefined : (mde.AggressorSide == 1 ? api.Side.Buy : api.Side.Sell));
+                     //convert to a fill entry to write
+                     var ord = new OrderEntry(mde.Symbol, "Market");
+                     FillEntry ent = new FillEntry(md.timestamp(), ord, side, mde.MdEntryPx, mde.LastQty);
+                     var b = shared_book[sec];
+                     ent.bidsize = b.current_book.bidsize_at(0);
+                     ent.bid = b.current_book.bid_at(0);
+                     ent.ask = b.current_book.ask_at(0);
+                     ent.asksize = b.current_book.asksize_at(0);
+                     _market_tradelog.Enqueue(ent);
+                  }
+               }
+            };
+         }
+         algo.on_fill += 
+            f => _tradelog.Enqueue(f);
+         _symbol = options.replaysymbol;
+         _securitytype = (SecurityType)Enum.Parse(typeof(SecurityType), options.replaysecuritytype);
+         _tradedate = DateTime.Parse(options.replaystart);
+         var path = DataDownloader.data_path(_tradedate.Date);
+         new FileInfo(path).Directory.Create();
+         _fs = new StreamWriter(
+             File.Open(path + $"/{_symbol}_{_securitytype}_{algo.GetType().Name}_{algo.bucket.LowerBound.ToString("0") }_trades.txt", FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite) );
+      }
+            
+      void do_stats()
+      {
+         foreach (var pair in _stats)
+         {
+            var stats = pair.Value;
+            var buys = _tradelog.Where(x => x.side == Side.Buy && x.order.security_desc == pair.Key);
+            var nbuys = buys.Count();
+            var buysize = buys.Sum(x => (int)x.size);
+            var buyprice = buys.Sum(x => x.price);
+
+            var sells = _tradelog.Where(x => x.side == Side.Sell && x.order.security_desc == pair.Key);
+            var nsells = sells.Count();
+            var sellsize = sells.Sum(x => (int)x.size);
+            var sellprice = sells.Sum(x => x.price);
+
+            stats.our_average_buy_piq = stats.our_average_buy_piq * stats.our_total_buys + buys.Where(x => x.piq > 0).Sum(x => (double)x.piq);
+            stats.our_average_buy_q_len = stats.our_average_buy_q_len * stats.our_total_buys + buys.Where(x => x.queue_len > 0).Sum(x => (double)x.queue_len);
+            stats.our_average_buy_price = stats.our_average_buy_price * stats.our_total_buys + buys.Sum(x => x.price);
+            stats.our_total_buys += nbuys;
+            if (stats.our_total_buys > 0)
+            {
+               stats.our_average_buy_piq /= stats.our_total_buys;
+               stats.our_average_buy_q_len /= stats.our_total_buys;
+               stats.our_average_buy_price /= stats.our_total_buys;
+            }
+
+            stats.our_average_sell_piq = stats.our_average_sell_piq * stats.our_total_sells + sells.Where(x => x.piq > 0).Sum(x => (double)x.piq);
+            stats.our_average_sell_q_len = stats.our_average_sell_q_len * stats.our_total_sells + sells.Where(x => x.queue_len > 0).Sum(x => (double)x.queue_len);
+            stats.our_average_sell_price = stats.our_average_sell_price * stats.our_total_sells + sells.Sum(x => x.price);
+            stats.our_total_sells += nsells;
+            if (stats.our_total_sells > 0)
+            {
+               stats.our_average_sell_piq /= stats.our_total_sells;
+               stats.our_average_sell_q_len /= stats.our_total_sells;
+               stats.our_average_sell_price /= stats.our_total_sells;
+            }
+
+            stats.our_total_volume += _tradelog.Where(x => x.order.security_desc == pair.Key).Sum(x => (int)x.size);
+
+            var pnl = compute_pnl();
+            stats.latest_pnl = pnl;
+         }         
+      }
+
+      public void reset()
+      {
+         do_stats();
+         
+         List<FillEntry> fills = new List<FillEntry>();
+         while(_tradelog.Count > 0)
+         {
+            var t = _tradelog.Dequeue();
+            fills.Add(t);
+         }
+         while (_market_tradelog.Count > 0)
+         {
+            var t = _market_tradelog.Dequeue();
+            fills.Add(t);
+         }
+         fills.Sort( Comparer<FillEntry>.Create( (x, y) => x.timestamp.compare(y.timestamp) ) );
+         foreach(var t in fills)
+            _fs.WriteLine(t.ToString());
+         _fs.Flush();
+         _algo.reset();
+      }
+
+      public void keep_md_stats(IOrderBook ob)
+      {
+         var stats = _stats[ob.security_desc];
+         stats.total_exchange_volume = ob.total_exchange_volume;
+         stats.total_exchange_bought = ob.total_exchange_bought;
+         stats.total_exchange_sold = ob.total_exchange_sold;
+         stats.last_trade_size = ob.last_trade_size;
+         stats.last_trade_price = ob.last_trade_price;
+         stats.security_timestamp = ob.security_timestamp;
+         stats.security_desc = ob.security_desc;
+         stats.best_bid = ob.current_book.bid_at(0);
+         stats.best_ask = ob.current_book.ask_at(0);
+         stats.best_bid_size = ob.current_book.bidsize_at(0);
+         stats.best_ask_size = ob.current_book.asksize_at(0);
+      }
+
+      public double compute_pnl()
+      {
+         double cost = 0.0, proceeds = 0.0;
+         ulong buys = 0, sells = 0;
+         foreach (var t in _tradelog)
+         {
+            if (t.side == Side.Buy)
+            {
+               cost += t.price * t.size;
+               buys += t.size;
+            }
+            else if (t.side == Side.Sell)
+            {
+               proceeds += t.price * t.size;
+               sells += t.size;
+            }
+         }
+
+         var pnl = proceeds - cost;
+         return pnl;
+      }
+   }
+}

src/client/spark_historical/Algos/CompositeAlgo.cs

@@ -0,0 +1,266 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   abstract class CompositeBase<Child, Hedge> : AlgoBase
+      where Hedge : AlgoBase
+   {
+      protected List<Handle> _children = new List<Handle>();
+      protected Handle _child 
+      {
+         get 
+         {
+            Log.assert(_children.Count <= 1);
+            return _children.FirstOrDefault(); 
+         } 
+         set 
+         { 
+            Log.assert(_children.Count <= 1);
+            if(value == null)
+            {
+               if(_children.Count == 1)            
+                  _children.RemoveAt(0);
+            }
+            else
+            {
+               if(_children.Count == 0)
+                  _children.Add(value);
+               else
+                  _children[0] = value;
+            }
+         }
+      }
+      protected CompositeBase(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) 
+      {
+      }
+
+      public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof, 
+                                         Action<Handle, FillEntry> more_on_fill = null, 
+                                         Action<Handle, OrderStatus> more_on_status = null)     
+      {         
+         if(typeof(C) == this.GetType())
+         {
+            return AlgoBase.activate_self(this, side, price, size, asof, more_on_fill, more_on_status);
+         }         
+         else
+         {
+            Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => on_child_fill(h, f));
+            if (more_on_fill != null)
+               of += more_on_fill;
+            Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) =>
+            {
+               on_child_status(h, s);
+               if (s.is_terminal())
+                  _children.Remove(h);
+            });
+            if (more_on_status != null)
+               os += more_on_status;
+
+            var handle = base_activate<C>(security_list, side, price, size, asof, of, os);
+            _children.Add(handle);
+            return handle;
+         }
+      }
+
+      protected Handle base_activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof, 
+                                        Action<Handle, FillEntry> of, Action<Handle, OrderStatus> os)
+         where C : AlgoBase
+      {
+         return base.activate<C>(security_list, side, price, size, asof, of, os);
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         foreach(var child in _children)
+            child?.update(asof, price, size);
+      }
+
+      protected override void do_cancel(Timestamp asof)
+      {
+         var kids = new List<Handle>(_children);
+         foreach (var child in kids)
+            child?.cancel(asof);
+      }      
+      protected virtual void on_child_fill(Handle child, FillEntry fill) { }
+      protected virtual void on_child_status(Handle child, OrderStatus stat) { }
+   }
+
+   abstract class CompositeAlgo<Child> : CompositeBase<Child, NullAlgo>
+   {
+      protected CompositeAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+
+      public override bool is_completed()
+      {
+         bool comp = true;
+         foreach (var child in _children)
+            comp &= child.algo.is_completed();
+         return comp;
+      }
+      
+      public override object report()
+      {
+         string c = "";
+         foreach (var child in _children)
+            c+= child.algo.report();
+         return $"{c}";
+      }
+
+      public override void set_param(string name, string val)
+      {
+         base.set_param(name, val);
+         foreach (var child in _children)
+            child?.algo.set_param(name, val);
+      }
+
+      public override void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                        MarketDataIncrementalRefresh packet)
+      {
+         base.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var child in _children)
+            child?.algo.process_book(security_desc, security_key, current_book, working_book, packet);
+      }
+
+      public override void execute_with_latency(MarketDataIncrementalRefresh packet)
+      {
+         base.execute_with_latency(packet);
+         foreach (var child in _children)
+            child?.algo.execute_with_latency(packet);
+      }   
+   }   
+
+   abstract class CompositeHedgeAlgo<Child, Hedge> : CompositeBase<Child, Hedge>
+      where Hedge : AlgoBase
+   {
+      protected int _payupcount;
+      protected Dictionary<Handle, Hedge> _hedges = new Dictionary<Handle, Hedge>();
+      protected int _hedges_in_transit;
+      protected CompositeHedgeAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) {}
+
+      protected virtual bool are_hedges_out()
+      {
+         return _hedges_in_transit > 0;
+      }
+
+      public override bool is_completed() 
+      {
+         bool comp = true;
+         foreach (var child in _children)
+            comp &= child.algo.is_completed();
+         foreach (var sub in _hedges.Values)
+         {
+            var hc = sub.is_completed();
+            if(!hc)
+               Log.assert(true);
+            comp &= hc;
+         }
+         return comp && !are_hedges_out();
+      }
+
+      public override object report() 
+      {
+         string c = "";
+         foreach (var child in _children)
+            c+= child.algo.report();
+         
+         int ih = 0;
+         string sh = string.Empty;
+         foreach (var sub in _hedges.Values)
+         {
+            if(sub.report() is string sr)
+               sh += sr;
+            if(sub.report() is int ir)
+               ih += ir;
+         }
+         return $"{c} Hedge Payups {sh} {ih+_payupcount} Incomplete {_hedges.Values.Sum( x => x.is_completed() ? 0u : 1u) } ";
+      }
+
+      public override void set_param(string name, string val)
+      {
+         base.set_param(name, val);
+         foreach (var child in _children)
+            child?.algo.set_param(name, val);
+         foreach (var sub in _hedges.Values)
+            sub.set_param(name, val);
+      }
+
+      public override void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                                MarketDataIncrementalRefresh packet)
+      {
+         base.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var child in _children)
+            child?.algo.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var sub in _hedges.Values)
+            sub.process_book(security_desc, security_key, current_book, working_book, packet);
+      }
+
+      public override void execute_with_latency(MarketDataIncrementalRefresh packet)
+      {
+         base.execute_with_latency(packet);
+         foreach (var child in _children)
+            child?.algo.execute_with_latency(packet);
+         foreach (var sub in _hedges.Values)
+            sub.execute_with_latency(packet);
+      }
+
+      public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof, 
+                                         Action<Handle, FillEntry> more_on_fill = null, 
+                                         Action<Handle, OrderStatus> more_on_status = null)
+      {
+         if (typeof(Hedge) == typeof(C))
+         {
+            Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => {} );
+            if (more_on_fill != null)
+               of += more_on_fill;
+            Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) => {} );
+            if (more_on_status != null)
+               os += more_on_status;
+
+            var handle = base_activate<Hedge>(security_list, side, price, size, asof, of, os);                                         
+            _hedges[handle] = (Hedge)handle.algo;
+            return handle;
+         }
+         else
+            return base.activate<C>(security_list, side, price, size, asof, more_on_fill, more_on_status);
+      }
+
+
+      protected void queue_hedge(MarketDataIncrementalRefresh packet, string security_desc, object param = null)
+      {
+         _hedges_in_transit++;
+         queue_reaction(send_hedge, packet, security_desc, param);
+      }
+
+      private void send_hedge(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof)
+      {
+         Action<Handle, FillEntry> hedge_on_fill = (hand, f) =>
+         {
+            if (f.fully_filled)
+            {
+               if (_hedges.TryGetValue(hand, out Hedge h))
+               {
+                  if (h.report() is int pay)
+                     _payupcount += pay;
+                  _hedges.Remove(hand);
+               }
+            }
+         };
+         Action<Handle, OrderStatus> hedge_on_status = (hand,s) =>
+         {
+            if (s.is_terminal())
+               _hedges.Remove(hand);
+         };
+
+         ValueTuple<string[], api.Side, double, uint> tup = (ValueTuple<string[], api.Side, double, uint>)param;
+         var handle = this.activate<Hedge>(tup.Item1, tup.Item2, tup.Item3, tup.Item4, asof, 
+                                           hedge_on_fill, hedge_on_status);
+
+         var piq = _fill_model.get_my_piq(security_desc, handle.order_key);
+         _hedges_in_transit--;
+      }
+   }
+   
+}

src/client/spark_historical/Algos/CoverAlgo.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using mbts.co2;
+
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   class CoverAlgo<Child, Hedge> : CompositeHedgeAlgo<Child, Hedge>
+      where Child : AlgoBase
+      where Hedge : AlgoBase
+   {
+      protected CoverAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+      
+      //algo state      
+      private MarketDataIncrementalRefresh _packet;
+      string[] _security_list;
+
+      //algo parameters
+      public uint hedge_ticks { get; set; } = 1;
+      public double hedge_factor { get; set; } = 1.0;
+
+      public double tick_size(string security_desc)
+      {
+         var b = _fill_model.tick_size(security_desc);
+         return b;
+      }
+
+      protected override void do_launch(string[] security_list, Side side, double price, uint size, Timestamp asof, ulong order_key) 
+      {
+         _security_list = security_list;
+         _child = this.activate<Child>(security_list, side, price, size, asof);
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue) 
+      {
+         _child.update(asof, price, size);
+      }
+
+      protected override void do_cancel(Timestamp asof) 
+      {
+         _child.cancel(asof);
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet) 
+      {
+         _packet = packet;
+      }
+
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+         base.on_child_fill(child, fill);
+                  
+         var side = Utils.opposite_side(fill.order.side);
+         uint size = (uint)(fill.size*this.hedge_factor);
+         double price = (side == Side.Buy) ?
+            fill.price - this.tick_size(fill.order.security_desc) * this.hedge_ticks :
+            fill.price + this.tick_size(fill.order.security_desc) * this.hedge_ticks;
+
+         queue_hedge(_packet, fill.order.security_desc, 
+                     (new[]{fill.order.security_desc}, side, price, size));
+      }
+   }
+}

src/client/spark_historical/Algos/IDGenerator.cs

@@ -0,0 +1,13 @@
+
+
+namespace bts.history
+{
+   static class IDGenerator
+   {
+      static uint __myid = 999_999;
+      static public uint next_order_id()
+      {
+         return (++__myid);
+      }
+   }
+}

src/client/spark_historical/Algos/JoinAlgo.cs

@@ -0,0 +1,158 @@
+
+using System;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   class JoinAlgo<Child> : CompositeAlgo<Child>
+      where Child : AlgoBase
+   {
+      enum AlgoState
+      {
+         Begin, WaitingForJoin, Joined, Done
+      }
+
+      //algo state
+      AlgoState _state;
+      OrderStatus _status;
+      api.Side _myside;
+      uint _mysize;
+      Timestamp _myasof;
+      
+      co2.BookMessage _snapshot, _currentbook;
+      int _turncount, _pilecount;      
+      (int, int) _piq;
+
+      protected JoinAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+      //algo parameters      
+      public bool initial_pile { get; set; } = false;
+      public bool should_chase { get; set; } = true;
+      public double join_factor { get; set; } = 1.0;
+      public uint join_quantity{ get; set; } = 0;
+
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _state = AlgoState.Begin;
+         _myside = side;
+         _mysize = size;
+         _myasof = asof;
+         _child = null;
+      }
+
+      public override bool is_completed()
+      {
+         return _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         var b = base.report();
+         return $"{b} Turns,Piles,Join(Size,Price,piq)={_turncount},{_pilecount},({_status?.state?.Quantity},{_status?.state?.Price},{_piq})";
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _status = stat;
+         if(stat.is_terminal())
+            _state = AlgoState.Done;
+      }
+      
+      bool has_turned()
+      {
+         bool turn = ((_myside == api.Side.Buy &&
+                       _currentbook.side_at(_myside, 0) > _snapshot.side_at(_myside, 0)) || //new top level bid price greater than old, market turn
+                      (_myside == api.Side.Sell &&
+                       _currentbook.side_at(_myside, 0) < _snapshot.side_at(_myside, 0))); //new top level ask price less than old, market turn            
+         return turn;
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         if(_snapshot == null)
+            _snapshot = current_book.Clone();            
+         _currentbook = current_book;
+         switch (_state)
+         {
+            case AlgoState.Begin:
+               {                  
+                  if(initial_pile)
+                  {
+                     var thresh = (this.join_quantity > 0 ? this.join_quantity : this.join_factor * _mysize);
+                     if(_currentbook.sidesize_at(_myside, 0) >= thresh)
+                     {
+                        _pilecount++;
+                        queue_reaction(deploy_join_function, packet, 
+                                       security_desc, _currentbook.side_at(_myside, 0));
+                        _state = AlgoState.Joined;
+                        _snapshot = _currentbook.Clone();
+                     }
+                  }
+                  else
+                     _state = AlgoState.WaitingForJoin;
+               }
+               break;
+            case AlgoState.WaitingForJoin:
+               {
+                  bool turn = has_turned();                     
+                  var thresh = (this.join_quantity > 0 ? this.join_quantity : this.join_factor * _mysize);
+                  if(turn &&_currentbook.sidesize_at(_myside, 0) >= thresh)
+                  {
+                     _turncount++;                     
+                     queue_reaction(deploy_join_function, packet,  
+                                    security_desc, _currentbook.side_at(_myside, 0));
+                     _state = AlgoState.Joined;
+                  }
+                  _snapshot = _currentbook.Clone();
+               }               
+               break;
+            case AlgoState.Joined:
+               {
+                  if (this.should_chase)
+                  {
+                     bool turn = has_turned();
+                     var thresh = (this.join_quantity > 0 ? this.join_quantity : this.join_factor * _mysize);
+                     if (turn && _currentbook.sidesize_at(_myside, 0) >= thresh)
+                     {
+                        _turncount++;
+                        queue_reaction(deploy_join_function, packet,  
+                                       security_desc, _currentbook.side_at(_myside, 0));
+                        _snapshot = _currentbook.Clone();
+                     }                  
+                  }
+               }
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+
+
+      void deploy_join_function(MarketDataIncrementalRefresh packet, 
+                                string security_desc, object param, Timestamp asof)
+      {
+         if(_state == AlgoState.Joined || _state == AlgoState.WaitingForJoin)
+         {
+            Log.Testing.info("deploy_join_function " + this.bucket.UpperBound);
+
+            double joinprice = (double) param;
+            if(_child == null)
+            {
+               _child = this.activate<Child>(new[]{security_desc}, _myside, joinprice, _mysize, asof);
+               _piq = _fill_model.get_my_piq(security_desc, _child.order_key);
+            }
+            else
+            {
+               _child.update(asof, joinprice);               
+               _piq = _fill_model.get_my_piq(security_desc, _child.order_key);
+            }
+         }
+      }
+   }
+}

src/client/spark_historical/Algos/LeanAlgo.cs

@@ -0,0 +1,122 @@
+
+using System;
+using System.Collections.Generic;
+using mbts.co2;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   class LeanAlgo<Child> : CompositeAlgo<Child>
+      where Child : AlgoBase
+   {
+      enum AlgoState
+      {
+         LeanWatch, Payup, Done
+      }
+
+      //algo state
+      AlgoState _state;
+      OrderStatus _status;
+      api.Side _myside;
+      double _myprice;
+      uint _mysize;      
+            
+      co2.BookMessage _snapshot, _currentbook;
+      int _payupcount;
+
+      protected LeanAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+      //algo parameters
+      public double lean_factor { get; set; } = 3.0;
+      public uint lean_quantity { get; set; } = 0;
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _state = AlgoState.LeanWatch;
+         _myside = side;
+         _mysize = size;
+         _myprice = price;         
+         _child = this.activate<Child>(security_list, side, price, size, asof);
+      }
+      
+      public override bool is_completed()
+      {
+         return _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         return _payupcount;
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _status = stat;
+         if (stat.is_terminal())
+            _state = AlgoState.Done;
+      } 
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         if (_snapshot == null)
+            _snapshot = current_book.Clone();
+         _currentbook = current_book;
+
+         switch (_state)
+         {
+            case AlgoState.LeanWatch:
+               {
+                  bool payupprice = false, payupsize = false;
+                  var leansize = (this.lean_quantity == 0) ? this.lean_factor * this._mysize : this.lean_quantity;
+                  if (_myside == api.Side.Buy) 
+                  {
+                     double best_ask = _snapshot.ask_at(0);
+                     var best_ask_size = _snapshot.asksize_at(0);
+                     payupprice = _currentbook.ask_at(0) > best_ask;
+                     payupsize = best_ask_size < leansize;
+                  }
+                  else
+                  {
+                     double best_bid = _snapshot.bid_at(0);
+                     var best_bid_size = _snapshot.bidsize_at(0);
+                     payupprice = _currentbook.bid_at(0) < best_bid;
+                     payupsize = best_bid_size < leansize;
+                  }
+                  if (payupprice || payupsize)
+                  {
+                     _payupcount++;
+                     _state = AlgoState.Payup;
+                     queue_reaction(payup_function, packet, security_desc);
+                  }
+               }
+               break;
+            case AlgoState.Payup:
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+
+      void payup_function(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof)
+      {
+         if(_state == AlgoState.Payup)
+         {
+            double newprice = 0.0;
+            //plow through
+            if (_myside == api.Side.Sell) 
+               newprice = _currentbook.all_bids().Min();
+            else
+               newprice = _currentbook.all_asks().Max();
+            //this should precipitate a fill
+            _child.update(asof, newprice);     
+            
+            Log.Testing.info($"payup_function {this.bucket.UpperBound}  ");
+         }
+      }      
+   }
+}

src/client/spark_historical/Algos/LimitOrder.cs

@@ -0,0 +1,80 @@
+using System;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   class LimitOrder : AlgoBase
+   {
+      protected LimitOrder(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+
+      protected OrderEntry _myorder;
+      OrderStatus _status;
+      uint _mysize;
+
+      public override object report() { return string.Empty; }
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)      
+      {
+         _mysize = size;
+         Log.assert(security_list.Length == 1);
+         var security_desc = security_list[0];
+         _status = new OrderStatus(asof);
+         _status.security_desc = security_desc;
+         _status.side = side;
+         _status.state = new api.orderman.OrderState
+            {
+               Status = api.orderman.OrderStatus.Working,
+               Quantity = size,
+               Price = price
+            };
+         Action<FillEntry> fill_action = f =>
+         {
+            _status.last_change = f.timestamp;
+            _status.state.LastPrice = f.price;
+            _status.state.AvgPrice = _status.state.AvgPrice * _status.state.FilledQuantity + f.price * f.size;
+            _status.state.FilledQuantity += (uint)f.size;
+            _status.state.AvgPrice /= _status.state.FilledQuantity;
+            if (f.fully_filled)
+               _status.state.Status = api.orderman.OrderStatus.Completed;
+
+            if (this.on_fill != null)
+               this.on_fill(f);
+            if (this.on_status != null)
+               this.on_status(_status.clone());
+         };
+         OrderEntry myorder = new OrderEntry(asof, order_key, security_desc, side, price, size, fill_action);
+         _fill_model.send(myorder);
+         if (this.on_status != null)
+            this.on_status(_status.clone());
+      }
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         if(size == _mysize)
+            size = uint.MinValue;
+         var ord = _fill_model.update(_myorder.security_desc, asof, _myorder.order_id, price, size);
+         if (this.on_status != null && ord != null)
+         {
+            _status.last_change = asof;
+            _status.state.Quantity = (uint)ord.size;
+            _status.state.Price = ord.price;
+            this.on_status(_status.clone());
+         }
+      }
+      protected override void do_cancel(Timestamp asof)
+      {
+         if(_myorder == null)
+            return;
+         _fill_model.cancel(_myorder.security_desc, asof, _myorder.order_id);
+         if(!is_completed())
+         {
+            _status.last_change = asof;
+            _status.state.Status = api.orderman.OrderStatus.Canceled;
+            this.on_status(_status.clone());
+         }
+      }
+
+      public override bool is_completed() => _status.is_terminal();
+   }
+}

src/client/spark_historical/Algos/MarketOrder.cs

@@ -0,0 +1,18 @@
+using System;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   class MarketOrder : LimitOrder
+   {
+      protected MarketOrder(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)      
+      {
+         base.do_launch(security_list, side, price, size, asof, order_key);
+         _myorder.market_order = true;
+      }
+   }
+}

src/client/spark_historical/Algos/Native/AlgoTestNativeMethods.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace bts.history
+{
+   static class AlgoTestNativeMethods
+   {
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_launch_callback(byte* OrderDefinition, ulong length, uint order_key, double price, uint size);
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_update_callback(byte* Update, ulong length, uint order_key);
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_cancel_callback(uint order_key);
+
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_status_callback(byte* OrderState, ulong length, uint order_key);
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_fill_callback(byte* Fill, ulong length, uint order_key);
+
+      [DllImport("libspark")]
+      internal static extern IntPtr try_create_algo(api.orderman.CustomOrderDefinition.CustomOneofCase algo_type);
+
+      [DllImport("libspark")]
+      internal static extern IntPtr create_algo_driver(byte[] Security, int seclength,
+                                                       byte[] OrderDefinition, int deflength, 
+                                                       child_launch_callback launch_cb,
+                                                       child_update_callback update_cb,
+                                                       child_cancel_callback cancel_cb,
+                                                       
+                                                       child_status_callback stat_cb,
+                                                       child_fill_callback fill_cb);
+      [DllImport("libspark")]
+      internal static extern void destroy_algo_driver(IntPtr algo_driver);
+      [DllImport("libspark")]
+      internal static extern void destroy_algo(IntPtr algo_driver);
+
+      [DllImport("libspark")]
+      internal static extern void launch_algo(IntPtr algo_driver, uint order_key, double price, uint size);
+
+      [DllImport("libspark")]
+      internal static extern void update_price(IntPtr algo_driver, double price);
+      [DllImport("libspark")]
+      internal static extern void update_size(IntPtr algo_driver, uint size);
+      [DllImport("libspark")]
+      internal static extern void cancel(IntPtr algo_driver);
+
+
+      [DllImport("libspark")]
+      internal static extern void advance_time(IntPtr algo_driver, ulong microsecond, uint steps);
+      [DllImport("libspark")]
+      internal static extern void update_book(IntPtr algo_driver, uint security_id, byte[] BookMessage, int length);
+      [DllImport("libspark")]
+      internal static extern void update_ticker(IntPtr algo_driver, uint security_id, byte[] TradeMessage, int length);
+
+      [DllImport("libspark")]
+      internal static extern void fill(IntPtr algo_driver, uint order_key, double price, uint quantity);
+
+
+      //see AlgoTestDriver.h
+      internal enum AlgoDriverState
+      {
+         CREATED,
+         // before_Launch() ->
+         READY,
+         // launch() ->
+         RUNNING,
+         CANCELED,
+         TERMINATED,
+         COMPLETED
+      };
+      
+      [DllImport("libspark")]
+      internal static extern AlgoDriverState algo_driver_state(IntPtr algo_driver);
+      
+      internal static bool state_is_terminal(AlgoDriverState st)
+      {
+         return st == AlgoDriverState.CANCELED || st == AlgoDriverState.TERMINATED || st == AlgoDriverState.COMPLETED;
+      }
+   }
+}

src/client/spark_historical/Algos/Native/NativeAlgo.cs

@@ -0,0 +1,321 @@
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using Google.Protobuf;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   class NativeAlgo : CompositeAlgo<AlgoBase>
+   {
+      readonly Dictionary<ulong, api.orderman.OrderState> _order_key_vs_state = new Dictionary<ulong, api.orderman.OrderState>();
+      api.Side _myside;
+      double _myprice;
+      uint _mysize;
+      Timestamp _myasof;
+
+      api.Security _sec;
+      readonly api.orderman.OrderDefinition _def;      
+      IntPtr _proxy;
+
+      readonly AlgoTestNativeMethods.child_launch_callback _launchcb;
+      readonly AlgoTestNativeMethods.child_update_callback _updatecb;
+      readonly AlgoTestNativeMethods.child_cancel_callback _cancelcb;
+      
+      readonly AlgoTestNativeMethods.child_status_callback _statcb;
+      readonly AlgoTestNativeMethods.child_fill_callback _fillcb;
+
+      MessageParser<api.orderman.OrderDefinition> _defparser = new MessageParser<api.orderman.OrderDefinition>(() => new api.orderman.OrderDefinition());
+      MessageParser<api.orderman.Update> _updateparser = new MessageParser<api.orderman.Update>(() => new api.orderman.Update());
+      MessageParser<api.orderman.OrderState> _stateparser = new MessageParser<api.orderman.OrderState>(() => new api.orderman.OrderState());
+      MessageParser<api.orderman.Fill> _fillparser = new MessageParser<api.orderman.Fill>(() => new api.orderman.Fill());
+      
+      public NativeAlgo(IFillModel fill_model, Bucket bucket, string[] security_list, 
+                        api.orderman.OrderDefinition def) : base(fill_model, bucket, security_list) 
+      {
+         _def = def;
+         unsafe
+         {
+            _launchcb = 
+               (byte * OrderDefinition, ulong length, uint order_key, double price, uint size) => { on_native_launch(OrderDefinition, length, order_key, price, size); } ;
+            _updatecb =
+               (byte* Update, ulong length, uint order_key) => { on_native_update(Update, length, order_key); };
+            _cancelcb =
+               (uint order_key) => { on_native_cancel(order_key); };
+
+            _statcb = 
+               (byte * OrderState, ulong length, uint order_key) => { on_native_status(OrderState, length, order_key); } ;
+            _fillcb = 
+               (byte * Fill, ulong length, uint order_key) => { on_native_fill(Fill, length, order_key); } ;
+         }
+         init_proxy(security_list);
+      }
+      
+      void init_proxy(string[] security_list)
+      {
+         Log.assert(security_list != null && security_list.Length == 1);
+         _sec = new api.Security { Id = _def.SecurityKey, Symbol = security_list[0] };
+         if (_sec.Symbol.Contains(':') || _sec.Symbol.Contains('-'))
+            _sec.Spread = new api.SpreadContract() { Normalized = true };
+         else
+            _sec.Future = new api.FutureContract();
+         reset();
+      }
+      
+      public override void reset()
+      {
+         unsafe
+         {
+            if(_proxy != IntPtr.Zero)
+            {
+               AlgoTestNativeMethods.destroy_algo_driver(_proxy);
+               _order_key_vs_state.Clear();
+            }
+            
+            var secser = _sec.ToByteArray();
+            var defser = _def.ToByteArray();
+            _proxy = AlgoTestNativeMethods.create_algo_driver(secser, secser.Length, defser, defser.Length,
+                                                              _launchcb, _updatecb, _cancelcb,
+                                                              _statcb, _fillcb);
+            Log.assert(_proxy != IntPtr.Zero);
+         }
+      }
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         Log.assert(security_list != null && security_list.Length == 1);
+         Log.assert(side == _def.Side);         
+         _myside = side;         
+         _myprice = price;
+         _mysize = size;
+         _myasof = asof;
+         AlgoTestNativeMethods.launch_algo(_proxy, (uint)order_key, price, size);
+      }
+
+      Handle hookup_child(AlgoBase child, string[] security_list, api.Side side, 
+                          double price, uint size, Timestamp asof, uint order_key) 
+      {
+         Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => 
+            on_child_fill(h, f));
+         Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) =>
+         {
+            on_child_status(h, s);
+            if (s.is_terminal())
+               _children.Remove(h);
+         });
+         var handle = this.hookup_child(child, security_list, side, price, size, asof, of, os, order_key);
+         _children.Add(handle);
+         return handle;
+      }
+
+      // callback when this child launches other guys
+      unsafe void on_native_launch(byte* OrderDefinition, ulong length, uint order_key, double price, uint size)
+      {
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)OrderDefinition, arr, 0, (int)length);
+
+         api.orderman.OrderDefinition def = _defparser.ParseFrom(arr);         
+         switch(def.CustomDef.CustomCase)
+         {            
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Market:
+               {
+                  var child = AlgoBase.algo_factory(typeof(MarketOrder), _fill_model, bucket, this.security_list);
+                  hookup_child(child, this.security_list, def.Side, price, size, _myasof, order_key);
+               }
+               break;
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Limit:
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.GlbxIceberg:
+               {
+                  var child = AlgoBase.algo_factory(typeof(LimitOrder), _fill_model, bucket, this.security_list);
+                  hookup_child(child, this.security_list, def.Side, price, size, _myasof, order_key);
+               }
+               break;
+
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Stop:
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.StopLimit:
+               Log.assert(false);
+               break;
+
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Dynamic:
+               Log.assert(false);
+               break;
+            default:
+               {
+                  IntPtr native = AlgoTestNativeMethods.try_create_algo(def.CustomDef.CustomCase);
+                  if(native != IntPtr.Zero)
+                  {
+                     AlgoTestNativeMethods.destroy_algo(native); //TODO, boolean
+                     var child = new NativeAlgo(_fill_model, bucket, this.security_list, def);
+                     hookup_child(child, this.security_list, def.Side, price, size, _myasof, order_key);
+                     // var handle = this.hookup_child(child, security_list, def.Side, price, size, _myasof, null, null, order_key);
+                     // _children.Add(handle);
+                  }               
+               }
+            break;
+         }
+      }
+
+      unsafe void on_native_update(byte* Update, ulong length, uint order_key)
+      {
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)Update, arr, 0, (int)length);
+         api.orderman.Update upd = _updateparser.ParseFrom(arr);
+
+         queue_reaction( (md, secdesc, par, timestamp) => 
+         {
+            (uint order_key_, api.orderman.Update upd_) = ( ValueTuple<uint, api.orderman.Update> )par;
+            var f = _children.FirstOrDefault(h => h.order_key == order_key_);
+            if(f != null)
+               f.update(timestamp, upd_.Price, upd_.Quantity);
+            
+         }, _lastmd, _sec.Symbol, (order_key, upd) );
+      }
+
+      unsafe void on_native_cancel(uint order_key)
+      {
+         queue_reaction((md, secdesc, par, timestamp) =>
+        {
+           var ok = (uint)par;
+           var f = _children.FirstOrDefault(h => h.order_key == ok);
+           if(f != null)
+              f.cancel(timestamp);
+        }, _lastmd, _sec.Symbol, order_key);         
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _order_key_vs_state[child.order_key] = stat.state;
+         process_status(stat.state);
+      }
+
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+         AlgoTestNativeMethods.fill(_proxy, (uint)fill.order.order_id, fill.price, (uint)fill.size);
+      }
+
+      unsafe void on_native_status(byte* OrderState, ulong length, uint order_key)
+      {
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)OrderState, arr, 0, (int)length);
+
+         api.orderman.OrderState curr = _stateparser.ParseFrom(arr);
+         _order_key_vs_state[order_key] = curr;
+        process_status(curr);
+      }
+      
+
+      private void process_status(api.orderman.OrderState curr)
+      {
+         bool done = is_completed();
+         api.orderman.OrderState aggstate = new api.orderman.OrderState
+         {
+            Status = done ? api.orderman.OrderStatus.Completed : curr.Status,
+            Price = curr.Price,
+            Quantity = curr.Quantity,
+            AvgPrice = _order_key_vs_state.Values.Average(x => x.Price),
+            FilledQuantity = _order_key_vs_state.Values.Sum(x => x.FilledQuantity),
+            LastChange = curr.LastChange,
+            LastMsg = curr.LastMsg,
+            LastPrice = curr.LastPrice
+         };
+         OrderStatus agg = new OrderStatus(curr.LastChange);
+         agg.security_desc = this.security_list[0];
+         agg.side =_myside;
+         agg.state = aggstate;
+         if (this.on_status != null)
+            this.on_status(agg);
+      }
+
+      unsafe void on_native_fill(byte* Fill, ulong length, uint order_key)
+      {         
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)Fill, arr, 0, (int)length);
+
+         api.orderman.Fill fill = _fillparser.ParseFrom(arr);
+         //this method is called as a result of us sending a fill down in AlgoTestNativeMethods.fill so we do nothing
+                  
+      }
+      
+      ~NativeAlgo()
+      {
+         AlgoTestNativeMethods.destroy_algo_driver(_proxy);
+      }
+
+      public override bool is_completed()
+      {
+         bool done = false;
+         if(_order_key_vs_state.Count > 0)
+         {
+            var astate = AlgoTestNativeMethods.algo_driver_state(_proxy);
+            done = AlgoTestNativeMethods.state_is_terminal(astate);
+            foreach(var st in _order_key_vs_state.Values)
+               done &= OrderStatus.is_terminal(st);
+         }
+         return done;
+      }
+
+      public override object report()
+      {         
+         return $"Native";
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         base.do_update(asof, price, size);
+         AlgoTestNativeMethods.update_price(_proxy, price);
+         if(size != uint.MinValue)
+         {
+            AlgoTestNativeMethods.update_size(_proxy, size);
+         }
+      }
+      
+      protected override void do_cancel(Timestamp asof)
+      {
+         base.do_cancel(asof);
+         AlgoTestNativeMethods.cancel(_proxy);         
+      }
+
+      MarketDataIncrementalRefresh _lastmd;
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                      MarketDataIncrementalRefresh packet)
+      {
+         security_key = _def.SecurityKey; //to fake this up for the AlgoTestDriver
+         var book = current_book.ToByteArray();
+         AlgoTestNativeMethods.update_book(_proxy, security_key, book, book.Length);
+
+         advance_time(packet);
+      }
+
+      protected override void on_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                                       MarketDataIncrementalRefresh packet)
+      {
+         security_key = _def.SecurityKey;
+         foreach(var trade in trades)
+         {
+            var tr = trade.ToByteArray();
+            AlgoTestNativeMethods.update_ticker(_proxy, security_key, tr, tr.Length);
+         }
+
+         advance_time(packet);            
+      }
+
+      void advance_time(MarketDataIncrementalRefresh packet)
+      {
+         if (_lastmd != null)
+         {
+            var dt = packet.timestamp() - _lastmd.timestamp();
+            if (dt.Seconds >= 0 && dt.Nanos > 0)
+            {
+               ulong us = (ulong)dt.Seconds * 1000_000ul + ((ulong)dt.Nanos / 1000ul);
+               AlgoTestNativeMethods.advance_time(_proxy, us, 1);
+            }
+         }
+         _lastmd = packet;
+      }            
+   }
+}

src/client/spark_historical/Algos/NullAlgo.cs

@@ -0,0 +1,22 @@
+
+using System;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   class NullAlgo : AlgoBase
+   {
+      protected NullAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+      public override bool is_completed() => true;
+
+      public override object report() { return string.Empty; }
+
+      protected override void do_launch(string[] security_list, Side side, double price, uint size, Timestamp asof, ulong order_key) { }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue) { }
+
+      protected override void do_cancel(Timestamp asof) { }
+   }
+}

src/client/spark_historical/Algos/RoundTripAlgo.cs

@@ -0,0 +1,211 @@
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   class RoundTripAlgo<Child, Hedge> : CompositeHedgeAlgo<Child, Hedge>
+      where Child : AlgoBase
+      where Hedge : AlgoBase
+   {
+      enum AlgoState
+      {
+         Begin, Quoting, Done
+      }
+
+      //algo state
+      string[] _security_list;
+      AlgoState _state;
+      uint _mysize;
+      
+      SortedList<string, co2.BookMessage> _current_book = new SortedList<string, co2.BookMessage>();
+      SortedList<string, co2.BookMessage> _working_book = new SortedList<string, co2.BookMessage>();
+      OrderStatus _buystatus, _sellstatus;
+      Handle _buychild;
+      Handle _sellchild;
+      
+      private MarketDataIncrementalRefresh _packet;
+
+      protected RoundTripAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+      //algo parameters      
+      public double lean_factor { get; set; } = 1.0;      
+      public int profit_ticks { get; set; } = 1;      
+      public double[] tick_size { get; set; }
+      public bool[] should_quote { get; set; } 
+      public double[] hedge_factors { get; set; }
+
+      SortedList<string, double> _tick_size = new SortedList<string, double>();
+      SortedList<string, double> _hedge_factors = new SortedList<string, double>();
+      string _quote_leg;
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _security_list = security_list;    
+         for(int i = 0; i < _security_list.Length; i++)
+         {
+            _current_book[_security_list[i]] = null;
+            _working_book[_security_list[i]] = null;
+            _tick_size[_security_list[i]] = tick_size[i];
+            _hedge_factors[_security_list[i]] = hedge_factors[i];
+            _quote_leg = _security_list[i]; //last
+         }     
+         _state = AlgoState.Begin;
+         _mysize = size;
+         _buychild = null;
+         _sellchild = null;
+         _buystatus = new OrderStatus(asof);
+         _sellstatus = new OrderStatus(asof);
+      }
+
+      public override bool is_completed()
+      {
+         return base.is_completed() && _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         var b = base.report();
+         string qtleg = _security_list.Last();
+         var buypiq = _fill_model.get_my_piq(qtleg, _buychild?.order_key ?? 0);
+         var sellpiq = _fill_model.get_my_piq(qtleg, _sellchild?.order_key ?? 0);
+         return $"{b} Quoting {_quote_leg} (bsz,bpx,bpiq ssz,spx,spiq) = ( {_buystatus?.state?.Quantity},{_buystatus?.state?.Price},{buypiq} at {_sellstatus?.state?.Quantity},{_sellstatus?.state?.Price},{sellpiq} )";
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         if(stat.side == api.Side.Buy)
+         {
+            _buystatus = stat;
+            if(stat.is_terminal())
+               _sellchild?.cancel(_buystatus.last_change);
+         }
+         else 
+         {
+            _sellstatus = stat;
+            if(stat.is_terminal())
+               _buychild?.cancel(_sellstatus.last_change);
+         }
+         //hedges are looked after elsewhere         
+         if( (_buystatus?.is_terminal() ?? true) && (_sellstatus?.is_terminal() ?? true) )
+            _state = AlgoState.Done;
+      }
+      
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+         var ff = factor_side(this._hedge_factors[_quote_leg]);
+         bool invert = ff != fill.order.side;
+
+         foreach(var sec in _security_list)
+         {
+            if(sec == _quote_leg)
+               continue;
+                     
+            var hedgeside = factor_side(this._hedge_factors[sec], invert);
+               
+            uint size = (uint)(fill.size * Math.Abs(this._hedge_factors[sec]));
+            double price = (hedgeside == api.Side.Buy) ? _current_book[sec].ask_at(0) : _current_book[sec].bid_at(0);
+            queue_hedge(_packet, sec, 
+                       (new[] { sec }, hedgeside, price, size));
+         }
+      }
+
+      api.Side factor_side(double mult, bool invert = false)
+      {
+         api.Side sd = api.Side.Undefined;
+         if(mult >= 0.0)
+            sd = api.Side.Buy;
+         else
+            sd = api.Side.Sell;
+         if(invert)
+            sd = Utils.opposite_side(sd);
+         return sd;
+      }
+      
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         _packet = packet;
+         _current_book[security_desc] = current_book;
+         _working_book[security_desc] = working_book;
+         var look_book = _current_book;
+         switch (_state)
+         {
+            case AlgoState.Begin:
+               {                  
+                  if(!_current_book.Values.Any(x => x == null) && !_working_book.Values.Any(x => x == null))
+                     _state = AlgoState.Quoting;
+               }
+               break;
+            case AlgoState.Quoting:
+               {
+                  uint sz = 0;
+                  foreach(var invert in new[] { false, true})
+                  {
+                     double qt = 0.0;  
+                     api.Side quoteside = factor_side(this._hedge_factors[_quote_leg], invert);
+                     foreach(var sec in _security_list)
+                     {
+                        var book = look_book[sec];                        
+                        if (sec != _quote_leg)
+                        {                
+                           var hedge_side = factor_side(this._hedge_factors[sec], invert); 
+                           var cross_side = Utils.opposite_side(hedge_side);
+                           qt += this._hedge_factors[sec] * book.side_at(cross_side, 0);
+                           sz = Math.Min(_mysize, (uint) (book.sidesize_at(cross_side, 0) / Math.Abs(this._hedge_factors[sec])) ); 
+                        }
+                        else
+                        {
+                           if(quoteside == api.Side.Sell)
+                           {
+                              qt += profit_ticks*_tick_size[sec];
+                              //qt = Math.Max(qt, current_book.ask_at(0));
+                           }
+                           else
+                           {
+                              qt -= profit_ticks*_tick_size[sec];
+                              //qt = Math.Min(qt, current_book.bid_at(0));
+                           }
+                        }
+                     }
+                     var param = (quoteside, qt, sz);
+                     if (!double.IsNaN(qt) && sz > 0 && security_desc == _quote_leg)
+                     {
+                        if(quoteside == api.Side.Buy && (_buystatus?.state?.Price != qt || _buystatus?.state?.Quantity != sz) && !_buystatus.is_terminal())
+                           queue_reaction(quote_function, packet, _quote_leg, param);
+                        else if(quoteside == api.Side.Sell && (_sellstatus?.state?.Price != qt || _sellstatus?.state?.Quantity != sz) && !_sellstatus.is_terminal())
+                           queue_reaction(quote_function, packet, _quote_leg, param);
+                     }
+                  }
+               }               
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+      void quote_function(MarketDataIncrementalRefresh packet, string qtleg, object param, Timestamp asof)
+      {
+         (api.Side quoteside, double qt, uint sz) tup = (ValueTuple<api.Side, double, uint>) param;
+         if (tup.quoteside == api.Side.Buy)
+         {
+            if (_buychild == null)
+               _buychild = this.activate<Child>(new[] { qtleg }, tup.quoteside, tup.qt, tup.sz, asof);
+            else 
+               _buychild.update(asof, tup.qt, tup.sz);
+         }         
+         else 
+         {
+            if (_sellchild == null)
+               _sellchild = this.activate<Child>(new[] { qtleg }, tup.quoteside, tup.qt, tup.sz, asof);
+            else 
+               _sellchild.update(asof, tup.qt, tup.sz);            
+         }
+      }
+   }
+}

src/client/spark_historical/Algos/TopAlgo.cs

@@ -0,0 +1,95 @@
+
+using System;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   class TopAlgo<Child> : CompositeAlgo<Child>
+      where Child : AlgoBase
+   {
+      enum AlgoState
+      {
+         WaitingToCancel, CancelRequested, Done
+      }
+
+      //algo state
+      AlgoState _state;
+      OrderStatus _status;
+      api.Side _myside;
+      uint _mysize;
+      
+      co2.BookMessage _currentbook;
+      int _topcount;
+
+      protected TopAlgo(IFillModel fill_model, Bucket bucket, string[] security_list) : base(fill_model, bucket, security_list) { }
+      //algo parameters      
+      public double top_factor { get; set; } = 1.0;
+      public uint top_quantity { get; set; } = 0;
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _state = AlgoState.WaitingToCancel;
+         _myside = side;
+         _mysize = size;
+         _child = this.activate<Child>(security_list, side, price, size, asof);
+      }      
+
+      public override bool is_completed()
+      {
+         return _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         var b = base.report();
+         return $"{b} Cancels { _topcount }";
+      }
+      
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _status = stat;
+         if (stat.is_terminal())
+            _state = AlgoState.Done;
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         _currentbook = current_book;
+         switch (_state)
+         {
+            case AlgoState.WaitingToCancel: 
+               {
+                  bool top = _currentbook.side_at(_myside, 0) == _status?.state?.Price;
+                  var thresh = (this.top_quantity > 0 ? this.top_quantity : this.top_factor * _mysize);
+                  if(top && _currentbook.sidesize_at(_myside, 0) < thresh)
+                  {
+                     queue_reaction(cancel_top_function , packet, 
+                                    security_desc, _currentbook.side_at(_myside, 0));                     
+                     _state = AlgoState.CancelRequested;
+                     _topcount++;
+                  }
+               }               
+               break;
+            case AlgoState.CancelRequested:
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+
+      void cancel_top_function(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof)
+      {
+         Log.assert(_state == AlgoState.CancelRequested || _state == AlgoState.Done);
+         Log.Testing.info("cancel_top_function " + this.bucket.UpperBound);
+
+         _child.cancel(asof);
+         _state = AlgoState.Done;
+      }
+   }
+}

src/client/spark_historical/Algos/TopLevelAlgos.cs

@@ -0,0 +1,73 @@
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   
+   class JoinCoverLean : CoverAlgo<JoinAlgo<TopAlgo<LimitOrder>>, LeanAlgo<LimitOrder>>
+   {
+      protected JoinCoverLean(IFillModel fill_model, Bucket bucket, string[] security_list) 
+         : base(fill_model, bucket, security_list) 
+      {
+      }
+   }
+   
+   class PegCoverLean : CoverAlgo<JoinAlgo<TopAlgo<LimitOrder>>, LeanAlgo<LimitOrder>>
+   {
+      protected PegCoverLean(IFillModel fill_model, Bucket bucket, string[] security_list) 
+         : base(fill_model, bucket, security_list) 
+      {
+         this.set_param(nameof(JoinAlgo<LimitOrder>.initial_pile), true.ToString());
+      }
+   }
+   class RoundTrip : RoundTripAlgo<LimitOrder, LeanAlgo<LimitOrder>>
+   {
+      protected RoundTrip(IFillModel fill_model, Bucket bucket, string[] security_list) 
+         : base(fill_model, bucket, security_list) 
+      {          
+      }
+   }
+   
+   class TWAP : NativeAlgo
+   {
+      protected TWAP(IFillModel fill_model, Bucket bucket, string[] security_list) 
+         : base(fill_model, bucket, security_list, build_def())
+      {
+      }      
+      static api.orderman.OrderDefinition build_def()
+      {
+         api.orderman.OrderDefinition def = new api.orderman.OrderDefinition();
+         def.SecurityKey = 51880u;
+         def.Side = api.Side.Buy;
+         def.CustomDef = new api.orderman.CustomOrderDefinition { Twap = new api.orderman.TWAPOrderDefinition() };
+         def.CustomDef.Twap.Duration = new api.orderman.TimeDuration() { DurationInMinutes = 6, IntervalInSeconds = 10 } ;
+         def.CustomDef.Twap.Aggression = new api.orderman.OrderAggression() { AllowedScheduleSlippage = 1 };
+         
+         def.CustomDef.Twap.TacticalChild = new api.orderman.CustomOrderDefinition
+         {
+            Peg = new api.orderman.PegOrderDefinition()
+            {
+               Child = new api.orderman.CustomOrderDefinition { Limit = new api.orderman.LimitOrderDefinition() } ,
+               TickOffset = 0,
+            }
+         };      
+         
+         def.CustomDef.Twap.AggressiveChild = new api.orderman.CustomOrderDefinition
+         {
+            Lean = new api.orderman.LeanOrderDefinition()
+            {
+               Child = new api.orderman.CustomOrderDefinition { Limit = new api.orderman.LimitOrderDefinition() },
+               CrossThreshold = new spark.QuantityMeasure { Ratio = 3.0 } ,
+               TradeTowardBehavior = api.orderman.LeanOrderDefinition.Types.OnToward.NoChange,
+               MaxChaseTicks = 10,
+            }
+         };
+         return def;
+      }
+   }
+}

src/client/spark_historical/Algos/TradeStats.cs

@@ -0,0 +1,33 @@
+using System;
+
+namespace bts.history
+{
+   internal class TradeStats
+   {
+      public DateTime security_timestamp { get; set; }
+      public string security_desc { get; set; }
+      public double our_average_buy_piq;
+      public double our_average_buy_q_len;
+      public double our_average_buy_price { get; set; }
+
+      public double our_average_sell_piq;
+      public double our_average_sell_q_len;
+      public double our_average_sell_price { get; set; }
+
+      public int our_total_buys { get; set; }
+      public int our_total_sells { get; set; }
+      public int our_total_volume { get; set; }
+      public double latest_pnl { get; set; }
+
+      public uint total_exchange_bought { get; set; }
+      public uint total_exchange_sold { get; set; }
+      public uint total_exchange_volume { get; set; }
+
+      public uint best_bid_size { get; set; }
+      public double best_bid { get; set; }
+      public double best_ask { get; set; }
+      public uint best_ask_size { get; set; }
+      public double last_trade_price { get; set; }
+      public uint last_trade_size { get; set; }
+   }
+}

src/client/spark_historical/Analyzers/AlgoAnalyzer.cs

@@ -0,0 +1,147 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   sealed class AlgoAnalyzer : IAnalyzer
+   {
+      const double _max_nanos = 500_000.0;
+      readonly DistributionEstimator _dist = new DistributionEstimator();
+      readonly Histogram _hist = new Histogram();
+
+      readonly Dictionary<Bucket, AlgoStatsWrapper> _algos = new Dictionary<Bucket, AlgoStatsWrapper>();
+      readonly SortedDictionary<string, IOrderBook> _shared_order_book = new SortedDictionary<string, IOrderBook>();
+      
+      string[] _security_list;    
+      public AlgoAnalyzer(HistoricalDataOptions options, System.Type algotype, string algo_params)
+      {
+         Log.assert(!string.IsNullOrEmpty(options.algo_security));
+         _security_list = new string[] { options.algo_security };
+         if (options.algo_security.Contains(';'))
+            _security_list = options.algo_security.Split(';');
+            
+         foreach(var sec in _security_list)
+         {
+            _shared_order_book[sec] = new OrderBook(sec);
+         }         
+         foreach (var hist in new[] { _hist } )
+         {
+            hist.AddBucket(new Bucket(0, 1000.0.Decrement()));
+            // hist.AddBucket(new Bucket(1000, 5000.0.Decrement()));
+            // hist.AddBucket(new Bucket(5000, 10000.0.Decrement()));
+            // hist.AddBucket(new Bucket(10000, 20000.0.Decrement()));
+            // hist.AddBucket(new Bucket(20000, 30000.0.Decrement()));
+            // hist.AddBucket(new Bucket(30000, 40000.0.Decrement()));
+            // hist.AddBucket(new Bucket(40000, 50000.0.Decrement()));
+            // hist.AddBucket(new Bucket(50000, 80000.0.Decrement()));
+            // hist.AddBucket(new Bucket(80000, (_max_nanos/2).Decrement()));
+            // hist.AddBucket(new Bucket(_max_nanos/2,  _max_nanos.Decrement()));
+            hist.LazySort();
+            for(int i = 0; i < hist.BucketCount; i++)
+            {
+               var buck = hist[i];
+               var fill_model = new FIFOModel(_shared_order_book);
+               var algo = AlgoBase.algo_factory(algotype, fill_model, buck, _security_list, algo_params);
+               _algos.Add(buck, new AlgoStatsWrapper( algo, _shared_order_book, options ) );
+               activate_algo(algo);               
+            }
+         }
+      }   
+      
+      static void activate_algo(AlgoBase algo) //relies on CompositeAlgo to know I'm trying to activate my same type
+      {
+         api.Side side = api.Side.Buy; double price = 0.0; uint size = 0;
+         if (algo.algo_params.TryGetValue(nameof(side), out string sside))
+            side = (api.Side) System.Enum.Parse(typeof(api.Side), sside, true);
+            
+         if (algo.algo_params.TryGetValue(nameof(price), out string sprice))
+            price = double.Parse(sprice);
+         if (algo.algo_params.TryGetValue(nameof(size), out string ssize))
+            size = uint.Parse(ssize);
+         AlgoBase.activate_self(algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow));
+      }
+
+      public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
+      {
+         process_md(md, debug_fix);
+      }
+      
+      int _samples;
+      MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh();
+      public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
+      {
+         //get the books right
+         process_md(md, debug_fix);
+         
+         foreach(var ob in _shared_order_book)
+         {
+            //the last known good book from the last quote event indicator
+            co2.BookMessage working_book = ob.Value.working_book;
+            //the working book, with all msgs between working_book and now 
+            co2.BookMessage current_book = ob.Value.current_book;                  
+            List<co2.TradeMessage> trades = ((OrderBook)ob.Value).process_market_trades(md, (mde, ore) => ob.Key == mde.Symbol);
+            foreach (var algo in _algos)
+            {
+               //check the market data for a 'catalyst' with (potential) latency
+               algo.Value.algo.process_book(ob.Key, ob.Value.security_key, current_book, working_book, md);
+               algo.Value.algo.process_trades(ob.Key, ob.Value.security_key, trades, md);
+               algo.Value.keep_md_stats(ob.Value);
+            }
+         }
+         foreach (var algo in _algos)
+         {
+            //maybe do the action if enough time has passed, update order book with my order 
+            algo.Value.algo.execute_with_latency(md);
+         }
+         match_our_trades(md);
+
+         if (_samples++ % 1000 == 0)
+         {
+            var dt = md.timestamp().ToDateTime();
+            Console.WriteLine($"{_security_list.Aggregate( (x,y) => x + " " + y)} {_samples } Trade date  {dt.ToShortDateString()} {dt.ToLongTimeString()}");
+            var first = _algos.First().Value; 
+            var last = _algos.Last().Value; 
+            var lf = Environment.NewLine;
+            var group = new []{"best","total","our", "security"};
+            Console.WriteLine($"{first.GetType().Name} Best Stats: {_algos.First().Key.UpperBound} {first.algo.report()}{lf}{Stringizer.all_to_string(first.stats, group)} ");            
+            Console.WriteLine($"{last.GetType().Name} Worst Stats: {_algos.Last().Key.UpperBound} {last.algo.report()}{lf}{Stringizer.all_to_string(last.stats, group)}");
+            Console.WriteLine($"{ _hist.ToString() }");
+            Console.WriteLine($"-----------------------------------------------------------------------------");
+         }
+         foreach (var algo in _algos)
+         {            
+            if (algo.Value.algo.is_completed())
+            {
+               Log.Testing.info("cleanup_function " + algo.Key.UpperBound);
+               var pnl = algo.Value.compute_pnl();
+               algo.Value.reset();
+               algo.Key.Count += pnl;
+               activate_algo(algo.Value.algo);
+            }            
+         }    
+         _lastpacket = md;     
+      }
+      void process_md(MarketDataIncrementalRefresh md, string debug_fix = null)
+      {
+         foreach (var ob in _shared_order_book)
+         {
+            OrderBook book = (OrderBook)ob.Value;
+            book.process_md(md, 
+               (mde, ord) => mde.Symbol == ob.Key, 
+            debug_fix);
+         }
+      }
+      void match_our_trades(MarketDataIncrementalRefresh md)
+      {
+         foreach (var ob in _shared_order_book)
+         {
+            OrderBook book = (OrderBook)ob.Value;
+            book.match_crosses(md.timestamp());
+            book.match_our_trades(md);            
+         }
+      }      
+   }
+}

src/client/spark_historical/Analyzers/BookAnalyzer.cs

@@ -0,0 +1,113 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System.Text;
+using System.IO;
+using mbts.config;
+using bts.config;
+
+namespace bts.history
+{
+   class BookAnalyzer : IAnalyzer
+   {
+      class SecData{
+         public uint n_updates = 0;
+         public System.IO.FileStream str;
+         public SecData (System.IO.FileStream str_a){
+            str = str_a;
+         }
+         public void inc_count(){
+            n_updates++;
+         }
+      }
+      private readonly Dictionary<string, co2.BookMessage> _books = 
+                     new Dictionary<string, co2.BookMessage>();
+      private readonly string _book_path;
+      IDictionary<string, SecData> _book_stream_map = 
+                              new Dictionary<string, SecData>();
+
+      public BookAnalyzer(string symbol,
+                          HistoricalDataOptions options) {
+         _book_path = get_book_path(symbol, DateTime.Parse(options.replaystart));
+         System.IO.Directory.CreateDirectory(_book_path);
+      }
+
+      public void pre_analyze(MarketDataIncrementalRefresh md, 
+                              string debug_fix = null)
+      {}
+
+      public void analyze(MarketDataIncrementalRefresh md,
+                          string debug_fix = null)
+      {
+         foreach(var entry in md.MdEntries)
+         {
+            if(entry.MdEntryType == MDEntryType.Bid || 
+               entry.MdEntryType == MDEntryType.Offer)
+            {
+               if(!_books.TryGetValue(entry.Symbol, out co2.BookMessage book))
+               {
+                  book = new co2.BookMessage();
+                  Utils.init_book(book);
+                  _books[entry.Symbol] = book;
+               }
+               Utils.keep_book(book, entry);
+               if (Utils.quote_event_done(md))
+                  on_new_book(book, entry, md);
+            }
+            // else if (entry.MdEntryType == MDEntryType.Trade)
+            //    handle trades here
+         }
+      }
+
+      public static string get_book_path (string symbol,
+                                          DateTime date){
+         return String.Format(@"{0}{1}/{2}/{3}/{4}/{5}/",
+                           Configuration.get<bts.config.System>().data_path,
+                           "trade_analysis",
+                           "market_data",
+                           date.ToShortDateString().Replace("/","_"),
+                           "books",
+                           symbol);
+      }
+
+      public static string get_book_path (string symbol,
+                                          string sec_name,
+                                          DateTime date){         
+         return String.Format(@"{0}{1}",
+                              get_book_path(symbol, date),
+                              sec_name);
+      }
+      
+      public static string get_algo_input_path (string algo_name){
+         return String.Format(@"{0}{1}/{2}/{3}/",
+                           Configuration.get<bts.config.System>().data_path,
+                           "trade_analysis",
+                           "algos",
+                           "algo_name");
+      }
+
+      private void on_new_book(co2.BookMessage book,
+                               MDEntry entry,
+                               MarketDataIncrementalRefresh md_inc){
+         ProcessedBook pb = new ProcessedBook();
+         pb.MsgSeqNum = md_inc.MsgSeqNum;
+         pb.SendingTime = md_inc.SendingTime;
+         pb.TransactTime = md_inc.TransactTime;         
+         pb.Security = entry.Symbol;
+         pb.Book = book;        
+         SecData s;
+         if(!_book_stream_map.TryGetValue(pb.Security, out s)){
+            string file_name = pb.Security;
+            file_name = file_name.Replace(":",".");
+            file_name = String.Format ("{0}{1}",_book_path,file_name);
+            var str = new FileStream(file_name, FileMode.OpenOrCreate);
+            s = new SecData (str);
+            _book_stream_map.Add(pb.Security,s);
+         }         
+         pb.BookSeqNum = s.n_updates;
+         s.inc_count();
+         Google.Protobuf.MessageExtensions.WriteDelimitedTo(pb,s.str);
+      }
+   }
+}

src/client/spark_historical/Analyzers/OrderBook.cs

@@ -0,0 +1,232 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.co2;
+using mbts.co2;
+
+using QueueType = System.Collections.Generic.SortedList<ulong, bts.history.OrderEntry>;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public delegate void OnMarketTrade(MarketDataIncrementalRefresh md);
+   interface IOrderBook
+   {
+      string security_desc { get; }
+      uint security_key { get; }
+      DateTime security_timestamp { get; }
+      BookMessage working_book { get; }
+      BookMessage current_book { get; }
+      
+      event OnMarketTrade on_market_trade;
+
+      uint total_exchange_volume { get; }
+      uint total_exchange_bought { get; }
+      uint total_exchange_sold { get; }
+      uint last_trade_size { get; }
+      double last_trade_price { get; }          
+
+      (int, int) get_my_piq(ulong orderid);
+      OrderEntry insert_my_order(OrderEntry order);
+      OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = uint.MinValue);
+      bool cancel_my_order(ulong my_order_id);
+      void validate_new_queue(AlgoBase algo, MarketDataIncrementalRefresh current_book_packet, Func<MDEntry, OrderIDEntry, bool> filter);
+      void match_our_trades(MarketDataIncrementalRefresh md);
+      void match_crosses(Timestamp ts);
+      MBOBook mbo_book {get;}
+   }
+   
+   class OrderBook : IOrderBook
+   {     
+      
+      public string security_desc { get; private set; }
+      public uint security_key { get; private set; }
+      
+      BookMessage _working_book = new BookMessage(), _current_book = new BookMessage();
+      public BookMessage current_book => _current_book;
+      public BookMessage working_book => _working_book;
+      public event OnMarketTrade on_market_trade;
+
+      public uint total_exchange_volume { get; private set; }
+      public uint total_exchange_bought { get; private set; }
+      public uint total_exchange_sold { get; private set; }
+      public uint last_trade_size { get; private set; }
+      public double last_trade_price { get; private set; }
+      public DateTime security_timestamp { get; private set; }
+      
+      readonly MBOBook _mbo;
+      public MBOBook mbo_book => _mbo;
+
+      public OrderBook(string security_desc)
+      {
+         this.security_desc = security_desc;
+         _mbo = new MBOBook(security_desc);
+         Utils.init_book(_working_book);
+         Utils.init_book(_current_book);
+      }
+
+      public void process_md(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter, string debug_fix = null)
+      {
+         bool bidoffer = false, trade = false;
+         foreach (var entry in md.MdEntries)
+         {
+            if(!filter(entry, null))
+               continue;
+            if (this.security_key == 0)
+               this.security_key = entry.SecurityId;
+            if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
+            {
+               Utils.keep_book(_working_book, entry);
+               if (Utils.quote_event_done(md)) //end of event https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Incremental+Refresh
+               {
+                  _current_book = _working_book.Clone();
+                  Utils.validate_book(_current_book, debug_fix);
+               }
+               bidoffer = true;
+            }
+            else if(entry.MdEntryType == MDEntryType.Trade)
+            {
+               trade = true;
+            }
+         }
+         this.security_timestamp = md.timestamp().ToDateTime();
+         Log.assert( !(bidoffer && trade), "Saw trade and bids / offers in same md packet");
+         if(bidoffer)
+            process_orders(md, filter);
+      }
+
+
+      void process_orders(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter)
+      {
+         foreach (var order in md.OrderIdEntries)
+         {
+            if (order.ReferenceId == 0)
+               continue;
+            // entries is indexed by 0 but ref is indexed by 1  :facepalm:
+            int mdidx = ((int)order.ReferenceId) - 1;
+            Log.assert((mdidx <= md.MdEntries.Count));
+
+            MDEntry mdval = md.MdEntries[mdidx];
+            if (!filter(mdval, order))
+               continue;
+            // //now merge this order stuff into an MDEntry because we can't do templates....no I don't miss c++
+            // mdval = mdval.Clone();
+            // mdval.LastQty = order.LastQty;
+            // mdval.MdDisplayQty = order.MdDisplayQty;
+            // mdval.MdOrderPriority = order.MdOrderPriority;
+            // mdval.OrderId = order.OrderId;
+            // mdval.MdUpdateAction = order.OrderUpdateAction;
+            // mdval.ReferenceId = order.ReferenceId;            
+            _mbo.process_one_mbo(md.timestamp(), MBOBook.make_mbo(mdval, order));
+         }
+         foreach (var mdval in md.MdEntries)
+         {
+            if (mdval.OrderId > 0 && md.OrderIdEntries.Count == 0)
+               _mbo.process_one_mbo(md.timestamp(), MBOBook.make_mbo(mdval, null));
+         }
+      }      
+
+      public List<TradeMessage> process_market_trades(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter)
+      {
+         bool has_trade = false;
+         List<TradeMessage> list = new List<TradeMessage>();
+         foreach (var entry in md.MdEntries)
+         {
+            if (!filter(entry, null))
+               continue;
+            bool trade = entry.MdEntryType == MDEntryType.Trade;
+            if (trade)
+            {
+               this.total_exchange_volume += entry.MdEntrySize;
+               if (entry.AggressorSide == 1)
+                  this.total_exchange_bought += entry.MdEntrySize;
+               else if (entry.AggressorSide == 2)
+                  this.total_exchange_sold += entry.MdEntrySize;
+
+               this.last_trade_size = entry.MdEntrySize;
+               this.last_trade_price = entry.MdEntryPx;
+
+               var tm = create_trade(md, entry);
+               list.Add(tm);
+            }
+            has_trade |= trade;
+         }
+         if (this.on_market_trade != null)
+            this.on_market_trade(md);
+         return list;
+      }
+      
+      static public TradeMessage create_trade(MarketDataIncrementalRefresh md, MDEntry entry)
+      {
+         Log.assert(entry.MdEntryType == MDEntryType.Trade);
+         var tm = new TradeMessage();
+         tm.Key = entry.SecurityId;
+         tm.Time = md.timestamp().Seconds;
+         var ip = IntPrice.from_double(entry.MdEntryPx);
+         tm.Mantissa = ip.mantissa;
+         tm.Exponent = ip.exponent;
+         tm.Size = entry.MdEntrySize;
+         tm.AggressorSide = (AggressorSide)entry.AggressorSide;
+         tm.OrderId = entry.OrderId;
+         return tm;
+      }
+
+      public (int, int) get_my_piq(ulong orderid)
+      {
+         return _mbo.get_my_piq(orderid);
+      }
+
+      public OrderEntry insert_my_order(OrderEntry order)
+      {
+         return _mbo.insert_my_order(order);
+      }
+
+      public OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = 0)
+      {
+         return _mbo.change_my_order(asof, order_id, price, size);
+      }
+
+      public bool cancel_my_order(ulong my_order_id)
+      {
+         return _mbo.cancel_my_order(my_order_id);
+      }
+
+      public void validate_new_queue(AlgoBase algo, MarketDataIncrementalRefresh current_book_packet, Func<MDEntry, OrderIDEntry, bool> filter)
+      {
+         _mbo.validate_new_queue(algo, current_book_packet, filter);
+      }
+      
+      public void match_our_trades(MarketDataIncrementalRefresh md)
+      {
+         foreach (var trade in md.OrderIdEntries)
+         {
+            MDEntry tradeentry = null;
+            if (trade.ReferenceId == 0)
+            {
+               tradeentry = md.MdEntries[0];
+               tradeentry.OrderId = trade.OrderId;
+               tradeentry.LastQty = trade.LastQty;
+               Log.assert(tradeentry.MdEntryType == MDEntryType.Trade);
+            }
+            else
+            {
+               int mdidx = ((int)trade.ReferenceId) - 1;
+               Log.assert((mdidx <= md.MdEntries.Count));
+               tradeentry = md.MdEntries[mdidx];
+               tradeentry.OrderId = trade.OrderId;
+               tradeentry.LastQty = trade.LastQty;
+               if (tradeentry.MdEntryType != MDEntryType.Trade)
+                  continue;
+            }
+            _mbo.process_one_trade(md.timestamp(), create_trade(md, tradeentry));
+         }         
+      }
+
+      public void match_crosses(Timestamp ts)
+      {
+         _mbo.match_crosses(ts);
+      }
+   }
+}

src/client/spark_historical/Analyzers/QueueAnalyzer.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   interface IAnalyzer 
+   {
+      //this data comes in before analysis time, but is useful to create complete books etc
+      void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null);
+      void analyze(MarketDataIncrementalRefresh md, string debug_fix = null);
+   }
+   
+   class QueueAnalyzer : IAnalyzer
+   {
+      const double _max_nanos = 200_000.0;      
+      readonly DistributionEstimator _turndist = new DistributionEstimator();
+      readonly DistributionEstimator _stackdist = new DistributionEstimator();
+      readonly MathNet.Numerics.Statistics.Histogram _turnhist = new MathNet.Numerics.Statistics.Histogram();
+      readonly MathNet.Numerics.Statistics.Histogram _stackhist = new MathNet.Numerics.Statistics.Histogram();
+
+      MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh();
+      Google.Protobuf.WellKnownTypes.Timestamp _laststacktime = new Google.Protobuf.WellKnownTypes.Timestamp();
+      Google.Protobuf.WellKnownTypes.Timestamp _lastturntime = new Google.Protobuf.WellKnownTypes.Timestamp();      
+      MDEntry _lastturn = new MDEntry();
+      MDEntry _lastentry = new MDEntry();
+      long _samples = 0;
+      long _stacksamples = 0;
+
+
+      readonly string _security_desc;
+      readonly MDEntryType _bid_or_ask;
+      public QueueAnalyzer(string security_desc) : this(security_desc, MDEntryType.Bid) {}
+      public QueueAnalyzer(string security_desc, MDEntryType bid_or_ask = MDEntryType.Bid)
+      {
+         _security_desc = security_desc;
+         _bid_or_ask = bid_or_ask;
+         foreach (var hist in new[] { _turnhist, _stackhist } )
+         {
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(0, 1000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(1000, 2000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(2000, 3000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(3000, 4000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(4000, 5000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(5000, 6000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(6000, 7000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(7000, 8000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(8000, 9000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(9000, 10000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(10000, 20000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(20000, 30000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(30000, 40000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(40000, 50000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(50000, 60000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(60000, 70000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(70000, 80000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(80000, 90000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(90000,  _max_nanos.Decrement()));
+            hist.LazySort();
+         }
+      }
+
+      public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
+      {
+      }
+
+      public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null)      
+      {            
+         var entries = md.MdEntries.Where(x => x.Symbol == _security_desc &&
+                                         (x.MdEntryType == _bid_or_ask) &&
+                                         (x.MdPriceLevel == 1) 
+                                    );
+         
+         foreach(var x in entries)
+         {
+            if( (_bid_or_ask == MDEntryType.Bid && x.MdEntryPx > _lastentry.MdEntryPx)  || //new top level bid price greater than old, market turn
+                (_bid_or_ask == MDEntryType.Offer && x.MdEntryPx < _lastentry.MdEntryPx) ) //new top level ask price less than old, market turn
+            {           
+               record_data(md, x, ref _lastturntime, _lastpacket.timestamp(), ref _samples, _turndist, _turnhist, "TURN");
+               _lastturn = x;
+            }
+            else if (x.MdEntryPx.AlmostEqual(_lastentry.MdEntryPx))
+            {
+               if (x.NumberOfOrders > _lastentry.NumberOfOrders) //price is same, norders increase
+               {
+                  record_data(md, x, ref _laststacktime, _lastturntime, ref _stacksamples, _stackdist, _stackhist, "STACK");
+               }
+            }
+            _lastentry = x;
+            _lastpacket = md; //inside the loop so we only get filtered packets
+         }                  
+      }
+
+      static void record_data(MarketDataIncrementalRefresh md, MDEntry entry,
+                              ref Google.Protobuf.WellKnownTypes.Timestamp last_time,
+                              Google.Protobuf.WellKnownTypes.Timestamp ref_time, ref long samples,
+                              DistributionEstimator dist, Histogram hist, string desc)
+      {
+         if(ref_time == null) 
+            return;
+         const double sumthresh = 10_000;
+         const double sumthresh2 = 25_000;
+         last_time = md.timestamp();
+         var delta = (md.timestamp() - ref_time);
+         double nanos = delta.Seconds * 1e9 + delta.Nanos;
+         if (delta.Seconds == 0 && delta.Nanos > 0 && delta.Nanos <= _max_nanos)
+         {
+            if (nanos > 100_000.0 || nanos < 1000)
+               nanos++; //for breakpoint                  
+            dist.add(nanos);
+            hist.AddData(nanos);
+            if (samples++ % 10 == 0)
+            {
+               Console.WriteLine($"{entry.Symbol} { entry.MdEntryType }  Trade date  {md.TradeDate.ToDateTime().ToShortDateString()}");
+               Console.WriteLine($"{desc} Current Sample {nanos}, Max filter { _max_nanos } { Environment.NewLine }");
+               Console.WriteLine($"{desc} Distribution {dist.ToString()}");
+               Console.WriteLine($"{desc} Histogram {Environment.NewLine + hist.ToString() }");
+               Console.WriteLine($"% < {sumthresh} {Environment.NewLine + (sumif(sumthresh, hist) / hist.DataCount).ToString("0.0%") }");
+               Console.WriteLine($"% < {sumthresh2} {Environment.NewLine + (sumif(sumthresh2, hist) / hist.DataCount).ToString("0.0%") }");
+               Console.WriteLine($"----------------------------------------------------------------------------- {Environment.NewLine}");
+            }
+         }
+      }
+
+      static double sumif(double sumthresh, Histogram hist)
+      {
+         double sum = 0.0;
+         for(int i = 0; i < hist.BucketCount; i++)
+         {
+            if(hist[i].UpperBound <= sumthresh)
+               sum += hist[i].Count;
+         }
+         return sum;
+      }
+   }
+}

src/client/spark_historical/BackTester.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+using Entry = System.Tuple<bts.api.SecurityType, bts.history.MarketDataIncrementalRefresh, string>;
+
+namespace bts.history
+{
+   class BackTester
+   {
+      readonly DataStreamer _stream;
+      MarketDataIncrementalRefresh _last = new MarketDataIncrementalRefresh() { SendingTime = new Timestamp(), TransactTime = new Timestamp() } ;
+      public BackTester()
+      {
+         _stream = new DataStreamer();
+      }
+
+      async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
+                               HistoricalDataOptions o)
+      {
+         start = start.ToUniversalTime();
+         end = end.ToUniversalTime();
+         var query = new MDQuery
+         {
+            Exchange = Venue.Glbx,
+            Symbol = symbol,
+            SecurityType = st,
+            Begin = Timestamp.FromDateTime(start),
+            End = Timestamp.FromDateTime(end),
+         };
+         if(DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
+            query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
+            
+         List<IAnalyzer> anal = new List<IAnalyzer>();
+         string algo_security = o.algo_security ?? string.Empty;
+         var t = System.Type.GetType(o.algo_type);
+         if (t == null)
+            t = System.Type.GetType("bts.history." + o.algo_type);
+         if (t != null)
+         {
+            if(typeof(AlgoBase).IsAssignableFrom(t))
+               anal.Add(new AlgoAnalyzer(o, t, o.algo_params));
+            else 
+               anal.Add((IAnalyzer)Activator.CreateInstance(t, new[] { algo_security }));
+         }
+         else
+            anal.Add(new BookAnalyzer(algo_security, o));
+
+         int count = 0;
+         var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y) 
+            => x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime()));
+         await _stream.process_md_stream<MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
+        {           
+           if (md.timestamp().ToDateTime() < end)
+           {                            
+              bool preanal = (md.timestamp().ToDateTime() < start);
+              if(o.skip_preanalysis)
+                 preanal = false;
+              if(preanal && count++ % 50000 == 1)
+                 Console.WriteLine("Pre analysis " + md.timestamp().ToDateTime());
+               do_anal(md, anal, preanal, line);
+               Log.assert(md.timestamp().greater_or_equal( _last.timestamp() ));
+               _last = md;
+           }
+           else
+              throw new System.IO.EndOfStreamException();
+        }, comparer);
+      }
+      
+
+      void do_anal(MarketDataIncrementalRefresh md, IEnumerable<IAnalyzer> anal, bool is_before_analysis_time, string debug_fix = null)
+      {
+         if (is_before_analysis_time)
+         {
+            foreach (var d in anal)
+               d.pre_analyze(md, debug_fix);
+         }
+         else
+         {
+            foreach (var d in anal)
+               d.analyze(md, debug_fix);
+         }
+      }
+   }
+}

src/client/spark_historical/DataDownloader.cs

@@ -0,0 +1,187 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Threading.Tasks;
+using Amazon;
+using Amazon.S3;
+using Amazon.S3.Model;
+using bts.api;
+
+namespace bts.history
+{
+   class DataDownloader
+   {
+      private static AmazonS3Client _client;
+      private const string _access_key = "AKIAJ4XUDWPQ7M42352A";
+      private const string _secret_key = "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";
+      static DataDownloader()
+      {
+         AmazonS3Config config = new AmazonS3Config();
+         //config.ServiceURL = "http://cme-bts-datamine.s3.us-east-1.amazonaws.com/";
+         config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
+         _client = new AmazonS3Client(_access_key, _secret_key, config);
+      }
+
+      private string _path;
+      private string _bucket_name; //http://cme-bts-datamine.s3.us-east-1.amazonaws.com/
+      public DataDownloader(string bucket_name = "cme-bts-datamine")
+      {
+         _path = mbts.config.Configuration.get<bts.config.System>().data_path;
+         _bucket_name = bucket_name;
+      }
+      bool is_in_cache(string key)
+      {
+         return File.Exists(_path + "/" + key);
+      }
+      Stream cached(string key)
+      {
+         if (is_in_cache(key))
+            return new FileStream(_path + "/" + key, FileMode.Open, FileAccess.Read, FileShare.Read);
+         else
+            return null;
+      }
+      async Task<Stream> download_s3(string key)
+      {
+         bts.Log.assert(!is_in_cache(key), $"key {key} already exists in cache!");
+         bts.Log.Gen.info($"downloading {key} from s3");
+         try
+         {
+            // var checkresp = await _client.ListObjectsAsync(new ListObjectsRequest { BucketName = _bucket_name} );
+            // foreach (S3Object o in checkresp.S3Objects)
+            // {
+            //    Console.WriteLine("{0}\t{1}\t{2}", o.Key, o.Size, o.LastModified);
+            // }
+
+            GetObjectRequest request = new GetObjectRequest { BucketName = _bucket_name, Key = key };
+            using (GetObjectResponse response = await _client.GetObjectAsync(request))
+            {
+               string title = response.Metadata["x-amz-meta-title"]; // Assume you have "title" as medata added to the object.
+               string contentType = response.Headers["Content-Type"];
+               bts.Log.Gen.info($"Object metadata, Title: {title}");
+               Console.WriteLine($"Downloading Object metadata, Title: {title} Key { key }");
+               Console.WriteLine($"Content type: {contentType}");
+               using (Stream response_stream = response.ResponseStream)
+               {
+                  var fullpath = _path + "/" + key;
+                  new FileInfo(fullpath).Directory.Create();                     
+                  FileStream writer = new FileStream(fullpath, FileMode.CreateNew);
+                  response_stream.CopyTo(writer);
+                  writer.Seek(0, SeekOrigin.Begin);
+                  return writer;
+               }
+            }
+         }
+         catch (AmazonS3Exception e)
+         {
+            bts.Log.Gen.error($"S3 Error encountered ***. Message:'{e.Message}' when writing an object {key}");            
+            throw;
+         }
+         catch (Exception e)
+         {
+            bts.Log.Gen.error($"Unknown encountered on server. Message:'{e.Message}' when writing an object {key}");
+            throw;
+         }
+      }
+      string map_mic(api.CMEMics mic)
+      {
+         switch (mic)
+         {
+            case CMEMics.Cbt:
+               return "xcbt";
+            case CMEMics.Cme:
+               return "xcme";
+            case CMEMics.Comex:
+               return "xcec";
+            case CMEMics.Dme:
+               return "dme";
+            case CMEMics.Nymex:
+               return "xnym";
+            default:
+               bts.Log.assert(false, "unknown mic map DataDownloader");
+               return string.Empty;
+         }
+      }
+      public string make_key(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
+      {
+         string micstring = map_mic(mic);
+         string date = dt.ToString("yyyyMMdd");
+         string key = date + "-MBO_" + micstring.ToLower() + "_" + symbol.ToLower();
+         if (st == api.SecurityType.Future)
+            key += "_fut_0-eth.gz";
+         else if (st == api.SecurityType.Spread)
+            key += "_fut_1-eth.gz";
+         string prefix = $"mbo/mbo/{date}/";
+         return prefix + key;
+      }
+      public static string data_path(DateTime dt)
+      {
+         string date = dt.ToString("yyyyMMdd");
+         string prefix = $"mbo/mbo/{date}/";
+         string p = mbts.config.Configuration.get<bts.config.System>().data_path + "/" + prefix;
+         return p;
+      }
+      public async Task<Stream> download(DateTime dt, string symbol, api.SecurityType st)
+      {
+         var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
+         var micmap = micconf.cme_mic;
+         if(micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic))
+         {
+            return await download(dt, themic, symbol, st);
+         }
+         throw new NotSupportedException("Can't find mic for symbol " + symbol);
+      }
+      public async Task<Stream> download(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
+      {         
+         string key = make_key(dt, mic, symbol, st);
+         return await download(key);
+      }
+
+      async Task<Stream> download_s3_uncompress(string key)
+      {                  
+         var gz = await download(key);
+         var fullpath = _path + "/" + key.Replace(".gz", "");
+         FileStream writer = new FileStream(fullpath, FileMode.CreateNew);
+         using ( GZipStream ungz = new GZipStream(gz, CompressionMode.Decompress))
+         {
+            ungz.CopyTo(writer);
+            writer.Seek(0, SeekOrigin.Begin);
+            return writer;
+         }
+      }
+
+      public async Task<Stream> download(string key)
+      {
+         if (!is_in_cache(key))
+            return await download_s3(key);
+         return cached(key);
+      }
+
+      public async Task<Stream> download_uncompressed(string key)
+      {
+         var unkey = key.Replace(".gz", "");
+         if (!is_in_cache(unkey))
+            return await download_s3_uncompress(key);
+         return cached(unkey);
+      }
+
+      public async Task<Stream> download_uncompressed(DateTime dt, string symbol, api.SecurityType st)
+      {
+         var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
+         var micmap = micconf.cme_mic;
+         if (micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic))
+         {
+            return await download_uncompressed(dt, themic, symbol, st);
+         }
+         throw new NotSupportedException("Can't find mic for symbol " + symbol);
+      }
+
+      public async Task<Stream> download_uncompressed(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
+      {
+         string key = make_key(dt, mic, symbol, st);
+         return await download_uncompressed(key);
+      }
+
+   }
+
+}

src/client/spark_historical/DataSender.cs

@@ -0,0 +1,171 @@
+using System;
+using System.Collections.Generic;
+using bts.api.core;
+using System.Linq;
+using System.Threading.Tasks;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.co2;
+
+namespace bts.history
+{
+   class DataSender
+   {
+      readonly DataStreamer _stream;
+      readonly bts.co2.MarketDataReceiverService _mdservice;
+
+      Google.Protobuf.WellKnownTypes.Timestamp _time;
+      Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
+
+      public DataSender(ServiceContext context)
+      {
+         _stream = new DataStreamer();
+         _mdservice = context.locate<bts.co2.MarketDataReceiverService>();
+      }
+
+      bts.api.ResultMessage send()
+      {
+         co2.MDReply md = new co2.MDReply();
+         md.Channels.AddRange(_channels.Values);
+         var x = _mdservice.push_md(md);
+         return x;
+      }
+
+      co2.ChannelUpdate create(Product prod, MDEntry md)
+      {
+         var chanup = new co2.ChannelUpdate()
+         {
+            Key = md.SecurityId, //TODO join with the secdefs that I don't have
+            Security = new Security()
+            {
+               Id = md.SecurityId,
+               Symbol = prod.Symbol,
+               Exchange = Venue.Glbx,
+               ContractSpec = new ContractSpecification()
+               {
+                  MainFraction = 32,
+                  SubFraction = 2,
+                  PriceBreak = 0,
+                  LowTick = 0.015625,
+                  HighTick = 0.015625,
+                  CashPerPoint = 1000.0,
+                  PriceDisplayFormat = 3,
+                  PriceMultiplier = 1.0,
+               },
+               Glbx = new GLBXExchangeData()
+               {
+                  SecurityDesc = md.Symbol,
+                  ExchangeId = md.SecurityId,
+                  CmeSymbol = prod.Symbol,
+               }
+            }
+         };
+         if (prod.Type == SecurityType.Future)
+         {
+            chanup.Security.Future = new FutureContract()
+            {
+               ExpirationDay = 1,
+               ExpirationMonth = 9,
+               ExpirationYear = 2019
+            };
+         }
+         else if (prod.Type == SecurityType.Spread)
+         {
+            chanup.Security.Spread = new SpreadContract()
+            {
+               Normalized = true
+            };
+         }
+         else
+            System.Diagnostics.Trace.Assert(false, "unknown contract type");
+         return chanup;
+      }
+
+      void process(Product prod, MarketDataIncrementalRefresh mdr, string fix)
+      {
+         if (!_channels.TryGetValue(prod, out co2.ChannelReply chan))
+         {
+            chan = new co2.ChannelReply()
+            {
+               Platform = co2.Platform.Mdp3Onixs,
+               Channel = new co2.SecurityChannel()
+               {
+                  Entitled = true,
+                  Exchange = prod.Exchange,
+                  Type = prod.Type,
+                  Symbol = prod.Symbol,
+                  Flags = (uint)co2.MDRequest.Types.Flags.Direct
+               },
+            };
+            _channels.Add(prod, chan);
+         }
+         int count = 0;
+
+         foreach (var md in mdr.MdEntries)
+         {
+            var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId);
+            if (upd == null)
+            {
+               upd = create(prod, md);
+               upd.Direct = new BookMessage();
+               Utils.init_book(upd.Direct);
+               chan.Updates.Add(upd);
+            }
+            Utils.keep_book(upd.Direct, md);
+            count++;
+            if (Utils.quote_event_done( mdr ))
+               Utils.validate_book(upd.Direct, fix);
+         }
+      }
+
+      async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
+                              HistoricalDataOptions o)
+      {         
+         int us = 20;//o.dt_us;
+         var query = new MDQuery
+         {
+            Exchange = Venue.Glbx,
+            Symbol = symbol,
+            SecurityType = st,
+            Begin = Timestamp.FromDateTime(DateTime.SpecifyKind(start, DateTimeKind.Utc)),
+            End = Timestamp.FromDateTime(DateTime.SpecifyKind(end, DateTimeKind.Utc))
+         };
+         api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol };
+         int count = 0;
+         await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
+        {
+           if (md.timestamp().ToDateTime() < end)
+           {
+               process(prod, md, line);
+               maybe_send(md, us);
+               count++;
+           }
+        });
+      }
+      
+      void maybe_send(history.MarketDataIncrementalRefresh md, int us)
+      {
+         if (_time == null)
+         {
+            if (Utils.quote_event_done(md))
+            {
+               send(); //send the initial books
+               _time = md.timestamp();
+            }
+            else
+               return;
+         }       
+         else
+         {
+            var ts = (md.timestamp() - _time).ToTimeSpan();
+            var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
+            if (diff > us && Utils.quote_event_done(md))
+            {
+               send(); //send the next book
+               _time = md.timestamp();
+            }            
+         }  
+      }
+
+   }
+}

src/client/spark_historical/DataStreamer.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using bts.api;
+
+namespace bts.history
+{
+   class DataStreamer 
+   {
+      readonly DataDownloader _dl = new DataDownloader();
+      public DataStreamer()
+      {
+      }
+
+
+      #pragma warning disable CS0183
+      async public Task process_md_stream<T>(MDQuery request, Action<SecurityType, T, string> action, IComparer<T> comp = null) 
+         where T : Google.Protobuf.IMessage, new()
+      {
+         DateTime d = request.Begin.ToDateTime();
+         while (d < request.End.ToDateTime())
+         {
+            try
+            {
+               if(request.SecurityType == SecurityType.Synthetic)
+               {
+                  var s3f = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Future);
+                  var s3s = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Spread);
+                  using (MBOParser fparser = new MBOParser(s3f))
+                  {
+                     using(MBOParser sparser = new MBOParser(s3s))
+                     {
+                        var fut = fparser.next<T>(null, null, request.Skip).GetEnumerator();
+                        var spr = sparser.next<T>(null, null, request.Skip).GetEnumerator();                        
+                        bool futgo = fut.MoveNext(), sprgo = spr.MoveNext();  
+                        while(futgo || sprgo)
+                        {
+                           if(comp.Compare(fut.Current.Item1, spr.Current.Item1) < 0)
+                           {
+                              action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2.Replace('\u0001', '|'));
+                              if(futgo)
+                                 futgo = fut.MoveNext();                              
+                           }
+                           else if(comp.Compare(fut.Current.Item1, spr.Current.Item1) > 0)
+                           {
+                              action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2.Replace('\u0001', '|'));
+                              if (sprgo)
+                                 sprgo = spr.MoveNext();
+                           }
+                           else
+                           {
+                              action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2.Replace('\u0001', '|'));
+                              if (futgo)
+                                 futgo = fut.MoveNext();
+                              action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2.Replace('\u0001', '|'));
+                              if (sprgo)
+                                 sprgo = spr.MoveNext();
+                           }
+                        }                
+                     }
+                  }
+               }
+               else
+               {
+                  var s3 = await _dl.download_uncompressed(d, request.Symbol, request.SecurityType);
+                  using (MBOParser parser = new MBOParser(s3))
+                  {
+                     foreach (var (mbo,line) in parser.next<T>(null, null, request.Skip))
+                     {
+                        action(request.SecurityType, mbo, line.Replace('\u0001', '|'));
+                     }
+                  }
+               }
+            }
+            catch (Amazon.S3.AmazonS3Exception) { }
+            catch (System.IO.EndOfStreamException) { }
+            d = d.AddDays(1.0).Date;
+         }
+      }
+
+
+      async public Task process_md_stream_by_line(MDQuery request, Action<SecurityType, string, string> action, int peek_field = 0)
+      {
+         DateTime d = request.Begin.ToDateTime();
+         while (d < request.End.ToDateTime())
+         {
+            try
+            {
+               if (request.SecurityType == SecurityType.Synthetic)
+               {
+                  var s3f = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Future);
+                  var s3s = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Spread);
+                  using (MBOParser fparser = new MBOParser(s3f))
+                  {
+                     using (MBOParser sparser = new MBOParser(s3s))
+                     {
+                        var fut = fparser.next_line(peek_field).GetEnumerator();
+                        var spr = sparser.next_line(peek_field).GetEnumerator();
+                        bool futgo = fut.MoveNext(), sprgo = spr.MoveNext();
+                        while (futgo || sprgo)
+                        {
+                           action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2);
+                           if (futgo)
+                              futgo = fut.MoveNext();
+                              
+                           action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2);
+                           if (sprgo)
+                              sprgo = spr.MoveNext();
+                        }
+                     }
+                  }
+               }
+               else
+               {
+                  var s3 = await _dl.download_uncompressed(d, request.Symbol, request.SecurityType);
+                  using (MBOParser parser = new MBOParser(s3))
+                  {
+                     foreach (var (peek, line) in parser.next_line(peek_field))
+                        action(request.SecurityType, peek, line);
+                  }
+               }
+            }
+            catch (Amazon.S3.AmazonS3Exception) { }
+            catch (System.IO.EndOfStreamException) { }
+            d = d.AddDays(1.0).Date;
+         }
+      }
+   }   
+}

src/client/spark_historical/DistributionEstimator.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Linq;
+
+namespace bts.history
+{
+   class DistributionEstimator
+   {
+      List<double> _vals = new List<double>();
+      NormalEstimator _norm = new NormalEstimator();
+      public DistributionEstimator() 
+      {
+         
+      }
+      public void add(double x)
+      {
+         _vals.Add(x);
+         _norm.add(x);
+      }
+      public void delete(double x)
+      {
+         _vals.Remove(x);
+         _norm.delete(x);
+      }
+      public long num_samples => _vals.Count;
+      public double mean() => _norm.mean();
+      // {
+      //    if (num_samples == 0)
+      //       return 0.0;
+      //    double sum = _vals.Sum();
+      //    return sum / num_samples;
+      // }
+      public double sum_sq()
+      {
+         double m = mean();
+         double sq = 0.0;
+         for (int i = 0; i < num_samples; i++)
+            sq += (_vals[i] - m) * (_vals[i] - m);
+         return sq;
+      }
+      public double variance() => _norm.variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / num_samples;
+      // }
+      public double unbiased_variance() => _norm.unbiased_variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / (num_samples - 1);
+      // }
+      public double std_deviation() => _norm.std_deviation();
+      // {
+      //    return Math.Sqrt(variance());
+      // }
+      public double unbiased_std_deviation() => _norm.unbiased_std_deviation();
+      // {
+      //    return Math.Sqrt(unbiased_variance());
+      // }
+      public double percentile(int p)
+      {
+         _vals.Sort();
+         double percentile = p / 100.0;
+         double n = (_vals.Count - 1) * percentile + 1;
+         if (n <= 1.0)
+            return _vals[0];
+         else if (n >= _vals.Count)
+            return _vals[_vals.Count - 1];
+         else
+         {
+            int k = (int)n;
+            double d = n - k;
+            return _vals[k - 1] + d * (_vals[k] - _vals[k - 1]);
+         }
+      }
+      public override string ToString()
+      {
+         var f = percentile(5);
+         var t = percentile(10);
+         var tw = percentile(25);
+         var h = percentile(50);
+         var s = percentile(75);
+         var nn = percentile(99);
+         var sum50 = f + t + tw + h;
+         return $@"
+   Normal( mean {mean()}, std_dev {std_deviation()} ) 
+   5th { f }, 
+   10th { t } 
+   25th { tw } 
+   50th { h } 
+   75th { s } 
+   99th { nn } 
+   [numSamples={ num_samples } ]
+";
+      }
+   }
+}

src/client/spark_historical/FillModel/FIFOModel.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   class FIFOModel : IFillModel
+   {
+      IDictionary<string, MBOBook> _books = new Dictionary<string, MBOBook>();
+      public FIFOModel()
+      {
+      }
+
+      public double tick_size(string security_desc)
+      {
+         var book = get_book(security_desc);
+         return book.ask_at(0) - book.bid_at(0);
+      }
+      public FIFOModel(IDictionary<string, IOrderBook> obs)
+      {
+         foreach(var val in obs)
+         {
+            _books[val.Key] = val.Value.mbo_book;
+         }
+      }
+      public void send(OrderEntry order)
+      {
+         var book = get_book(order.security_desc);
+         book.insert_my_order(order);
+         book.match_crosses(order.timestamp);
+      }
+
+      public OrderEntry update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = 0)
+      {
+         var book = get_book(security_desc);
+         var ord = book.change_my_order(asof, order_id, price, size);
+         book.match_crosses(asof);
+         return ord;
+      }
+
+      public bool cancel(string security_desc, Timestamp asof, ulong order_id)
+      {
+         return get_book(security_desc).cancel_my_order(order_id);
+      }
+
+      public (int, int) get_my_piq(string security_desc, ulong orderid)
+      {
+         return get_book(security_desc).get_my_piq(orderid);
+      }
+
+      public void process_external_order(Timestamp ts, MBOMessage mbo)
+      {
+         get_book(mbo.SecurityDesc).process_one_mbo(ts, mbo);
+      }
+
+      public void process_external_trade(Timestamp ts, string security_desc, co2.TradeMessage trade)
+      {
+         get_book(security_desc).process_one_trade(ts, trade);
+      }
+      
+      MBOBook get_book(string security_desc)
+      {
+         if(!_books.TryGetValue(security_desc, out MBOBook book))
+         {
+            _books[security_desc] = book = new MBOBook(security_desc);
+         }
+         return book;
+      }      
+   }
+
+}

src/client/spark_historical/FillModel/FillEntry.cs

@@ -0,0 +1,40 @@
+using System;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public class FillEntry
+   {
+      static uint _fillid = 0;
+      public FillEntry(Timestamp ts, OrderEntry order, api.Side side, double price, ulong size)
+      {
+         this.timestamp = ts;
+         this.order = order;
+         this.side = side;
+         this.price = price;
+         this.size = size;
+         this.piq = piq;
+         this.queue_len = queue_len;
+         this.fill_id = ++_fillid;
+      }
+      public Timestamp timestamp { get; }
+      public OrderEntry order { get; }
+      public api.Side side { get; }
+      public double price { get; }
+      public ulong size { get; }
+      public bool fully_filled { get; set; }
+      public int piq { get; set; }
+      public int queue_len { get; set; }
+      public uint fill_id { get; private set; }
+
+      public uint bidsize { get; set; }
+      public double bid { get; set; }
+      public uint asksize { get; set; }
+      public double ask { get; set; }
+
+      public override string ToString()
+      {
+         return $"{timestamp.ToDateTime().ToLongDateString()} {timestamp.ToDateTime().ToString("H:mm:ss.ffffff")} {order.tag ?? "Our   "} {order.security_desc} {side} {price} {size}       --       {bidsize} {bid} at {ask} { asksize }    queue: {piq} of {queue_len}";
+      }
+   }
+}

src/client/spark_historical/FillModel/IFillModel.cs

@@ -0,0 +1,20 @@
+using System;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+
+public interface IFillModel
+{   
+   void send(OrderEntry order);
+   OrderEntry update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = uint.MinValue);
+   bool cancel(string security_desc, Timestamp asof, ulong order_id);
+   (int, int) get_my_piq(string security_desc, ulong orderid);
+
+   void process_external_order(Timestamp ts, MBOMessage mbo);
+   void process_external_trade(Timestamp ts, string security_desc, co2.TradeMessage trade);
+   double tick_size(string security_desc);
+}
+
+
+}

src/client/spark_historical/FillModel/MBOBook.cs

@@ -0,0 +1,446 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.co2;
+using mbts.co2;
+
+using QueueType = System.Collections.Generic.SortedList<ulong, bts.history.OrderEntry>;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   class MBOBook
+   {
+      public string security_desc { get; private set; }
+
+      //price, priority
+      readonly Dictionary<double, QueueType> _mbobuy = new Dictionary<double, QueueType>();
+      readonly Dictionary<double, QueueType> _mbosell = new Dictionary<double, QueueType>();
+      //orderid to { price, priority }
+      readonly Dictionary<ulong, Tuple<api.Side, double, ulong>> _mbokey = new Dictionary<ulong, Tuple<api.Side, double, ulong>>();
+      readonly SortedList<ulong, OrderEntry> _myorders = new SortedList<ulong, OrderEntry>();
+      
+      public double bid_at(int level)
+      {
+         return _mbobuy.Keys.OrderBy( x => x).Skip(level).First();
+      }
+
+      public double ask_at(int level)
+      {
+         return _mbosell.Keys.OrderBy(x => x).Skip(level).First();
+      }
+
+      uint bidsize_at(int level)
+      {
+         double bid = bid_at(level);
+         int sz = _mbobuy[bid].Values.Sum( x => (int)x.size );
+         return (uint)sz;
+      }
+
+      uint asksize_at(int level)
+      {
+         double ask = ask_at(level);
+         int sz = _mbosell[ask].Values.Sum(x => (int)x.size);
+         return (uint)sz;
+      }
+
+
+      public MBOBook(string security_desc)
+      {
+         this.security_desc = security_desc;
+      }      
+
+      public void validate_new_queue(AlgoBase algo, MarketDataIncrementalRefresh current_book_packet, Func<MDEntry, OrderIDEntry, bool> filter)
+      {
+         foreach (var entry in current_book_packet.MdEntries)
+         {
+            if (!filter(entry, null))
+               continue;
+            if (entry.MdPriceLevel == 1 && entry.MdEntryType == MDEntryType.Bid)
+            {
+               var mbo = get_mbo(Utils.same_side(entry.MdEntryType));
+               Log.assert(mbo.TryGetValue(entry.MdEntryPx, out QueueType q));
+               //at times the crossed orders take a few lines to get deleted from queue.
+               if (q.Where(x => x.Value.side == api.Side.Sell).Count() > (current_book_packet.OrderIdEntries.Count + current_book_packet.MdEntries.Count))
+                  Log.Testing.warn("q should be empty");
+               if (algo.bucket.LowerBound == 0 && q.Count > 1)
+                  Log.Testing.warn("q should be empty");
+            }
+         }
+      }
+
+
+      public (int, int) get_my_piq(ulong orderid)
+      {
+         int piq = 0, count = 0;
+         if (_myorders.TryGetValue(orderid, out OrderEntry myorder))
+         {
+            var mbo = get_mbo(myorder.side);
+            if (mbo.TryGetValue(myorder.price, out QueueType q))
+            {
+               foreach (var ent in q.Values)
+               {
+                  if (!ent.deleted)
+                  {
+                     if (ent.order_priority < myorder.order_priority)
+                        piq++;
+                     count++;
+                  }
+               }
+               return (piq, count);
+            }
+         }
+         return (piq, count);
+      }
+
+      //once you're in, they can't get in front of you
+      const ulong _first_in_q = 999ul;
+      public OrderEntry insert_my_order(OrderEntry myorder)
+      {
+         QueueType queue = get_mbo(myorder.side).GetOrAdd(myorder.price, x =>
+         {
+            Log.Testing.error("FIRST in Q at " + myorder.price);
+            return new QueueType();
+         });
+
+         Log.assert(myorder.order_id != 0);
+
+         var key = queue.LastOrDefault(kvp => kvp.Value.timestamp.less_or_equal(myorder.timestamp), new KeyValuePair<ulong, OrderEntry>(0u, null));
+         if (key.Value != null)
+            myorder.order_priority = key.Key;
+         else
+            myorder.order_priority = _first_in_q;
+
+         Log.Testing.info($"insert myorder {key} at price { myorder.price } piq { queue.Count } ");
+
+         _myorders.Add(myorder.order_id, myorder);
+         return myorder;
+      }
+
+      public OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = uint.MinValue)
+      {
+         if (!_myorders.TryGetValue(order_id, out OrderEntry myorder))
+            return null;
+
+         myorder.price = price;
+         if (size > uint.MinValue)
+         {
+            if (size > myorder.size)
+            {
+               var mbo = get_mbo(myorder.side);
+               if (mbo.TryGetValue(price, out QueueType queue))
+               {
+                  var key = queue.LastOrDefault(kvp => kvp.Value.timestamp.less_or_equal(asof), new KeyValuePair<ulong, OrderEntry>(0u, null));
+                  if (key.Value != null)
+                     myorder.order_priority = key.Key;
+                  else
+                     myorder.order_priority = _first_in_q;
+                  Log.Testing.info($"update myorder {order_id} at price { price } piq { queue.Count } ");
+               }
+            }
+            myorder.size = size;
+         }
+         return myorder;
+      }
+
+      bool remove_my_order(ulong my_order_id)
+      {
+         return _myorders.Remove(my_order_id);
+      }
+
+      public bool cancel_my_order(ulong my_order_id)
+      {
+         return remove_my_order(my_order_id);
+      }
+
+      public void process_one_trade(Timestamp ts, co2.TradeMessage tradeentry)
+      {
+         //look for the order that got filled, if we have it (could have been here since last Tuesday)
+         var q = find_queue(tradeentry.OrderId, out Side side, out double price, out ulong priority_key);
+         if (q != null)
+         {
+            if (!q.TryGetValue(priority_key, out OrderEntry order))
+               Log.assert(false);
+            //if the priority is greater than the priority of any of my orders, fill me first
+            //the real order still gets decremented later by CME which isn't quite right TODO 
+            List<FillEntry> tfills = new List<FillEntry>();
+            foreach (var myorder in _myorders.Values)
+            {
+               if (order.price == myorder.price && side == myorder.side)
+               {
+                  if (side == Side.Buy && tradeentry.price() <= order.price ||
+                      side == Side.Sell && tradeentry.price() >= order.price)
+                  {
+                     if (priority_key >= myorder.order_priority)
+                     {
+                        Log.assert(!order.deleted);
+                        var (piq, count) = get_my_piq(myorder.order_id);
+                        // int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count;
+                        var f = generate_fill(ts, myorder, order.price, (uint)Math.Min(tradeentry.Size, myorder.size),
+                                             piq, count, $" {piq} of {count} ");
+                        tfills.Add(f);
+                     }
+                  }
+               }
+            }
+            post_process_fill(tfills);
+         }
+      }
+
+      public void process_one_mbo(Timestamp ts, IMBOMessage order)
+      {
+         process_mbo(ts, order);
+      }      
+
+      public void match_crosses(Timestamp ts)
+      {
+         List<FillEntry> fills = new List<FillEntry>();
+         foreach (var myorder in _myorders.Values)
+         {
+            if (myorder.market_order)
+            {
+               double px = myorder.side == Side.Buy ? ask_at(0) : bid_at(0);
+               var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0);
+               fills.Add(f);
+            }
+            else
+               fills.AddRange(check_my_cross(ts, myorder));
+         }
+         post_process_fill(fills);
+      }
+
+      static public IDictionary<string, List<MBOMessage>> extract_mbo(MarketDataIncrementalRefresh md)
+      {
+         Dictionary<string, List<MBOMessage>> cache = new Dictionary<string, List<MBOMessage>>();
+         foreach (var order in md.OrderIdEntries)
+         {
+            if (order.ReferenceId == 0)
+               continue;
+            // entries is indexed by 0 but ref is indexed by 1  :facepalm:
+            int mdidx = ((int)order.ReferenceId) - 1;
+            Log.assert((mdidx <= md.MdEntries.Count));
+
+            MDEntry mdval = md.MdEntries[mdidx];
+            cache_mbo(make_mbo(mdval, order));
+         }
+         foreach (var mdval in md.MdEntries)
+         {
+            if (mdval.OrderId > 0 && md.OrderIdEntries.Count == 0)
+               cache_mbo(make_mbo(mdval, null));
+         }
+
+         return cache;
+
+         void cache_mbo(MBOMessage mbo)
+         {
+            if (!cache.TryGetValue(mbo.SecurityDesc, out List<MBOMessage> list))
+               cache[mbo.SecurityDesc] = list = new List<MBOMessage>();
+            list.Add(mbo);
+         }
+      }
+
+      //each order in here can be for a different secdesc
+      static public MBOMessage make_mbo(MDEntry mdval, OrderIDEntry order)
+      {
+         MBOMessage mbo = new MBOMessage() { MdEntry = mdval, OrderEntry = order };
+         return mbo;
+      }
+
+
+      Dictionary<double, QueueType> get_mbo(api.Side side)
+      {
+         if (side == Side.Buy)
+            return _mbobuy;
+         else
+            return _mbosell;
+      }
+
+      FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string dbg = null)
+      {
+         var f = new FillEntry(ts, myorder, myorder.side, price, qty);
+         f.piq = piq;
+         f.queue_len = qlen;
+         f.bidsize = bidsize_at(0);
+         f.bid = bid_at(0);
+         f.ask = ask_at(0);
+         f.asksize = asksize_at(0);
+         Log.assert(qty <= myorder.size);
+         myorder.size -= qty;
+         if (myorder.size == 0)
+         {
+            Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (dbg ?? ""));
+            f.fully_filled = true;
+         }
+         else
+         {
+            Log.Testing.info("Partial FILL " + Stringizer.to_string(f));
+         }
+         return f;
+      }
+
+      void post_process_fill(IEnumerable<FillEntry> fills)
+      {
+         foreach (var f in fills)
+         {
+            if (f.fully_filled)
+               remove_my_order(f.order.order_id);
+            if (f.order.on_fill != null)
+               f.order.on_fill(f);
+         }
+      }
+
+      OrderEntry find_order(ulong order_id)
+      {
+         OrderEntry order = null;
+         find_queue(order_id, out Side _, out double _, out ulong prior)?.TryGetValue(prior, out order);
+         return order;
+      }
+
+      QueueType find_queue(ulong order_id, out api.Side side, out double price, out ulong priority_key)
+      {
+         QueueType q = null; side = Side.Undefined; price = double.NaN; priority_key = 0;
+         if (_mbokey.TryGetValue(order_id, out Tuple<api.Side, double, ulong> tup))
+         {
+            var mbo = get_mbo(tup.Item1);
+            mbo.TryGetValue(tup.Item2, out q);
+            side = tup.Item1;
+            price = tup.Item2;
+            priority_key = tup.Item3;
+         }
+         return q;
+      }
+
+      List<FillEntry> check_my_cross(Timestamp ts, OrderEntry myorder)
+      {
+         List<FillEntry> fills = new List<FillEntry>();
+         if (myorder.side == Side.Buy)
+         {
+            int level = 0; int nasks = Math.Min(_mbosell.Keys.Count, 10);
+            while (myorder.size > 0 && level < nasks)
+            {
+               var currask = ask_at(level);
+               if (myorder.price >= currask)
+               {
+                  var qty = Math.Min(myorder.size, asksize_at(level));
+                  if (qty > 0)
+                     fills.Add(generate_fill(ts, myorder, currask, (uint)qty, 0, 0, "CROSS"));
+               }
+               else
+                  break;
+               level++;
+            }
+         }
+         else
+         {
+            int level = 0; int nbids = Math.Min(_mbobuy.Keys.Count, 10);
+            while (myorder.size > 0 && level < nbids)
+            {
+               var currbid = bid_at(level);
+               if (myorder.price <= currbid)
+               {
+                  var qty = Math.Min(myorder.size, bidsize_at(level));
+                  if (qty > 0)
+                     fills.Add(generate_fill(ts, myorder, currbid, (uint)qty, 0, 0, "CROSS"));
+               }
+               else
+                  break;
+               level++;
+            }
+         }
+         return fills;
+      }
+
+      void process_mbo(Timestamp ts, IMBOMessage order)
+      {
+         if (order.MdEntryType != MDEntryType.Bid && order.MdEntryType != MDEntryType.Offer)
+            return;
+         var mbo = get_mbo(Utils.same_side(order.MdEntryType));
+         if (order.OrderUpdateAction == MDUpdateAction.New)
+         {
+            var q = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
+            var side = Utils.same_side(order.MdEntryType);
+            var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, order.ReferenceId,
+                                          this.security_desc, side, order.MdEntryPx, order.MdDisplayQty);
+            q.Add(order.MdOrderPriority, neworder);
+            _mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority);
+         }
+         else if (order.OrderUpdateAction == MDUpdateAction.Change)
+         {
+            var currq = find_queue(order.OrderId, out Side _, out double _, out ulong priority_key);
+            if (currq != null)
+               currq.Remove(priority_key); //this could have been previously deleted
+            var newq = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
+            newq.AddOrUpdate(order.MdOrderPriority,
+               x =>
+               {
+                  var side = Utils.same_side(order.MdEntryType); var refid = 0u;
+                  var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, refid,
+                                                this.security_desc, side, order.MdEntryPx, order.MdDisplayQty);
+
+                  Log.assert(neworder.order_priority == order.MdOrderPriority, "bad priority");
+                  _mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority);
+                  return neworder;
+               },
+               (id, x) =>
+               {
+                  x.timestamp = ts;
+                  if (x.order_priority != order.MdOrderPriority)
+                  {
+                     var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority);
+                     if (myorder != null)
+                     {
+                        //target order got better, go with
+                        if (order.MdOrderPriority < myorder.order_priority)
+                        {
+                           _myorders.Remove(myorder.order_id);
+                           myorder.order_priority = order.MdOrderPriority;
+                           _myorders[myorder.order_id] = myorder;
+                        }
+                        else //target order got worse, attach to prior
+                           slide_order_up(currq, order);
+                     }
+                     x.order_priority = myorder.order_priority;
+                  }
+                  x.price = order.MdEntryPx;
+                  x.size = order.MdDisplayQty;
+                  Log.assert(x.order_id == order.OrderId, "orderid invariant");
+                  _mbokey[x.order_id] = Tuple.Create(Utils.same_side(order.MdEntryType), x.price, x.order_priority);
+                  return x;
+               });
+         }
+         else if (order.OrderUpdateAction == MDUpdateAction.Delete)
+         {
+            var currq = find_queue(order.OrderId, out Side _, out double _, out ulong _);
+            if (currq == null)
+               Log.assert(true);
+            if (currq != null && currq.TryGetValue(order.MdOrderPriority, out OrderEntry removed_order))
+            {
+               removed_order.deleted = true;
+               slide_order_up(currq, order);
+               //currq.Remove(order.MdOrderPriority);
+               //we can't delete the orders because we don't know if we are going to get the trade or the delete first
+            }
+         }
+      }
+
+      void slide_order_up(QueueType currq, IMBOMessage order)
+      {
+         var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == order.MdOrderPriority);
+         if (myorder != null)
+         {
+            _myorders.Remove(myorder.order_id);
+            var idx = currq.IndexOfKey(order.MdOrderPriority);
+            if (idx > 0)
+            {
+               var prev = currq.ElementAt(idx - 1);
+               myorder.order_priority = prev.Key;
+            }
+            else
+               myorder.order_priority = _first_in_q;
+            _myorders[myorder.order_id] = myorder;
+         }
+      }
+   }
+}

src/client/spark_historical/FillModel/OrderEntry.cs

@@ -0,0 +1,54 @@
+using System;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public class OrderEntry
+   {
+      public OrderEntry(string security_desc, string tag) { this.security_desc = security_desc; this.tag = tag; }
+      public OrderEntry(Timestamp timestamp, ulong order_id, ulong order_priority, uint reference_id,
+                        string security_desc, api.Side side, double price, ulong size, Action<FillEntry> on_fill = null, string tag = null)
+      {
+         this.timestamp = timestamp;
+         this.order_id = order_id;
+         this.order_priority = order_priority;
+         this.reference_id = reference_id;
+         this.security_desc = security_desc;
+         this.side = side;
+         this.price = price;
+         this.size = size;
+         this.on_fill = on_fill;
+         this.tag = tag;
+      }
+
+      public OrderEntry(Timestamp timestamp, ulong order_id, string security_desc, api.Side side, double price, ulong size, 
+                        Action<FillEntry> on_fill = null, string tag = null)
+      {
+         this.timestamp = timestamp;
+         this.order_id = order_id;
+         this.order_priority = order_priority;
+         this.reference_id = reference_id;
+         this.security_desc = security_desc;
+         this.side = side;
+         this.price = price;
+         this.size = size;
+         this.on_fill = on_fill;
+         this.tag = tag;
+      }
+
+      public Timestamp timestamp { get; set; }
+      public ulong order_id { get; }
+      public ulong order_priority { get; set; }
+      public uint reference_id { get; }
+
+      public string security_desc { get; }
+      public api.Side side { get; }
+      public double price { get; set; }
+      public ulong size { get; set; }
+      public Action<FillEntry> on_fill { get; }
+      public string tag { get; }
+      public bool deleted { get; set; }
+      public bool market_order { get; set; }
+   }
+
+}

src/client/spark_historical/FillModel/OrderStatus.cs

@@ -0,0 +1,38 @@
+using System;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public class OrderStatus
+   {
+      public Timestamp last_change;
+      public string security_desc;
+      public api.Side side;
+      public api.orderman.OrderState state;
+
+      public OrderStatus clone()
+      {
+         return (OrderStatus)this.MemberwiseClone();
+      }
+
+      public OrderStatus(Timestamp ts)
+      {
+         last_change = ts;
+      }
+
+      public bool is_terminal()
+      {
+         return is_terminal(state);
+      }
+
+      static public bool is_terminal(api.orderman.OrderState state)
+      {
+         if (state == null)
+            return false;
+         return state.Status == api.orderman.OrderStatus.Canceled ||
+                state.Status == api.orderman.OrderStatus.Completed ||
+                state.Status == api.orderman.OrderStatus.Error ||
+                state.Status == api.orderman.OrderStatus.RejectedBySpark;
+      }
+   }   
+}

src/client/spark_historical/Histogram.cs

@@ -0,0 +1,533 @@
+    
+// <copyright file="Histogram.cs" company="Math.NET">
+// Math.NET Numerics, part of the Math.NET Project
+// http://numerics.mathdotnet.com
+// http://github.com/mathnet/mathnet-numerics
+//
+// Copyright (c) 2009-2015 Math.NET
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Text;
+using System.Linq;
+
+namespace MathNet.Numerics.Statistics
+{   
+   static class MathNetExtensionsSoItBuilds
+   {
+      public static bool AlmostEqual(this double val, double other)
+      {
+         return Math.Abs(val - other) < double.Epsilon;
+      }
+      public static double Decrement(this double val)
+      {
+         return val - double.Epsilon;
+      }
+   }
+   
+   /// <summary>
+   /// A <see cref="Histogram"/> consists of a series of <see cref="Bucket"/>s,
+   /// each representing a region limited by a lower bound (exclusive) and an upper bound (inclusive).
+   /// </summary>
+   /// <remarks>
+   /// This type declares a DataContract for out of the box ephemeral serialization
+   /// with engines like DataContractSerializer, Protocol Buffers and FsPickler,
+   /// but does not guarantee any compatibility between versions.
+   /// It is not recommended to rely on this mechanism for durable persistence.
+   /// </remarks>
+   [Serializable]
+   [DataContract(Namespace = "urn:MathNet/Numerics")]
+   public class Bucket : IComparable<Bucket>
+#if !NETSTANDARD1_3
+        , ICloneable
+#endif
+   {
+      /// <summary>
+      /// This <c>IComparer</c> performs comparisons between a point and a bucket.
+      /// </summary>
+      private sealed class PointComparer : IComparer<Bucket>
+      {
+         /// <summary>
+         /// Compares a point and a bucket. The point will be encapsulated in a bucket with width 0.
+         /// </summary>
+         /// <param name="bkt1">The first bucket to compare.</param>
+         /// <param name="bkt2">The second bucket to compare.</param>
+         /// <returns>-1 when the point is less than this bucket, 0 when it is in this bucket and 1 otherwise.</returns>
+         public int Compare(Bucket bkt1, Bucket bkt2)
+         {
+            return bkt2.IsSinglePoint
+                ? -bkt1.Contains(bkt2.UpperBound)
+                : -bkt2.Contains(bkt1.UpperBound);
+         }
+      }
+
+      private static readonly PointComparer Comparer = new PointComparer();
+
+      /// <summary>
+      /// Lower Bound of the Bucket.
+      /// </summary>
+      [DataMember(Order = 1)]
+      public double LowerBound { get; set; }
+
+      /// <summary>
+      /// Upper Bound of the Bucket.
+      /// </summary>
+      [DataMember(Order = 2)]
+      public double UpperBound { get; set; }
+
+      /// <summary>
+      /// The number of datapoints in the bucket.
+      /// </summary>
+      /// <remarks>
+      /// Value may be NaN if this was constructed as a <see cref="IComparer{Bucket}"/> argument.
+      /// </remarks>
+      [DataMember(Order = 3)]
+      public double Count { get; set; }
+
+      /// <summary>
+      /// Initializes a new instance of the Bucket class.
+      /// </summary>
+      public Bucket(double lowerBound, double upperBound, double count = 0.0)
+      {
+         if (lowerBound > upperBound)
+         {
+            throw new ArgumentException("Resources.ArgumentUpperBoundMustBeLargerThanOrEqualToLowerBound");
+         }
+
+         if (count < 0.0)
+         {
+            throw new ArgumentOutOfRangeException(nameof(count), "Resources.ArgumentMustBePositive");
+         }
+
+         LowerBound = lowerBound;
+         UpperBound = upperBound;
+         Count = count;
+      }
+
+      /// <summary>
+      /// Constructs a Bucket that can be used as an argument for a <see cref="IComparer{Bucket}"/>
+      /// like <see cref="DefaultPointComparer"/> when performing a Binary search.
+      /// </summary>
+      /// <param name="targetValue">Value to look for</param>
+      public Bucket(double targetValue)
+      {
+         LowerBound = targetValue;
+         UpperBound = targetValue;
+         Count = double.NaN;
+      }
+
+      /// <summary>
+      /// Creates a copy of the Bucket with the lowerbound, upperbound and counts exactly equal.
+      /// </summary>
+      /// <returns>A cloned Bucket object.</returns>
+      public object Clone()
+      {
+         return new Bucket(LowerBound, UpperBound, Count);
+      }
+
+      /// <summary>
+      /// Width of the Bucket.
+      /// </summary>
+      public double Width
+      {
+         get { return UpperBound - LowerBound; }
+      }
+
+      /// <summary>
+      /// True if this is a single point argument for <see cref="IComparer{Bucket}"/>
+      /// when performing a Binary search.
+      /// </summary>
+      private bool IsSinglePoint
+      {
+         get { return double.IsNaN(Count); }
+      }
+
+      /// <summary>
+      /// Default comparer.
+      /// </summary>
+      public static IComparer<Bucket> DefaultPointComparer
+      {
+         get { return Comparer; }
+      }
+
+      /// <summary>
+      /// This method check whether a point is contained within this bucket.
+      /// </summary>
+      /// <param name="x">The point to check.</param>
+      /// <returns>
+      ///  0 if the point falls within the bucket boundaries;
+      /// -1 if the point is smaller than the bucket,
+      /// +1 if the point is larger than the bucket.</returns>
+      public int Contains(double x)
+      {
+         if (LowerBound < x)
+         {
+            if (UpperBound >= x)
+            {
+               return 0;
+            }
+
+            return 1;
+         }
+
+         return -1;
+      }
+
+      /// <summary>
+      /// Comparison of two disjoint buckets. The buckets cannot be overlapping.
+      /// </summary>
+      /// <returns>
+      ///  0 if <c>UpperBound</c> and <c>LowerBound</c> are bit-for-bit equal
+      ///  1 if This bucket is lower that the compared bucket
+      /// -1 otherwise
+      /// </returns>
+      public int CompareTo(Bucket bucket)
+      {
+         if (UpperBound > bucket.LowerBound && LowerBound < bucket.LowerBound)
+         {
+            throw new ArgumentException("Resources.PartialOrderException");
+         }
+
+         if (UpperBound.Equals(bucket.UpperBound)
+             && LowerBound.Equals(bucket.LowerBound))
+         {
+            return 0;
+         }
+
+         if (bucket.UpperBound <= LowerBound)
+         {
+            return 1;
+         }
+
+         return -1;
+      }
+
+      /// <summary>
+      /// Checks whether two Buckets are equal.
+      /// </summary>
+      /// <remarks>
+      /// <c>UpperBound</c> and <c>LowerBound</c> are compared bit-for-bit, but This method tolerates a
+      /// difference in <c>Count</c> given by  <seealso cref="Precision.AlmostEqual(double,double)"/>.
+      /// </remarks>
+      public override bool Equals(object obj)
+      {
+         if (!(obj is Bucket))
+         {
+            return false;
+         }
+
+         var bucket = (Bucket)obj;
+         return LowerBound.Equals(bucket.LowerBound)
+             && UpperBound.Equals(bucket.UpperBound)
+             && Count.AlmostEqual(bucket.Count);
+      }
+
+      /// <summary>
+      /// Provides a hash code for this bucket.
+      /// </summary>
+      public override int GetHashCode()
+      {
+         return LowerBound.GetHashCode() ^ UpperBound.GetHashCode() ^ Count.GetHashCode();
+      }
+
+      /// <summary>
+      /// Formats a human-readable string for this bucket.
+      /// </summary>
+      public override string ToString()
+      {
+         return "(" + LowerBound + ";" + UpperBound + "] = " + Count;
+      }
+   }
+
+   /// <summary>
+   /// A class which computes histograms of data.
+   /// </summary>
+   [Serializable]
+   [DataContract(Namespace = "urn:MathNet/Numerics")]
+   public class Histogram
+   {
+      /// <summary>
+      /// Contains all the <c>Bucket</c>s of the <c>Histogram</c>.
+      /// </summary>
+      [DataMember(Order = 1)]
+      private readonly List<Bucket> _buckets;
+
+      /// <summary>
+      /// Indicates whether the elements of <c>buckets</c> are currently sorted.
+      /// </summary>
+      [DataMember(Order = 2)]
+      private bool _areBucketsSorted;
+
+      /// <summary>
+      /// Initializes a new instance of the Histogram class.
+      /// </summary>
+      public Histogram()
+      {
+         _buckets = new List<Bucket>();
+         _areBucketsSorted = true;
+      }
+
+      /// <summary>
+      /// Constructs a Histogram with a specific number of equally sized buckets. The upper and lower bound of the histogram
+      /// will be set to the smallest and largest datapoint.
+      /// </summary>
+      /// <param name="data">The data sequence to build a histogram on.</param>
+      /// <param name="nbuckets">The number of buckets to use.</param>
+      public Histogram(IEnumerable<double> data, int nbuckets)
+          : this()
+      {
+         if (nbuckets < 1)
+         {
+            throw new ArgumentOutOfRangeException(nameof(data), "The number of bins in a histogram should be at least 1.");
+         }
+
+         double lower = data.Min(); //KD .Minimum();
+         double upper = data.Max(); //KD .Maximum();
+         double width = (upper - lower) / nbuckets;
+
+         if (double.IsNaN(width))
+         {
+            throw new ArgumentException("Data must contain at least one entry.", nameof(data));
+         }
+
+         // Add buckets for each bin; the smallest bucket's lowerbound must be slightly smaller
+         // than the minimal element.
+         double fNextLowerBound = lower + width;
+         AddBucket(new Bucket(lower.Decrement(), fNextLowerBound));
+         for (int n = 1; n < nbuckets; n++)
+         {
+            AddBucket(new Bucket(
+                fNextLowerBound,
+                fNextLowerBound = (lower + (n + 1) * width)));
+         }
+
+         AddData(data);
+      }
+
+      /// <summary>
+      /// Constructs a Histogram with a specific number of equally sized buckets.
+      /// </summary>
+      /// <param name="data">The data sequence to build a histogram on.</param>
+      /// <param name="nbuckets">The number of buckets to use.</param>
+      /// <param name="lower">The histogram lower bound.</param>
+      /// <param name="upper">The histogram upper bound.</param>
+      public Histogram(IEnumerable<double> data, int nbuckets, double lower, double upper)
+          : this()
+      {
+         if (lower > upper)
+         {
+            throw new ArgumentOutOfRangeException(nameof(upper), "The histogram lower bound must be smaller than the upper bound.");
+         }
+
+         if (nbuckets < 1)
+         {
+            throw new ArgumentOutOfRangeException(nameof(nbuckets), "The number of bins in a histogram should be at least 1.");
+         }
+
+         double width = (upper - lower) / nbuckets;
+
+         // Add buckets for each bin.
+         for (int n = 0; n < nbuckets; n++)
+         {
+            AddBucket(new Bucket(lower + n * width, lower + (n + 1) * width));
+         }
+
+         AddData(data);
+      }
+
+      /// <summary>
+      /// Add one data point to the histogram. If the datapoint falls outside the range of the histogram,
+      /// the lowerbound or upperbound will automatically adapt.
+      /// </summary>
+      /// <param name="d">The datapoint which we want to add.</param>
+      public void AddData(double d)
+      {
+         // Sort if needed.
+         LazySort();
+
+         if (d <= LowerBound)
+         {
+            // Make the lower bound just slightly smaller than the datapoint so it is contained in this bucket.
+            _buckets[0].LowerBound = d.Decrement();
+            _buckets[0].Count++;
+         }
+         else if (d > UpperBound)
+         {
+            _buckets[BucketCount - 1].UpperBound = d;
+            _buckets[BucketCount - 1].Count++;
+         }
+         else
+         {
+            _buckets[GetBucketIndexOf(d)].Count++;
+         }
+      }
+
+      /// <summary>
+      /// Add a sequence of data point to the histogram. If the datapoint falls outside the range of the histogram,
+      /// the lowerbound or upperbound will automatically adapt.
+      /// </summary>
+      /// <param name="data">The sequence of datapoints which we want to add.</param>
+      public void AddData(IEnumerable<double> data)
+      {
+         foreach (double d in data)
+         {
+            AddData(d);
+         }
+      }
+
+      /// <summary>
+      /// Adds a <c>Bucket</c> to the <c>Histogram</c>.
+      /// </summary>
+      public void AddBucket(Bucket bucket)
+      {
+         _buckets.Add(bucket);
+         _areBucketsSorted = false;
+      }
+
+      /// <summary>
+      /// Sort the buckets if needed.
+      /// </summary>
+      public void LazySort()
+      {
+         if (!_areBucketsSorted)
+         {
+            _buckets.Sort();
+            _areBucketsSorted = true;
+         }
+      }
+
+      /// <summary>
+      /// Returns the <c>Bucket</c> that contains the value <c>v</c>.
+      /// </summary>
+      /// <param name="v">The point to search the bucket for.</param>
+      /// <returns>A copy of the bucket containing point <paramref name="v"/>.</returns>
+      public Bucket GetBucketOf(double v)
+      {
+         return (Bucket)_buckets[GetBucketIndexOf(v)].Clone();
+      }
+
+      /// <summary>
+      /// Returns the index in the <c>Histogram</c> of the <c>Bucket</c>
+      /// that contains the value <c>v</c>.
+      /// </summary>
+      /// <param name="v">The point to search the bucket index for.</param>
+      /// <returns>The index of the bucket containing the point.</returns>
+      public int GetBucketIndexOf(double v)
+      {
+         // Sort if needed.
+         LazySort();
+
+         // Binary search for the bucket index.
+         int index = _buckets.BinarySearch(new Bucket(v), Bucket.DefaultPointComparer);
+
+         if (index < 0)
+         {
+            throw new ArgumentException("Resources.ArgumentHistogramContainsNot");
+         }
+
+         return index;
+      }
+
+      /// <summary>
+      /// Returns the lower bound of the histogram.
+      /// </summary>
+      public double LowerBound
+      {
+         get
+         {
+            LazySort();
+            return _buckets[0].LowerBound;
+         }
+      }
+
+      /// <summary>
+      /// Returns the upper bound of the histogram.
+      /// </summary>
+      public double UpperBound
+      {
+         get
+         {
+            LazySort();
+            return _buckets[_buckets.Count - 1].UpperBound;
+         }
+      }
+
+      /// <summary>
+      /// Gets the <c>n</c>'th bucket.
+      /// </summary>
+      /// <param name="n">The index of the bucket to be returned.</param>
+      /// <returns>A copy of the <c>n</c>'th bucket.</returns>
+      public Bucket this[int n]
+      {
+         get
+         {
+            LazySort();
+            //return (Bucket)_buckets[n].Clone();
+            return _buckets[n];
+         }
+      }
+
+      /// <summary>
+      /// Gets the number of buckets.
+      /// </summary>
+      public int BucketCount
+      {
+         get { return _buckets.Count; }
+      }
+
+      /// <summary>
+      /// Gets the total number of datapoints in the histogram.
+      /// </summary>
+      public double DataCount
+      {
+         get
+         {
+            double totalCount = 0;
+
+            for (int i = 0; i < BucketCount; i++)
+            {
+               totalCount += this[i].Count;
+            }
+
+            return totalCount;
+         }
+      }
+
+      /// <summary>
+      /// Prints the buckets contained in the <see cref="Histogram"/>.
+      /// </summary>
+      public override string ToString()
+      {
+         var sb = new StringBuilder();
+         foreach (Bucket b in _buckets)
+         {
+            sb.Append("   " + b + "  " + (b.Count / this.DataCount).ToString("0.0%") + Environment.NewLine);
+         }
+
+         return sb.ToString();
+      }
+   }
+}

src/client/spark_historical/HistoricalDataOptions.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using bts.api.core;
+using bts.md;
+using CommandLine;
+using System.Linq;
+using bts.history;
+using System.IO;
+using System.Threading.Tasks;
+using bts.api;
+using System.Threading;
+
+namespace bts.history
+{
+
+[Verb("historical", HelpText = "Historical Data Service command")]
+public class HistoricalDataOptions
+{
+   [Option('!', "server", Required = false, Default = false, HelpText = "Run HistoricalData Server")]
+   public bool server { get; set; }
+
+   [Option('p', "parse", Required = false, HelpText = "Parse a cme gzip file and print to screen")]
+   public string parsefile { get; set; }
+
+   [Option('d', "download", Required = false, HelpText = "Download and Parse a cme gzipped mbo file from an S3 bucket")]
+   public string s3key { get; set; }
+
+
+   [Option('$', "rskip", Required = false, HelpText = "Skip ahead to this date/time for replay backtesting")]
+   public string replayskip { get; set; }
+
+   [Option('b', "rstart", Required = false, HelpText = "Start date for replay backtesting")]
+   public string replaystart { get; set; }
+
+   [Option('e', "rend", Required = false, HelpText = "End date for replay backtesting")]
+   public string replayend { get; set; }
+
+   [Option('s', "rsymbol", Required = false, HelpText = "Symbol for replay backtesting")]
+   public string replaysymbol { get; set; }
+
+   [Option('t', "rsecuritytype", Required = false, HelpText = "SecurityType for replay backtesting")]
+   public string replaysecuritytype { get; set; }
+
+
+   [Option('1', "algo_security", Required = false, HelpText = "Security Desc i.e. ZCZ9 on which to run stat analysis")]
+   public string algo_security { get; set; }
+
+   [Option('2', "algo_type", Required = false, Default = "", HelpText = "which algo to run, leave blank for BookAnalyzer")]
+   public string algo_type { get; set; }
+
+   [Option('3', "algo_params", Required = false, Default = "size=1;", HelpText = "semicolon dellimited list of key=value pairs to parameterize algo")]
+   public string algo_params { get; set; }
+
+   [Option('4', "skip_preanalysis", Required = false, Default = false, HelpText = "don't send data before midnight on rstart to pre-analysis")]
+   public bool skip_preanalysis { get; set; }
+
+
+   [Option('5', "post_process_db", Required = false, Default = "", HelpText = "Name of the postgres db to generate from post processing")]
+   public string post_process_db { get; set; }
+   [Option('6', "post_process_db_host", Required = false, Default = "localhost", HelpText = "IP of the db ")]
+   public string post_process_db_host { get; set; }
+   [Option('7', "post_process_db_port", Required = false, Default = "27017", HelpText = "Port of the db ")]
+   public string post_process_db_port { get; set; }
+   [Option('8', "post_process_db_user", Required = false, Default = "mongo", HelpText = "User of the db ")]
+   public string post_process_db_user { get; set; }
+   [Option('9', "post_process_db_pass", Required = false, Default = "enter", HelpText = "Password of the db ")]
+   public string post_process_db_pass { get; set; }
+
+   public override string ToString()
+   {
+      return Stringizer.to_string(this);
+   }
+}
+
+}

src/client/spark_historical/HistoricalDataServiceServer.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using bts.api.core;
+using bts.md;
+using System.Linq;
+using System.Threading.Tasks;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   class HistoricalDataServiceServer : HistoricalDataServiceBase
+   {
+      readonly DataStreamer _stream = new DataStreamer();
+      public HistoricalDataServiceServer(ServiceContext context) : base(context, "")
+      {
+      }
+
+      async public override Task<MDStream> get_md_stream(MDQuery request)
+      {
+         if(request.Exchange != Venue.Glbx)
+            throw new NotSupportedException("HistoricalDataService only supports glbx right now");
+         var str = new MDStream();
+         await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(request, (sectype, mbo, _) => str.Md.Add(mbo));
+         return str;
+      }
+
+      async public override Task<TradingDistribution> get_trading_distribution(TradingDistributionQuery request)
+      {
+         var str = await get_md_stream(new MDQuery { Symbol = request.Symbol, Exchange = request.Exchange, SecurityType = request.SecurityType, 
+                                                     Begin = request.Begin, End = request.End } );
+         TradingDistribution dist = new TradingDistribution();
+         foreach(var md in str.Md)
+         {
+            if(md.timestamp().Seconds >= request.Begin.Seconds && md.timestamp().Seconds <= request.End.Seconds)
+            {
+               //foreach(
+            }
+         }
+         return dist;
+      }
+
+      async public override Task<Slices> get_trading_slices(SlicesQuery request)
+      {
+         var str = await get_md_stream(new MDQuery
+         {
+            Symbol = request.Symbol,
+            Exchange = request.Exchange,
+            SecurityType = request.SecurityType,
+            Begin = request.Begin,
+            End = request.End
+         });
+
+         throw new NotImplementedException();
+      }
+   }   
+}

src/client/spark_historical/HistoricalExtensions.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   static class HistoricalExtensions
+   {
+      public static Timestamp timestamp(this MarketDataIncrementalRefresh md)
+      {
+         return md.SendingTime;
+      }
+      public static string security_desc(this MDEntry md)
+      {
+         return md.Symbol;
+      }
+      public static string security_desc(this HistoryRow md)
+      {
+         return md.SecurityDesc;
+      }
+      public static bool less(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) < 0;
+      }
+      public static bool less_or_equal(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) <= 0;
+      }
+      public static bool greater(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) > 0;
+      }
+      public static bool greater_or_equal(this Timestamp lhs, Timestamp rhs)
+      {
+         return lhs.compare(rhs) >= 0;
+      }
+      public static int compare(this Timestamp lhs, Timestamp rhs)
+      {
+         int secdiff = lhs.Seconds.CompareTo(rhs.Seconds);
+         return (secdiff != 0) ? secdiff : (lhs.Nanos.CompareTo(rhs.Nanos));
+      }
+
+      public static int IndexOfMin<T>(this IList<T> self, IComparer<T> comparer) 
+      {
+         if (self == null)
+            throw new ArgumentNullException("self");
+
+         if (self.Count == 0)
+            throw new ArgumentException("List is empty.", "self");
+
+         T min = self[0];
+         int minIndex = 0;
+
+         for (int i = 1; i < self.Count; ++i)
+         {
+            if (comparer.Compare(self[i], min) < 0)
+            {
+               min = self[i];
+               minIndex = i;
+            }
+         }
+         return minIndex;
+      }
+   }
+   
+}

src/client/spark_historical/MBOParser.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   class MBOParser : IDisposable
+   {
+      Stream _stream = null;
+      StreamReader _reader;
+      QuickFix.IMessageFactory _defaultMsgFactory = new QuickFix.DefaultMessageFactory();
+      public MBOParser(string path) : this(File.Open(path, FileMode.Open)) {}
+      public MBOParser(Stream s3stream)
+      {
+         //_stream = new GZipStream( s3stream, CompressionMode.Decompress );
+         _reader = new StreamReader(s3stream, System.Text.Encoding.UTF8);
+      }
+
+      int _line = 0;
+      public IEnumerable<Tuple<T, string>> next<T>(string msgtype = null, Func<T, bool> pred = null, 
+                                                   Timestamp skip_before = null) where T : Google.Protobuf.IMessage, new()
+      {
+         int count = 0;
+         while (_reader.Peek() != -1)
+         {
+            _line++;
+            string line = _reader.ReadLine();
+            if(msgtype != null)
+            {
+               var type = QuickFix.Message.IdentifyType(line);            
+               if (type.getValue() != msgtype)
+                  continue;
+            }
+            if(skip_before != null)
+            {
+               var ts = ProtoFix.peek_timestamp(line);
+               if(ts == null)
+               {
+                  Console.WriteLine("Skipping null timestamp");
+                  continue;
+               }
+               else if(ts.less(skip_before))
+               {
+                  if (count++ % 50000 == 1)
+                     Console.WriteLine("Skipping " + ts.ToDateTime());
+                  continue;
+               }
+               else
+                  skip_before = null;
+            }
+            T md = default(T);
+            try
+            {
+               md = ProtoFix.parse_fix<T>(line);
+            }
+            catch(QuickFix.MessageParseError) {}
+            if(md == null)
+            {
+               Console.WriteLine("FIX Parse Error");
+               continue;
+            }
+            else if(pred != null && !pred(md))
+               continue;
+            
+            yield return Tuple.Create(md, line);
+         }
+      }
+
+      public IEnumerable<Tuple<string, string>> next_line(int peek_field = 0) 
+      {
+         while (_reader.Peek() != -1)
+         {
+            _line++;
+            string line = _reader.ReadLine();
+            string peek = string.Empty;
+            if(peek_field != 0)
+               peek = ProtoFix.peek_field(line, peek_field);
+            yield return Tuple.Create(peek, line);
+         }
+      }
+
+      public void Dispose()
+      {
+         if(_stream != null)
+            _stream.Dispose();
+      }
+   }
+}

src/client/spark_historical/NormalEstimator.cs

@@ -0,0 +1,74 @@
+using System;
+
+namespace bts.history
+{
+   class NormalEstimator
+   {
+      private long _n;
+      private double _mu;
+      private double _sq;
+      private double _min = double.MaxValue;
+      private double _max = double.MinValue;
+
+      public NormalEstimator() { }
+      public void add(double x)
+      {
+         _n++;
+         double next_mu = _mu + (x - _mu) / _n;
+         _sq += (x - _mu) * (x - next_mu);
+         _mu = next_mu;
+
+         if (x < _min)
+            _min = x;
+         if (x > _max)
+            _max = x;
+      }
+      public void delete(double x)
+      {
+         if (_n == 0)
+            throw new ArgumentOutOfRangeException("No elements to delete");
+         else if (_n == 1)
+         {
+            _n = 0;
+            _mu = 0.0;
+            _sq = 0.0;
+         }
+         else
+         {
+            double mu = (_n * _mu - x) / (_n - 1);
+            _sq -= (x - _mu) * (x - mu);
+            _mu = mu;
+            --_n;
+         }
+      }
+      public long num_samples => _n;
+      public double mean() => _mu;
+      public double variance()
+      {
+         if (_n > 1)
+            return _sq / _n;
+         else
+            return 0.0;
+      }
+      public double unbiased_variance()
+      {
+         if (_n > 1)
+            return _sq / (_n - 1);
+         else
+            return 0.0;
+      }
+      public double std_deviation()
+      {
+         return Math.Sqrt(variance());
+      }
+      public double unbiased_std_deviation()
+      {
+         return Math.Sqrt(unbiased_variance());
+      }
+      public override string ToString()
+      {
+         return $"Normal( mean {mean()}, std_dev {std_deviation()} ) [min {_min}, max { _max} ] [numSamples={num_samples}]";
+      }
+   }
+
+}

src/client/spark_historical/Notebooks/.ipynb_checkpoints/oil_11_20-checkpoint.ipynb

@@ -0,0 +1,307 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"oil_nov\", \"bigblue\");\n",
+    "//Timestamp ts = new Timestamp();\n",
+    "//ts.Seconds = 15;\n",
+    "//DateTime dt = ts.ToDateTime();\n",
+    "//display(dt.Kind);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var f = q.collection<HistoryRow>(\"CLF0-CLJ0\");\n",
+    "var j = q.collection<HistoryRow>(\"CLJ0-CLM0\");\n",
+    "var m = q.collection<HistoryRow>(\"CLF0-CLM0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 19, 0, 0, 0), new DateTime(2019, 11, 21, 23, 59, 59));\n",
+    "var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 19, 0, 0, 0), new DateTime(2019, 11, 21, 23, 59, 59));\n",
+    "var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 19, 0, 0, 0), new DateTime(2019, 11, 21, 23, 59, 59));\n",
+    "\n",
+    "//var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "317961"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "402409"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "438568"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(ft.Count());\n",
+    "display(jt.Count());\n",
+    "display(mt.Count());\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var ff = ft.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var jf = jt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var mf = mt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "\n",
+    "var ftop = ff.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var jtop = jf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var mtop = mf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "//display(h0f.Take(1));\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{ftop, jtop, mtop}, new[] {1.0, 1.0, -1.0}, new[] {1.0, 1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>77608908</td><td>2019-11-19 23:00:00.200092</td><td>2019-11-19 23:00:00.010916</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77608393 88 71.00 |77608908 4 141.00 |</td><td>77608433 1 77.00 |77608393 82 73.00 |77608908 0 0.00 |</td></tr><tr><td>1</td><td>77608909</td><td>2019-11-19 23:00:00.200105</td><td>2019-11-19 23:00:00.010918</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77608393 88 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77608393 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>2</td><td>77609069</td><td>2019-11-19 23:00:00.202060</td><td>2019-11-19 23:00:00.016665</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77609069 88 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77609069 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>3</td><td>77609070</td><td>2019-11-19 23:00:00.202068</td><td>2019-11-19 23:00:00.016666</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77609070 89 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77609070 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>4</td><td>77609115</td><td>2019-11-19 23:00:00.202598</td><td>2019-11-19 23:00:00.017703</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>5</td><td>77609123</td><td>2019-11-19 23:00:00.202678</td><td>2019-11-19 23:00:00.017813</td><td>0</td><td>141</td><td>9</td><td>1</td><td>77609123 2 70.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609123 1 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>6</td><td>77609124</td><td>2019-11-19 23:00:00.202687</td><td>2019-11-19 23:00:00.017813</td><td>0</td><td>141</td><td>9</td><td>4</td><td>77609124 2 70.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609124 4 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>7</td><td>77609369</td><td>2019-11-19 23:00:00.205493</td><td>2019-11-19 23:00:00.020957</td><td>0</td><td>142</td><td>9</td><td>4</td><td>77609369 3 71.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609369 4 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>8</td><td>77609370</td><td>2019-11-19 23:00:00.205506</td><td>2019-11-19 23:00:00.020959</td><td>0</td><td>142</td><td>6</td><td>3</td><td>77609370 3 71.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609370 3 74.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>9</td><td>77609460</td><td>2019-11-19 23:00:00.206467</td><td>2019-11-19 23:00:00.021838</td><td>0</td><td>142</td><td>5</td><td>3</td><td>77609370 3 71.00 |77609115 89 71.00 |77609460 3 142.00 |</td><td>77609370 3 74.00 |77609115 82 73.00 |77609460 0 0.00 |</td></tr><tr><td>10</td><td>127750158</td><td>2019-11-20 22:47:47.470815</td><td>2019-11-20 22:47:47.470477</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750158 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750158 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>11</td><td>127750159</td><td>2019-11-20 22:47:47.470854</td><td>2019-11-20 22:47:47.470478</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750159 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750159 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>12</td><td>127750160</td><td>2019-11-20 22:47:47.470864</td><td>2019-11-20 22:47:47.470479</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750160 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750160 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>13</td><td>127750161</td><td>2019-11-20 22:47:47.470875</td><td>2019-11-20 22:47:47.470480</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750161 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750161 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>14</td><td>127750162</td><td>2019-11-20 22:47:47.470885</td><td>2019-11-20 22:47:47.470481</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750162 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750162 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>15</td><td>127750163</td><td>2019-11-20 22:47:47.470900</td><td>2019-11-20 22:47:47.470483</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750163 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750163 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>16</td><td>127750164</td><td>2019-11-20 22:47:47.470909</td><td>2019-11-20 22:47:47.470484</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750164 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750164 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>17</td><Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.209544500Z\" active_for:210100 ns -0.01:8\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211537800Z\" active_for:11000 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211548800Z\" active_for:42400 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211591200Z\" active_for:296400 ns -0.01:7\n",
+    "display(synth.Take(5000).OrderBy(x => x.sending_time));\n",
+    "//Console.WriteLine(synth.Take(10).Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/.ipynb_checkpoints/oil_11_20_zfg-checkpoint.ipynb

@@ -0,0 +1,295 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"oil_11_20\", \"bigblue\");\n",
+    "//Timestamp ts = new Timestamp();\n",
+    "//ts.Seconds = 15;\n",
+    "//DateTime dt = ts.ToDateTime();\n",
+    "//display(dt.Kind);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var c1 = q.collection<HistoryRow>(\"CLZ9-CLF0\");\n",
+    "var c2 = q.collection<HistoryRow>(\"CLF0-CLG0\");\n",
+    "var c3 = q.collection<HistoryRow>(\"CLZ9-CLG0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var begin = DateTime.SpecifyKind(new DateTime(2019, 11, 19, 0, 0, 0),  DateTimeKind.Utc);\n",
+    "var end = DateTime.SpecifyKind(new DateTime(2019, 11, 22, 23, 59, 59), DateTimeKind.Utc);\n",
+    "var c1t = q.timeseries<HistoryRow>(c1, begin, end);\n",
+    "var c2t = q.timeseries<HistoryRow>(c2, begin, end);\n",
+    "var c3t = q.timeseries<HistoryRow>(c3, begin, end);\n",
+    "\n",
+    "//var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "18844"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "83773"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "35257"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(c1t.Count());\n",
+    "display(c2t.Count());\n",
+    "display(c3t.Count());\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th>TopBook</th><th>value</th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>MsgSeqNum</th></tr></thead><tbody><tr><td>{ bts.co2.TopBookMessage: Bid: { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 9 }, Ask: { bts.co2.PriceSizeMessage: Mantissa: 4, Exponent: 0, Size: 17 } }</td><td>{ bts.history.HistoryRow: trade_date: 2019-11-20 00:00:00Z, sending_time: 2019-11-20 16:23:37Z, transact_time: 2019-11-20 16:23:37Z, TradeDate: { Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574208000, Nanos: 0 }, MsgSeqNum: 106692306, SendingTime: { Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574267017, Nanos: 559710300 }, TransactTime: { Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574267017, Nanos: 559610900 }, SecurityDesc: CLZ9-CLG0, Book: { bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 9 }, { bts.co2.PriceSizeMessage: Mantissa: -1, Exponent: 0, Size: 14 }, { bts.co2.PriceSizeMessage: Mantissa: -2, Exponent: 0, Size: 50 }, { bts.co2.PriceSizeMessage: Mantissa: -3, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -4, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -5, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -6, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -7, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -8, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -9, Exponent: 0, Size: 10 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 4, Exponent: 0, Size: 17 }, { bts.co2.PriceSizeMessage: Mantissa: 5, Exponent: 0, Size: 15 }, { bts.co2.PriceSizeMessage: Mantissa: 6, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 7, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 8, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 9, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 10, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 11, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 0 }, { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 0 } ] }, TopBook: &lt;null&gt;, Trade: &lt;null&gt;, Order: &lt;null&gt; }</td><td>2019-11-20 00:00:00Z</td><td>2019-11-20 16:23:37Z</td><td>2019-11-20 16:23:37Z</td><td>106692306</td></tr></tbody></table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var c1f = c1t.Where(x => x.Book != null && (x.Book.Bids.Count > 0 || x.Book.Asks.Count > 0));\n",
+    "var c2f = c2t.Where(x => x.Book != null && (x.Book.Bids.Count > 0 || x.Book.Asks.Count > 0));\n",
+    "var c3f = c3t.Where(x => x.Book != null && (x.Book.Bids.Count > 0 || x.Book.Asks.Count > 0));\n",
+    "\n",
+    "var c1top = c1f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var c2top = c2f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var c3top = c3f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "display(c3top.Last());\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string trade_date => _synth.trade_date.ToString(\"yyyy-MM-dd\");\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{c1top, c2top, c3top}, new[] {1.0, 1.0, -1.0}, new[] {1.0, 1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>77632899</td><td>2019-11-20</td><td>2019-11-19 23:01:29.444834</td><td>2019-11-19 23:01:29.444674</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77632359 599 10.00 |77632899 0 0.00 |</td><td>77623170 9 -12.00 |77632359 809 11.00 |77632899 1 22.00 |</td></tr><tr><td>1</td><td>77632900</td><td>2019-11-20</td><td>2019-11-19 23:01:29.444935</td><td>2019-11-19 23:01:29.444830</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77632359 599 10.00 |77632900 0 0.00 |</td><td>77623170 9 -12.00 |77632359 809 11.00 |77632900 1 22.00 |</td></tr><tr><td>2</td><td>77633139</td><td>2019-11-20</td><td>2019-11-19 23:01:31.008894</td><td>2019-11-19 23:01:31.008777</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633139 599 10.00 |77632900 0 0.00 |</td><td>77623170 9 -12.00 |77633139 809 11.00 |77632900 1 22.00 |</td></tr><tr><td>3</td><td>77633233</td><td>2019-11-20</td><td>2019-11-19 23:01:31.507462</td><td>2019-11-19 23:01:31.507346</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77632900 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77632900 1 22.00 |</td></tr><tr><td>4</td><td>77633265</td><td>2019-11-20</td><td>2019-11-19 23:01:32.157546</td><td>2019-11-19 23:01:32.157409</td><td>1</td><td>-10</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633265 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633265 1 7.00 |</td></tr><tr><td>5</td><td>77633266</td><td>2019-11-20</td><td>2019-11-19 23:01:32.157639</td><td>2019-11-19 23:01:32.157536</td><td>1</td><td>-10</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633266 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633266 1 7.00 |</td></tr><tr><td>6</td><td>77633967</td><td>2019-11-20</td><td>2019-11-19 23:01:38.955779</td><td>2019-11-19 23:01:38.955632</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633967 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633967 1 3.00 |</td></tr><tr><td>7</td><td>77633968</td><td>2019-11-20</td><td>2019-11-19 23:01:38.955866</td><td>2019-11-19 23:01:38.955760</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633968 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633968 1 3.00 |</td></tr><tr><td>8</td><td>77634067</td><td>2019-11-20</td><td>2019-11-19 23:01:41.509218</td><td>2019-11-19 23:01:41.509092</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634067 599 10.00 |77633968 0 0.00 |</td><td>77623170 9 -12.00 |77634067 809 11.00 |77633968 1 3.00 |</td></tr><tr><td>9</td><td>77634146</td><td>2019-11-20</td><td>2019-11-19 23:01:42.008457</td><td>2019-11-19 23:01:42.008346</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634146 599 10.00 |77633968 0 0.00 |</td><td>77623170 9 -12.00 |77634146 809 11.00 |77633968 1 3.00 |</td></tr><tr><td>10</td><td>77634662</td><td>2019-11-20</td><td>2019-11-19 23:01:44.039520</td><td>2019-11-19 23:01:44.039346</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634146 599 10.00 |77634662 0 0.00 |</td><td>77623170 9 -12.00 |77634146 809 11.00 |77634662 1 1.00 |</td></tr><tr><td>11</td><td>77634663</td><td>2019-11-20</td><td>2019-11-19 23:01:44.039606</td><td>2019-11-19 23:01:44.039493</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634146 599 10.00 |77634663 0 0.00 |</td><td>77623170 9 -12.00 |77634146 809 11.00 |77634663 1 1.00 |</td></tr><tr><td>12</td><td>77635060</td><td>2019-11-20</td><td>2019-11-19 23:01:52.009603</td><td>2019-11-19 23:01:52.009478</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635060 599 10.00 |77634663 0 0.00 |</td><td>77623170 9 -12.00 |77635060 809 11.00 |77634663 1 1.00 |</td></tr><tr><td>13</td><td>77635104</td><td>2019-11-20</td><td>2019-11-19 23:01:52.509048</td><td>2019-11-19 23:01:52.508929</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635104 599 10.00 |77634663 0 0.00 |</td><td>77623170 9 -12.00 |77635104 809 11.00 |77634663 1 1.00 |</td></tr><tr><td>14</td><td>77635195</td><td>2019-11-20</td><td>2019-11-19 23:01:54.107257</td><td>2019-11-19 23:01:54.107128</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635104 599 10.00 |77635195 0 0.00 |</td><td>77623170 9 -12.00 |77635104 809 11.00 |77635195 1 1.00 |</td></tr><tr><td>15</td><td>77635196</td><td>2019-11-20</td><td>2019-11-19 23:01:54.107340</td><td>2019-11-19 23:01:54.107238</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635104 599 10.00 |77635196 0 0.00 |</td><td>77623170 9 -12.00 |77635104 809 11.00 |77635196 Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(synth.OrderBy(x => x.sending_time));\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/.ipynb_checkpoints/oil_12_4_2019-checkpoint.ipynb

@@ -0,0 +1,309 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"oil_12_4_2019\", \"bigblue\");\n",
+    "//Timestamp ts = new Timestamp();\n",
+    "//ts.Seconds = 15;\n",
+    "//DateTime dt = ts.ToDateTime();\n",
+    "//display(dt.Kind);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var f = q.collection<HistoryRow>(\"CLH0-CLM0\");\n",
+    "var j = q.collection<HistoryRow>(\"CLM0-CLZ0\");\n",
+    "var m = q.collection<HistoryRow>(\"CLH0-CLZ0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var start = new DateTime(2019, 12, 4, 0, 0, 0);\n",
+    "var end = new DateTime(2019, 12, 5, 23, 59, 59);\n",
+    "var ft = q.timeseries<HistoryRow>(f, start, end);\n",
+    "var jt = q.timeseries<HistoryRow>(j, start, end);\n",
+    "var mt = q.timeseries<HistoryRow>(m,start, end);\n",
+    "\n",
+    "//var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "223173"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "273808"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "223254"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(ft.Count());\n",
+    "display(jt.Count());\n",
+    "display(mt.Count());\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var ff = ft.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var jf = jt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var mf = mt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "\n",
+    "var ftop = ff.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var jtop = jf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var mtop = mf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "//display(h0f.Take(1));\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{ftop, jtop, mtop}, new[] {1.0, 1.0, -1.0}, new[] {1.0, 1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>113288047</td><td>2019-12-04 15:09:12.008300</td><td>2019-12-04 15:09:12.008131</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288047 106 116.00 |112831258 32 200.00 |113288020 3 319.00 |</td><td>113288047 30 117.00 |112831258 17 201.00 |113288020 40 321.00 |</td></tr><tr><td>1</td><td>113288049</td><td>2019-12-04 15:09:12.008331</td><td>2019-12-04 15:09:12.008144</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288049 106 116.00 |112831258 32 200.00 |113288020 3 319.00 |</td><td>113288049 30 117.00 |112831258 17 201.00 |113288020 40 321.00 |</td></tr><tr><td>2</td><td>113288065</td><td>2019-12-04 15:09:12.008554</td><td>2019-12-04 15:09:12.008279</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288065 106 116.00 |112831258 32 200.00 |113288020 3 319.00 |</td><td>113288065 30 117.00 |112831258 17 201.00 |113288020 40 321.00 |</td></tr><tr><td>3</td><td>113288069</td><td>2019-12-04 15:09:12.008614</td><td>2019-12-04 15:09:12.008330</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288069 106 116.00 |112831258 32 200.00 |113288020 3 319.00 |</td><td>113288069 30 117.00 |112831258 17 201.00 |113288020 40 321.00 |</td></tr><tr><td>4</td><td>113288070</td><td>2019-12-04 15:09:12.008633</td><td>2019-12-04 15:09:12.008330</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288070 106 116.00 |112831258 32 200.00 |113288020 3 319.00 |</td><td>113288070 30 117.00 |112831258 17 201.00 |113288020 40 321.00 |</td></tr><tr><td>5</td><td>113288079</td><td>2019-12-04 15:09:12.008763</td><td>2019-12-04 15:09:12.008346</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288070 106 116.00 |112831258 32 200.00 |113288079 3 319.00 |</td><td>113288070 30 117.00 |112831258 17 201.00 |113288079 40 321.00 |</td></tr><tr><td>6</td><td>113288081</td><td>2019-12-04 15:09:12.008783</td><td>2019-12-04 15:09:12.008350</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288081 106 116.00 |112831258 32 200.00 |113288079 3 319.00 |</td><td>113288081 30 117.00 |112831258 17 201.00 |113288079 40 321.00 |</td></tr><tr><td>7</td><td>113288090</td><td>2019-12-04 15:09:12.008903</td><td>2019-12-04 15:09:12.008420</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288090 106 116.00 |112831258 32 200.00 |113288079 3 319.00 |</td><td>113288090 30 117.00 |112831258 17 201.00 |113288079 40 321.00 |</td></tr><tr><td>8</td><td>113288099</td><td>2019-12-04 15:09:12.009058</td><td>2019-12-04 15:09:12.008476</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288090 106 116.00 |112831258 32 200.00 |113288099 3 319.00 |</td><td>113288090 30 117.00 |112831258 17 201.00 |113288099 45 321.00 |</td></tr><tr><td>9</td><td>113288100</td><td>2019-12-04 15:09:12.009071</td><td>2019-12-04 15:09:12.008477</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288090 106 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288090 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>10</td><td>113288122</td><td>2019-12-04 15:09:12.009381</td><td>2019-12-04 15:09:12.008909</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288122 106 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288122 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>11</td><td>113288123</td><td>2019-12-04 15:09:12.009399</td><td>2019-12-04 15:09:12.008910</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288123 106 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288123 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>12</td><td>113288124</td><td>2019-12-04 15:09:12.009407</td><td>2019-12-04 15:09:12.008916</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288124 106 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288124 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>13</td><td>113288133</td><td>2019-12-04 15:09:12.009536</td><td>2019-12-04 15:09:12.009143</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288133 106 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288133 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>14</td><td>113288147</td><td>2019-12-04 15:09:12.009857</td><td>2019-12-04 15:09:12.009582</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288147 100 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288147 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>15</td><td>113288152</td><td>2019-12-04 15:09:12.010024</td><td>2019-12-04 15:09:12.009665</td><td>32</td><td>-5</td><td>-1</td><td>3</td><td>113288152 100 116.00 |112831258 32 200.00 |113288100 3 319.00 |</td><td>113288152 30 117.00 |112831258 17 201.00 |113288100 45 321.00 |</td></tr><tr><td>16</td><td>113288155</td><td>2019-12-04 15:09:12.010201</td><td>201Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.209544500Z\" active_for:210100 ns -0.01:8\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211537800Z\" active_for:11000 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211548800Z\" active_for:42400 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211591200Z\" active_for:296400 ns -0.01:7\n",
+    "display(synth.Take(15000).OrderBy(x => x.sending_time));\n",
+    "//Console.WriteLine(synth.Take(10).Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/.ipynb_checkpoints/roll-checkpoint.ipynb

@@ -0,0 +1,278 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"roll\", \"bigblue\");"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var cz9 = q.collection<HistoryRow>(\"ZNZ9\");\n",
+    "var ch0 = q.collection<HistoryRow>(\"ZNH0\");\n",
+    "var cspread = q.collection<HistoryRow>(\"ZNZ9-ZNH0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var z9 = q.timeseries<HistoryRow>(cz9, new DateTime(2019, 11, 25, 14,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n",
+    "var h0 = q.timeseries<HistoryRow>(ch0, new DateTime(2019, 11, 25, 14,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n",
+    "var spread = q.timeseries<HistoryRow>(cspread, new DateTime(2019, 11, 25, 14 ,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>TradeDate</th><th>MsgSeqNum</th><th>SendingTime</th><th>TransactTime</th><th>SecurityDesc</th><th>Book</th><th>TopBook</th><th>Trade</th><th>Order</th></tr></thead><tbody><tr><td>0</td><td>2019-11-26 00:00:00Z</td><td>2019-11-25 22:54:30Z</td><td>2019-11-25 22:54:30Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574726400, Nanos: 0 }</td><td>11851041</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667725300 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667299500 }</td><td>ZNZ9</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td><td>{ bts.history.MBOMessage: OrderUpdateAction: { bts.history.MDUpdateAction: value__: 48 }, OrderId: 846787349208, SecurityId: 59121, SecurityDesc: ZNZ9, MdOrderPriority: 12370689694, MdEntryPx: 129.34375, MdPriceLevel: 0, MdDisplayQty: 72 }</td></tr><tr><td>1</td><td>2019-11-26 00:00:00Z</td><td>2019-11-25 22:54:30Z</td><td>2019-11-25 22:54:30Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574726400, Nanos: 0 }</td><td>11851041</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667725300 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667299500 }</td><td>ZNZ9</td><td>{ bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 2 }, { bts.co2.PriceSizeMessage: Mantissa: 129546875000, Exponent: -9, Size: 54 }, { bts.co2.PriceSizeMessage: Mantissa: 129531250000, Exponent: -9, Size: 426 }, { bts.co2.PriceSizeMessage: Mantissa: 129515625000, Exponent: -9, Size: 897 }, { bts.co2.PriceSizeMessage: Mantissa: 129500000000, Exponent: -9, Size: 1181 }, { bts.co2.PriceSizeMessage: Mantissa: 129484375000, Exponent: -9, Size: 1558 }, { bts.co2.PriceSizeMessage: Mantissa: 129468750000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129453125000, Exponent: -9, Size: 1555 }, { bts.co2.PriceSizeMessage: Mantissa: 129437500000, Exponent: -9, Size: 1628 }, { bts.co2.PriceSizeMessage: Mantissa: 129421875000, Exponent: -9, Size: 1582 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 76 }, { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 144 }, { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 666 }, { bts.co2.PriceSizeMessage: Mantissa: 129609375000, Exponent: -9, Size: 1054 }, { bts.co2.PriceSizeMessage: Mantissa: 129625000000, Exponent: -9, Size: 1590 }, { bts.co2.PriceSizeMessage: Mantissa: 129640625000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129656250000, Exponent: -9, Size: 1877 }, { bts.co2.PriceSizeMessage: Mantissa: 129671875000, Exponent: -9, Size: 1594 }, { bts.co2.PriceSizeMessage: Mantissa: 129687500000, Exponent: -9, Size: 2246 }, { bts.co2.PriceSizeMessage: Mantissa: 129703125000, Exponent: -9, Size: 2006 } ] }</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td></tr><tr><td>2</td><td>2019-11-26 00:00:00Z</td><td>2019-11-25 22:54:30Z</td><td>2019-11-25 22:54:30Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574726400, Nanos: 0 }</td><td>11851041</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667725300 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667299500 }</td><td>ZNZ9</td><td>{ bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 129546875000, Exponent: -9, Size: 54 }, { bts.co2.PriceSizeMessage: Mantissa: 129531250000, Exponent: -9, Size: 426 }, { bts.co2.PriceSizeMessage: Mantissa: 129515625000, Exponent: -9, Size: 897 }, { bts.co2.PriceSizeMessage: Mantissa: 129500000000, Exponent: -9, Size: 1181 }, { bts.co2.PriceSizeMessage: Mantissa: 129484375000, Exponent: -9, Size: 1558 }, { bts.co2.PriceSizeMessage: Mantissa: 129468750000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129453125000, Exponent: -9, Size: 1555 }, { bts.co2.PriceSizeMessage: Mantissa: 129437500000, Exponent: -9, Size: 1628 }, { bts.co2.PriceSizeMessage: Mantissa: 129421875000, Exponent: -9, Size: 1582 }, { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 0 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 76 }, { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 144 }, { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 666 }, { bts.co2.PriceSizeMessage: Mantissa: 129609375000, Exponent: -9, Size: 1054 }, { bts.co2.PriceSizeMessage: Mantissa: 129625000000, Exponent: -9, Size: 1590 }, { bts.co2.PriceSizeMessage: Mantissa: 129640625000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129656250000, Exponent: -9, Size: 1877 }, { bts.co2.PriceSizeMessage: Mantissa: 129671875000, Exponent: -9, Size: 1594 }, { bts.co2.PriceSizeMessage: ManLine truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var zz = z9.Where(x => x.MsgSeqNum == 11851041 );\n",
+    "//display(z9.Count());\n",
+    "//display(h0.Count());\n",
+    "//display(spread.Count());\n",
+    "display(zz);\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>TradeDate</th><th>MsgSeqNum</th><th>SendingTime</th><th>TransactTime</th><th>SecurityDesc</th><th>Book</th><th>TopBook</th><th>Trade</th><th>Order</th></tr></thead><tbody><tr><td>0</td><td>2019-11-25 00:00:00Z</td><td>2019-11-25 19:00:00Z</td><td>2019-11-25 19:00:00Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574640000, Nanos: 0 }</td><td>9955014</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574708400, Nanos: 47819600 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574708400, Nanos: 47703400 }</td><td>ZNH0</td><td>{ bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 1114 }, { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 2349 }, { bts.co2.PriceSizeMessage: Mantissa: 129546875000, Exponent: -9, Size: 2129 }, { bts.co2.PriceSizeMessage: Mantissa: 129531250000, Exponent: -9, Size: 3452 }, { bts.co2.PriceSizeMessage: Mantissa: 129515625000, Exponent: -9, Size: 1822 }, { bts.co2.PriceSizeMessage: Mantissa: 129500000000, Exponent: -9, Size: 1940 }, { bts.co2.PriceSizeMessage: Mantissa: 129484375000, Exponent: -9, Size: 2178 }, { bts.co2.PriceSizeMessage: Mantissa: 129468750000, Exponent: -9, Size: 1869 }, { bts.co2.PriceSizeMessage: Mantissa: 129453125000, Exponent: -9, Size: 1959 }, { bts.co2.PriceSizeMessage: Mantissa: 129437500000, Exponent: -9, Size: 2074 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 1479 }, { bts.co2.PriceSizeMessage: Mantissa: 129609375000, Exponent: -9, Size: 2141 }, { bts.co2.PriceSizeMessage: Mantissa: 129625000000, Exponent: -9, Size: 2139 }, { bts.co2.PriceSizeMessage: Mantissa: 129640625000, Exponent: -9, Size: 2071 }, { bts.co2.PriceSizeMessage: Mantissa: 129656250000, Exponent: -9, Size: 1817 }, { bts.co2.PriceSizeMessage: Mantissa: 129671875000, Exponent: -9, Size: 2216 }, { bts.co2.PriceSizeMessage: Mantissa: 129687500000, Exponent: -9, Size: 2034 }, { bts.co2.PriceSizeMessage: Mantissa: 129703125000, Exponent: -9, Size: 1950 }, { bts.co2.PriceSizeMessage: Mantissa: 129718750000, Exponent: -9, Size: 1896 }, { bts.co2.PriceSizeMessage: Mantissa: 129734375000, Exponent: -9, Size: 1952 } ] }</td><td>{ bts.co2.TopBookMessage: Bid: { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 1114 }, Ask: { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 1479 } }</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td></tr></tbody></table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var z9f = z9.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var h0f = h0.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var spreadf = spread.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "\n",
+    "var z9t = z9f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var h0t = h0f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var spreadt = spreadf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "display(h0f.Take(1));\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.000000 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.000000 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{z9t, h0t, spreadt}, new[] {1.0, -1.0, -1.0}, new[] {1.0, -1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>11851041</td><td>2019-11-25 22:54:30.667725</td><td>2019-11-25 22:54:30.667299</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851041 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851041 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>1</td><td>11851042</td><td>2019-11-25 22:54:30.668019</td><td>2019-11-25 22:54:30.667773</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851042 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851042 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>2</td><td>11851042</td><td>2019-11-25 22:54:30.668019</td><td>2019-11-25 22:54:30.667773</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851042 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851042 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>3</td><td>11851043</td><td>2019-11-25 22:54:30.668036</td><td>2019-11-25 22:54:30.667774</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851043 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851043 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>4</td><td>11851043</td><td>2019-11-25 22:54:30.668036</td><td>2019-11-25 22:54:30.667774</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851043 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851043 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>5</td><td>11851044</td><td>2019-11-25 22:54:30.668054</td><td>2019-11-25 22:54:30.667777</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851044 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851044 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>6</td><td>11851044</td><td>2019-11-25 22:54:30.668054</td><td>2019-11-25 22:54:30.667777</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851044 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851044 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>7</td><td>11851045</td><td>2019-11-25 22:54:30.668078</td><td>2019-11-25 22:54:30.667779</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851045 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851045 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>8</td><td>11851045</td><td>2019-11-25 22:54:30.668078</td><td>2019-11-25 22:54:30.667779</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851045 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851045 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>9</td><td>11851046</td><td>2019-11-25 22:54:30.668091</td><td>2019-11-25 22:54:30.667782</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851046 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851046 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>10</td><td>11851046</td><td>2019-11-25 22:54:30.668091</td><td>2019-11-25 22:54:30.667782</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851046 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851046 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>11</td><td>11851047</td><td>2019-11-25 22:54:30.668105</td><td>2019-11-25 22:54:30.667787</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851047 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851047 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>12</td><td>11851047</td><td>2019-11-25 22:54:30.668105</td><td>2019-11-25 22:54:30.667787</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851047 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851047 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>13</td><td>11851048</td><td>2019-11-25 22:54:30.668137</td><td>2019-11-25 22:54:30.667789</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851048 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851048 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>14</td><td>11851048</td><td>2019-11-25 22:54:30.668137</td><td>2019-11-25 22:54:30.667789</td><td>193</td><td>-Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(synth.Take(10000).OrderBy(x => x.sending_time));\n",
+    "//Console.WriteLine(synth.Take(10).Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/oil_11_20.ipynb

@@ -0,0 +1,307 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"oil_nov\", \"bigblue\");\n",
+    "//Timestamp ts = new Timestamp();\n",
+    "//ts.Seconds = 15;\n",
+    "//DateTime dt = ts.ToDateTime();\n",
+    "//display(dt.Kind);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var f = q.collection<HistoryRow>(\"CLF0-CLJ0\");\n",
+    "var j = q.collection<HistoryRow>(\"CLJ0-CLM0\");\n",
+    "var m = q.collection<HistoryRow>(\"CLF0-CLM0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 19, 0, 0, 0), new DateTime(2019, 11, 21, 23, 59, 59));\n",
+    "var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 19, 0, 0, 0), new DateTime(2019, 11, 21, 23, 59, 59));\n",
+    "var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 19, 0, 0, 0), new DateTime(2019, 11, 21, 23, 59, 59));\n",
+    "\n",
+    "//var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "317961"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "402409"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "438568"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(ft.Count());\n",
+    "display(jt.Count());\n",
+    "display(mt.Count());\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var ff = ft.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var jf = jt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var mf = mt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "\n",
+    "var ftop = ff.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var jtop = jf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var mtop = mf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "//display(h0f.Take(1));\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{ftop, jtop, mtop}, new[] {1.0, 1.0, -1.0}, new[] {1.0, 1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>77608908</td><td>2019-11-19 23:00:00.200092</td><td>2019-11-19 23:00:00.010916</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77608393 88 71.00 |77608908 4 141.00 |</td><td>77608433 1 77.00 |77608393 82 73.00 |77608908 0 0.00 |</td></tr><tr><td>1</td><td>77608909</td><td>2019-11-19 23:00:00.200105</td><td>2019-11-19 23:00:00.010918</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77608393 88 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77608393 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>2</td><td>77609069</td><td>2019-11-19 23:00:00.202060</td><td>2019-11-19 23:00:00.016665</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77609069 88 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77609069 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>3</td><td>77609070</td><td>2019-11-19 23:00:00.202068</td><td>2019-11-19 23:00:00.016666</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77609070 89 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77609070 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>4</td><td>77609115</td><td>2019-11-19 23:00:00.202598</td><td>2019-11-19 23:00:00.017703</td><td>0</td><td>140</td><td>9</td><td>1</td><td>77608433 1 69.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77608433 1 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>5</td><td>77609123</td><td>2019-11-19 23:00:00.202678</td><td>2019-11-19 23:00:00.017813</td><td>0</td><td>141</td><td>9</td><td>1</td><td>77609123 2 70.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609123 1 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>6</td><td>77609124</td><td>2019-11-19 23:00:00.202687</td><td>2019-11-19 23:00:00.017813</td><td>0</td><td>141</td><td>9</td><td>4</td><td>77609124 2 70.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609124 4 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>7</td><td>77609369</td><td>2019-11-19 23:00:00.205493</td><td>2019-11-19 23:00:00.020957</td><td>0</td><td>142</td><td>9</td><td>4</td><td>77609369 3 71.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609369 4 77.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>8</td><td>77609370</td><td>2019-11-19 23:00:00.205506</td><td>2019-11-19 23:00:00.020959</td><td>0</td><td>142</td><td>6</td><td>3</td><td>77609370 3 71.00 |77609115 89 71.00 |77608909 4 141.00 |</td><td>77609370 3 74.00 |77609115 82 73.00 |77608909 0 0.00 |</td></tr><tr><td>9</td><td>77609460</td><td>2019-11-19 23:00:00.206467</td><td>2019-11-19 23:00:00.021838</td><td>0</td><td>142</td><td>5</td><td>3</td><td>77609370 3 71.00 |77609115 89 71.00 |77609460 3 142.00 |</td><td>77609370 3 74.00 |77609115 82 73.00 |77609460 0 0.00 |</td></tr><tr><td>10</td><td>127750158</td><td>2019-11-20 22:47:47.470815</td><td>2019-11-20 22:47:47.470477</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750158 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750158 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>11</td><td>127750159</td><td>2019-11-20 22:47:47.470854</td><td>2019-11-20 22:47:47.470478</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750159 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750159 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>12</td><td>127750160</td><td>2019-11-20 22:47:47.470864</td><td>2019-11-20 22:47:47.470479</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750160 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750160 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>13</td><td>127750161</td><td>2019-11-20 22:47:47.470875</td><td>2019-11-20 22:47:47.470480</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750161 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750161 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>14</td><td>127750162</td><td>2019-11-20 22:47:47.470885</td><td>2019-11-20 22:47:47.470481</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750162 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750162 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>15</td><td>127750163</td><td>2019-11-20 22:47:47.470900</td><td>2019-11-20 22:47:47.470483</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750163 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750163 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>16</td><td>127750164</td><td>2019-11-20 22:47:47.470909</td><td>2019-11-20 22:47:47.470484</td><td>42</td><td>5</td><td>10</td><td>14</td><td>85514251 42 71.00 |127750164 85 82.00 |85514263 44 146.00 |</td><td>85514251 14 72.00 |127750164 45 84.00 |85514263 72 148.00 |</td></tr><tr><td>17</td><Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.209544500Z\" active_for:210100 ns -0.01:8\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211537800Z\" active_for:11000 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211548800Z\" active_for:42400 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211591200Z\" active_for:296400 ns -0.01:7\n",
+    "display(synth.Take(5000).OrderBy(x => x.sending_time));\n",
+    "//Console.WriteLine(synth.Take(10).Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/oil_11_20_zfg.ipynb

@@ -0,0 +1,295 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"oil_11_20\", \"bigblue\");\n",
+    "//Timestamp ts = new Timestamp();\n",
+    "//ts.Seconds = 15;\n",
+    "//DateTime dt = ts.ToDateTime();\n",
+    "//display(dt.Kind);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var c1 = q.collection<HistoryRow>(\"CLZ9-CLF0\");\n",
+    "var c2 = q.collection<HistoryRow>(\"CLF0-CLG0\");\n",
+    "var c3 = q.collection<HistoryRow>(\"CLZ9-CLG0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var begin = DateTime.SpecifyKind(new DateTime(2019, 11, 19, 0, 0, 0),  DateTimeKind.Utc);\n",
+    "var end = DateTime.SpecifyKind(new DateTime(2019, 11, 22, 23, 59, 59), DateTimeKind.Utc);\n",
+    "var c1t = q.timeseries<HistoryRow>(c1, begin, end);\n",
+    "var c2t = q.timeseries<HistoryRow>(c2, begin, end);\n",
+    "var c3t = q.timeseries<HistoryRow>(c3, begin, end);\n",
+    "\n",
+    "//var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "18844"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "83773"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "35257"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(c1t.Count());\n",
+    "display(c2t.Count());\n",
+    "display(c3t.Count());\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th>TopBook</th><th>value</th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>MsgSeqNum</th></tr></thead><tbody><tr><td>{ bts.co2.TopBookMessage: Bid: { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 9 }, Ask: { bts.co2.PriceSizeMessage: Mantissa: 4, Exponent: 0, Size: 17 } }</td><td>{ bts.history.HistoryRow: trade_date: 2019-11-20 00:00:00Z, sending_time: 2019-11-20 16:23:37Z, transact_time: 2019-11-20 16:23:37Z, TradeDate: { Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574208000, Nanos: 0 }, MsgSeqNum: 106692306, SendingTime: { Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574267017, Nanos: 559710300 }, TransactTime: { Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574267017, Nanos: 559610900 }, SecurityDesc: CLZ9-CLG0, Book: { bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 9 }, { bts.co2.PriceSizeMessage: Mantissa: -1, Exponent: 0, Size: 14 }, { bts.co2.PriceSizeMessage: Mantissa: -2, Exponent: 0, Size: 50 }, { bts.co2.PriceSizeMessage: Mantissa: -3, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -4, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -5, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -6, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -7, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -8, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: -9, Exponent: 0, Size: 10 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 4, Exponent: 0, Size: 17 }, { bts.co2.PriceSizeMessage: Mantissa: 5, Exponent: 0, Size: 15 }, { bts.co2.PriceSizeMessage: Mantissa: 6, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 7, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 8, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 9, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 10, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 11, Exponent: 0, Size: 10 }, { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 0 }, { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 0 } ] }, TopBook: &lt;null&gt;, Trade: &lt;null&gt;, Order: &lt;null&gt; }</td><td>2019-11-20 00:00:00Z</td><td>2019-11-20 16:23:37Z</td><td>2019-11-20 16:23:37Z</td><td>106692306</td></tr></tbody></table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var c1f = c1t.Where(x => x.Book != null && x.Trade == null && (x.Book.Bids.Count > 0 || x.Book.Asks.Count > 0));\n",
+    "var c2f = c2t.Where(x => x.Book != null && x.Trade == null && (x.Book.Bids.Count > 0 || x.Book.Asks.Count > 0));\n",
+    "var c3f = c3t.Where(x => x.Book != null && x.Trade == null && (x.Book.Bids.Count > 0 || x.Book.Asks.Count > 0));\n",
+    "\n",
+    "var c1top = c1f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var c2top = c2f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var c3top = c3f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "display(c3top.Last());\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string trade_date => _synth.trade_date.ToString(\"yyyy-MM-dd\");\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{c1top, c2top, c3top}, new[] {1.0, 1.0, -1.0}, new[] {1.0, 1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>77632899</td><td>2019-11-20</td><td>2019-11-19 23:01:29.444834</td><td>2019-11-19 23:01:29.444674</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77632359 599 10.00 |77632899 0 0.00 |</td><td>77623170 9 -12.00 |77632359 809 11.00 |77632899 1 22.00 |</td></tr><tr><td>1</td><td>77632900</td><td>2019-11-20</td><td>2019-11-19 23:01:29.444935</td><td>2019-11-19 23:01:29.444830</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77632359 599 10.00 |77632900 0 0.00 |</td><td>77623170 9 -12.00 |77632359 809 11.00 |77632900 1 22.00 |</td></tr><tr><td>2</td><td>77633139</td><td>2019-11-20</td><td>2019-11-19 23:01:31.008894</td><td>2019-11-19 23:01:31.008777</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633139 599 10.00 |77632900 0 0.00 |</td><td>77623170 9 -12.00 |77633139 809 11.00 |77632900 1 22.00 |</td></tr><tr><td>3</td><td>77633233</td><td>2019-11-20</td><td>2019-11-19 23:01:31.507462</td><td>2019-11-19 23:01:31.507346</td><td>1</td><td>-25</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77632900 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77632900 1 22.00 |</td></tr><tr><td>4</td><td>77633265</td><td>2019-11-20</td><td>2019-11-19 23:01:32.157546</td><td>2019-11-19 23:01:32.157409</td><td>1</td><td>-10</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633265 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633265 1 7.00 |</td></tr><tr><td>5</td><td>77633266</td><td>2019-11-20</td><td>2019-11-19 23:01:32.157639</td><td>2019-11-19 23:01:32.157536</td><td>1</td><td>-10</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633266 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633266 1 7.00 |</td></tr><tr><td>6</td><td>77633967</td><td>2019-11-20</td><td>2019-11-19 23:01:38.955779</td><td>2019-11-19 23:01:38.955632</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633967 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633967 1 3.00 |</td></tr><tr><td>7</td><td>77633968</td><td>2019-11-20</td><td>2019-11-19 23:01:38.955866</td><td>2019-11-19 23:01:38.955760</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77633233 599 10.00 |77633968 0 0.00 |</td><td>77623170 9 -12.00 |77633233 809 11.00 |77633968 1 3.00 |</td></tr><tr><td>8</td><td>77634067</td><td>2019-11-20</td><td>2019-11-19 23:01:41.509218</td><td>2019-11-19 23:01:41.509092</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634067 599 10.00 |77633968 0 0.00 |</td><td>77623170 9 -12.00 |77634067 809 11.00 |77633968 1 3.00 |</td></tr><tr><td>9</td><td>77634146</td><td>2019-11-20</td><td>2019-11-19 23:01:42.008457</td><td>2019-11-19 23:01:42.008346</td><td>1</td><td>-6</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634146 599 10.00 |77633968 0 0.00 |</td><td>77623170 9 -12.00 |77634146 809 11.00 |77633968 1 3.00 |</td></tr><tr><td>10</td><td>77634662</td><td>2019-11-20</td><td>2019-11-19 23:01:44.039520</td><td>2019-11-19 23:01:44.039346</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634146 599 10.00 |77634662 0 0.00 |</td><td>77623170 9 -12.00 |77634146 809 11.00 |77634662 1 1.00 |</td></tr><tr><td>11</td><td>77634663</td><td>2019-11-20</td><td>2019-11-19 23:01:44.039606</td><td>2019-11-19 23:01:44.039493</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77634146 599 10.00 |77634663 0 0.00 |</td><td>77623170 9 -12.00 |77634146 809 11.00 |77634663 1 1.00 |</td></tr><tr><td>12</td><td>77635060</td><td>2019-11-20</td><td>2019-11-19 23:01:52.009603</td><td>2019-11-19 23:01:52.009478</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635060 599 10.00 |77634663 0 0.00 |</td><td>77623170 9 -12.00 |77635060 809 11.00 |77634663 1 1.00 |</td></tr><tr><td>13</td><td>77635104</td><td>2019-11-20</td><td>2019-11-19 23:01:52.509048</td><td>2019-11-19 23:01:52.508929</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635104 599 10.00 |77634663 0 0.00 |</td><td>77623170 9 -12.00 |77635104 809 11.00 |77634663 1 1.00 |</td></tr><tr><td>14</td><td>77635195</td><td>2019-11-20</td><td>2019-11-19 23:01:54.107257</td><td>2019-11-19 23:01:54.107128</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635104 599 10.00 |77635195 0 0.00 |</td><td>77623170 9 -12.00 |77635104 809 11.00 |77635195 1 1.00 |</td></tr><tr><td>15</td><td>77635196</td><td>2019-11-20</td><td>2019-11-19 23:01:54.107340</td><td>2019-11-19 23:01:54.107238</td><td>1</td><td>-4</td><td>-1</td><td>0</td><td>77623170 26 -13.00 |77635104 599 10.00 |77635196 0 0.00 |</td><td>77623170 9 -12.00 |77635104 809 11.00 |77635196 Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(synth.OrderBy(x => x.sending_time));\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/oil_12_4_2019.ipynb

@@ -0,0 +1,289 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"oil_12_4_2019\", \"bigblue\");\n",
+    "//Timestamp ts = new Timestamp();\n",
+    "//ts.Seconds = 15;\n",
+    "//DateTime dt = ts.ToDateTime();\n",
+    "//display(dt.Kind);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var f = q.collection<HistoryRow>(\"CLH0-CLM0\");\n",
+    "var j = q.collection<HistoryRow>(\"CLM0-CLZ0\");\n",
+    "var m = q.collection<HistoryRow>(\"CLH0-CLZ0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var start = new DateTime(2019, 12, 4, 0, 0, 0);\n",
+    "var end = new DateTime(2019, 12, 5, 23, 59, 59);\n",
+    "var ft = q.timeseries<HistoryRow>(f, start, end);\n",
+    "var jt = q.timeseries<HistoryRow>(j, start, end);\n",
+    "var mt = q.timeseries<HistoryRow>(m,start, end);\n",
+    "\n",
+    "//var ft = q.timeseries<HistoryRow>(f, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var jt = q.timeseries<HistoryRow>(j, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));\n",
+    "//var mt = q.timeseries<HistoryRow>(m, new DateTime(2019, 11, 20, 21, 21, 22), new DateTime(2019, 11, 20, 21, 21, 23));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "545413"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "705878"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "548884"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(ft.Count());\n",
+    "display(jt.Count());\n",
+    "display(mt.Count());\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var ff = ft.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var jf = jt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var mf = mt.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "\n",
+    "var ftop = ff.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var jtop = jf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var mtop = mf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "//display(h0f.Take(1));\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.00 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{ftop, jtop, mtop}, new[] {1.0, 1.0, -1.0}, new[] {1.0, 1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>125079130</td><td>2019-12-04 16:38:53.314544</td><td>2019-12-04 16:38:53.313920</td><td>17</td><td>-3</td><td>-1</td><td>15</td><td>125078783 195 117.00 |125078780 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125078780 15 207.00 |125079130 17 326.00 |</td></tr><tr><td>1</td><td>125079130</td><td>2019-12-04 16:38:53.314544</td><td>2019-12-04 16:38:53.313920</td><td>17</td><td>-3</td><td>-1</td><td>15</td><td>125078783 195 117.00 |125078780 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125078780 15 207.00 |125079130 17 326.00 |</td></tr><tr><td>2</td><td>125079130</td><td>2019-12-04 16:38:53.314544</td><td>2019-12-04 16:38:53.313920</td><td>17</td><td>-3</td><td>-1</td><td>15</td><td>125078783 195 117.00 |125078780 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125078780 15 207.00 |125079130 17 326.00 |</td></tr><tr><td>3</td><td>125079130</td><td>2019-12-04 16:38:53.314544</td><td>2019-12-04 16:38:53.313920</td><td>24</td><td>-4</td><td>-1</td><td>15</td><td>125078783 195 117.00 |125078780 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125078780 15 207.00 |125079130 24 327.00 |</td></tr><tr><td>4</td><td>125079130</td><td>2019-12-04 16:38:53.314544</td><td>2019-12-04 16:38:53.313920</td><td>24</td><td>-4</td><td>-1</td><td>15</td><td>125078783 195 117.00 |125078780 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125078780 15 207.00 |125079130 24 327.00 |</td></tr><tr><td>5</td><td>125079133</td><td>2019-12-04 16:38:53.314690</td><td>2019-12-04 16:38:53.314367</td><td>24</td><td>-4</td><td>-1</td><td>15</td><td>125078783 195 117.00 |125079133 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079133 15 207.00 |125079130 24 327.00 |</td></tr><tr><td>6</td><td>125079134</td><td>2019-12-04 16:38:53.314776</td><td>2019-12-04 16:38:53.314367</td><td>24</td><td>-4</td><td>-1</td><td>14</td><td>125078783 195 117.00 |125079134 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079134 14 207.00 |125079130 24 327.00 |</td></tr><tr><td>7</td><td>125079134</td><td>2019-12-04 16:38:53.314776</td><td>2019-12-04 16:38:53.314401</td><td>24</td><td>-4</td><td>-1</td><td>14</td><td>125078783 195 117.00 |125079134 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079134 14 207.00 |125079130 24 327.00 |</td></tr><tr><td>8</td><td>125079134</td><td>2019-12-04 16:38:53.314776</td><td>2019-12-04 16:38:53.314401</td><td>24</td><td>-4</td><td>-1</td><td>12</td><td>125078783 195 117.00 |125079134 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079134 12 207.00 |125079130 24 327.00 |</td></tr><tr><td>9</td><td>125079135</td><td>2019-12-04 16:38:53.314875</td><td>2019-12-04 16:38:53.314410</td><td>24</td><td>-4</td><td>-1</td><td>12</td><td>125078783 195 117.00 |125079135 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079135 12 207.00 |125079130 24 327.00 |</td></tr><tr><td>10</td><td>125079135</td><td>2019-12-04 16:38:53.314875</td><td>2019-12-04 16:38:53.314410</td><td>24</td><td>-4</td><td>-1</td><td>11</td><td>125078783 195 117.00 |125079135 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079135 11 207.00 |125079130 24 327.00 |</td></tr><tr><td>11</td><td>125079155</td><td>2019-12-04 16:38:53.315284</td><td>2019-12-04 16:38:53.314515</td><td>24</td><td>-4</td><td>-1</td><td>11</td><td>125078783 195 117.00 |125079155 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079155 11 207.00 |125079130 24 327.00 |</td></tr><tr><td>12</td><td>125079156</td><td>2019-12-04 16:38:53.315308</td><td>2019-12-04 16:38:53.314515</td><td>24</td><td>-4</td><td>-1</td><td>6</td><td>125078783 195 117.00 |125079156 71 206.00 |125079130 35 326.00 |</td><td>125078783 47 118.00 |125079156 6 207.00 |125079130 24 327.00 |</td></tr><tr><td>13</td><td>125079164</td><td>2019-12-04 16:38:53.315434</td><td>2019-12-04 16:38:53.314520</td><td>24</td><td>-4</td><td>-1</td><td>6</td><td>125079164 195 117.00 |125079156 71 206.00 |125079130 35 326.00 |</td><td>125079164 47 118.00 |125079156 6 207.00 |125079130 24 327.00 |</td></tr><tr><td>14</td><td>125079165</td><td>2019-12-04 16:38:53.315444</td><td>2019-12-04 16:38:53.314520</td><td>24</td><td>-4</td><td>-1</td><td>6</td><td>125079165 195 117.00 |125079156 71 206.00 |125079130 35 326.00 |</td><td>125079165 42 118.00 |125079156 6 207.00 |125079130 24 327.00 |</td></tr><tr><td>15</td><td>125079200</td><td>2019-12-04 16:38:53.315936</td><td>2019-12-04 16:38:53.314590</td><td>24</td><td>-4</td><td>-1</td><td>6</td><td>125079165 195 117.00 |125079156 71 206.00 |125079200 35 326.00 |</td><td>125079165 42 118.00 |125079156 6 207.00 |125079200 24 327.00 |</td></tr><tr><td>16</td><td>125079201</td><td>2019-12-04 16:Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.209544500Z\" active_for:210100 ns -0.01:8\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211537800Z\" active_for:11000 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211548800Z\" active_for:42400 ns -0.01:7\n",
+    "//BUY:CL_F0J0M0 t:\"2019-11-20T21:21:22.211591200Z\" active_for:296400 ns -0.01:7\n",
+    "display(synth.Take(15000).OrderBy(x => x.sending_time));\n",
+    "//Console.WriteLine(synth.Take(10).Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Notebooks/roll.ipynb

@@ -0,0 +1,278 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dllInvalid: Command: #r C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll"
+     ]
+    }
+   ],
+   "source": [
+    "//#r \"nuget:Google.Protobuf, 3.6.1\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/google.protobuf/3.6.1/lib/netstandard1.0/Google.Protobuf.dll\"\n",
+    "//#r \"nuget:Microsoft.CSharp, 4.5.0\"\n",
+    "//#r \"C:/Users/darby/.nuget/packages/microsoft.netcore.app.runtime.linux-x64/3.0.0/runtimes/linux-x64/lib/netcoreapp3.0/Microsoft.CSharp.dll\"    \n",
+    "//#r \"nuget:Newtonsoft.Json, 12.0.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/newtonsoft.json/12.0.3/lib/netstandard2.0/Newtonsoft.Json.dll\"\n",
+    "//#r \"nuget:System.Linq.Dynamic, 1.0.7\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/system.linq.dynamic.core/1.0.19/lib/netstandard2.0/System.Linq.Dynamic.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Driver, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver/2.9.3/lib/netstandard1.5/MongoDB.Driver.dll\"    \n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.driver.core/2.9.3/lib/netstandard1.5/MongoDB.Driver.Core.dll\"\n",
+    "//#r \"nuget:MongoDB.Bson, 2.9.3\"\n",
+    "#r \"C:/Users/darby/.nuget/packages/mongodb.bson/2.9.3/lib/netstandard1.5/MongoDB.Bson.dll\"        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Invalid: Command: #r c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll"
+     ]
+    }
+   ],
+   "source": [
+    "#r \"c:/code/bts/spark/build/Debug/bin/netstandard2.0/spark_historical.dll\"   \n",
+    "using bts.history;\n",
+    "using MongoDB.Driver;\n",
+    "using MongoDB.Driver.Linq;\n",
+    "using Google.Protobuf.WellKnownTypes;\n",
+    "using static bts.history.Query;\n",
+    "using mbts.co2;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var q = new Query(\"roll\", \"bigblue\");"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "var cz9 = q.collection<HistoryRow>(\"ZNZ9\");\n",
+    "var ch0 = q.collection<HistoryRow>(\"ZNH0\");\n",
+    "var cspread = q.collection<HistoryRow>(\"ZNZ9-ZNH0\");\n",
+    "\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "//var s = q.series<HistoryRow>(\"ZNZ9-ZNH0\").Where( x=> x.Book != null  );\n",
+    "//var s1 = s.Where( x => x.Book.Bids[0].Size > 0);\n",
+    "//display(s.Count())\n",
+    "//var s2 = s.Where( x => x.sending_time >= new DateTime(2019, 11, 25));\n",
+    "//display(s2.Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "var z9 = q.timeseries<HistoryRow>(cz9, new DateTime(2019, 11, 25, 14,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n",
+    "var h0 = q.timeseries<HistoryRow>(ch0, new DateTime(2019, 11, 25, 14,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));\n",
+    "var spread = q.timeseries<HistoryRow>(cspread, new DateTime(2019, 11, 25, 14 ,0, 0), new DateTime(2019, 11, 25, 23, 59, 59));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>TradeDate</th><th>MsgSeqNum</th><th>SendingTime</th><th>TransactTime</th><th>SecurityDesc</th><th>Book</th><th>TopBook</th><th>Trade</th><th>Order</th></tr></thead><tbody><tr><td>0</td><td>2019-11-26 00:00:00Z</td><td>2019-11-25 22:54:30Z</td><td>2019-11-25 22:54:30Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574726400, Nanos: 0 }</td><td>11851041</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667725300 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667299500 }</td><td>ZNZ9</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td><td>{ bts.history.MBOMessage: OrderUpdateAction: { bts.history.MDUpdateAction: value__: 48 }, OrderId: 846787349208, SecurityId: 59121, SecurityDesc: ZNZ9, MdOrderPriority: 12370689694, MdEntryPx: 129.34375, MdPriceLevel: 0, MdDisplayQty: 72 }</td></tr><tr><td>1</td><td>2019-11-26 00:00:00Z</td><td>2019-11-25 22:54:30Z</td><td>2019-11-25 22:54:30Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574726400, Nanos: 0 }</td><td>11851041</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667725300 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667299500 }</td><td>ZNZ9</td><td>{ bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 2 }, { bts.co2.PriceSizeMessage: Mantissa: 129546875000, Exponent: -9, Size: 54 }, { bts.co2.PriceSizeMessage: Mantissa: 129531250000, Exponent: -9, Size: 426 }, { bts.co2.PriceSizeMessage: Mantissa: 129515625000, Exponent: -9, Size: 897 }, { bts.co2.PriceSizeMessage: Mantissa: 129500000000, Exponent: -9, Size: 1181 }, { bts.co2.PriceSizeMessage: Mantissa: 129484375000, Exponent: -9, Size: 1558 }, { bts.co2.PriceSizeMessage: Mantissa: 129468750000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129453125000, Exponent: -9, Size: 1555 }, { bts.co2.PriceSizeMessage: Mantissa: 129437500000, Exponent: -9, Size: 1628 }, { bts.co2.PriceSizeMessage: Mantissa: 129421875000, Exponent: -9, Size: 1582 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 76 }, { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 144 }, { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 666 }, { bts.co2.PriceSizeMessage: Mantissa: 129609375000, Exponent: -9, Size: 1054 }, { bts.co2.PriceSizeMessage: Mantissa: 129625000000, Exponent: -9, Size: 1590 }, { bts.co2.PriceSizeMessage: Mantissa: 129640625000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129656250000, Exponent: -9, Size: 1877 }, { bts.co2.PriceSizeMessage: Mantissa: 129671875000, Exponent: -9, Size: 1594 }, { bts.co2.PriceSizeMessage: Mantissa: 129687500000, Exponent: -9, Size: 2246 }, { bts.co2.PriceSizeMessage: Mantissa: 129703125000, Exponent: -9, Size: 2006 } ] }</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td></tr><tr><td>2</td><td>2019-11-26 00:00:00Z</td><td>2019-11-25 22:54:30Z</td><td>2019-11-25 22:54:30Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574726400, Nanos: 0 }</td><td>11851041</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667725300 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574722470, Nanos: 667299500 }</td><td>ZNZ9</td><td>{ bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 129546875000, Exponent: -9, Size: 54 }, { bts.co2.PriceSizeMessage: Mantissa: 129531250000, Exponent: -9, Size: 426 }, { bts.co2.PriceSizeMessage: Mantissa: 129515625000, Exponent: -9, Size: 897 }, { bts.co2.PriceSizeMessage: Mantissa: 129500000000, Exponent: -9, Size: 1181 }, { bts.co2.PriceSizeMessage: Mantissa: 129484375000, Exponent: -9, Size: 1558 }, { bts.co2.PriceSizeMessage: Mantissa: 129468750000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129453125000, Exponent: -9, Size: 1555 }, { bts.co2.PriceSizeMessage: Mantissa: 129437500000, Exponent: -9, Size: 1628 }, { bts.co2.PriceSizeMessage: Mantissa: 129421875000, Exponent: -9, Size: 1582 }, { bts.co2.PriceSizeMessage: Mantissa: 0, Exponent: 0, Size: 0 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 76 }, { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 144 }, { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 666 }, { bts.co2.PriceSizeMessage: Mantissa: 129609375000, Exponent: -9, Size: 1054 }, { bts.co2.PriceSizeMessage: Mantissa: 129625000000, Exponent: -9, Size: 1590 }, { bts.co2.PriceSizeMessage: Mantissa: 129640625000, Exponent: -9, Size: 1561 }, { bts.co2.PriceSizeMessage: Mantissa: 129656250000, Exponent: -9, Size: 1877 }, { bts.co2.PriceSizeMessage: Mantissa: 129671875000, Exponent: -9, Size: 1594 }, { bts.co2.PriceSizeMessage: ManLine truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var zz = z9.Where(x => x.MsgSeqNum == 11851041 );\n",
+    "//display(z9.Count());\n",
+    "//display(h0.Count());\n",
+    "//display(spread.Count());\n",
+    "display(zz);\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>trade_date</th><th>sending_time</th><th>transact_time</th><th>TradeDate</th><th>MsgSeqNum</th><th>SendingTime</th><th>TransactTime</th><th>SecurityDesc</th><th>Book</th><th>TopBook</th><th>Trade</th><th>Order</th></tr></thead><tbody><tr><td>0</td><td>2019-11-25 00:00:00Z</td><td>2019-11-25 19:00:00Z</td><td>2019-11-25 19:00:00Z</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574640000, Nanos: 0 }</td><td>9955014</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574708400, Nanos: 47819600 }</td><td>{ Google.Protobuf.WellKnownTypes.Timestamp: Seconds: 1574708400, Nanos: 47703400 }</td><td>ZNH0</td><td>{ bts.co2.BookMessage: Bids: [ { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 1114 }, { bts.co2.PriceSizeMessage: Mantissa: 129562500000, Exponent: -9, Size: 2349 }, { bts.co2.PriceSizeMessage: Mantissa: 129546875000, Exponent: -9, Size: 2129 }, { bts.co2.PriceSizeMessage: Mantissa: 129531250000, Exponent: -9, Size: 3452 }, { bts.co2.PriceSizeMessage: Mantissa: 129515625000, Exponent: -9, Size: 1822 }, { bts.co2.PriceSizeMessage: Mantissa: 129500000000, Exponent: -9, Size: 1940 }, { bts.co2.PriceSizeMessage: Mantissa: 129484375000, Exponent: -9, Size: 2178 }, { bts.co2.PriceSizeMessage: Mantissa: 129468750000, Exponent: -9, Size: 1869 }, { bts.co2.PriceSizeMessage: Mantissa: 129453125000, Exponent: -9, Size: 1959 }, { bts.co2.PriceSizeMessage: Mantissa: 129437500000, Exponent: -9, Size: 2074 } ], Asks: [ { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 1479 }, { bts.co2.PriceSizeMessage: Mantissa: 129609375000, Exponent: -9, Size: 2141 }, { bts.co2.PriceSizeMessage: Mantissa: 129625000000, Exponent: -9, Size: 2139 }, { bts.co2.PriceSizeMessage: Mantissa: 129640625000, Exponent: -9, Size: 2071 }, { bts.co2.PriceSizeMessage: Mantissa: 129656250000, Exponent: -9, Size: 1817 }, { bts.co2.PriceSizeMessage: Mantissa: 129671875000, Exponent: -9, Size: 2216 }, { bts.co2.PriceSizeMessage: Mantissa: 129687500000, Exponent: -9, Size: 2034 }, { bts.co2.PriceSizeMessage: Mantissa: 129703125000, Exponent: -9, Size: 1950 }, { bts.co2.PriceSizeMessage: Mantissa: 129718750000, Exponent: -9, Size: 1896 }, { bts.co2.PriceSizeMessage: Mantissa: 129734375000, Exponent: -9, Size: 1952 } ] }</td><td>{ bts.co2.TopBookMessage: Bid: { bts.co2.PriceSizeMessage: Mantissa: 129578125000, Exponent: -9, Size: 1114 }, Ask: { bts.co2.PriceSizeMessage: Mantissa: 129593750000, Exponent: -9, Size: 1479 } }</td><td>&lt;null&gt;</td><td>&lt;null&gt;</td></tr></tbody></table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var z9f = z9.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var h0f = h0.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "var spreadf = spread.Where(x => x.Book != null && x.Book.Bids.Count > 0 && x.Book.Asks.Count > 0);\n",
+    "\n",
+    "var z9t = z9f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var h0t = h0f.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "var spreadt = spreadf.AsEnumerable().Select(x => new GetTopOfBook(x)).OrderBy(x => x.sending_time);\n",
+    "\n",
+    "display(h0f.Take(1));\n",
+    "//display(z9f.Take(1));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class TopReport<T> where T : IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>\n",
+    "{\n",
+    "   readonly T[] _legs;\n",
+    "   readonly T _synth;\n",
+    "   public TopReport(T[] legs, T synth)\n",
+    "   {\n",
+    "       _legs = (T[])legs.Clone();\n",
+    "       _synth = synth.Clone();\n",
+    "   }\n",
+    "   public uint seq_num => _synth.MsgSeqNum;\n",
+    "   public string sending_time => _synth.sending_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public string transact_time => _synth.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
+    "   public uint bidsize => _synth.TopBook.Bid.Size;\n",
+    "   public double bid => _synth.TopBook.Bid.price();\n",
+    "   public double ask => _synth.TopBook.Ask.price();\n",
+    "   public uint asksize => _synth.TopBook.Ask.Size;\n",
+    "   \n",
+    "   public string[] legbids => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Bid.Size + \" \" +  \n",
+    "                                                x.TopBook.Bid.price().ToString(\"0.000000 |\") ).ToArray();\n",
+    "   public string[] legasks => _legs.Select( x=> x.MsgSeqNum + \" \" + x.TopBook.Ask.Size + \" \" + \n",
+    "                                                x.TopBook.Ask.price().ToString(\"0.000000 |\") ).ToArray();\n",
+    "}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "var synth = create_synthetic_top( new []{z9t, h0t, spreadt}, new[] {1.0, -1.0, -1.0}, new[] {1.0, -1.0, -1.0}, \n",
+    "                                  x => x.TopBook.Bid.price() > 0.0 || x.TopBook.Ask.price() < 0.0, \n",
+    "                                  (legs, synth)=> new TopReport<GetTopOfBook>(legs, synth));"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table><thead><tr><th><i>index</i></th><th>seq_num</th><th>sending_time</th><th>transact_time</th><th>bidsize</th><th>bid</th><th>ask</th><th>asksize</th><th>legbids</th><th>legasks</th></tr></thead><tbody><tr><td>0</td><td>11851041</td><td>2019-11-25 22:54:30.667725</td><td>2019-11-25 22:54:30.667299</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851041 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851041 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>1</td><td>11851042</td><td>2019-11-25 22:54:30.668019</td><td>2019-11-25 22:54:30.667773</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851042 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851042 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>2</td><td>11851042</td><td>2019-11-25 22:54:30.668019</td><td>2019-11-25 22:54:30.667773</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851042 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851042 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>3</td><td>11851043</td><td>2019-11-25 22:54:30.668036</td><td>2019-11-25 22:54:30.667774</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851043 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851043 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>4</td><td>11851043</td><td>2019-11-25 22:54:30.668036</td><td>2019-11-25 22:54:30.667774</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851043 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851043 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>5</td><td>11851044</td><td>2019-11-25 22:54:30.668054</td><td>2019-11-25 22:54:30.667777</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851044 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851044 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>6</td><td>11851044</td><td>2019-11-25 22:54:30.668054</td><td>2019-11-25 22:54:30.667777</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851044 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851044 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>7</td><td>11851045</td><td>2019-11-25 22:54:30.668078</td><td>2019-11-25 22:54:30.667779</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851045 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851045 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>8</td><td>11851045</td><td>2019-11-25 22:54:30.668078</td><td>2019-11-25 22:54:30.667779</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851045 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851045 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>9</td><td>11851046</td><td>2019-11-25 22:54:30.668091</td><td>2019-11-25 22:54:30.667782</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851046 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851046 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>10</td><td>11851046</td><td>2019-11-25 22:54:30.668091</td><td>2019-11-25 22:54:30.667782</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851046 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851046 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>11</td><td>11851047</td><td>2019-11-25 22:54:30.668105</td><td>2019-11-25 22:54:30.667787</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851047 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851047 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>12</td><td>11851047</td><td>2019-11-25 22:54:30.668105</td><td>2019-11-25 22:54:30.667787</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851047 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851047 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>13</td><td>11851048</td><td>2019-11-25 22:54:30.668137</td><td>2019-11-25 22:54:30.667789</td><td>193</td><td>-0.0390625</td><td>-0.0078125</td><td>60</td><td>11851048 482 129.531250 |11851040 156 129.593750 |11850702 60 -0.054688 |</td><td>11851048 72 129.531250 |11851040 193 129.625000 |11850702 14833 -0.054688 |</td></tr><tr><td>14</td><td>11851048</td><td>2019-11-25 22:54:30.668137</td><td>2019-11-25 22:54:30.667789</td><td>193</td><td>-Line truncated
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(synth.Take(10000).OrderBy(x => x.sending_time));\n",
+    "//Console.WriteLine(synth.Take(10).Count());"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".NET (C#)",
+   "language": "C#",
+   "name": ".net-csharp"
+  },
+  "language_info": {
+   "file_extension": ".cs",
+   "mimetype": "text/x-csharp",
+   "name": "C#",
+   "pygments_lexer": "csharp",
+   "version": "8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/client/spark_historical/Properties/AssemblyAttributes.cs

@@ -0,0 +1,4 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("sparkcmd")]
+

src/client/spark_historical/ProtoFix.cs

@@ -0,0 +1,259 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using Google.Protobuf;
+using Google.Protobuf.Reflection;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts
+{
+   public static class ProtoFix
+   {
+      static readonly QuickFix.IMessageFactory _factory = new QuickFix.DefaultMessageFactory();
+      public static T parse_fix<T>(string fixstring) where T : IMessage, new()
+      {
+         return (T) parse_fix(new T().Descriptor, fixstring);
+      }
+
+      public static IMessage parse_fix(MessageDescriptor desc, string fixstring)
+      {
+         var fix = load_message(desc, fixstring);
+         var msg = create_message(desc, fix);
+         return msg;
+      }
+
+      static string get_between(string source, string start, string end)
+      {
+         int spos = source.IndexOf(start, 0);
+         if(spos < 0)
+            return string.Empty;
+         spos += start.Length;
+         int epos = source.IndexOf(end, spos);
+         if(epos < 0)
+            return string.Empty;
+         return source.Substring(spos, epos - spos);
+      }      
+
+      public static Timestamp peek_timestamp(string fixstring, int field = 60) 
+      {
+         string ts = get_between(fixstring, field.ToString() + "=", "\u0001");
+         if(!string.IsNullOrEmpty(ts))
+         {
+            var stamp = parse_timestamp(ts);
+            return stamp;
+         }
+         return null;
+      }
+
+      public static string peek_field(string fixstring, int field)
+      {
+         string ts = get_between(fixstring, field.ToString() + "=", "\u0001");
+         return ts;
+      }
+
+      public static string peek_symbol(string fixstring)
+      {
+         return peek_field(fixstring, 55);
+      }
+
+      static QuickFix.Message load_message(MessageDescriptor desc, string fixstring)
+      {
+         QuickFix.Message fix = new QuickFix.Message(fixstring, false);
+         return fix;
+
+         // QuickFix.Fields.MsgType msgtype = QuickFix.Message.IdentifyType(fixstring);
+         // string begin = QuickFix.Message.ExtractBeginString(fixstring);
+         // var dd = new QuickFix.DataDictionary.DataDictionary();
+
+         // var ddmap = new QuickFix.DataDictionary.DDMap();
+         // ddmap.AddGroup(new QuickFix.DataDictionary.DDGrp { NumFld = 268, Delim = 279 });
+         // dd.Messages["X"] = ddmap;
+
+         // var fix = _factory.Create(begin, msgtype.Obj);
+         // fix.FromString(fixstring, false, dd, dd, _factory);
+         // return fix;               
+      }
+
+      static IMessage create_message(MessageDescriptor desc, QuickFix.FieldMap fix)
+      {
+         object obj = Activator.CreateInstance(desc.ClrType);
+         IMessage msg = (IMessage)obj;
+         foreach (var fd in desc.Fields.InDeclarationOrder())
+         {
+            set_field(msg, fd, fix);
+         }
+         return msg;
+      }
+
+      static void set_field(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         if (!fix.IsSetField(fd.FieldNumber))
+         {
+            if(fix is QuickFix.Message fm && fm.Header.IsSetField(fd.FieldNumber))
+               fix = fm.Header;
+            else
+               return;
+         }
+         switch (fd.FieldType)
+         {
+            case FieldType.Bool:
+               {
+                  var data = fix.GetBoolean(fd.FieldNumber);
+                  fd.Accessor.SetValue(msg, data);
+               }
+               break;
+            case FieldType.Float:
+               {
+                  var data = fix.GetDecimal(fd.FieldNumber);
+                  object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                  fd.Accessor.SetValue(msg, conv);
+               }
+               break;
+            case FieldType.Double:
+               {
+                  decimal data = fix.GetDecimal(fd.FieldNumber);
+                  double d = decimal.ToDouble(data);
+                  fd.Accessor.SetValue(msg, d);
+               }
+               break;
+            case FieldType.Int32:
+            case FieldType.SInt32:
+            case FieldType.UInt32:
+               {
+                  var data = fix.GetInt(fd.FieldNumber);
+                  object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                  fd.Accessor.SetValue(msg, conv);
+               }
+               break;
+            case FieldType.Int64:
+            case FieldType.SInt64:
+            case FieldType.UInt64:
+               {
+                  var data = fix.GetDecimal(fd.FieldNumber);
+                  object conv = Convert.ChangeType(data, fd.Accessor.GetValue(msg).GetType());
+                  fd.Accessor.SetValue(msg, conv);
+               }
+               break;
+            case FieldType.Enum:
+               {
+                  char data = fix.GetChar(fd.FieldNumber);
+
+                  EnumValueDescriptor val = null;
+                  val = fd.EnumType.FindValueByNumber((int)data);
+                  if(val == null)
+                     val = fd.EnumType.FindValueByNumber(0); //unset
+                  fd.Accessor.SetValue(msg, val.Number);
+               }
+               break;
+            case FieldType.String:
+               {
+                  var data = fix.GetString(fd.FieldNumber);
+                  fd.Accessor.SetValue(msg, data);
+               }
+               break;
+            case FieldType.Message:
+               {
+                  handle_message(msg, fd, fix);
+               }
+               break;
+            default:
+               throw new NotSupportedException("unsupported field type " + fd.FieldType.ToString());
+         }
+      }
+      public static Timestamp parse_timestamp(string data)
+      {
+         DateTime dt = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+         try
+         {
+            if (data.Length <= 8)
+            {
+               dt = DateTime.ParseExact(data, "yyyyMMdd", System.Globalization.CultureInfo.InvariantCulture);
+            }
+            else
+            {
+               int sep = 14;
+               var yyMMddhhmmss = data.Substring(0, sep);
+               dt = DateTime.ParseExact(yyMMddhhmmss, "yyyyMMddHHmmss", System.Globalization.CultureInfo.InvariantCulture);
+               var nnanos = data.Substring(sep, data.Length - sep);
+               long nano = long.Parse(nnanos);
+               dt = dt.AddTicks(nano / 100);
+            }
+         }
+         catch (Exception _)
+         {
+            System.Diagnostics.Debug.Assert(_ != null); //UNUSED(_)
+         }
+         dt = DateTime.SpecifyKind(dt, DateTimeKind.Utc);
+         var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(dt);
+         return ts;
+      }
+      static void handle_timestamp(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         var data = fix.GetString(fd.FieldNumber);
+         Timestamp ts = parse_timestamp(data);         
+         fd.Accessor.SetValue(msg, ts);         
+      }
+      static void handle_message(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
+      {
+         if (fd.MessageType.ClrType == typeof(Google.Protobuf.WellKnownTypes.Timestamp))
+         {
+            handle_timestamp(msg, fd, fix);
+         }
+         else
+         {
+            if (!fd.IsRepeated)
+            {
+               var submsg = create_message(fd.MessageType, fix);
+               fd.Accessor.SetValue(msg, submsg);
+            }
+            else //for repeating groups we assume that the fd.FieldNumber corresponds to the "no" (number) tag in the fix
+            {
+               int group = fd.FieldNumber;
+               int nvals = fix.GetInt(group);
+               IList list = (IList)fd.Accessor.GetValue(msg);
+               if (nvals == 1)
+               {
+                  var submsg = create_message(fd.MessageType, fix);
+                  list.Add(submsg);
+               }
+               else
+               {
+                  var delimproto = fd.MessageType.Fields.InDeclarationOrder()[0];
+                  var delim = delimproto.FieldNumber;
+
+                  List<QuickFix.Fields.IField> repeated = new List<QuickFix.Fields.IField>();
+                  var forder = fix.RepeatedTags.Select(x => x.Tag).Distinct();
+                  var dict = fix.ToDictionary();
+                  foreach(int fo in forder)
+                  {
+                     if(dict.TryGetValue(fo, out QuickFix.Fields.IField field))
+                        repeated.Add(field);
+                  }
+                  repeated.AddRange(fix.RepeatedTags);
+                  QuickFix.FieldMap fm = null;
+                  foreach (var rep in repeated)
+                  {
+                     if (rep.Tag == delim)
+                     {
+                        if (fm != null)
+                        {
+                           var submsg = create_message(fd.MessageType, fm);
+                           list.Add(submsg);
+                        }
+                        fm = new QuickFix.FieldMap();
+                     }
+                     if (fm != null)
+                        fm.SetField(rep);
+                  } //the last one
+                  if (fm != null && fm.Count() > 0)
+                  {
+                     var submsg = create_message(fd.MessageType, fm);
+                     list.Add(submsg);
+                  }
+               }
+            }
+         }         
+      }
+   }
+}