Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis

changed the keys
change aws keys
2020-04-06 16:00:24 -04:00 · 2020-04-06 16:00:12 -04:00 · 2020-03-31 11:00:26 -05:00 · 2020-03-03 17:09:56 -05:00 · 2020-03-02 18:32:19 -05:00 · 2020-03-02 09:46:10 -05:00
296 changed files with 116208 additions and 356 deletions
@@ -1,5 +1,7 @@
 nohup.out
 build/
+build
+cme_data/
 cmake-build-debug/
 .idea/
 .dir-locals.el
@@ -34,3 +36,5 @@ trace_data/
 /doc/user/public/*
 .ionide/*
 /.ignore
+/build
+/analysis_data
@@ -46,7 +46,7 @@
            "sourceFileMap": {
                "/home/bts/spark/": "${workspaceRoot}"
            }
-        },
+        },     
        {
            "name": "(gdb) Launch",
            "type": "cppdbg",
@@ -98,6 +98,46 @@
            "externalConsole": true,
            "requireExactSource": false
        },
+            {
+         "name": "sparkcmd Launch",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "historical",
+            "--rstart", "11/20/2019",
+            "--rend", "11/29/2019",
+            "--rsymbol", "NG",
+            "--rsecuritytype", "Synthetic",
+            "--post_process_db", "NG_201911_W4",
+            "--post_process_db_host", "bigblue",
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },  
+      {
+         "name": "sparkcmd simulation",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": [
+            "historical",
+            "--rstart", "11/26/2019",
+            "--rend", "11/26/2019",
+            "--rsymbol", "ZN",
+            "--rsecuritytype", "Synthetic",
+            "--algo_security", "ZNH0;ZNZ9-ZNH0;ZNZ9",
+            "--algo_type", "RoundTrip",
+            "--algo_params", "size=10;tick_size=0.015625,0.0078125,0.015625;hedge_factors=1,1,-1",
+            "--skip_preanalysis", "true",         
+         ],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },       
        {
            "name": "Excel Launch",
            "type": "clr",
@@ -131,6 +171,18 @@
            "request": "attach",
            "processId": "${command:pickProcess}"
        },
+        {
+            "name": ".NET Framework Attach",
+            "type": "clr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+        },
+         {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+         },        
        {
            "name": "Spreader Tests",
            "type": "cppdbg",
@@ -342,15 +394,19 @@
            "justMyCode": true
        },
        {
-            "name": ".NET Core Remote Launch",
+            "name": "sparkcmd vsdbg (c#)",
            "type": "coreclr",
            "request": "launch",
-            "program": "/usr/bin/dotnet",
+            "program": "sparkcmd",
            "args": [
-                "sparkcmd.dll",
-                "test",
-                "test_join_create_and_takeout_market"
+            "historical",
+            "--rstart", "11/20/2019",
+            "--rend", "11/20/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Spread",
+            "--post_process_db", "mbo_history",
            ],
+            "env": {"LD_LIBRARY_PATH" : "."},
            "stopAtEntry": false,
            "cwd": "/home/bts/spark/build/ninja_debug/server",
            "pipeTransport": {
@@ -361,11 +417,77 @@
                "debuggerPath": "/home/bts/vsdbg/vsdbg",
                "pipeCwd": "${workspaceRoot}",
                "quoteArgs": true
-            },
+            },           
            "sourceFileMap": {
                "/home/bts/spark/": "${workspaceRoot}"
            },
            "justMyCode": false
-        },
+        },                  
+         {
+            "name": "sparkcmd gdb (c++)",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/home/bts/spark/build/ninja_debug/server/sparkcmd",
+            "args": [
+            "historical",
+            "--rstart", "10/21/2019",
+            "--rend", "10/21/2019",
+            "--rsymbol", "CL",
+            "--rsecuritytype", "Future",
+            "--algo_security", "CLX9",
+            "--algo_type", "TWAP",
+            "--algo_params", "'size=100;profit_ticks=1;tick_size=1,1,1;;hedge_factors=1,1,-1'",
+            "--skip_preanalysis", "true",
+            ],   
+            "stopAtEntry": false,
+            "cwd": "/home/bts/spark/build/ninja_debug/server/",
+            "environment": [],
+            "externalConsole": true,
+            "MIMode": "gdb",
+            "preLaunchTask": "",
+            "logging": {
+               "engineLogging": false
+            },
+            "setupCommands": [
+               {
+                  "description": "Enable pretty-printing for gdb",
+                  "text": "enable pretty-printer",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "Skip STL",
+                  "text": "skip -gfi /usr/include/c++/7/bits/*.h",
+                  "ignoreFailures": true
+               },
+               {
+                  "description": "LD LIBRARY",
+                  "text": "set env LD_LIBRARY_PATH=.",
+                  "ignoreFailures": false
+               }
+            ],
+            "pipeTransport": {
+               "pipeCwd": "${workspaceRoot}",
+               "pipeProgram": "c:\\program files\\git\\usr\\bin\\ssh.exe",
+               "pipeArgs": [
+                  "bts@sparkvm"
+               ],
+               "debuggerPath": "/usr/bin/gdb"
+            },
+            "sourceFileMap": {
+               "/home/bts/spark/": "${workspaceRoot}"
+            }
+         },
+         {
+            "name": "Jupyter notebook",
+            "type": "coreclr",
+            "request": "launch",
+            "program": "jupyter",
+            "args": ["notebook", "--notebook-dir=c:\\code\\bts\\spark\\src\\client\\spark_historical\\Notebooks"],
+            "cwd": "${workspaceFolder}",
+            "stopAtEntry": false,
+            "console": "internalConsole",
+            
+         },
+         
    ]
 }
@@ -48,6 +48,7 @@ else ()
  add_subdirectory(doc/user)
  add_subdirectory(src/sdk)
  add_subdirectory(src/data_files)
+  add_subdirectory(src/client)

  add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/server
    COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
@@ -59,7 +60,16 @@ else ()
    COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
    COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
    COMMENT "Creating server runtime layout"
-    DEPENDS sparkd data_files_dir)  
+    DEPENDS sparkd data_files_dir spark_client_dir)  

  add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
+  
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.0/linux-x64/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+else()
+   add_custom_command(TARGET server_layout POST_BUILD
+      COMMAND ln -fs ${CMAKE_BINARY_DIR}/../${CMAKE_BUILD_TYPE}/bin/netcoreapp3.0/linux-x64/publish/sparkcmd ${CMAKE_BINARY_DIR}/server/ ) 
+endif()
+      
 endif()
@@ -23,7 +23,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\sp
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_excel", "src\client\spark_excel\spark_excel.csproj", "{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_fix", "src\client\spark_fix\spark_fix.csproj", "{D8031C13-7BEA-430A-BB60-40927D15CFC3}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_historical", "src\client\spark_historical\spark_historical.csproj", "{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}"
+EndProject
 Global
+	GlobalSection(Performance) = preSolution
+		HasPerformanceSessions = true
+	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Debug|x64 = Debug|x64
@@ -219,6 +226,50 @@ Global
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x64.Build.0 = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.ActiveCfg = Release|x64
 		{BE9F27C3-B583-47DF-95FA-1F8BBC89DCCB}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x64.Build.0 = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.MinSizeRel|x86.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.Release|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|Any CPU.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x64.Build.0 = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.ActiveCfg = Release|x64
+		{D8031C13-7BEA-430A-BB60-40927D15CFC3}.RelWithDebInfo|x86.Build.0 = Release|x64
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Debug|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.Release|x86.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
+		{5F64A360-43F4-4AD8-8B04-3C3F4424BCBC}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -0,0 +1,123 @@
+drop table if exists msgs ;
+create table msgs (
+       MsgID Int4,
+       TradeDate varchar,
+       SeqNum Int4,
+       SendingTime varchar,
+       TransactTime varchar,
+       MatchEventIndicator varchar,
+       NoMDEntries Int2,
+       NoOrderIDEntries Int2,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+drop table if exists books ;
+create table books (
+       MsgID Int4,
+       SendingTime varchar,
+       TransactTime varchar,
+       SeqNum Int4,
+       bid_p_0 Real,
+       bid_sz_0 Int4,
+       bid_p_1 Real,
+       bid_sz_1 Int4,
+       bid_p_2 Real,
+       bid_sz_2 Int4,
+       bid_p_3 Real,
+       bid_sz_3 Int4,
+       bid_p_4 Real,
+       bid_sz_4 Int4,
+       bid_p_5 Real,
+       bid_sz_5 Int4,
+       bid_p_6 Real,
+       bid_sz_6 Int4,
+       bid_p_7 Real,
+       bid_sz_7 Int4,
+       bid_p_8 Real,
+       bid_sz_8 Int4,
+       bid_p_9 Real,
+       bid_sz_9 Int4,
+       Symbol Varchar,
+       ask_p_0 Real,
+       ask_sz_0 Int4,
+       ask_p_1 Real,
+       ask_sz_1 Int4,
+       ask_p_2 Real,
+       ask_sz_2 Int4,
+       ask_p_3 Real,
+       ask_sz_3 Int4,
+       ask_p_4 Real,
+       ask_sz_4 Int4,
+       ask_p_5 Real,
+       ask_sz_5 Int4,
+       ask_p_6 Real,
+       ask_sz_6 Int4,
+       ask_p_7 Real,
+       ask_sz_7 Int4,
+       ask_p_8 Real,
+       ask_sz_8 Int4,
+       ask_p_9 Real,
+       ask_sz_9 Int4,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+drop table if exists trades;
+create table trades (
+       MsgID Int4,
+       Symbol varchar,
+       SendingTime varchar,
+       TransactTime varchar,
+       TradeDate varchar,
+       MDEntryPx Real,
+       MDnetrySize Int4,
+       NumberOfOrders Int4,
+       AgressorSide Int4,
+       MDTradeEntryID Int4,
+       AgressorOID varchar,
+       RestingOID varchar,
+       PartialQty Int4,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+drop table if exists orders;
+create table orders (
+       MsgID Int4,
+       Symbol varchar,
+       SendingTime varchar,
+       TransactTime varchar,
+       MDEntryPx Real,
+       OrderID varchar,
+       MDDisplayQty Int4,
+       MDOrderPriority Int8,
+       OrderUpdateAction Int4,
+       Side Int4,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+-- create unique index msg_msg_id on msgs msgid;
+
+-- create view ranges as ( 
+--   select
+--      sendingtime,
+--      transacttime,
+--      bid_p_0 price,
+--      case when lag(bid_p_0) over (order by SendingTime) = bid_p_0 then null else 1 end seq 
+--   from
+--      u0v0_books);
+     
+-- create view groups as (
+--   select
+--      sendingtime,
+--      transacttime,
+--      price,
+--      sum(seq) over (order by SendingTime) grp
+--   from
+--      ranges);
+     
+     
+       
+   
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+DB=$1
+TABLE=$2
+FILE=$3
+
+zcat $FILE | psql -U postgres -d $DB -c "copy $TABLE from STDIN with delimiter as ',' CSV HEADER" 
@@ -0,0 +1,4 @@
+#!/bin/bash
+docker exec -it postgres psql -U postgres "$@"
+
+
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#docker run --net host -e PGADMIN_DEFAULT_EMAIL=bts -e PGADMIN_DEFAULT_PASSWORD=BT5u53r dpage/pgadmin4
+
+docker run \
+       --rm \
+       -d \
+       -p 8000:80 \
+       -e PGADMIN_DEFAULT_EMAIL=bts \
+       -e PGADMIN_DEFAULT_PASSWORD=BT5u53r \
+       --name pgadmin \
+       dpage/pgadmin4 
@@ -0,0 +1,24 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SRC_ROOT=$DIR/../../..
+DATA_DIR=`readlink -f $SRC_ROOT/analysis_data`
+DB_DIR=$DATA_DIR/db
+
+mkdir -p $DB_DIR
+
+docker run \
+       --rm \
+       -d \
+       --name postgres \
+       --net host \
+       -v $DB_DIR:/var/lib/postgresql/data \
+       -v /home/$USER:/home/$USER \
+       -v $DATA_DIR:$DATA_DIR \
+       -v /etc/group:/etc/group \
+       -v /run/user/:/run/user/ \
+       -u $UID \
+       -t postgres \
+       postgres 
+
+
+
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SRC_ROOT=$DIR/../../
+DATE=$1
+DB=md_$DATE
+
+$DIR/db/psql.sh -c "drop database if exists $DB"
+$DIR/db/psql.sh -c "create database $DB"
+$DIR/db/psql.sh -d md_$DATE -f $DIR/db/create_db.sql
+
+PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE
+
+ALL_COMMANDS=""
+for TABLE in trades orders msgs books 
+do
+    for ITEM in $PROCESSED_DATA/*.$TABLE.gz
+    do
+	ALL_COMMANDS+="docker exec -t postgres /bin/bash -c \""
+	ALL_COMMANDS+="$DIR/db/import_csv.sh $DB $TABLE $ITEM"
+	ALL_COMMANDS+="\"\n"
+    done
+done
+
+echo -e $ALL_COMMANDS | parallel --will-cite
@@ -0,0 +1,29 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DATE=$1
+
+SRC_ROOT=$DIR/../../
+SPARK_BIN_ROOT=`readlink -f $SRC_ROOT/build`/ninja_release/server
+RAW_DATA=$SRC_ROOT/analysis_data/raw/$DATE
+PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE
+
+mkdir -p $PROCESSED_DATA
+rm $PROCESSED_DATA/* >/dev/null
+
+ALL_COMMANDS=""
+for IN_PATH in $RAW_DATA/*.gz
+do
+    BASE_NAME=`basename $IN_PATH`
+    BASE_NAME_STEM=${BASE_NAME%.*}
+    for CMD in trades orders msgs books 
+    do
+	OUT_PATH=$PROCESSED_DATA/$BASE_NAME_STEM.$CMD.gz
+	ALL_COMMANDS+="(cd $SPARK_BIN_ROOT && zcat $IN_PATH | ./sparkd parsehistoricalfix $CMD | gzip >$OUT_PATH)\n"
+    done
+done
+echo -e $ALL_COMMANDS | parallel --will-cite
+
+
+	 
+
+    
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+d=$1
+end=${2-`date +%Y%m%d`}
+until [[ $d > $end ]]; do 
+    echo "$d"
+    d=$(date -d "$d + 1 day" +%Y%m%d)
+done
@@ -0,0 +1,8 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DATE=${1-`date -d "yesterday" +%Y%m%d`}
+
+$DIR/sync.sh $DATE &&
+$DIR/parse.sh $DATE &&
+$DIR/load.sh $DATE 
+    
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DATE=$1
+SRC_ROOT=$DIR/../../
+RAW_DATA_ROOT=$SRC_ROOT/analysis_data/raw/$DATE
+mkdir -p $RAW_DATA_ROOT
+AWS_ACCESS_KEY_ID=AKIAIFIGOOWGBLEMO3PQ AWS_SECRET_ACCESS_KEY=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 AWS_REGION=us-east-1 aws s3 sync s3://cme-bts-datamine/mbo/mbo/$DATE/ $RAW_DATA_ROOT
@@ -0,0 +1,19 @@
+
+cmake_minimum_required(VERSION 3.5)
+project(spark_client)
+include(${CMAKE_SOURCE_DIR}/tools/safe_dep_glob.cmake)
+
+file(GLOB_RECURSE spark_client_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cs)
+file(GLOB_RECURSE spark_client_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.csproj)
+file(GLOB_RECURSE spark_ui_sources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.cs)
+file(GLOB_RECURSE spark_ui_csproj RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} spark/*.csproj)
+list(REMOVE_ITEM spark_ui_sources ${spark_client_sources} )
+list(REMOVE_ITEM spark_ui_csproj ${spark_client_csproj} )
+
+add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/../
+   COMMAND /bin/bash ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh
+   ARGS ${CMAKE_BUILD_TYPE} ${CMAKE_BINARY_DIR}
+   COMMENT "Building sparkcmd in netcore3"
+   DEPENDS libspark ${spark_client_sources} ${spark_client_csproj} ${CMAKE_CURRENT_SOURCE_DIR}/build_spark_dotnetcore.sh)
+
+   add_custom_target(spark_client_dir ALL DEPENDS ${CMAKE_BINARY_DIR}/../)   
@@ -5,6 +5,7 @@
    <SparkRoot>$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\..))</SparkRoot>
    <SourceRoot>$([System.IO.Path]::GetFullPath($(SparkRoot)\src))</SourceRoot>
    <Externals>$(SparkRoot)\externals\windows</Externals>
+    <Externals Condition=" '$(OS)' != 'Windows_NT' ">$(SparkRoot)/externals/linux</Externals>
    <DepotName>$([System.IO.Path]::GetFileName($([System.IO.Path]::GetFullPath($(SparkRoot)))))</DepotName>
    <ErrorReport>None</ErrorReport>
    <Prefer32Bit>False</Prefer32Bit>
@@ -22,17 +23,20 @@
    <GenDir>$(BaseOutputPath)\gen\$(MSBuildProjectName)</GenDir>
 </PropertyGroup>

+
 <Import Project="$(ComputerSpecificConfig)" Condition="Exists($(ComputerSpecificConfig))" />

 <ItemGroup>
    <AvailableItemName Include="Proto2">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto2</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto2Linux</Targets>
    </AvailableItemName>
    <AvailableItemName Include="Proto3">
-        <Targets>CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' " >CompileProto3</Targets>
+        <Targets Condition=" '$(OS)' != 'Windows_NT' " >CompileProto3Linux</Targets>
    </AvailableItemName>
    <AvailableItemName Include="Proto3Excel">
-        <Targets>CompileProto3Excel</Targets>
+        <Targets Condition=" '$(OS)' == 'Windows_NT' ">CompileProto3Excel</Targets>
    </AvailableItemName>    
 </ItemGroup>

@@ -43,6 +47,14 @@
    <protogen>$(Externals)\bin\protogen.exe</protogen>
 </PropertyGroup>

+<PropertyGroup Label="ProtoBuf" Condition=" '$(OS)' != 'Windows_NT' ">
+    <ProtocPluginBase>$(SparkRoot)/src/lib/api/service_compiler</ProtocPluginBase>
+    <ProtoInclude>$(Externals)/include</ProtoInclude>
+    <protoc>$(Externals)/bin/protoc</protoc>
+    <protoc_rename>$(Externals)/bin/lowercase_files.sh</protoc_rename>
+    <protogen>$(Externals)/bin/protogen</protogen>
+</PropertyGroup>
+
 <ItemDefinitionGroup>
    <Proto2>
    </Proto2>
@@ -3,7 +3,7 @@


 <Target Name="CompileProto2" 
-    Condition="'@(Proto2)'!=''"
+    Condition="'@(Proto2)'!='' AND '$(OS)' == 'Windows_NT' "
    Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
    Outputs="$(GenDir)\proto2\%(Proto2.filename).cs"
    AfterTargets="PreBuildEvent">
@@ -28,7 +28,7 @@
    passes wipes out the additional compile includes that we add.
 -->
 <Target Name="CompileProto3" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3)'!=''"
+    Condition="'@(Proto3)'!=''  AND '$(OS)' == 'Windows_NT' "
    Inputs="$(protoc);
    %(Proto3.Identity);$(ProtocPluginBase)\plugin.py;
    $(ProtocPluginBase)\templates\c#\client.pb.cs;
@@ -58,7 +58,7 @@
 </Target>

 <Target Name="CompileProto3Excel" AfterTargets="GetAssemblyVersion"
-    Condition="'@(Proto3Excel)'!=''"
+    Condition="'@(Proto3Excel)'!='' AND '$(OS)' == 'Windows_NT' "
    Inputs="$(protoc);
    %(Proto3Excel.Identity);$(ProtocPluginBase)\plugin.py;
    $(ProtocPluginBase)\templates\excel\excel.pb.cs;"
@@ -82,12 +82,74 @@
    </ItemGroup>
 </Target>

+
+<Target Name="CompileProto2Linux" 
+    Condition="'@(Proto2)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(MSBuildProjectFile);$(protogen);%(Proto2.FullPath)" 
+    Outputs="$(GenDir)/proto2/%(Proto2.filename).cs"
+    AfterTargets="PreBuildEvent">
+    <PropertyGroup>
+        <SourceRelativeFile>$([System.String]::Copy('%(proto2.fullpath)').Replace($(SourceRoot), ''))</SourceRelativeFile>
+        <SourceRelativeDir>$([System.IO.Path]::GetDirectoryName($(SourceRelativeFile)))</SourceRelativeDir>
+        <WorkingDirectory>$(GenDir)/proto2</WorkingDirectory>
+    </PropertyGroup>
+    <MakeDir Directories="$(WorkingDirectory)" />
+    <Copy SourceFiles="%(Proto2.FullPath)" DestinationFolder="$(WorkingDirectory)" SkipUnchangedFiles="True" UseHardLinksIfPossible="True" />
+    <Error Text="protogen.exe does not exist. If you forgot to clone with externals try: git submodule update --init --recursive" Condition="!Exists($(protogen))" />
+    <Message Importance="High" Text="Doing Proto2 %(proto2.filename)" />
+    <Exec Command="PYTHONPATH=$(Externals)/python $(Protogen) %(Proto2.filename)%(proto2.extension) --csharp_out=. -I. -I$(SourceRoot) -I$(Externals)/include"
+          WorkingDirectory="$(WorkingDirectory)" />
+    <ItemGroup>
+        <Compile Include="$(WorkingDirectory)/%(Proto2.filename).cs" />
+    </ItemGroup>
+</Target>
+
+<Target Name="CompileProto3Linux" AfterTargets="GetAssemblyVersion"
+    Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' "
+    Inputs="$(protoc);
+    %(Proto3.Identity);$(ProtocPluginBase)/plugin.py;
+    $(ProtocPluginBase)/templates/c#/client.pb.cs;
+    $(ProtocPluginBase)/templates/c#/server.pb.cs;
+    $(ProtocPluginBase)/templates/c#/immutable.pb.cs;"
+    Outputs="%(Proto3.OutCS);%(Proto3.OutCSServer);%(Proto3.OutCSClient);%(Proto3.OutCSImmutable)">
+    <MakeDir Directories="%(Proto3.OutBase)" />
+    <MakeDir Directories="$(IntermediateOutputPath)/gen" />
+   
+    <Message Importance="High" Text="Generating service %(Proto3.FileName)" />
+    <Exec Command="
+       PYTHONPATH=$(Externals)/python $(protoc) \
+       %(Proto3.FullPath) \
+       --error_format=gcc \
+       --proto_path=$(protoinc):$(SourceRoot) \
+       --plugin=&apos;protoc-gen-service=$(ProtocPluginBase)/plugin.py&apos; \
+       --service_out=lang=c#,strip_out_path=1:%(Proto3.OutBase) \
+       --csharp_out=%(CompileOptions)%(Proto3.OutBase)"
+      WorkingDirectory="%(relativedir)" />    
+      
+    <!-- <Exec Command="$(protoc_rename) %(Proto3.OutBase)"  />    -->
+            
+    <ItemGroup>
+       <Compile Include="%(Proto3.OutCS)" />
+       <Compile Include="%(Proto3.OutCSClient)" />
+       <Compile Include="%(Proto3.OutCSServer)" />
+       <Compile Include="%(Proto3.OutCSImmutable)" />
+    </ItemGroup>
+</Target>
+
+<!-- <Target Name="RenameProto3Linux" AfterTargets="CompileProto3Linux"
+   Condition="'@(Proto3)'!='' AND '$(OS)' != 'Windows_NT' ">
+   <Move SourceFiles="$(OutBase)\Apicore.cs" DestinationFiles="$(OutBase)\apicore.cs" />  
+   <Move SourceFiles="$(OutBase)\Apioptions.cs" DestinationFiles="$(OutBase)\apioptions.cs" />  
+   <Move SourceFiles="$(OutBase)\Co2.cs" DestinationFiles="$(OutBase)\co2.cs" />  
+</Target> -->
+
+
 <Target Name="AfterClean" AfterTargets="Clean">
    <Delete Files="%(Proto3.OutCS)" />
    <Delete Files="%(Proto3.OutCSClient)" />
    <Delete Files="%(Proto3.OutCSServer)" />
    <Delete Files="%(Proto3.OutCSImmutable)" />
-    <Delete Files="$(GenDir)\proto2\*" />
+    <Delete Files="$(GenDir)/proto2/*" />
    <Delete Files="%(Proto3Excel.OutCSExcel)" />
 </Target>

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#$1 = Debug / Release (CMAKE_BUILD_TYPE), $2 = CMAKE_BINARY_DIR
+cd "$(dirname "$0")"
+echo $PWD dotnet build -c $1 ./sparkcmd/
+dotnet build -c $1 ./sparkcmd/ 
+
+if [ "$1" == "Release" ]
+then
+   dotnet publish -c $1 ./sparkcmd/ 
+   ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.0/linux-x64/publish
+fi
+
+ln -fs $2/src/lib/libspark.so $2/../$1/bin/netcoreapp3.0/linux-x64
@@ -1,6 +1,7 @@
 using bts.api;
 using bts.co2;
 using bts.collections;
+using bts.utils.csv;
 using mbts.persist;
 using mbts.spark.filters;
 using mbts.ui;
@@ -52,7 +53,7 @@ namespace mbts.spark {
            columns.Add(new RenderTextColumn<Row, TimeAndSalesView.Columns>(col_key, r => r[col_key], filter, filter_template, r => _get_aggressor_brush(r))
            { id = col_id(col_key), Header = header, Width = width, sort_member_path = sort_member_path, horizontal_alignment = h_alignment  }
         );
-         
+
         // be sure to load GridSettings AFTER the columns collection has been populated, the grid settings manager looks columns by id to apply the appropriate settings,
         // if the collection is not populated, the column settings will not be applied
         grid_settings_manager = new SimpleGridSettingsManager<Row>(columns, new Dictionary<string, GridColumnSettingsFeatureFlags> { }, row_vms);
@@ -173,6 +174,19 @@ namespace mbts.spark {
            }
         }

+         public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(
+               typeof(TimeAndSalesVM.Row),
+               new string[] {
+                  nameof(security),
+                  nameof(time),
+                  nameof(price),
+                  nameof(size),
+                  nameof(aggressor_side),
+                  nameof(security_key),
+               });
+
+
         public void raise_invalidated() => invalidated?.Invoke();
      }
   }
@@ -3,6 +3,7 @@ using bts.api;
 using bts.shell;
 using bts.shell.fills;
 using bts.spark;
+using bts.utils.csv;
 using mbts.co2;
 using mbts.persist;
 using mbts.ui;
@@ -63,6 +64,7 @@ namespace mbts.spark {
         var current_smd_source = connectable_builder.create(vm => vm.current_security_md_observable(_bus)).OnDispatcher();
         var selected_secs_source = connectable_builder.create(vm => vm.selected_securities_observable()).OnDispatcher();
         _commands = new SparkCommand[] {
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.value.row_vms.ToCsv(TimeAndSalesVM.Row.csv_ordered_properties), "TaS")),
            SparkCommands.ShowLadder(SparkCommands.KeyGestures.ShowLadder).complete(current_smd_source, md => md.HasValue, ConvertUtils.ValueOrDefault),
            SparkCommands.ShowOrderTicketFromSecurity(SparkCommands.KeyGestures.ShowOrderTicket).complete(current_smd_source, md => md.HasValue, ConvertUtils.ValueOrDefault),
            SparkCommands.ShowFills(SparkCommands.KeyGestures.ShowFills).complete(selected_secs_source, always_true, get_fills_startup_data),
@@ -5,6 +5,7 @@ using bts.collections;
 using bts.shell.fills;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.collections;
 using mbts.persist;
 using mbts.spark.filters;
@@ -23,6 +24,7 @@ using System.Linq;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
 using System.Windows.Data;
+using System.Windows.Input;
 using System.Windows.Media;
 using static bts.api.orderman.Fill.Types;
 using static bts.shell.FillsWindowFactory;
@@ -298,6 +300,7 @@ namespace mbts.spark {
         public FillStatus fill_status => _fill.FillStatus;
         public string fill_key => _fill.FillKey;
         public string order_key => _order.Key.ToString();
+         public string symbol => fill_tuple.security?.Symbol;

         // Root Algo
         public OrderType root_algo_order_type => _root_algo.HasValue ? _root_algo.Value.value.CustomCase : OrderType.None;
@@ -309,6 +312,28 @@ namespace mbts.spark {
         public override bool Equals(object obj) => Equals(obj as Row);
         public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();

+         public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(typeof(Row),
+               new string[] {
+                  nameof(security),
+                  nameof(symbol),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(side),
+                  nameof(fill_time),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(size),
+                  nameof(price),
+                  nameof(fill_kind),
+                  nameof(fill_status),
+                  nameof(fill_key),
+                  nameof(order_key),
+                  nameof(security_key)
+               });
+
         public event Action invalidated;
         void IRowVM.raise_invalidated() => invalidated?.Invoke();
      }
@@ -3,6 +3,7 @@ using bts.api;
 using bts.shell;
 using bts.shell.fills;
 using bts.spark;
+using bts.utils.csv;
 using mbts.persist;
 using mbts.ui;
 using mbts.ui.settings;
@@ -59,6 +60,7 @@ namespace mbts.spark {
         _view = new FillsView { DataContext = _vm };

         _commands = new List<SparkCommand> {
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(FillsVM.Row.csv_ordered_properties), "Fills")),
            // wrap the grid settings view factory in a dialogCommand here since the vm is destroyed/recreated when the security criteria changes
            SparkCommands.window_settings_command(new DialogCommand(() => {
               var vm = new ShellSettingsDialogVM<GridSettings, SimpleGridSettingsVM>(
@@ -4,6 +4,7 @@ using bts.api.orderman;
 using bts.collections;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.ui;
 using mbts.ui.controls;
 using System;
@@ -11,6 +12,7 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Runtime.CompilerServices;
+using System.Linq;
 using System.Windows.Input;
 using System.Windows.Media;
 using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
@@ -60,6 +62,35 @@ namespace mbts.spark {
      public event Action<(OrderSecurityRootPair oldval, OrderSecurityRootPair newval)> updated;
      public event PropertyChangedEventHandler PropertyChanged;

+      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(
+               typeof(OrderBookRowVM), 
+               new string[] {
+                  nameof(security_text),
+                  nameof(symbol),
+                  nameof(time_created),
+                  nameof(last_change),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(order_sub_type),
+                  nameof(side),
+                  nameof(quantity),
+                  nameof(filled_quantity),
+                  nameof(price),
+                  nameof(avg_price),
+                  nameof(last_price),
+                  nameof(piq),
+                  nameof(status),
+                  // last message (could contains commas, so skipping)
+                  nameof(security_key),
+                  nameof(order_key),
+                  nameof(parent_order_key),
+               });
+
      // Core
      public OrderSecurityRootPair order_tuple { get; private set; }
      public OrderRecordImmutable order => order_tuple.order.order;
@@ -4,6 +4,7 @@ using bts.shell;
 using bts.shell.fills;
 using bts.shell.orderbook;
 using bts.spark;
+using bts.utils.csv;
 using bts.view;
 using mbts.persist;
 using mbts.ui;
@@ -184,6 +185,7 @@ namespace mbts.spark {
         show_grid_settings_command.group = CommandGroups.Auxiliary;

         _commands = new SparkCommand[] {
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(OrderBookRowVM.csv_ordered_properties), "OrderBook")),
            toggle_show_right_panel_command,
            attach_selected_orders_command,
            detach_selected_orders_command,
@@ -653,54 +653,25 @@ namespace mbts.spark.utilities {
            return def;
         }
      }
+      
+      static CustomOrderDefinition.CustomOneofCase[] algo_config(
+         Func<bts.config.AlgoPermissions, List<string>> selector)
+      {
+         var conf = mbts.config.Configuration.get<bts.config.AlgoPermissions>();
+         return selector(conf).Select(x => 
+         {
+            return (CustomOrderDefinition.CustomOneofCase)System.Enum.Parse(typeof(CustomOrderDefinition.CustomOneofCase), x, true);
+         }).ToArray();
+      }

      public static readonly CustomOrderDefinition.CustomOneofCase[]

         // exchange_order_types are the types which can be submitted on an exchange security
-         exchange_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         spreader_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Spreader,
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover, 
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Splitter,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         attachable_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Iceberg,
-            CustomOrderDefinition.CustomOneofCase.Lean,
-            CustomOrderDefinition.CustomOneofCase.Join,
-            CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top,
-            CustomOrderDefinition.CustomOneofCase.Offset,
-            CustomOrderDefinition.CustomOneofCase.Scratch,
-            CustomOrderDefinition.CustomOneofCase.Prorata,
-         },
-         outright_order_types = new[] {
-            CustomOrderDefinition.CustomOneofCase.Market,
-            CustomOrderDefinition.CustomOneofCase.Limit,
-            CustomOrderDefinition.CustomOneofCase.Stop,
-            CustomOrderDefinition.CustomOneofCase.StopLimit,
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg,
-         };
+         exchange_order_types = algo_config(x => x.exchange_order_types),
+         spreader_order_types = algo_config(x => x.spreader_order_types),
+         attachable_order_types = algo_config(x => x.attachable_order_types),
+         outright_order_types = algo_config(x => x.outright_order_types);
+         
      public static readonly CustomOrderDefinition.CustomOneofCase[] all_order_types = exchange_order_types.Union(spreader_order_types).ToArray();

      public static void copy_values(CustomOrderDefinitionImmutable source, ref CustomOrderDefinition target) =>
@@ -54,13 +54,16 @@ namespace mbts.spark
   {
      readonly Func<TModel, TView> _convert;
      readonly Func<TView, TModel> _convert_back;
+      readonly Func<IMessage, IMessage> _message_override;
      TView _value;

-      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, Func<TView, TModel> convert_back) : 
+      public GenericOrderParamVM(FieldDescriptor field_descriptor, Func<TModel, TView> convert, 
+                                 Func<TView, TModel> convert_back, Func<IMessage, IMessage> message_override = null) : 
         base(field_descriptor) 
      {
         _convert = convert;
         _convert_back = convert_back;
+         _message_override = message_override;
      }

      public TView value 
@@ -68,7 +71,8 @@ namespace mbts.spark
         get => _value;
         set 
         {
-            this.descriptor.Accessor.SetValue(this.message, _convert_back(_value = value));
+            this.descriptor.Accessor.SetValue( _message_override == null ? this.message : _message_override(this.message), 
+                                              _convert_back(_value = value));
            RaisePropertyChanged();
         }
      }
@@ -84,7 +88,7 @@ namespace mbts.spark

      protected override void on_param_changed() 
      {
-         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(this.message)));
+         _value = _convert(ConvertUtils.try_convert<TModel>(this.descriptor.Accessor.GetValue(_message_override == null ? this.message : _message_override(this.message))));
         RaisePropertyChanged(nameof(value));
      }
   }
@@ -163,7 +167,6 @@ namespace mbts.spark
               RaisePropertyChanged();
         }
      }      
-
      public IObservable<QuantityMeasureImmutable?> subscribe_value() 
      {
         return Observable.Create<QuantityMeasureImmutable?>(o =>
@@ -174,10 +177,9 @@ namespace mbts.spark
               o.OnNext(quantity);
         });
      }
-
      protected override void on_param_changed()
      {
         RaisePropertyChanged(nameof(quantity));
      }
-   }   
+   }
 }
@@ -96,9 +96,41 @@ namespace mbts.spark {
            {               
               Google.Protobuf.Reflection.MessageDescriptor desc = new TMutable().Descriptor;               
               _parameters.SyncWithDictionary(
-                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), _mk_param_vm);
+                  desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), x => _mk_param_vm(x));
+               add_message_params(desc);
            }
         }
+         void add_message_params(Google.Protobuf.Reflection.MessageDescriptor desc)
+         {
+            var msgs = desc.Fields.InDeclarationOrder().Where(x => !_is_value_param_desc(x) && 
+                                                              x.MessageType.ClrType != typeof(CustomOrderDefinition) );
+            foreach(var message_field in msgs)
+            {
+               foreach (var fd in message_field.MessageType.Fields.InDeclarationOrder())
+               {
+                  if(_is_value_param_desc(fd)) //one level deep
+                  {
+                     var mk = _mk_param_vm(fd, 
+                     outside_object => 
+                     {                        
+                        var msgfield = message_field.Accessor.GetValue(outside_object);
+                        if(msgfield == null)
+                        {
+                           message_field.Accessor.SetValue(outside_object, Activator.CreateInstance(message_field.MessageType.ClrType));
+                           msgfield = message_field.Accessor.GetValue(outside_object);
+                        }
+                        return (IMessage)msgfield;
+                     });
+                     if(mk == null) 
+                        continue;
+                        
+                     if(!_parameters.Add(mk))
+                        _parameters.Add(message_field.Name + "_" + mk.descriptor.Name, mk);                     
+                  }
+               }
+            }            
+         }
+         
         static bool _is_value_param_desc(FieldDescriptor desc)
         {
            return (desc.FieldType != Google.Protobuf.Reflection.FieldType.Message || 
@@ -117,7 +149,7 @@ namespace mbts.spark {
            return v;
         }

-         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor) 
+         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor, Func<IMessage, IMessage> message_override = null) 
         {
            ExceptionAssert.Argument.NotNull(descriptor, nameof(descriptor));

@@ -134,33 +166,36 @@ namespace mbts.spark {
               return pvm;
            }
            else if (descriptor.FieldType == FieldType.Bool)
-               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
            else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float)
-               return mk<double, double>(descriptor, v => v, v => v, double.NaN);
+               return mk<double, double>(descriptor, v => v, v => v, message_override, double.NaN);
            else if (descriptor.FieldType == FieldType.Int32)
-               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
            else if (descriptor.FieldType == FieldType.UInt32)
-               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault());
+               return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault(), message_override);
            else if (descriptor.FieldType == FieldType.String)
-               return mk<string, string>(descriptor, v => v, v => v);
+               return mk<string, string>(descriptor, v => v, v => v, message_override);
            else
               return null;
         }
-         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, TView default_value = default)
+         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, 
+                                                                        Func<IMessage, IMessage> message_override = null, TView default_value = default)
         {
-            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back);
-            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param, default_value);
+            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back, message_override);
+            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param,
+                                                                                 message_override, default_value);

            TView get_from_param(IMessage param, FieldDescriptor fd) => convert(ConvertUtils.try_convert<TModel>(fdesc.Accessor.GetValue(param)));
            void assign_to_param(IMessage param, FieldDescriptor fd, TView value) => fdesc.Accessor.SetValue(param, convert_back(value));
         }

-         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
+         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, 
+                                    Func<IMessage, IMessage> message_override = null, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
         {
            add_property<T>(
               pvm.descriptor.Name,
-               mdef => mdef != null ? selector(mdef, pvm.descriptor) : default_value,
-               (mdef, v) => assign_to_param(mdef, pvm.descriptor, v),
+               mdef => mdef != null ? selector( message_override == null ?  mdef : message_override(mdef), pvm.descriptor) : default_value,
+               (mdef, v) => assign_to_param(message_override == null ? mdef : message_override(mdef), pvm.descriptor, v),
               () => pvm.value,
               v => pvm.value = v,
               pvm.subscribe_value(),
@@ -4,14 +4,14 @@ using OffsetDefi = bts.api.orderman.OffsetOrderDefinitionImmutable;
 namespace mbts.spark {
   static partial class OrderTypeVMFactory {
      class OffsetOrderVM : OrderTypeVMBase<OffsetDef, OffsetDefi> {
-         readonly Property<uint?> _tick_offset_on_deploy;
+         readonly Property<int?> _tick_offset_on_deploy;

         public OffsetOrderVM(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions)
            : base(key, display_name, editing_permissions) {
-            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0u);
+            _tick_offset_on_deploy = add_nullable_property(nameof(tick_offset_on_deploy), OffsetDef.TickOffsetOnDeployFieldNumber, 0);
         }

-         public uint? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
+         public int? tick_offset_on_deploy { get => _tick_offset_on_deploy.Value; set => _tick_offset_on_deploy.set(value); }
      }
   }
 }
@@ -9,7 +9,6 @@
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
    <Configurations>Debug;Release</Configurations>
    <Platforms>x64</Platforms>
-    <RuntimeIdentifiers>win7-x64</RuntimeIdentifiers>
    <PlatformTarget>AnyCPU</PlatformTarget>
    <LanguageTargets>$(MSBuildExtensionsPath)\$(VisualStudioVersion)\Bin\Microsoft.CSharp.targets</LanguageTargets>
    <ApplicationIcon>logo.ico</ApplicationIcon>
@@ -35,7 +34,7 @@
    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
    <PackageReference Include="IronPython" Version="2.7.7" />
    <PackageReference Include="NAudio" Version="1.8.4" />
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
    <PackageReference Include="NUnit" Version="3.10.1" />
    <PackageReference Include="NUnit3TestAdapter" Version="3.8.0" />
@@ -64,6 +64,25 @@ namespace mbts.ui {
            }
         }, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));

+      public static IDialogCommand export_csv(
+         Func<string> get_file_content,
+         string source_window_name) =>
+         new AnonymousFileDialogCommand<object>(_ =>
+            _create_save_file_dialog(
+               Environment.GetFolderPath(Environment.SpecialFolder.Desktop),
+               $"BTS_Spark_{source_window_name}_{DateTime.Now.ToString("yyyyMMdd_HHmm")}.csv",
+               new[] { ("CSV (comma delimited)", new string[] { "csv" }) }
+            ),
+            (_, file) => {
+               try {
+                  File.WriteAllText(file, get_file_content.Invoke());
+               } catch (Exception ex) {
+                  BTSMessageBox.Show($"Error exporting data to csv: {ex}.", "CSV Export Error", BTSMessageBoxButtons.OK, System.Windows.MessageBoxImage.Error, true, false, true);
+               }
+            },
+            // we are always able to perform an export, even when no rows (it will just be a header)
+            (_) => true);
+
      static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
         var rv = new TDialog();
         if (!string.IsNullOrEmpty(initial_dir))
@@ -4,6 +4,8 @@ using System.Runtime.InteropServices;
 using System.Security;
 using Google.Protobuf;
 using security_key = System.UInt32;
+using bts.co2;
+using System.Linq;

 namespace mbts.co2
 {
@@ -85,6 +87,7 @@ namespace mbts.co2

      [DllImport("libspark")]
      internal unsafe static extern void post_on_scan_security_thread(Action callback);
+      
   }

   internal static class Co2NativeTestMethods
@@ -133,5 +136,59 @@ namespace mbts.co2
      [DllImport("libspark")]
      internal static extern void co2_test_get_constants(out ulong max_depth, out ulong max_implied_depth, out sbyte nan_exponent);

+      internal unsafe delegate void raw_receive_pb_or_string(byte* buffer, int buffer_size);
+      [DllImport("libspark")]
+      private static unsafe extern void raw_co2_compute_synthetic_book(
+         byte[] security_buffer,
+         int security_length,
+         IntPtr[] leg_books,
+         int[] leg_lengths,
+         int leg_books_length,
+         raw_receive_pb_or_string callback);
+
+      internal static unsafe BookMessageImmutable co2_compute_synthetic_book(SecurityImmutable synthetic, params BookMessageImmutable[] books) {
+         BookMessage result = null;
+         string error = null;
+
+         var security_buffer = synthetic.ToByteArray();
+         // var h = GCHandle.Alloc(null, GCHandleType.Pinned).
+         var book_buffers = 
+            books
+            .Select(b => b.ToByteArray())
+            .ToArray();
+         GCHandle[] book_buffer_handles = 
+            book_buffers
+            .Select(b => GCHandle.Alloc(b,GCHandleType.Pinned))
+            .ToArray();
+         IntPtr[] book_buffer_pointers = 
+            book_buffer_handles
+            .Select(h => h.AddrOfPinnedObject())
+            .ToArray();
+         var lengths = 
+            book_buffers
+            .Select(b => b.Length)
+            .ToArray();
+
+         raw_co2_compute_synthetic_book(
+            security_buffer,
+            security_buffer.Length,
+            book_buffer_pointers,
+            lengths,
+            book_buffers.Length,
+            (byte* buffer, int length) => {
+               if (length < 0) {
+                  error =  new string((char*)buffer,0,-1 * length);
+               }
+               else {
+                  result = Co2NativeMethods.deserialize<BookMessage>(buffer, (ulong)length);
+               }
+            });
+
+         if(error != null) {
+            // C# style multiple return values
+            throw new Exception(error);
+         }
+         return new BookMessageImmutable(result);
+      }
   }
 }
@@ -52,7 +52,49 @@ namespace mbts.co2 {
         return _mantissa.GetHashCode();
      }

+      public static IntPrice from_double(double price, sbyte precision) {
+         if (double.IsNaN(price)) return NaN;
+         if (Math.Abs(price%1) != 0) {
+            double rounding = 0.5 * Math.Pow(10,-precision);
+            price += (price > 0) ? rounding : -rounding;
+            price *= Math.Pow(10,precision);
+            return new IntPrice((long)price, (sbyte)-precision);
+         }
+         return new IntPrice((long)price, 0);
+      }
+
      public static IntPrice NaN = new IntPrice(0,127);
+
+      static double[] powers_10 = {
+  1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10, 1E-9, 1E-8, 1E-7, 1E-6, 1E-5, 1E-4, 1E-3, 1E-2, 1E-1,
+  1E0,
+  1E1, 1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8, 1E9, 1E10, 1E11, 1E12, 1E13, 1E14, 1E15, 1E16, 1E17, 1E18, 1E19, 1E20};
+      static int max_abs_power_10 = powers_10.Length / 2;
+      static double pow10(int power)
+      {
+         int offset = power + max_abs_power_10;
+         int powers_10_sz = powers_10.Length;
+         bts.Log.assert(offset >= 0 && offset < powers_10_sz);
+         if (offset >= 0 && offset < powers_10_sz)
+         {
+            return powers_10[offset];
+         }
+         return double.NaN;
+      }      
+      
+      public static IntPrice from_double(double price, int precision = 9)
+      {
+         if (Double.IsNaN(price)) 
+            return NaN;
+         if (Math.Abs(price - Math.Truncate(price)) != 0.0)
+         {
+            double rounding = 0.5 * pow10(-precision);
+            price += (price > 0.0) ? rounding : -rounding;
+            price *= pow10(precision);
+            return new IntPrice((long)price, Convert.ToSByte(-precision));
+         }
+         return new IntPrice((long)price, 0);
+      }
   }
   
 }
@@ -1,4 +1,6 @@
 using bts;
+using System.Collections.Generic;
+using System.Linq;

 namespace mbts.co2 {
   public static class NativePriceHelpers {
@@ -38,6 +40,60 @@ namespace mbts.co2 {
      public const uint COCONUT_MAX_DEPTH = 10;
      public const uint COCONUT_MAX_IMPLIED_DEPTH = 2;
      internal const sbyte COCONUT_NAN_EXPONENT = 127;
+
+      public static double side_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if(side == bts.api.Side.Buy)
+            return bid_at(book, level);
+         else
+            return ask_at(book, level);
+      }
+      public static uint sidesize_at(this bts.co2.BookMessage book, bts.api.Side side, int level)
+      {
+         if (side == bts.api.Side.Buy)
+            return bidsize_at(book, level);
+         else
+            return asksize_at(book, level);
+      }
+      public static double bid_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static double ask_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return NativePriceHelpers.to_double(lev.Mantissa, (sbyte)lev.Exponent);
+      }
+      public static uint bidsize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Bids[level];
+         return lev.Size;
+      }
+      public static uint asksize_at(this bts.co2.BookMessage book, int level)
+      {
+         var lev = book.Asks[level];
+         return lev.Size;
+      }
+      public static double price(this bts.co2.PriceSizeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }
+      public static double price(this bts.co2.TradeMessage p)
+      {
+         return NativePriceHelpers.to_double(p.Mantissa, (sbyte)p.Exponent);
+      }      
+
+      public static IList<double> all_bids(this bts.co2.BookMessage book)
+      {
+         var lev = book.Bids.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }
+      public static IList<double> all_asks(this bts.co2.BookMessage book)
+      {
+         var lev = book.Asks.Select(x => NativePriceHelpers.to_double(x.Mantissa, (sbyte)x.Exponent)).ToList();
+         return lev;
+      }      
   }
   
 }
@@ -3,4 +3,6 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark_core")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_fix")]
+[assembly: InternalsVisibleTo("spark_historical")]

@@ -0,0 +1,147 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf;
+using Google.Protobuf.Collections;
+using Google.Protobuf.WellKnownTypes;
+using mbts.co2;
+
+namespace bts.history
+{
+   public interface IHasMsgSeqNum 
+   {
+      uint MsgSeqNum { get; }
+   }
+   
+   public interface IHasDateTimeStamp 
+   {
+      DateTime trade_date { get; set;}
+      DateTime sending_time { get; set;}
+      DateTime transact_time { get; set; }      
+   }
+
+   public interface IHasTimestamp 
+   {
+      Timestamp SendingTime { get; set; }
+      Timestamp TransactTime { get; set; }
+   }
+
+   public interface IHasMdPacket
+   {
+      MarketDataIncrementalRefresh Md{ get; }
+   }
+
+   public interface IHasMdEntry
+   {
+      MDEntry MdEntry { get; }
+      int EntryIndex { get; }
+   }
+   
+   public interface IHasTopBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.TopBookMessage TopBook { get; set; }
+   }
+
+   public interface IHasBook : IHasMdEntry, IHasMdPacket
+   {
+      co2.BookMessage Book { get; set; }
+   }
+
+   public interface IHasTrade : IHasMdEntry, IHasMdPacket
+   {
+      co2.TradeMessage Trade { get; set; }      
+   }   
+
+   public interface IHasMBO : IHasMdPacket
+   {
+      RepeatedField<OrderIDEntry> Orders { get;  }
+   }
+   
+   public sealed partial class HistoryRow : IHasTopBook, IHasMBO, IHasTrade, IHasBook, IHasDateTimeStamp, IHasTimestamp, IHasMsgSeqNum
+   {
+      public DateTime trade_date { get { return DateTime.SpecifyKind(this.TradeDate.ToDateTime(), DateTimeKind.Utc); } 
+                                   set { this.TradeDate = Timestamp.FromDateTime(value); } }
+      public DateTime sending_time { get { return DateTime.SpecifyKind(this.SendingTime.ToDateTime(), DateTimeKind.Utc); } 
+                                     set { this.SendingTime = Timestamp.FromDateTime(value); } }
+      public DateTime transact_time { get { return DateTime.SpecifyKind(this.TransactTime.ToDateTime(), DateTimeKind.Utc); } 
+                                      set { this.TransactTime = Timestamp.FromDateTime(value); } }
+      
+      public HistoryRow add_orders(IEnumerable<OrderIDEntry> orderentries)
+      {
+         this.Orders.AddRange(orderentries);
+         return this;
+      }                                         
+   }
+   
+   public interface IMBOMessage
+   {
+      MDUpdateAction OrderUpdateAction {get;}
+      ulong OrderId {get;}
+      uint SecurityId {get;}
+      string SecurityDesc {get;}
+      ulong MdOrderPriority {get;}
+      double MdEntryPx {get;}
+      uint MdPriceLevel {get;}
+      ulong MdDisplayQty {get;}
+      uint LastQty {get;}
+      MDEntryType MdEntryType {get;}
+      uint ReferenceId {get;}      
+   }
+   
+   public sealed partial class MBOMessage : IMBOMessage
+   {
+      public MDUpdateAction OrderUpdateAction => OrderEntry != null ? OrderEntry.OrderUpdateAction : MdEntry.MdUpdateAction;
+      public ulong OrderId  => OrderEntry != null ? OrderEntry.OrderId : MdEntry.OrderId;
+      public uint SecurityId => MdEntry.SecurityId;
+      public string SecurityDesc => MdEntry.Symbol;
+      public ulong MdOrderPriority => OrderEntry != null ? OrderEntry.MdOrderPriority : MdEntry.MdOrderPriority;
+      public double MdEntryPx => MdEntry.MdEntryPx;
+      public uint MdPriceLevel => MdEntry.MdPriceLevel;
+      public ulong MdDisplayQty => OrderEntry != null ? OrderEntry.MdDisplayQty : MdEntry.MdDisplayQty;
+      public uint LastQty => OrderEntry != null ? OrderEntry.LastQty : MdEntry.LastQty;
+      public MDEntryType MdEntryType => MdEntry.MdEntryType;
+      public uint ReferenceId => OrderEntry != null ? OrderEntry.ReferenceId : MdEntry.ReferenceId;
+      public uint NumberOfOrders => MdEntry.NumberOfOrders;
+   }
+
+   public sealed partial class MarketDataIncrementalRefresh
+   {
+      public MarketDataIncrementalRefresh add_entries(IEnumerable<MDEntry> mdentries, IEnumerable<OrderIDEntry> orderentries)
+      {
+         this.MdEntries.AddRange(mdentries);
+         this.OrderIdEntries.AddRange(orderentries);
+         return this;
+      }
+   }
+   public interface ITradeDetail
+   {
+      ulong OrderId { get; set; }
+      uint LastQty { get; set; }
+      ITradeDetail clone();
+   }
+
+   public sealed partial class OrderIDEntry : ITradeDetail
+   {
+      ITradeDetail ITradeDetail.clone()
+      {
+         return this.Clone();
+      }
+   }
+}
+
+namespace bts.co2
+{
+   public sealed partial class BookMessage
+   {
+      public BookMessage add_bids_and_asks(IEnumerable<PriceSizeMessage> bids, IEnumerable<PriceSizeMessage> asks)
+      {
+         this.Bids.AddRange(bids);
+         this.Asks.AddRange(asks);
+         return this;
+      }
+   }
+   
+   public sealed partial class TradeMessage 
+   {
+      public double price => NativePriceHelpers.to_double(this.Mantissa, (sbyte)this.Exponent);
+   }     
+}
@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using static bts.api.Security;

 namespace bts.api {
-   struct FutureLeg {
+   public struct FutureLeg {
      public readonly Venue exchange;
      public readonly string symbol, exchange_desc;
      public readonly FutureContractImmutable future;
@@ -4,7 +4,7 @@ using System.Linq;
 using static bts.api.Security;

 namespace bts.api {
-   internal static class SecurityHelper {
+   public static class SecurityHelper {
      public static IEnumerable<FutureLeg> get_flat_future_legs(SecurityImmutable sec) {
         switch (sec.ContractCase) {
            case ContractOneofCase.Spread: return _get_flat_future_legs(sec.Spread.Legs, 1);
@@ -25,7 +25,7 @@
    <Proto3 Include="../../lib/api/*.proto" />
    <Proto3 Include="../../lib/api/services/*.proto" />
    <!--Proto3 Remove="../../lib/api/services/MarketDataRelay.proto" /-->
-    <Proto3 Include="../../lib/md/protos/co2.proto" />
+    <Proto3 Include="../../lib/md/protos/Co2.proto" />
    <Proto3 Include="../../lib/domain_model/Security.proto" />
    <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
    <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">
@@ -28,5 +28,8 @@ namespace bts.utils

      [DllImport("libspark")]
      public static extern bool initialize_nothrow(StringBuilder buffer, int length);
+
+      //public static string bts_data_path()
+      //use mbts.config.Configuration.get<bts.config.System>().data_path;
   }
 }
@@ -36,6 +36,7 @@ namespace bts {
      Plugin,
      Sdk,
      Http,
+      Fix
      // Please keep this list in sync with logging.h/logging.cpp
   }
   
@@ -102,9 +103,10 @@ namespace bts {
      internal static Logger Assert => new Logger(Subsystem.Assert);
      internal static Logger Ui => new Logger(Subsystem.Ui);
      internal static Logger Testing => new Logger(Subsystem.Testing);
-      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);
+      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);      
      public static Logger Sdk => new Logger(Subsystem.Sdk);
      public static Logger Http => new Logger(Subsystem.Http);
+      internal static Logger FIX => new Logger(Subsystem.Fix);

      internal static void log_at(this Logger logger, Level level, string message) {
         switch (level) {
@@ -2,4 +2,5 @@ using System.Runtime.CompilerServices;
 [assembly: InternalsVisibleTo("spark")]
 [assembly: InternalsVisibleTo("sparkcmd")]
 [assembly: InternalsVisibleTo("spark_test")]
+[assembly: InternalsVisibleTo("spark_historical")]

@@ -70,10 +70,13 @@ namespace bts.test {
         psi.RedirectStandardOutput = true;
         psi.RedirectStandardError = true;
         psi.UseShellExecute = false;
-
+ 
         try{
            var p = Process.Start(psi);
            p.WaitForExit((int)time_out.TotalMilliseconds);
+            var errorlevel = p.ExitCode;
+            string stdout = p.StandardOutput.ReadToEnd();
+            string stderr = p.StandardError.ReadToEnd();
            if (p.HasExited) {
               return 0;
            }
@@ -0,0 +1,64 @@
+using bts;
+using bts.api.admin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.api.core
+{
+   public class AdminModelWrapper
+   {
+      readonly AdminDataImmutable? _admin_data;
+      public AdminDataImmutable? admin_data => _admin_data;
+      readonly string _user_name;
+      readonly List<BTSAccountImmutable> _allowed_accounts;
+      readonly Dictionary<string, string> _allowed_account_dict = new Dictionary<string, string>();
+      public AdminModelWrapper(AdminDataImmutable? admin_data = null, string user_name = null)
+      {
+         _admin_data = admin_data;
+         _user_name = user_name ?? string.Empty;
+         _allowed_accounts = search_admin(u => u.Name == _user_name).Where(x =>
+         {
+              return (x.Item2 != null && x.Item2.HasPermissions) || (x.Item3 != null && x.Item3.HasPermissions);
+         }).Select(xx => xx.Item1).ToList();
+         foreach (var val in _allowed_accounts)
+            _allowed_account_dict[val.Name] = val.Name;
+      }
+      public IEnumerable<BTSAccountImmutable> viewable_accounts()
+      {
+         return _allowed_accounts;
+      }
+      public bool can_view_account(string account)
+      {
+         return _allowed_account_dict.ContainsKey(account) || _user_name.ToLower().Equals("bts");
+      }
+
+      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>> 
+         search_admin(Func<BTSUserImmutable, bool> predicate)
+      {
+         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>>();
+         if (_admin_data.HasValue)
+         {
+            var users = _admin_data.Value.Users.Where(predicate);
+            if (users != null && users.Count() == 1)
+            {
+               var user = users.ElementAt(0);
+               if (user.HasUserData && user.UserData.HasAccountPermissions)
+               {
+                  foreach (var kv in user.UserData.AccountPermissions.AccountPermissions)
+                  {
+                     var account_id = kv.Key;
+                     var accounts = _admin_data.Value.Accounts.Where(a => a.Id == account_id);
+                     if (accounts != null && accounts.Count() == 1)
+                     {
+                        var account = accounts.ElementAt(0);
+                        rv.Add(Tuple.Create(account, kv.Value.TradingPermission, kv.Value.AdminPermission));
+                     }
+                  }
+               }
+            }
+         }
+         return rv;
+      }
+   }
+}
@@ -0,0 +1,64 @@
+using bts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.position;
+using bts.spark;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Concurrency;
+using System.Reactive.Linq;
+
+namespace bts.api.core
+{
+   public static class PermissionFilters
+   {
+      public static IObservable<Update<PositionUnitImmutable, int>> 
+         PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<PositionUnitImmutable, bool>>(observer =>
+         {
+            return am.Subscribe(x =>
+            {
+               Func<PositionUnitImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+            });
+         });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<OrderRecordImmutable, uint>> 
+         PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<OrderRecordImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.Def.Account ?? string.Empty));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<FillImmutable, string>> 
+         PermissionFilter(this IObservable<Update<FillImmutable, string>> source, IObservable<AdminModelWrapper> am)
+      {
+         var pred = Observable.Create<Func<FillImmutable, bool>>(observer =>
+        {
+           return am.Subscribe(x =>
+           {
+               Func<FillImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+   }
+}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
    <Platforms>x64</Platforms>
    <PlatformTarget>AnyCPU</PlatformTarget>
@@ -0,0 +1,52 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reflection;
+
+namespace bts.utils.csv {
+   public static class CsvHelper {
+
+      public static string ToCsv<T>(this IEnumerable<T> items, OrderedDictionary properties = null) where T : class {
+         var output = "";
+         var delimiter = ",";
+         if (properties == null) {
+            properties = new OrderedDictionary();
+            foreach (var pi in typeof(T).GetProperties()) {
+               properties.Add(pi.Name, pi);
+            }
+         }
+
+         using (var sw = new StringWriter()) {
+            var dict_values = properties.Values.OfType<PropertyInfo>();
+            var header = dict_values.Select(n => n.Name).Aggregate((first, second) => first + delimiter + second);
+            sw.WriteLine(header);
+            foreach (var i in items) {
+               var row = dict_values.Select(x => x.GetValue(i, null))
+                  .Select(x => x == null ? "null" : x.ToString())
+                  .Aggregate((first, second) => first + delimiter + second);
+               sw.WriteLine(row);
+            }
+            output = sw.ToString();
+         }
+         return output;
+      }
+
+      public static OrderedDictionary create_ordering_maintainer(Type t, string[] property_names_in_order) {
+         OrderedDictionary od = new OrderedDictionary();
+         var props = t.GetProperties();
+         foreach (var n in property_names_in_order) {
+            var query = props.Where(pi => pi.Name == n);
+            if (query.Any()) {
+               var property_info = query.First();
+               if (!od.Contains(property_info)) {
+                  od.Add(n, property_info);
+               }
+            }
+         }
+         return od;
+      }
+
+   }
+}
@@ -66,7 +66,6 @@ namespace mbts {
            }
         }
      }
-
      public static void sync_with_models<T, TModel>(this IList<T> target, IEnumerable<TModel> models, Func<TModel, T> add_factory, Func<T, TModel, T> update_factory) {
         var model_list = models as IList<TModel> ?? models.ToList();
         if (model_list.Count == 0) {
@@ -25,7 +25,7 @@
    <PackageReference Include="System.Reactive" Version="3.1.1" />
    <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
 	 <PackageReference Include="NUnit" Version="3.10.1" />  
-    <PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
    <PackageReference Include="System.Linq.Dynamic" Version="1.0.7" />
    
    <PackageReference Include="ExcelDna.AddIn" Version="0.34.6" />  
@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+namespace bts.fix
+{
+   public class FixInit
+   {
+      static FixInit()
+      {
+         AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
+      }
+
+      static void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)
+      {
+         var exception = e.ExceptionObject as Exception;
+         Log.FIX.error($"Unhandled Exception: {e.ExceptionObject} {exception?.StackTrace}");
+         if (e.IsTerminating)
+         {
+            Log.system_is_crashing("Unhandled exception", e.ExceptionObject);
+         }
+      }
+      
+      public static int start_acceptor(string settings_file, bool orders_allowed, string http_address = null)
+      {
+         SessionSettings settings = new SessionSettings(settings_file);
+         IApplication app = new FixRouter( (th, sess) =>
+         {
+            if(orders_allowed)
+               return new OrderSession(th, sess);
+            else
+               return new DropCopySession(th, sess);
+         });
+         IMessageStoreFactory storeFactory = new FileStoreFactory(settings);
+         ILogFactory logFactory = new FileLogFactory(settings);
+         IAcceptor acceptor = new ThreadedSocketAcceptor(app, storeFactory, settings, logFactory);
+
+         acceptor.Start();
+         HttpServer srv = null;
+         if (!string.IsNullOrEmpty(http_address))
+         {
+            if(!http_address.StartsWith("http"))
+               http_address = "http://127.0.0.1:5080/";
+            srv = new HttpServer(http_address, settings);
+            srv.Start();
+         }
+         Console.WriteLine("press <q> to quit");
+         while (Console.Read() != 'q') { }
+         acceptor.Stop();
+         if(srv != null)
+            srv.Stop();
+         return 0;
+
+      }
+   }
+}
+
@@ -0,0 +1,231 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.md;
+using bts.api.admin;
+
+using security_key = System.UInt32;
+
+namespace bts.fix
+{
+   interface IFixRouter
+   {
+      IEnumerable<SecurityImmutable> get_securities();
+      SecurityImmutable? lookup(security_key key);
+      SecurityImmutable? lookup(string desc);
+
+      void send(Message mess, SessionID sess);
+      void send_session_reject(SessionID sess, string text, Message refmessage = null);
+      void send_biz_reject(SessionID sess, string text, Message refmessage, int reason_code = 0);
+   }
+
+   class FixRouter : QuickFix.MessageCracker, IFIXSession, IFixRouter
+   {
+      readonly ConcurrentDictionary<SessionID, QuickFix.IApplication> _sessions = new ConcurrentDictionary<SessionID, IApplication>();
+      readonly Func<IFixRouter, SessionID, IApplication> _factory;
+      readonly MarketDataClient _md = new MarketDataClient();
+      readonly object _lock = new object();
+
+      public FixRouter(Func<IFixRouter, SessionID, IApplication> factory)
+      {
+         _factory = factory;
+      }
+
+      IApplication session(SessionID sessionID)
+      {
+         IApplication app;
+         if (!_sessions.TryGetValue(sessionID, out app))
+         {
+            app = _factory(this, sessionID);
+            _sessions[sessionID] = app;
+         }
+         return app;
+      }
+
+      public void send(Message mess, SessionID sess)
+      {
+         lock (_lock)
+         {
+            try
+            {
+               Session.SendToTarget(mess, sess);
+            }
+            catch (SessionNotFound)
+            {
+               Log.assert(false, "session not found " + sess);
+            }
+            catch (Exception ex)
+            {
+               Log.FIX.error(sess.ToString() + " " + ex.Message + " " + sess);
+            }
+         }
+      }
+
+      public void send_session_reject(SessionID sess, string text, Message refmessage = null)
+      {
+         QuickFix.FIX44.Reject rej = null;
+         if (refmessage != null && refmessage.Header.IsSetField(Tags.MsgSeqNum))
+            rej = new QuickFix.FIX44.Reject(new QuickFix.Fields.RefSeqNum(refmessage.Header.GetInt(Tags.MsgSeqNum)));
+         else
+            rej = new QuickFix.FIX44.Reject();
+         rej.Text = new Text(text);
+         send(rej, sess);
+      }
+
+      public void send_biz_reject(SessionID sess, string text, Message refmessage, int reason_code = 0)
+      {
+         QuickFix.FIX44.BusinessMessageReject rej = new QuickFix.FIX44.BusinessMessageReject();
+
+         rej.RefMsgType = new QuickFix.Fields.RefMsgType(refmessage.Header.GetString(Tags.MsgType));
+         rej.RefSeqNum = new QuickFix.Fields.RefSeqNum(refmessage.Header.GetInt(Tags.MsgSeqNum));
+         rej.BusinessRejectReason = new QuickFix.Fields.BusinessRejectReason(reason_code);
+         rej.Text = new Text(text);
+         send(rej, sess);
+      }      
+
+      void handle_session_exception(Exception ex, SessionID sessionID, Message message = null)
+      {
+         string msg = "==ERROR in session " + sessionID.ToString() + "== ";
+         msg += message?.GetType().Name ?? string.Empty;
+         msg += ex?.ToString() ?? string.Empty;
+         if(!string.IsNullOrEmpty(ex.StackTrace))
+            msg += Environment.NewLine + ex.StackTrace;
+
+         bts.Log.FIX.error(msg);
+         Console.WriteLine(msg);
+         //now shut this fucker down
+         try
+         {
+            var sess = Session.LookupSession(sessionID);
+            if(sess != null)
+               sess.Disconnect(msg);
+         }
+         catch { }
+      }
+
+      void handle_field_not_found(FieldNotFoundException ex, SessionID sess, Message message)
+      {
+         if(ex.Field > 0)
+            send_biz_reject(sess, $"Field {ex.Field} was expected but not found", message, 5 );
+         //QF does not include the tag number....
+         //throw new FieldNotFoundException($"Field {ex.Field} was expected but not found", ex);
+         else
+            throw ex;
+      }
+
+      public void FromAdmin(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).FromAdmin(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void FromApp(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).FromApp(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void OnCreate(SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).OnCreate(sessionID);    
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }      
+      }
+
+      public void OnLogon(SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).OnLogon(sessionID);
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }
+      }
+
+      public void OnLogout(SessionID sessionID)
+      {
+         try
+         {
+            //TODO stackoverflow?
+            session(sessionID).OnLogout(sessionID);
+         }
+         catch (Exception ex)
+         {
+            handle_session_exception(ex, sessionID);
+         }
+      }
+
+      public void ToAdmin(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).ToAdmin(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) {  throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      public void ToApp(Message message, SessionID sessionID)
+      {
+         try
+         {
+            session(sessionID).ToApp(message, sessionID);
+         }
+         catch (FieldNotFoundException ex) { handle_field_not_found(ex, sessionID, message); }
+         catch (UnsupportedMessageType) { throw; }
+         catch (IncorrectTagValue) { throw; }
+         catch (Exception ex) { handle_session_exception(ex, sessionID, message); }
+      }
+
+      IEnumerable<SecurityImmutable> IFixRouter.get_securities()
+      {
+         return _md.securities;
+      }
+
+      SecurityImmutable? IFixRouter.lookup(uint key)
+      {
+         return _md.lookup(key);
+      }
+
+      SecurityImmutable? IFixRouter.lookup(string desc)
+      {
+         if (_md.lookup_security_key_from_desc(desc, out uint key))
+            return _md.lookup(key);
+         else
+            return null;
+      }
+   }
+}
@@ -0,0 +1,585 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Web;
+using QuickFix;
+
+namespace bts.fix
+{
+   public class HttpServer : IDisposable
+   {
+      private readonly HttpListener _httpListener;
+      private Thread _connectionThread;
+      private Boolean _running, _disposed;
+      private readonly SessionSettings _sessionSettings;
+      private StringBuilder _sbHtmlHeader;
+
+
+      public HttpServer(string prefix, SessionSettings settings)
+      {
+         if (!HttpListener.IsSupported)
+         {
+            // Requires at least a Windows XP with Service Pack 2
+            throw new NotSupportedException(
+                "The Http Server cannot run on this operating system.");
+         }
+
+         _httpListener = new HttpListener();
+         _httpListener.Prefixes.Add(prefix);
+         _sessionSettings = settings;
+      }
+
+      public void Start()
+      {
+         if (!_httpListener.IsListening)
+         {
+            _httpListener.Start();
+            _running = true;
+            // Use a thread to listen to the Http requests
+            _connectionThread = new Thread(ConnectionThreadStart);
+            _connectionThread.Start();
+         }
+      }
+
+      public void Stop()
+      {
+         if (_httpListener.IsListening)
+         {
+            _running = false;
+            _httpListener.Stop();
+         }
+      }
+
+      private void ConnectionThreadStart()
+      {
+         try
+         {
+            while (_running)
+            {
+               // Grab the context and pass it to the processor methods to handle it
+               HttpListenerContext context = _httpListener.GetContext();
+               HttpListenerRequest request = context.Request;
+               HttpListenerResponse response = context.Response;
+               string responseString;
+
+               _sbHtmlHeader = new StringBuilder(@"<HEAD><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>");
+               _sbHtmlHeader.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEADER><BODY>", request.Url.OriginalString);
+
+               switch (request.Url.AbsolutePath)
+               {
+                  case "/":
+                     responseString = ProcessRoot(request);
+                     break;
+                  case "/session":
+                     responseString = SessionDetails(request);
+                     break;
+                  case "/resetSession":
+                     responseString = ResetSession(request);
+                     break;
+                  case "/resetSessions":
+                     responseString = ResetSessions(request);
+                     break;
+                  case "/refreshSession":
+                     responseString = RefreshSession(request);
+                     break;
+                  case "/refreshSessions":
+                     responseString = RefreshSessions(request);
+                     break;
+                  case "/enableSessions":
+                     responseString = EnableSessions(request);
+                     break;
+                  case "/disableSessions":
+                     responseString = DisableSessions(request);
+                     break;
+                  default:
+                     responseString = ProcessRoot(request);
+                     break;
+               }
+
+               byte[] buffer = Encoding.UTF8.GetBytes(responseString);
+               response.ContentLength64 = buffer.Length;
+               Stream output = response.OutputStream;
+               output.Write(buffer, 0, buffer.Length);
+               output.Close();
+            }
+         }
+         catch (HttpListenerException)
+         {
+            Console.WriteLine("HTTP server was shut down.");
+         }
+      }
+
+      private string DisableSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Logout();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been disabled</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to disable all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, disable sessions</A>]&nbsp;[<A HREF='/'>NO, do not disable sessions</A>]</CENTER>", request.Url.OriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string EnableSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Logon();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been enabled</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to enable all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, enable sessions</A>]&nbsp;[<A HREF='/'>NO, do not enable sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string RefreshSession(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+
+         string url = "/session?" + GetParameterList(urlOriginalString);
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               sessionDetails.Refresh();
+               url = RemoveQueryStringByKey(urlOriginalString, "confirm");
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL=/session?{0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", GetParameterList(urlOriginalString));
+            sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEAD><BODY>", urlOriginalString);
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='/session?{0}'>{1}</A> has been refreshed</h2></CENTER>", GetParameterList(url), sessionId);
+         }
+         else
+         {
+            sbHtmlPageBody.AppendFormat("<CENTER><h2>Are you sure you want to refresh session <A HREF='{0}'>{1}</A>?</h2></CENTER>", url, sessionId);
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}&confirm=1'>YES, refresh session</A>]&nbsp;[<A HREF='{0}'>NO, do not refresh session</A>]</CENTER>", url);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string RefreshSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Refresh();
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been refreshed</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to refresh all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, refresh sessions</A>]&nbsp;[<A HREF='/'>NO, do not refresh sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string ResetSessions(HttpListenerRequest request)
+      {
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               foreach (SessionID session in _sessionSettings.GetSessions())
+               {
+                  Session sessionDetails = Session.LookupSession(session);
+                  sessionDetails.Reset("Reset from WebInterface");
+               }
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL={0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", "/");
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='{0}'>Sessions</A> have been reset</h2></CENTER>", "/");
+         }
+         else
+         {
+            sbHtmlPageBody.Append("<CENTER><h2>Are you sure you want to reset all sessions ?</h2></CENTER>");
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}?confirm=1'>YES, reset sessions</A>]&nbsp;[<A HREF='/'>NO, do not reset sessions</A>]</CENTER>", urlOriginalString);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+      private string ResetSession(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         bool confirm = false;
+         string urlOriginalString = request.Url.OriginalString;
+
+         string url = "/session?" + GetParameterList(urlOriginalString);
+
+         if (request.QueryString["confirm"] != null)
+         {
+            if (Convert.ToInt16(request.QueryString["confirm"]) != 0)
+            {
+               confirm = true;
+               sessionDetails.Reset("Reset from WebInterface");
+               url = RemoveQueryStringByKey(urlOriginalString, "confirm");
+            }
+         }
+
+         if (confirm)
+         {
+            sbHtmlPageBody = new StringBuilder();
+            sbHtmlPageBody.AppendFormat(@"<HEAD><META http-equiv='refresh' content=2;URL=/session?{0} /><CENTER><TITLE>QuickFIX Engine Web Interface</TITLE><H1>QuickFIX Engine Web Interface</H1></CENTER>", GetParameterList(urlOriginalString));
+            sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/'>HOME</A>]&nbsp;[<A HREF='{0}'>RELOAD</A>]</CENTER><HR/></HEAD><BODY>", urlOriginalString);
+            sbHtmlPageBody.AppendFormat("<CENTER><h2><A HREF='/session?{0}'>{1}</A> has been reset</h2></CENTER>", GetParameterList(url), sessionId);
+         }
+         else
+         {
+            sbHtmlPageBody.AppendFormat("<CENTER><h2>Are you sure you want to reset session <A HREF='{0}'>{1}</A>?</h2></CENTER>", url, sessionId);
+            sbHtmlPageBody.AppendFormat("<CENTER>[<A HREF='{0}&confirm=1'>YES, reset session</A>]&nbsp;[<A HREF='{0}'>NO, do not reset session</A>]</CENTER>", url);
+         }
+         return sbHtmlPageBody.ToString();
+      }
+
+
+      private string ProcessRoot(HttpListenerRequest request)
+      {
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         //Session count
+         sbHtmlPageBody.AppendFormat("<CENTER><b><i>{0} Sessions managed by QuickFIX</b></i></CENTER><HR/>", _sessionSettings.GetSessions().Count);
+
+         //session management links
+         sbHtmlPageBody.AppendFormat(@"<CENTER><A HREF='/resetSessions{0}'>RESET</A>&nbsp;<A HREF='/refreshSessions'>REFRESH</A>&nbsp;<A HREF='/enableSessions'>ENABLE</A>&nbsp;<A HREF='/disableSessions'>DISABLE</A></CENTER><HR/></HEADER><BODY>", GetParameterList(request.Url.OriginalString));
+
+         //Start the table generation
+         sbHtmlPageBody.Append("<table id=\"sessionlist\" style=\"border-width:1; padding:2; width:100%\">");
+
+         sbHtmlPageBody.Append("<tr>");
+         sbHtmlPageBody.Append(AddCell("Session", true));
+         sbHtmlPageBody.Append(AddCell("Type", true));
+         sbHtmlPageBody.Append(AddCell("Enabled", true));
+         sbHtmlPageBody.Append(AddCell("Session Time", true));
+         sbHtmlPageBody.Append(AddCell("Logged On", true));
+         sbHtmlPageBody.Append(AddCell("Next Incoming", true));
+         sbHtmlPageBody.Append(AddCell("Next Outgoing", true));
+         sbHtmlPageBody.Append("</tr>");
+
+         foreach (SessionID session in _sessionSettings.GetSessions())
+         {
+            Session sessionDetails = Session.LookupSession(session);
+            sbHtmlPageBody.Append("<tr>");
+            sbHtmlPageBody.Append(AddCell(String.Format(
+                "<a href=\"session?BeginString={0}&SenderCompID={1}&TargetCompID={2}\">{0}:{1}->{2}</a>",
+                session.BeginString, session.SenderCompID, session.TargetCompID)));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsInitiator ? "initiator" : "acceptor"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsEnabled ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsSessionTime ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.IsLoggedOn ? "yes" : "no"));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.NextTargetMsgSeqNum.ToString()));
+            sbHtmlPageBody.Append(AddCell(sessionDetails.NextSenderMsgSeqNum.ToString()));
+            sbHtmlPageBody.Append("</tr>");
+         }
+
+         sbHtmlPageBody.Append("</table>");
+         return sbHtmlPageBody.ToString();
+      }
+
+      public string SessionDetails(HttpListenerRequest request)
+      {
+         SessionID sessionId = new SessionID(request.QueryString["beginstring"], request.QueryString["sendercompid"], request.QueryString["targetcompid"]);
+         Session sessionDetails = Session.LookupSession(sessionId);
+         if (sessionDetails == null) throw new Exception("Session not found");
+         StringBuilder sbHtmlPageBody = _sbHtmlHeader;
+
+         string url = request.Url.OriginalString;
+         string urlOriginalString = request.Url.OriginalString;
+
+         if (request.QueryString["enabled"] != null)
+         {
+            if (!Convert.ToBoolean(request.QueryString["enabled"]))
+               sessionDetails.Logout();
+            else
+               sessionDetails.Logon();
+
+            url = RemoveQueryStringByKey(urlOriginalString, "Enabled");
+         }
+
+         if (request.QueryString["next incoming"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["next incoming"]);
+            sessionDetails.NextTargetMsgSeqNum = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "next incoming");
+         }
+
+         if (request.QueryString["Next Outgoing"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["Next Outgoing"]);
+            sessionDetails.NextSenderMsgSeqNum = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "Next Outgoing");
+         }
+
+         if (request.QueryString["SendRedundantResendRequests"] != null)
+         {
+            sessionDetails.SendRedundantResendRequests = Convert.ToBoolean(request.QueryString["SendRedundantResendRequests"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "SendRedundantResendRequests");
+         }
+
+         if (request.QueryString["CheckCompId"] != null)
+         {
+            sessionDetails.CheckCompID = Convert.ToBoolean(request.QueryString["CheckCompId"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "CheckCompId");
+         }
+
+         if (request.QueryString["CheckLatency"] != null)
+         {
+            sessionDetails.CheckLatency = Convert.ToBoolean(request.QueryString["CheckLatency"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "CheckLatency");
+         }
+
+         if (request.QueryString["MaxLatency"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["MaxLatency"]);
+            sessionDetails.MaxLatency = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "MaxLatency");
+         }
+
+         if (request.QueryString["LogonTimeout"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["LogonTimeout"]);
+            sessionDetails.LogonTimeout = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "LogonTimeout");
+         }
+
+         if (request.QueryString["LogoutTimeout"] != null)
+         {
+            int value = Convert.ToInt16(request.QueryString["LogoutTimeout"]);
+            sessionDetails.LogoutTimeout = value <= 0 ? 1 : value;
+            url = RemoveQueryStringByKey(urlOriginalString, "LogoutTimeout");
+         }
+
+         if (request.QueryString["ResetOnLogon"] != null)
+         {
+            sessionDetails.ResetOnLogon = Convert.ToBoolean(request.QueryString["ResetOnLogon"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnLogon");
+         }
+
+         if (request.QueryString["ResetOnLogout"] != null)
+         {
+            sessionDetails.ResetOnLogout = Convert.ToBoolean(request.QueryString["ResetOnLogout"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnLogout");
+         }
+
+         if (request.QueryString["ResetOnDisconnect"] != null)
+         {
+            sessionDetails.ResetOnDisconnect = Convert.ToBoolean(request.QueryString["ResetOnDisconnect"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "ResetOnDisconnect");
+         }
+
+         if (request.QueryString["RefreshOnLogon"] != null)
+         {
+            sessionDetails.RefreshOnLogon = Convert.ToBoolean(request.QueryString["RefreshOnLogon"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "RefreshOnLogon");
+         }
+
+         if (request.QueryString["MillisecondsInTimestamp"] != null)
+         {
+            sessionDetails.MillisecondsInTimeStamp = Convert.ToBoolean(request.QueryString["MillisecondsInTimestamp"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "MillisecondsInTimestamp");
+         }
+
+         if (request.QueryString["PersistMessages"] != null)
+         {
+            sessionDetails.PersistMessages = Convert.ToBoolean(request.QueryString["PersistMessages"]);
+            url = RemoveQueryStringByKey(urlOriginalString, "PersistMessages");
+         }
+
+
+         sbHtmlPageBody.AppendFormat(@"<CENTER>{0}</CENTER><HR/>", sessionId);
+         sbHtmlPageBody.AppendFormat(@"<CENTER>[<A HREF='/resetSession?{0}'>RESET</A>]&nbsp;[<A HREF='/refreshSession?{0}'>REFRESH</A>]</CENTER><HR/></HEADER><BODY>", GetParameterList(urlOriginalString));
+
+         sbHtmlPageBody.Append("<table id=\"session_details\" style=\"border-width:1; padding:2; width:100%\">");
+
+         sbHtmlPageBody.Append(AddRow("Enabled", sessionDetails.IsEnabled, url));
+         sbHtmlPageBody.Append(AddRow("ConnectionType", sessionDetails.IsInitiator ? "initiator" : "acceptor"));
+         sbHtmlPageBody.Append(AddRow("SessionTime", sessionDetails.IsSessionTime));
+         sbHtmlPageBody.Append(AddRow("LoggedOn", sessionDetails.IsLoggedOn));
+         sbHtmlPageBody.Append(AddRow("Next Incoming", sessionDetails.NextTargetMsgSeqNum, url));
+         sbHtmlPageBody.Append(AddRow("Next Outgoing", sessionDetails.NextSenderMsgSeqNum, url));
+         sbHtmlPageBody.Append(AddRow("SendRedundantResendRequests", sessionDetails.SendRedundantResendRequests, url));
+         sbHtmlPageBody.Append(AddRow("CheckCompId", sessionDetails.CheckCompID, url));
+         sbHtmlPageBody.Append(AddRow("CheckLatency", sessionDetails.CheckLatency, url));
+         sbHtmlPageBody.Append(AddRow("MaxLatency", sessionDetails.MaxLatency, url));
+         sbHtmlPageBody.Append(AddRow("LogonTimeout", sessionDetails.LogonTimeout, url));
+         sbHtmlPageBody.Append(AddRow("LogoutTimeout", sessionDetails.LogoutTimeout, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnLogon", sessionDetails.ResetOnLogon, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnLogout", sessionDetails.ResetOnLogout, url));
+         sbHtmlPageBody.Append(AddRow("ResetOnDisconnect", sessionDetails.ResetOnDisconnect, url));
+         sbHtmlPageBody.Append(AddRow("RefreshOnLogon", sessionDetails.RefreshOnLogon, url));
+         sbHtmlPageBody.Append(AddRow("MillisecondsInTimestamp", sessionDetails.MillisecondsInTimeStamp, url));
+         sbHtmlPageBody.Append(AddRow("PersistMessages", sessionDetails.PersistMessages, url));
+
+         sbHtmlPageBody.Append("</table>");
+         return sbHtmlPageBody.ToString();
+      }
+
+      public static string RemoveQueryStringByKey(string url, string key)
+      {
+         var uri = new Uri(url);
+
+         // this gets all the query string key value pairs as a collection
+         var newQueryString = HttpUtility.ParseQueryString(uri.Query);
+
+         // this removes the key if exists
+         newQueryString.Remove(key);
+
+         // this gets the page path from root without QueryString
+         string pagePathWithoutQueryString = uri.GetLeftPart(UriPartial.Path);
+
+         return newQueryString.Count > 0
+             ? $"{pagePathWithoutQueryString}?{newQueryString}"
+             : pagePathWithoutQueryString;
+      }
+
+      public static string GetParameterList(string url)
+      {
+         return HttpUtility.ParseQueryString((new Uri(url).Query)).ToString();
+      }
+
+      private static string AddRow(string colName, bool value, string url = "")
+      {
+         string valueAsStr = value ? "yes" : "no";
+         string innerHtml = url.Length > 0
+             ? $"<a href=\" {url}&{colName}={!value} \">toggle</a>"
+             : "";
+         return AddRow(colName, valueAsStr, innerHtml);
+      }
+
+      private static string AddRow(string colName, int value, string url = "")
+      {
+         string innerHtml = $"<a href=\" {url}&{colName}={value - 10} \"> << </a>" +
+                            $"<a href=\" {url}&{colName}={value - 1} \"> < </a>" +
+                            " | " +
+                            $"<a href=\" {url}&{colName}={value + 1} \"> > </a>" +
+                            $"<a href=\" {url}&{colName}={value + 10} \"> >> </a>";
+         return AddRow(colName, value.ToString(), innerHtml);
+      }
+      private static string AddRow(string colName, string value, string innerHtml = "")
+      {
+         StringBuilder row = new StringBuilder();
+         row.Append("<tr>");
+         row.Append(AddCell(colName));
+         row.Append(AddCell(value));
+         row.Append(AddCell(innerHtml));
+         row.Append("</tr>");
+         return row.ToString();
+      }
+
+      private static string AddCell(string cellContent, bool header = false)
+      {
+         string entryType = header ? "th" : "td";
+         return "<" + entryType + " align=\"left\">" + cellContent + "</" + entryType + ">";
+      }
+
+      public virtual void Dispose()
+      {
+         Dispose(true);
+         GC.SuppressFinalize(this);
+      }
+
+      private void Dispose(bool disposing)
+      {
+         if (_disposed)
+         {
+            return;
+         }
+         if (disposing)
+         {
+            if (_running)
+            {
+               Stop();
+            }
+            if (_connectionThread != null)
+            {
+               _connectionThread.Abort();
+               _connectionThread = null;
+            }
+         }
+         _disposed = true;
+      }
+   }
+}
@@ -0,0 +1,2 @@
+using System.Runtime.CompilerServices;
+
@@ -0,0 +1,2 @@
+# BTS Spark FIX Adapter
+
@@ -0,0 +1,269 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.api.admin;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+namespace bts.fix
+{
+   class DropCopySession : SessionBase
+   {
+      protected IDisposable _fillsdisp;
+      protected IDisposable _ordersdisp;
+      protected static Lazy<OrderService> _orderservice = new Lazy<OrderService>(() => bts.api.Locator.locate<OrderService>(), true);
+
+      protected readonly ConcurrentDictionary<uint, OrderRecordImmutable> _order_cache = new ConcurrentDictionary<uint, OrderRecordImmutable>();
+      protected readonly ConcurrentDictionary<uint, string> _fix_clorderids = new ConcurrentDictionary<uint, string>();
+      public DropCopySession(IFixRouter router, SessionID sess) : base(router, sess)
+      {
+      }
+
+      override public void OnLogon(SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         var msg = sessionID.ToString() + " Logged on";
+         log_info(msg);
+
+         subscribe_fills_and_orders(sessionID);
+      }
+      override public void OnLogout(SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         var msg = sessionID.ToString() + " Logged out";
+         log_info(msg);
+
+         if (_ordersdisp != null)
+            _ordersdisp.Dispose();
+         if (_fillsdisp != null)
+            _fillsdisp.Dispose();
+      }      
+
+      override public void FromApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.ORDER_STATUS_REQUEST:
+            case MsgType.SECURITYDEFINITIONREQUEST:
+               log_info("IN: " + message);
+               Crack(message, sessionID);
+               break;
+            default:
+               log_info("UNSUPPORTED MESSAGE IN: " + type.getValue());
+               throw new UnsupportedMessageType();
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.OrderStatusRequest req, SessionID s)
+      {
+         uint oid = uint.Parse(req.OrderID.getValue());
+         if (_order_cache.TryGetValue(oid, out OrderRecordImmutable order))
+         {
+            var ostat = build_order_status(s, order);
+            ostat.OrdStatusReqID = new OrdStatusReqID(req.OrdStatusReqID.getValue());
+            _router.send(ostat, s);
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.SecurityDefinitionRequest req, SessionID s)
+      {
+         if (req.SecurityRequestType.getValue() == SecurityRequestType.REQUEST_LIST_SECURITIES)
+         {
+            var secs = _router.get_securities();
+            if (req.IsSetSymbol())
+               secs = secs.Where(x => x.Symbol.ToLower() == req.Symbol.getValue().ToLower());
+            if (req.IsSetSecurityType())
+               secs = secs.Where(x => security_types_equal(x.security_type(), req.SecurityType));
+            if (req.IsSetSecurityExchange())
+               secs = secs.Where(x => exchanges_equal(x.Exchange, req.SecurityExchange));
+
+            var def = new QuickFix.FIX44.SecurityDefinition(req.SecurityReqID,
+                                                            new SecurityResponseID(Guid.NewGuid().ToString()),
+                                                            new SecurityResponseType(SecurityResponseType.LIST_OF_SECURITIES_RETURNED_PER_REQUEST));
+            _router.send(def, s);
+         }
+         else
+            throw new IncorrectTagValue(req.SecurityRequestType.Tag);
+      }
+
+
+
+      virtual protected void subscribe_fills_and_orders(SessionID sessionID)
+      {
+         var scheduler = TaskPoolScheduler.Default;
+         var interval = TimeSpan.FromMilliseconds(500);
+         var on_orders = _orderservice.Value.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(_adminobservable).Table().Cache();
+         var on_fills = _orderservice.Value.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(_adminobservable).Table().Cache();
+
+         _ordersdisp = on_orders.Subscribe(o =>
+         {
+            foreach (var order in o.added)
+            {
+               var ostat = build_order_status(sessionID, order);
+               _order_cache[order.Key] = order;
+               _router.send(ostat, sessionID);
+            }
+            foreach (var order in o.changed)
+            {
+               var ostat = build_order_status(sessionID, order, true);
+               _router.send(ostat, sessionID);
+            }
+         });
+
+         _fillsdisp = on_fills.Subscribe(f =>
+         {
+            foreach (var fill in f.added)
+            {
+               var fi = build_fill(sessionID, fill);
+               _router.send(fi, sessionID);
+            }
+            foreach (var fill in f.changed)
+            {
+               var fi = build_fill(sessionID, fill, true);
+               _router.send(fi, sessionID);
+            }
+         });
+      }
+
+
+      virtual protected void process_order_state(in OrderStateImmutable state, ref uint filled, ref uint leaves, ref double avgprx,
+                                                 ref QuickFix.Fields.OrdStatus fstatus, ref NativeOrderStateImmutable? native, FillImmutable? fill = null)
+      {
+         filled = state.FilledQuantity;
+         leaves = state.Quantity - state.FilledQuantity;
+         avgprx = state.AvgPrice;
+         fstatus = translate_status(state, ref native);         
+      }
+
+      protected bts.api.SecurityImmutable? resolve_execution_report_security(bts.api.SecurityImmutable? sec, QuickFix.FIX44.ExecutionReport report)
+      {
+         report.SecurityExchange = convert_exchange(sec.Value.Exchange);
+         var sectype = sec.Value.security_type();
+         report.SecurityType = convert_security_type(sectype);
+         if (sec.Value.HasGlbx)
+            report.SecurityDesc = new SecurityDesc(sec.Value.Glbx.SecurityDesc);
+
+         switch (sectype)
+         {
+            case bts.api.SecurityType.Future:
+               report.MaturityMonthYear = new MaturityMonthYear(sec.Value.EarliestExpiration().ToString("yyyyMMdd"));
+               break;
+            case bts.api.SecurityType.Spread: //TODO this is not quite right for spreads of spreads            
+               var leg = sec.Value.Spread.Legs.OrderBy(x => x.Security.EarliestExpiration()).First();
+               report.MaturityMonthYear = new MaturityMonthYear(leg.Security.EarliestExpiration().ToString("yyyyMMdd"));
+               break;
+            default:
+               throw new UnsupportedMessageType();               
+         }
+         return sec;
+      }
+
+      virtual protected QuickFix.FIX44.ExecutionReport build_fill(SessionID sess, in FillImmutable fill, bool change = false)
+      {
+         var exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.PARTIAL_FILL);
+         QuickFix.Fields.OrdStatus status = new QuickFix.Fields.OrdStatus();
+         uint filled = 0; uint leaves = 0; double prx = 0.0;
+         OrderRecordImmutable order;  NativeOrderStateImmutable? native = null;
+         if (_order_cache.TryGetValue(fill.OrderKey, out order))
+         {
+            process_order_state(order.State, ref filled, ref leaves, ref prx, ref status, ref native);
+            if(Convert.ToDouble(order.State.Quantity) == fill.LastShares)
+               exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.FILL);
+         }
+         if (change)
+            exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.TRADE_CORRECT);
+
+         var sec = _router.lookup(fill.SecurityKey);
+         QuickFix.FIX44.ExecutionReport report = new QuickFix.FIX44.ExecutionReport(
+             new OrderID(fill.OrderKey.ToString()),
+             new ExecID(fill.FillKey.ToString()), 
+             exec,
+             status,
+             new Symbol(sec?.Symbol ?? string.Empty),
+             new QuickFix.Fields.Side(fill.Side == bts.api.Side.Buy ? QuickFix.Fields.Side.BUY : QuickFix.Fields.Side.SELL),
+             new LeavesQty(leaves),
+             new CumQty(filled),
+             new AvgPx(Convert.ToDecimal(prx)));
+             
+         report.Account = new Account(fill.Account);
+         report.LastQty = new LastQty(Convert.ToDecimal(fill.LastShares));
+         report.LastPx = new LastPx(Convert.ToDecimal(fill.LastPrice));
+         var ts = fill.HasTimestamp ? fill.Timestamp.ToDateTime() : DateTime.MinValue;
+         report.TransactTime = new TransactTime(ts, true);
+         report.TradeDate = new TradeDate(ts.ToString("yyyyMMdd"));
+         sec = resolve_execution_report_security(sec, report);
+         if (native.HasValue && !string.IsNullOrEmpty(native.Value.ExchangeOrderId))
+            report.SecondaryOrderID = new SecondaryOrderID(native.Value.ExchangeOrderId);
+
+         if (sec.HasValue && sec.Value.security_type() == bts.api.SecurityType.Spread || 
+             sec.Value.security_type() == bts.api.SecurityType.Synthetic)
+         {
+            if (fill.FillKind == bts.api.orderman.Fill.Types.FillKind.Leg)
+               report.MultiLegReportingType = new MultiLegReportingType('2');
+            else
+               report.MultiLegReportingType = new MultiLegReportingType('3');
+         }
+         return report;
+      }
+
+      virtual protected QuickFix.FIX44.ExecutionReport build_order_status(SessionID sess, in OrderRecordImmutable order, bool change = false)
+      {
+         var exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.NEW);
+         if (_order_cache.TryGetValue(order.Key, out OrderRecordImmutable _))
+            exec = new QuickFix.Fields.ExecType(QuickFix.Fields.ExecType.ORDER_STATUS);
+            
+         QuickFix.Fields.OrdStatus status = new QuickFix.Fields.OrdStatus();         
+         uint filled = 0; uint leaves = 0; double prx = 0.0;  NativeOrderStateImmutable? native = null;
+         process_order_state(order.State, ref filled, ref leaves, ref prx, ref status, ref native);
+
+         var sec = _router.lookup(order.Def.SecurityKey);
+         QuickFix.FIX44.ExecutionReport report = new QuickFix.FIX44.ExecutionReport(
+             new OrderID(order.Key.ToString()),
+             new ExecID("0"),
+             exec,
+             status,
+             new Symbol(sec?.Symbol ?? string.Empty),
+             new QuickFix.Fields.Side(order.Def.Side == bts.api.Side.Buy ? QuickFix.Fields.Side.BUY : QuickFix.Fields.Side.SELL),
+             new LeavesQty(leaves),
+             new CumQty(filled),
+             new AvgPx(Convert.ToDecimal(prx)));
+
+         report.Account = new Account(order.Def.Account);
+         report.OrderQty = new OrderQty(order.State.Quantity);
+         report.Price = new Price(Convert.ToDecimal(order.State.Price));
+         report.TimeInForce = new QuickFix.Fields.TimeInForce((char)order.Def.TimeInForce);
+         var ts = order.State.LastChange.ToDateTime();
+         report.TransactTime = new TransactTime(ts, true);
+         if(!string.IsNullOrEmpty(order.State.LastMsg))
+            report.Text = new Text(order.State.LastMsg);
+
+         sec = resolve_execution_report_security(sec, report);
+         if (native.HasValue)
+         {
+            report.OrdType = new OrdType((char)native.Value.NativeType);        
+            if(!string.IsNullOrEmpty(native.Value.ExchangeOrderId))
+               report.SecondaryOrderID = new SecondaryOrderID(native.Value.ExchangeOrderId);            
+         }
+         //TODO Big race here, we need a correlation id of some sort
+         if (_fix_clorderids.TryGetValue(order.Key, out string clorderid))
+         {
+            report.ClOrdID = new ClOrdID(clorderid);
+         }
+         return report;         
+      }      
+
+   }
+}
@@ -0,0 +1,8 @@
+using System;
+
+namespace bts.fix
+{
+   interface IFIXSession : QuickFix.IApplication
+   {
+   }
+}
@@ -0,0 +1,167 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using bts.api;
+using bts.api.orderman;
+
+namespace bts.fix
+{
+   /* Some test fix
+   order for 4 lot CLZ8 
+   8=FIX.4.49=15635=D49=SPARK56=CLIENT134=150=3073797=Y52=20000809-20:20:5011=900010081=ACC A21=255=SIM:CLZ854=138=440=259=044=7460=20000809-18:20:32207=SIM10=045   
+   
+   order for 4 lot CLZ8-CLH9
+   8=FIX.4.49=15635=D49=SPARK56=CLIENT134=150=3073797=Y52=20000809-20:20:5011=900010081=ACC A21=255=SIM:CLZ8-CLH954=138=440=259=044=7460=20000809-18:20:32207=SIM10=045
+   
+    */
+
+   class OrderSession : DropCopySession
+   {
+      public OrderSession(IFixRouter router, SessionID sess) : base(router, sess)
+      {
+      }
+
+      override public void FromApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.ORDER_STATUS_REQUEST: //handled in DropCopySession
+            case MsgType.SECURITYDEFINITIONREQUEST: //handled in DropCopySession
+
+            case MsgType.NEWORDERSINGLE:
+            case MsgType.ORDERCANCELREPLACEREQUEST:
+            case MsgType.ORDERCANCELREQUEST:
+               log_info("IN: " + message);
+               Crack(message, sessionID);
+               break;
+            default:
+               log_info("UNSUPPORTED MESSAGE IN: " + type.getValue());
+               throw new UnsupportedMessageType();
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.OrderCancelRequest cancel, SessionID s)
+      {
+         var id = cancel.OrderID.getValue();
+         if(!uint.TryParse(id, out uint key))
+            throw new IncorrectTagValue(cancel.OrderID.Tag);            
+         var resp = _orderservice.Value.cancel_order(new Cancel { OrderKey = key });
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            send_cancel_reject(s, "Cancel Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, cancel);
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.OrderCancelReplaceRequest replace, SessionID s)
+      {
+         var id = replace.OrderID.getValue();
+         if (!uint.TryParse(id, out uint key))
+            throw new IncorrectTagValue(replace.OrderID.Tag);
+         double price = Convert.ToDouble(replace.Price.getValue());
+         uint sz = Convert.ToUInt32(Math.Abs(replace.OrderQty.getValue()));
+         var resp = _orderservice.Value.update_order(new Update
+         {
+            OrderKey = key,
+            Price = price,
+            Quantity = sz,
+         });
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            send_cancel_reject(s, "Cancel Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, replace);
+         }
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.NewOrderSingle create, SessionID s)
+      {
+         Venue ven = convert_exchange(create.SecurityExchange);
+         if (ven == bts.api.Venue.Unset)
+            throw new IncorrectTagValue(create.SecurityExchange.Tag);
+
+         var sec = _router.lookup(create.Symbol.getValue());
+         if (!sec.HasValue)
+            sec = _router.lookup(create.SecurityDesc.getValue());
+         if (!sec.HasValue)
+            throw new IncorrectTagValue(create.SecurityDesc.Tag);
+
+         OrderDefinition def = null;
+         try
+         {
+            string manual = "Y";
+            if(create.IsSetField(Tags.ManualOrderIndicator))
+               manual = create.GetField(Tags.ManualOrderIndicator);
+            def = new OrderDefinition()
+            {
+               Account = create.Account.getValue(),
+               AccountId = lookup_account(create.Account.getValue()),
+               SecurityKey = sec.Value.Id,
+               Side = (api.Side)(int)create.Side.getValue(),
+               TimeInForce = (api.orderman.TimeInForce)(int)create.TimeInForce.getValue(),
+               User = _user_name,
+               UserId = lookup_user(_user_name),
+               ManualOrderIndicator = manual == "Y" ?  bts.api.orderman.ManualOrderIndicator.Yes : bts.api.orderman.ManualOrderIndicator.No,
+            };
+         }
+         catch
+         {
+            throw new IncorrectTagValue(create.TimeInForce.Tag);
+         }
+
+         set_ordertype(def, create, out bool price_req);
+         Create cr = new Create();
+         cr.NewOrder = def;
+         try
+         {
+            if (price_req)
+               cr.Price = Convert.ToDouble(create.Price.getValue());
+         }
+         catch
+         {
+            throw new IncorrectTagValue(create.Price.Tag);
+         }
+         try
+         {
+            cr.Quantity = Convert.ToUInt32(Math.Abs(create.OrderQty.getValue()));
+         }
+         catch
+         {
+            throw new IncorrectTagValue(create.OrderQty.Tag);
+         }
+         var resp = _orderservice.Value.create_order(cr);
+         if (resp.ErrorCode != OrderError.Ok)
+         {
+            _router.send_biz_reject(s, "Order Rejected: " + resp.ErrorCode.ToString() + " - " + resp.Msg, create);
+         }
+      }
+
+      private void set_ordertype(OrderDefinition def, QuickFix.FIX44.NewOrderSingle req, out bool price_req)
+      {
+         price_req = false;
+         switch (req.OrdType.getValue())
+         {
+            case '1':
+               def.CustomDef = new CustomOrderDefinition { Market = new MarketOrderDefinition() };
+               break;
+            case '2':
+               def.CustomDef = new CustomOrderDefinition { Limit = new LimitOrderDefinition() };
+               price_req = true;
+               break;
+            case '3':
+               def.CustomDef = new CustomOrderDefinition { Stop = new StopOrderDefinition() {StopPrice = Convert.ToDouble(req.StopPx.getValue()) } };
+               break;
+            case '4':
+               def.CustomDef = new CustomOrderDefinition { StopLimit = new StopLimitOrderDefinition() { StopPrice = Convert.ToDouble(req.StopPx.getValue()) } };
+               price_req = true;
+               break;
+            default:
+               throw new IncorrectTagValue(req.OrdType.Tag);
+         }
+      }
+   }
+}
@@ -0,0 +1,361 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Concurrency;
+using System.Text;
+using System.Collections.Concurrent;
+using System.Runtime.CompilerServices;
+
+using QuickFix;
+using QuickFix.Fields;
+
+using mbts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.user;
+using bts.api.admin;
+
+
+namespace bts.fix
+{
+   abstract class SessionBase : QuickFix.MessageCracker, IFIXSession
+   {      
+      protected static AuthService _authservice = bts.api.Locator.locate<AuthService>();      
+      protected static Lazy<AdminService> _adminservice = new Lazy<AdminService>(() => bts.api.Locator.locate<AdminService>(), true);
+      protected static Lazy<bts.metrics.ShellLatencyService> _latencyservice = 
+         new Lazy<bts.metrics.ShellLatencyService>(() => bts.api.Locator.locate<bts.metrics.ShellLatencyService>(), true);
+
+      protected string _user_name;
+      protected AuthToken _token;
+
+      protected IDisposable _admindisp;
+      protected IObservable<AdminModelWrapper> _adminobservable;
+      protected AdminDataImmutable? _admin_data;      
+
+      protected readonly IFixRouter _router;
+      protected readonly SessionID _sess;
+      
+      public SessionBase(IFixRouter router, SessionID sess)
+      {
+         _router = router;
+         _sess = sess;
+
+         _clockdisp = _latencyservice.Value.subscribe_clock().Subscribe(x =>
+         {
+            var t = x.all.FirstOrDefault();
+            if(t != null)
+               _clock = t.Timestamp;
+         });
+
+         _adminobservable = _adminservice.Value.subscribei_admin_data().Select(x =>
+         {
+            return new AdminModelWrapper(x.all.TryFirst(), _user_name);
+         });
+         _admindisp = _adminobservable.Subscribe(x => _admin_data = x.admin_data);         
+      }
+
+      abstract public void FromApp(Message message, SessionID sessionID);
+      abstract public void OnLogon(SessionID sessionID); //subscribe to your stuff 
+      abstract public void OnLogout(SessionID sessionID); //unsub
+      
+      virtual public void ToApp(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         log_info("OUT: " + message);
+      }
+
+      readonly IDisposable _clockdisp; ulong _clock = 0;
+      virtual public void FromAdmin(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+
+         MsgType type = Message.IdentifyType(message.ToString());
+         switch (type.getValue())
+         {
+            case MsgType.LOGON:
+               {
+                  try
+                  {
+                     QuickFix.FIX44.Logon msg = (QuickFix.FIX44.Logon)message;
+                     _user_name = msg.Username.getValue();
+                     string pass = string.Empty;
+                     if(msg.IsSetPassword())
+                        pass = msg.Password.getValue();
+                     var resp = _authservice.authenticate_login(new AuthenticateLoginRequest { 
+                                                                Login = msg.Username.getValue(), Password = pass});
+                     if (resp != null && resp.IsValid)
+                        _token = resp.Token;
+                     else
+                        throw new RejectLogon("Logon rejected " + resp?.Reason ?? string.Empty);
+                  }
+                  catch (RejectLogon)
+                  {
+                     throw;
+                  }
+                  catch (Exception ex)
+                  {
+                     throw new RejectLogon("Logon rejected " + ex.ToString());
+                  }
+               }
+               break;
+            case MsgType.HEARTBEAT:
+               {
+                  QuickFix.FIX44.Heartbeat hb = (QuickFix.FIX44.Heartbeat)message;
+                  handle_heartbeat(hb, sessionID);
+               }
+               break;               
+            default:
+               log_info("ADMININ: " + message);
+               break;
+         }
+      }
+
+      async protected void handle_heartbeat(QuickFix.FIX44.Heartbeat hb, SessionID sessionID)
+      {
+         var resp = await _latencyservice.Value.report_async(new bts.metrics.Report { HostName = "BTSFIX_" + sessionID.ToString(), 
+                                                                                      Timestamp = _clock, User = _user_name });
+         if (resp.success)
+            _router.send(hb, sessionID);         
+      }
+
+      virtual public void ToAdmin(Message message, SessionID sessionID)
+      {
+         Log.assert_d(sessionID.Equals(_sess));
+         log_info("ADMINOUT: " + message);
+      }
+
+      virtual public void OnCreate(SessionID sessionID)
+      {
+         log_info("CREATE");
+      }
+
+      virtual public void OnMessage(QuickFix.FIX44.BusinessMessageReject n, SessionID s)
+      {
+         //Log.assert(false, "biz reject wtf " + n.EncodedText.getValue());
+         Log.FIX.warn(_sess.ToString() + " Business Reject " + n.BusinessRejectReason.getValue() + " " + n.Text.getValue());
+      }
+      virtual public void OnMessage(QuickFix.FIX44.News n, SessionID s)
+      {
+         log_info("news " + n.EncodedHeadline.getValue());
+      }      
+
+
+      protected void log_info(string message, [CallerFilePath] string file = "", [CallerLineNumber] int line = 0)
+      {
+         Log.FIX.info(_sess.ToString() + " " + message, file, line);
+      }
+
+      protected void log_error(string message, [CallerFilePath] string file = "", [CallerLineNumber] int line = 0)
+      {
+         Log.FIX.error(_sess.ToString() + " " + message, file, line);
+      }
+
+      protected void send_cancel_reject(SessionID sess, string text, Message refmessage)
+      {
+         QuickFix.FIX44.OrderCancelReject rej = new QuickFix.FIX44.OrderCancelReject();
+         if (refmessage is QuickFix.FIX44.OrderCancelRequest cancel)
+         {
+            if (cancel.IsSetClOrdID())
+               rej.SetField(cancel.ClOrdID);
+            if (cancel.IsSetOrigClOrdID())
+               rej.SetField(cancel.OrigClOrdID);
+            if (cancel.IsSetOrderID())
+               rej.SetField(cancel.OrderID);
+            rej.CxlRejResponseTo = new CxlRejResponseTo('1');
+            rej.Text = new Text(text);
+            _router.send(rej, sess);
+         }
+         else if (refmessage is QuickFix.FIX44.OrderCancelReplaceRequest replace)
+         {
+            if (replace.IsSetClOrdID())
+               rej.SetField(replace.ClOrdID);
+            if (replace.IsSetOrigClOrdID())
+               rej.SetField(replace.OrigClOrdID);
+            if (replace.IsSetOrderID())
+               rej.SetField(replace.OrderID);
+            rej.CxlRejResponseTo = new CxlRejResponseTo('2');
+            rej.Text = new Text(text);
+            _router.send(rej, sess);
+         }
+         else 
+            throw new UnsupportedMessageType();
+
+      }      
+
+      protected bool security_types_equal(bts.api.SecurityType btstype, QuickFix.Fields.SecurityType fixtype)
+      {
+         return btstype == convert_security_type(fixtype);
+      }
+
+      protected QuickFix.Fields.SecurityType convert_security_type(bts.api.SecurityType btstype)
+      {
+         switch (btstype)
+         {
+            case bts.api.SecurityType.Future:
+               return new QuickFix.Fields.SecurityType("FUT");
+            case bts.api.SecurityType.Option:
+               return new QuickFix.Fields.SecurityType("OPT");
+            case bts.api.SecurityType.Spread:
+               return new QuickFix.Fields.SecurityType("MLEG");
+            case bts.api.SecurityType.Stock:
+               return new QuickFix.Fields.SecurityType("CS");
+            case bts.api.SecurityType.Synthetic:
+               return new QuickFix.Fields.SecurityType("SYNTH");
+            default:
+               return new QuickFix.Fields.SecurityType();
+         }
+      }
+
+      protected bts.api.SecurityType convert_security_type(QuickFix.Fields.SecurityType fixtype)
+      {
+         switch (fixtype.getValue())
+         {
+            case "FUT":
+               return bts.api.SecurityType.Future;
+            case "OPT":
+               return bts.api.SecurityType.Option;
+            case "MLEG":
+               return bts.api.SecurityType.Spread;
+            case "CS":
+               return bts.api.SecurityType.Stock;
+            case "SYNTH":
+               return bts.api.SecurityType.Synthetic;
+            default:
+               return bts.api.SecurityType.Unset;
+         }
+      }
+
+      protected bool exchanges_equal(bts.api.Venue btstype, QuickFix.Fields.SecurityExchange fixtype)
+      {
+         return btstype == convert_exchange(fixtype);
+      }
+
+      protected QuickFix.Fields.SecurityExchange convert_exchange(bts.api.Venue btstype)
+      {
+         switch (btstype)
+         {
+            case bts.api.Venue.Bts:
+               return new QuickFix.Fields.SecurityExchange("BTS");
+            case bts.api.Venue.Cboe:
+               return new QuickFix.Fields.SecurityExchange("XCBO");
+            case bts.api.Venue.Cboecfe:
+               return new QuickFix.Fields.SecurityExchange("XCBF");
+            case bts.api.Venue.Glbx:
+               return new QuickFix.Fields.SecurityExchange("XCME");
+            case bts.api.Venue.Ice:
+               return new QuickFix.Fields.SecurityExchange("IEPA");
+            case bts.api.Venue.Sim:
+               return new QuickFix.Fields.SecurityExchange("SIM");
+            default:
+               return new QuickFix.Fields.SecurityExchange();
+         }
+      }
+
+      protected bts.api.Venue convert_exchange(QuickFix.Fields.SecurityExchange fixtype)
+      {
+         switch (fixtype.getValue())
+         {
+            case "BTS":
+               return bts.api.Venue.Bts;
+            case "XCBO":
+               return bts.api.Venue.Cboe;
+            case "XCBF":
+               return bts.api.Venue.Cboecfe;
+            case "GLBX":
+            case "XCME":
+            case "XCBT":
+            case "XCEC":
+            case "XNYM":
+               return bts.api.Venue.Glbx;
+            case "IEPA":
+            case "IFUS":
+            case "IFEU":
+            case "IFLL":
+               return bts.api.Venue.Ice;
+            case "SIM":
+               return bts.api.Venue.Sim;
+            default:
+               return bts.api.Venue.Unset;
+         }
+      }      
+
+      protected QuickFix.Fields.OrdStatus translate_status(in bts.api.orderman.OrderStateImmutable state, 
+                                                           ref bts.api.orderman.NativeOrderStateImmutable? native)
+      {
+         native = null;
+         if (state.HasMarket && state.Market.HasNativeState && state.Market.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Market.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasLimit && state.Limit.HasNativeState && state.Limit.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Limit.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasStop && state.Stop.HasNativeState && state.Stop.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.Stop.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else if (state.HasStopLimit && state.StopLimit.HasNativeState && state.StopLimit.NativeState.NativeStatus != NativeOrderStatus.Unacknowledged)
+         {
+            native = state.StopLimit.NativeState;
+            return new OrdStatus((char)native?.NativeStatus);
+         }
+         else
+         {
+            switch (state.Status)
+            {
+               case OrderStatus.Canceled:
+                  return new OrdStatus(OrdStatus.CANCELED);
+               case OrderStatus.Completed:
+                  return new OrdStatus(OrdStatus.FILLED);
+               case OrderStatus.Disconnected:
+                  return new OrdStatus(OrdStatus.STOPPED);
+               case OrderStatus.Error:
+                  return new OrdStatus(OrdStatus.REJECTED);
+               case OrderStatus.Inactive:
+                  return new OrdStatus(OrdStatus.PENDING_NEW);
+               case OrderStatus.PendingWorking:
+                  return new OrdStatus(OrdStatus.PENDING_NEW);
+               case OrderStatus.RejectedBySpark:
+                  return new OrdStatus(OrdStatus.REJECTED);
+               case OrderStatus.Working:
+                  {
+                     if (state.FilledQuantity < state.Quantity)
+                        return new OrdStatus(OrdStatus.PARTIALLY_FILLED);
+                     else
+                        return new OrdStatus(OrdStatus.NEW);
+                  }
+               default:
+                  return new OrdStatus();
+            }
+         }
+      }
+
+      protected uint lookup_account(string acct)
+      {
+         if (_admin_data.HasValue)
+         {
+            var ac = _admin_data.Value.Accounts.Where(x => x.Name == acct).TryFirst();
+            if(ac.HasValue)
+               return ac.Value.Id;
+         }
+         throw new IncorrectTagValue(new Account().Tag);
+      }
+      
+      protected uint lookup_user(string user)
+      {
+         if (_admin_data.HasValue)
+         {
+            var ac = _admin_data.Value.Users.Where(x => x.Name == user).TryFirst();
+            if (ac.HasValue)
+               return ac.Value.Id;
+         }
+         throw new IncorrectTagValue(new Username().Tag);
+      }
+   }
+}
@@ -0,0 +1,42 @@
+#http://www.quickfixengine.org/quickfix/doc/html/configuration.html
+#https://www.onixs.biz/fix-dictionary.html
+
+[DEFAULT]
+ConnectionType=acceptor
+SocketAcceptPort=5001
+SocketReuseAddress=Y
+StartTime=00:00:00
+EndTime=00:00:00
+FileLogPath=log
+UseDataDictionary=Y
+ResetOnLogon=Y
+ResetOnLogout=Y
+ResetOnDisconnect=Y
+
+HeartBtInt=30
+LogonTimeout=30
+
+SSLEnable=N
+# SSLEnable=Y
+# # It is recommended to install the certificate and refer to it by name instead of using filename + password
+# SSLCertificate=QuickFixn-TestServer.pfx
+# SSLCertificatePassword=QuickFixn-TestServer
+# # For production refer to certificate by name instead: SSLCertificate=CN=QuickFixn-TestServer
+# SSLCACertificate=QuickFixn-TestCA.cer
+# SSLCheckCertificateRevocation=N
+
+
+[SESSION]
+BeginString=FIX.4.4
+SenderCompID=SPARK
+TargetCompID=CLIENT1
+FileStorePath=store
+DataDictionary=./spec/fix/FIX44.xml
+
+[SESSION]
+BeginString=FIX.4.4
+SenderCompID=SPARK
+TargetCompID=CLIENT2
+FileStorePath=store
+DataDictionary=./spec/fix/FIX44.xml
+
@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>  
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Description />
+    <Platforms>x64</Platforms>
+    <PlatformTarget>x64</PlatformTarget>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <LangVersion>7.2</LangVersion>
+    <Version>1.0.0</Version>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.6.1" />
+    <PackageReference Include="System.Reactive" Version="3.1.1" />
+    <PackageReference Include="System.Reactive.PlatformServices" Version="3.1.1" />
+	 <PackageReference Include="NUnit" Version="3.10.1" />  
+    
+    <PackageReference Include="QuickFix.Net.NETCore" Version="1.8.1" />     
+    <PackageReference Include="QuickFix.Net.NETCore.FIX44" Version="1.8.1" />         
+  </ItemGroup>
+
+  <ItemGroup> 
+    <ProjectReference Include="../spark_api/spark_api.csproj"/>
+    <ProjectReference Include="../spark_core/spark_core.csproj"/>
+  </ItemGroup>
+  
+  <Target Name="copy_fix" AfterTargets="PostBuildEvent"  >
+    <Copy SourceFiles="./spark_fix.cfg" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX44.xml" DestinationFolder="$(OutputPath)/spec/fix/" SkipUnchangedFiles="true" />
+  </Target>  
+  
+  <Target Name="copy_fix471" AfterTargets="PostBuildEvent"  >
+    <Copy SourceFiles="./spark_fix.cfg" DestinationFolder="$(OutputPath)/../net471" SkipUnchangedFiles="true" />
+    <Copy SourceFiles="./spec/fix/FIX44.xml" DestinationFolder="$(OutputPath)/../net471/spec/fix/" SkipUnchangedFiles="true" />
+  </Target>  
+</Project>
@@ -0,0 +1,828 @@
+<fix major="4" minor="0">
+  <header>
+    <field name="BeginString" required="Y"/>
+    <field name="BodyLength" required="Y"/>
+    <field name="MsgType" required="Y"/>
+    <field name="SenderCompID" required="Y"/>
+    <field name="TargetCompID" required="Y"/>
+    <field name="OnBehalfOfCompID" required="N"/>
+    <field name="DeliverToCompID" required="N"/>
+    <field name="SecureDataLen" required="N"/>
+    <field name="SecureData" required="N"/>
+    <field name="MsgSeqNum" required="Y"/>
+    <field name="SenderSubID" required="N"/>
+    <field name="TargetSubID" required="N"/>
+    <field name="OnBehalfOfSubID" required="N"/>
+    <field name="DeliverToSubID" required="N"/>
+    <field name="PossDupFlag" required="N"/>
+    <field name="PossResend" required="N"/>
+    <field name="SendingTime" required="Y"/>
+    <field name="OrigSendingTime" required="N"/>
+  </header>
+  <trailer>
+    <field name="SignatureLength" required="N"/>
+    <field name="Signature" required="N"/>
+    <field name="CheckSum" required="Y"/>
+  </trailer>
+  <messages>
+    <message name="Heartbeat" msgtype="0" msgcat="admin">
+      <field name="TestReqID" required="N"/>
+    </message>
+    <message name="Logon" msgtype="A" msgcat="admin">
+      <field name="EncryptMethod" required="Y"/>
+      <field name="HeartBtInt" required="Y"/>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="TestRequest" msgtype="1" msgcat="admin">
+      <field name="TestReqID" required="Y"/>
+    </message>
+    <message name="ResendRequest" msgtype="2" msgcat="admin">
+      <field name="BeginSeqNo" required="Y"/>
+      <field name="EndSeqNo" required="Y"/>
+    </message>
+    <message name="Reject" msgtype="3" msgcat="admin">
+      <field name="RefSeqNum" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="SequenceReset" msgtype="4" msgcat="admin">
+      <field name="GapFillFlag" required="N"/>
+      <field name="NewSeqNo" required="Y"/>
+    </message>
+    <message name="Logout" msgtype="5" msgcat="admin">
+      <field name="Text" required="N"/>
+    </message>
+    <message name="Advertisement" msgtype="7" msgcat="app">
+      <field name="AdvId" required="Y"/>
+      <field name="AdvTransType" required="Y"/>
+      <field name="AdvRefID" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="AdvSide" required="Y"/>
+      <field name="Shares" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TransactTime" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="IndicationofInterest" msgtype="6" msgcat="app">
+      <field name="IOIid" required="Y"/>
+      <field name="IOITransType" required="Y"/>
+      <field name="IOIRefID" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="IOIShares" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="ValidUntilTime" required="N"/>
+      <field name="IOIQltyInd" required="N"/>
+      <field name="IOIOthSvc" required="N"/>
+      <field name="IOINaturalFlag" required="N"/>
+      <field name="IOIQualifier" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="News" msgtype="B" msgcat="app">
+      <field name="OrigTime" required="N"/>
+      <field name="Urgency" required="N"/>
+      <field name="RelatdSym" required="N" repeating="Y"/>
+      <group name="LinesOfText" required="Y">
+        <field name="Text" required="Y"/>
+      </group>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="Email" msgtype="C" msgcat="app">
+      <field name="EmailType" required="Y"/>
+      <field name="OrigTime" required="N"/>
+      <field name="RelatdSym" required="N" repeating="Y"/>
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="N"/>
+      <group name="LinesOfText" required="Y">
+        <field name="Text" required="Y"/>
+      </group>
+      <field name="RawDataLength" required="N"/>
+      <field name="RawData" required="N"/>
+    </message>
+    <message name="QuoteRequest" msgtype="R" msgcat="app">
+      <field name="QuoteReqID" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="N"/>
+      <field name="OrderQty" required="N"/>
+    </message>
+    <message name="Quote" msgtype="S" msgcat="app">
+      <field name="QuoteReqID" required="N"/>
+      <field name="QuoteID" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="BidPx" required="Y"/>
+      <field name="OfferPx" required="N"/>
+      <field name="BidSize" required="N"/>
+      <field name="OfferSize" required="N"/>
+      <field name="ValidUntilTime" required="N"/>
+    </message>
+    <message name="NewOrderSingle" msgtype="D" msgcat="app">
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="ProcessCode" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="LocateReqd" required="N"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="IOIid" required="N"/>
+      <field name="QuoteID" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ExecutionReport" msgtype="8" msgcat="app">
+      <field name="OrderID" required="Y"/>
+      <field name="ClOrdID" required="N"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="ListID" required="N"/>
+      <field name="ExecID" required="Y"/>
+      <field name="ExecTransType" required="Y"/>
+      <field name="ExecRefID" required="N"/>
+      <field name="OrdStatus" required="Y"/>
+      <field name="OrdRejReason" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="N"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="ExecInst" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="LastShares" required="Y"/>
+      <field name="LastPx" required="Y"/>
+      <field name="LastMkt" required="N"/>
+      <field name="LastCapacity" required="N"/>
+      <field name="CumQty" required="Y"/>
+      <field name="AvgPx" required="Y"/>
+      <field name="TradeDate" required="N"/>
+      <field name="TransactTime" required="N"/>
+      <field name="ReportToExch" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <group name="NoMiscFees" required="N">
+        <field name="MiscFeeAmt" required="N"/>
+        <field name="MiscFeeCurr" required="N"/>
+        <field name="MiscFeeType" required="N"/>
+      </group>
+      <field name="NetMoney" required="N"/>
+      <field name="SettlCurrAmt" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="DontKnowTrade" msgtype="Q" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ExecID" required="N"/>
+      <field name="DKReason" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="LastShares" required="Y"/>
+      <field name="LastPx" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelReplaceRequest" msgtype="G" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="OrigClOrdID" required="Y"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ListID" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelRequest" msgtype="F" msgcat="app">
+      <field name="OrigClOrdID" required="Y"/>
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ListID" required="N"/>
+      <field name="CxlType" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderCancelReject" msgtype="9" msgcat="app">
+      <field name="OrderID" required="Y"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="ListID" required="N"/>
+      <field name="CxlRejReason" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="OrderStatusRequest" msgtype="H" msgcat="app">
+      <field name="OrderID" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Side" required="Y"/>
+    </message>
+    <message name="Allocation" msgtype="J" msgcat="app">
+      <field name="AllocID" required="Y"/>
+      <field name="AllocTransType" required="Y"/>
+      <field name="RefAllocID" required="N"/>
+      <group name="NoOrders" required="Y">
+        <field name="ClOrdID" required="Y"/>
+        <field name="OrderID" required="N"/>
+        <field name="ListID" required="N"/>
+        <field name="WaveNo" required="N"/>
+      </group>
+      <group name="NoExecs" required="N">
+        <field name="ExecID" required="N"/>
+        <field name="LastShares" required="N"/>
+        <field name="LastPx" required="N"/>
+        <field name="LastMkt" required="N"/>
+      </group>
+      <field name="Side" required="Y"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="Shares" required="Y"/>
+      <field name="AvgPx" required="Y"/>
+      <field name="Currency" required="N"/>
+      <field name="AvgPrxPrecision" required="N"/>
+      <field name="TradeDate" required="Y"/>
+      <field name="TransactTime" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="NetMoney" required="N"/>
+      <group name="NoMiscFees" required="N">
+        <field name="MiscFeeAmt" required="N"/>
+        <field name="MiscFeeCurr" required="N"/>
+        <field name="MiscFeeType" required="N"/>
+      </group>
+      <field name="SettlCurrAmt" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="OpenClose" required="N"/>
+      <field name="Text" required="N"/>
+      <group name="NoAllocs" required="Y">
+        <field name="AllocAccount" required="Y"/>
+        <field name="AllocShares" required="Y"/>
+        <field name="ProcessCode" required="N"/>
+        <field name="ExecBroker" required="N"/>
+        <field name="ClientID" required="N"/>
+        <field name="Commission" required="N"/>
+        <field name="CommType" required="N"/>
+        <group name="NoDlvyInst" required="N">
+          <field name="BrokerOfCredit" required="N"/>
+          <field name="DlvyInst" required="N"/>
+        </group>
+      </group>
+    </message>
+    <message name="AllocationACK" msgtype="P" msgcat="app">
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="AllocID" required="Y"/>
+      <field name="TradeDate" required="Y"/>
+      <field name="TransactTime" required="N"/>
+      <field name="AllocStatus" required="Y"/>
+      <field name="AllocRejCode" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="NewOrderList" msgtype="E" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="ListSeqNo" required="Y"/>
+      <field name="ListNoOrds" required="Y"/>
+      <field name="ListExecInst" required="N"/>
+      <field name="ClOrdID" required="Y"/>
+      <field name="ClientID" required="N"/>
+      <field name="ExecBroker" required="N"/>
+      <field name="Account" required="N"/>
+      <field name="SettlmntTyp" required="N"/>
+      <field name="FutSettDate" required="N"/>
+      <field name="HandlInst" required="Y"/>
+      <field name="ExecInst" required="N"/>
+      <field name="MinQty" required="N"/>
+      <field name="MaxFloor" required="N"/>
+      <field name="ExDestination" required="N"/>
+      <field name="ProcessCode" required="N"/>
+      <field name="Symbol" required="Y"/>
+      <field name="SymbolSfx" required="N"/>
+      <field name="SecurityID" required="N"/>
+      <field name="IDSource" required="N"/>
+      <field name="Issuer" required="N"/>
+      <field name="SecurityDesc" required="N"/>
+      <field name="PrevClosePx" required="N"/>
+      <field name="Side" required="Y"/>
+      <field name="LocateReqd" required="N"/>
+      <field name="OrderQty" required="Y"/>
+      <field name="OrdType" required="Y"/>
+      <field name="Price" required="N"/>
+      <field name="StopPx" required="N"/>
+      <field name="Currency" required="N"/>
+      <field name="TimeInForce" required="N"/>
+      <field name="ExpireTime" required="N"/>
+      <field name="Commission" required="N"/>
+      <field name="CommType" required="N"/>
+      <field name="Rule80A" required="N"/>
+      <field name="ForexReq" required="N"/>
+      <field name="SettlCurrency" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListStatus" msgtype="N" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="NoRpts" required="Y"/>
+      <field name="RptSeq" required="Y"/>
+      <group name="NoOrders" required="Y">
+        <field name="ClOrdID" required="Y"/>
+        <field name="CumQty" required="Y"/>
+        <field name="CxlQty" required="Y"/>
+        <field name="AvgPx" required="Y"/>
+      </group>
+    </message>
+    <message name="ListExecute" msgtype="L" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListCancelRequest" msgtype="K" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+    <message name="ListStatusRequest" msgtype="M" msgcat="app">
+      <field name="ListID" required="Y"/>
+      <field name="WaveNo" required="N"/>
+      <field name="Text" required="N"/>
+    </message>
+  </messages>
+  <fields>
+    <field number="1" name="Account" type="STRING"/>
+    <field number="2" name="AdvId" type="INT"/>
+    <field number="3" name="AdvRefID" type="INT"/>
+    <field number="4" name="AdvSide" type="STRING">
+      <value enum="B" description="BUY"/>
+      <value enum="S" description="SELL"/>
+      <value enum="X" description="CROSS"/>
+      <value enum="T" description="TRADE"/>
+    </field>
+    <field number="5" name="AdvTransType" type="STRING">
+      <value enum="N" description="NEW"/>
+      <value enum="C" description="CANCEL"/>
+      <value enum="R" description="REPLACE"/>
+    </field>
+    <field number="6" name="AvgPx" type="FLOAT"/>
+    <field number="7" name="BeginSeqNo" type="INT"/>
+    <field number="8" name="BeginString" type="STRING"/>
+    <field number="9" name="BodyLength" type="INT"/>
+    <field number="10" name="CheckSum" type="STRING"/>
+    <field number="11" name="ClOrdID" type="STRING"/>
+    <field number="12" name="Commission" type="FLOAT"/>
+    <field number="13" name="CommType" type="STRING">
+      <value enum="1" description="PER_SHARE"/>
+      <value enum="2" description="PERCENTAGE"/>
+      <value enum="3" description="ABSOLUTE"/>
+    </field>
+    <field number="14" name="CumQty" type="INT"/>
+    <field number="15" name="Currency" type="STRING"/>
+    <field number="16" name="EndSeqNo" type="INT"/>
+    <field number="17" name="ExecID" type="INT"/>
+    <field number="18" name="ExecInst" type="MULTIPLEVALUESTRING">
+      <value enum="1" description="NOT_HELD"/>
+      <value enum="2" description="WORK"/>
+      <value enum="3" description="GO_ALONG"/>
+      <value enum="4" description="OVER_THE_DAY"/>
+      <value enum="5" description="HELD"/>
+      <value enum="6" description="PARTICIPATE_DONT_INITIATE"/>
+      <value enum="7" description="STRICT_SCALE"/>
+      <value enum="8" description="TRY_TO_SCALE"/>
+      <value enum="9" description="STAY_ON_BIDSIDE"/>
+      <value enum="0" description="STAY_ON_OFFERSIDE"/>
+      <value enum="A" description="NO_CROSS"/>
+      <value enum="B" description="OK_TO_CROSS"/>
+      <value enum="C" description="CALL_FIRST"/>
+      <value enum="D" description="PERCENT_OF_VOLUME"/>
+      <value enum="E" description="DO_NOT_INCREASE_DNI"/>
+      <value enum="F" description="DO_NOT_REDUCE_DNR"/>
+      <value enum="G" description="ALL_OR_NONE_AON"/>
+      <value enum="I" description="INSTITUTIONS_ONLY"/>
+      <value enum="L" description="LAST_PEG"/>
+      <value enum="M" description="MIDPRICE_PEG"/>
+      <value enum="N" description="NONNEGOTIABLE"/>
+      <value enum="O" description="OPENING_PEG"/>
+      <value enum="P" description="MARKET_PEG"/>
+      <value enum="R" description="PRIMARY_PEG"/>
+      <value enum="S" description="SUSPEND"/>
+    </field>
+    <field number="19" name="ExecRefID" type="INT"/>
+    <field number="20" name="ExecTransType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="CANCEL"/>
+      <value enum="2" description="CORRECT"/>
+      <value enum="3" description="STATUS"/>
+    </field>
+    <field number="21" name="HandlInst" type="STRING">
+      <value enum="1" description="AUTOMATED_EXECUTION_ORDER_PRIVATE_NO_BROKER_INTERVENTION"/>
+      <value enum="2" description="AUTOMATED_EXECUTION_ORDER_PUBLIC_BROKER_INTERVENTION_OK"/>
+      <value enum="3" description="MANUAL_ORDER_BEST_EXECUTION"/>
+    </field>
+    <field number="22" name="IDSource" type="STRING">
+      <value enum="1" description="CUSIP"/>
+      <value enum="2" description="SEDOL"/>
+      <value enum="3" description="QUIK"/>
+      <value enum="4" description="ISIN_NUMBER"/>
+      <value enum="5" description="RIC_CODE"/>
+    </field>
+    <field number="23" name="IOIid" type="INT"/>
+    <field number="24" name="IOIOthSvc" type="STRING"/>
+    <field number="25" name="IOIQltyInd" type="STRING">
+      <value enum="L" description="LOW"/>
+      <value enum="M" description="MEDIUM"/>
+      <value enum="H" description="HIGH"/>
+    </field>
+    <field number="26" name="IOIRefID" type="INT"/>
+    <field number="27" name="IOIShares" type="STRING"/>
+    <field number="28" name="IOITransType" type="STRING">
+      <value enum="N" description="NEW"/>
+      <value enum="C" description="CANCEL"/>
+      <value enum="R" description="REPLACE"/>
+    </field>
+    <field number="29" name="LastCapacity" type="STRING">
+      <value enum="1" description="AGENT"/>
+      <value enum="2" description="CROSS_AS_AGENT"/>
+      <value enum="3" description="CROSS_AS_PRINCIPAL"/>
+      <value enum="4" description="PRINCIPAL"/>
+    </field>
+    <field number="30" name="LastMkt" type="STRING"/>
+    <field number="31" name="LastPx" type="FLOAT"/>
+    <field number="32" name="LastShares" type="INT"/>
+    <field number="33" name="LinesOfText" type="INT"/>
+    <field number="34" name="MsgSeqNum" type="INT"/>
+    <field number="35" name="MsgType" type="STRING"/>
+    <field number="36" name="NewSeqNo" type="INT"/>
+    <field number="37" name="OrderID" type="STRING"/>
+    <field number="38" name="OrderQty" type="INT"/>
+    <field number="39" name="OrdStatus" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="PARTIALLY_FILLED"/>
+      <value enum="2" description="FILLED"/>
+      <value enum="3" description="DONE_FOR_DAY"/>
+      <value enum="4" description="CANCELED"/>
+      <value enum="5" description="REPLACED"/>
+      <value enum="6" description="PENDING_CANCELREPLACE"/>
+      <value enum="7" description="STOPPED"/>
+      <value enum="8" description="REJECTED"/>
+      <value enum="9" description="SUSPENDED"/>
+      <value enum="A" description="PENDING_NEW"/>
+      <value enum="B" description="CALCULATED"/>
+      <value enum="C" description="EXPIRED"/>
+    </field>
+    <field number="40" name="OrdType" type="STRING">
+      <value enum="1" description="MARKET"/>
+      <value enum="2" description="LIMIT"/>
+      <value enum="3" description="STOP"/>
+      <value enum="4" description="STOP_LIMIT"/>
+      <value enum="5" description="MARKET_ON_CLOSE"/>
+      <value enum="6" description="WITH_OR_WITHOUT"/>
+      <value enum="7" description="LIMIT_OR_BETTER"/>
+      <value enum="8" description="LIMIT_WITH_OR_WITHOUT"/>
+      <value enum="9" description="ON_BASIS"/>
+      <value enum="A" description="ON_CLOSE"/>
+      <value enum="B" description="LIMIT_ON_CLOSE"/>
+      <value enum="C" description="FOREX"/>
+      <value enum="D" description="PREVIOUSLY_QUOTED"/>
+      <value enum="E" description="PREVIOUSLY_INDICATED"/>
+      <value enum="P" description="PEGGED"/>
+    </field>
+    <field number="41" name="OrigClOrdID" type="STRING"/>
+    <field number="42" name="OrigTime" type="TIME"/>
+    <field number="43" name="PossDupFlag" type="STRING">
+      <value enum="Y" description="POSSIBLE_DUPLICATE"/>
+      <value enum="N" description="ORIGINAL_TRANSMISSION"/>
+    </field>
+    <field number="44" name="Price" type="FLOAT"/>
+    <field number="45" name="RefSeqNum" type="INT"/>
+    <field number="46" name="RelatdSym" type="STRING"/>
+    <field number="47" name="Rule80A" type="STRING">
+      <value enum="A" description="AGENCY_SINGLE_ORDER"/>
+      <value enum="D" description="PROGRAM_ORDER_INDEX_ARB_FOR_MEMBER_FIRMORG"/>
+      <value enum="C" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_MEMBER_FIRMORG"/>
+      <value enum="J" description="PROGRAM_ORDER_INDEX_ARB_FOR_INDIVIDUAL_CUSTOMER"/>
+      <value enum="K" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_INDIVIDUAL_CUSTOMER"/>
+      <value enum="U" description="PROGRAM_ORDER_INDEX_ARB_FOR_OTHER_AGENCY"/>
+      <value enum="Y" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_OTHER_AGENCY"/>
+      <value enum="M" description="PROGRAM_ORDER_INDEX_ARB_FOR_OTHER_MEMBER"/>
+      <value enum="N" description="PROGRAM_ORDER_NONINDEX_ARB_FOR_OTHER_MEMBER"/>
+      <value enum="W" description="ALL_OTHER_ORDERS_AS_AGENT_FOR_OTHER_MEMBER"/>
+    </field>
+    <field number="48" name="SecurityID" type="STRING"/>
+    <field number="49" name="SenderCompID" type="STRING"/>
+    <field number="50" name="SenderSubID" type="STRING"/>
+    <field number="52" name="SendingTime" type="TIME"/>
+    <field number="53" name="Shares" type="INT"/>
+    <field number="54" name="Side" type="STRING">
+      <value enum="1" description="BUY"/>
+      <value enum="2" description="SELL"/>
+      <value enum="3" description="BUY_MINUS"/>
+      <value enum="4" description="SELL_PLUS"/>
+      <value enum="5" description="SELL_SHORT"/>
+      <value enum="6" description="SELL_SHORT_EXEMPT"/>
+    </field>
+    <field number="55" name="Symbol" type="STRING"/>
+    <field number="56" name="TargetCompID" type="STRING"/>
+    <field number="57" name="TargetSubID" type="STRING"/>
+    <field number="58" name="Text" type="STRING"/>
+    <field number="59" name="TimeInForce" type="STRING">
+      <value enum="0" description="DAY"/>
+      <value enum="1" description="GOOD_TILL_CANCEL"/>
+      <value enum="2" description="AT_THE_OPENING"/>
+      <value enum="3" description="IMMEDIATE_OR_CANCEL"/>
+      <value enum="4" description="FILL_OR_KILL"/>
+      <value enum="5" description="GOOD_TILL_CROSSING"/>
+      <value enum="6" description="GOOD_TILL_DATE"/>
+    </field>
+    <field number="60" name="TransactTime" type="TIME"/>
+    <field number="61" name="Urgency" type="STRING">
+      <value enum="0" description="NORMAL"/>
+      <value enum="1" description="FLASH"/>
+      <value enum="2" description="BACKGROUND"/>
+    </field>
+    <field number="62" name="ValidUntilTime" type="TIME"/>
+    <field number="63" name="SettlmntTyp" type="STRING">
+      <value enum="0" description="REGULAR"/>
+      <value enum="1" description="CASH"/>
+      <value enum="2" description="NEXT_DAY"/>
+      <value enum="3" description="TPLUS2"/>
+      <value enum="4" description="TPLUS3"/>
+      <value enum="5" description="TPLUS4"/>
+      <value enum="6" description="FUTURE"/>
+      <value enum="7" description="WHEN_ISSUED"/>
+      <value enum="8" description="SELLERS_OPTION"/>
+      <value enum="9" description="TPLUS5"/>
+    </field>
+    <field number="64" name="FutSettDate" type="DATE"/>
+    <field number="65" name="SymbolSfx" type="STRING"/>
+    <field number="66" name="ListID" type="STRING"/>
+    <field number="67" name="ListSeqNo" type="INT"/>
+    <field number="68" name="ListNoOrds" type="INT"/>
+    <field number="69" name="ListExecInst" type="STRING"/>
+    <field number="70" name="AllocID" type="INT"/>
+    <field number="71" name="AllocTransType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="REPLACE"/>
+      <value enum="2" description="CANCEL"/>
+    </field>
+    <field number="72" name="RefAllocID" type="INT"/>
+    <field number="73" name="NoOrders" type="INT"/>
+    <field number="74" name="AvgPrxPrecision" type="INT"/>
+    <field number="75" name="TradeDate" type="DATE"/>
+    <field number="76" name="ExecBroker" type="STRING"/>
+    <field number="77" name="OpenClose" type="STRING"/>
+    <field number="78" name="NoAllocs" type="INT"/>
+    <field number="79" name="AllocAccount" type="STRING"/>
+    <field number="80" name="AllocShares" type="INT"/>
+    <field number="81" name="ProcessCode" type="STRING">
+      <value enum="0" description="REGULAR"/>
+      <value enum="1" description="SOFT_DOLLAR"/>
+      <value enum="2" description="STEPIN"/>
+      <value enum="3" description="STEPOUT"/>
+      <value enum="4" description="SOFTDOLLAR_STEPIN"/>
+      <value enum="5" description="SOFTDOLLAR_STEPOUT"/>
+      <value enum="6" description="PLAN_SPONSOR"/>
+    </field>
+    <field number="82" name="NoRpts" type="INT"/>
+    <field number="83" name="RptSeq" type="INT"/>
+    <field number="84" name="CxlQty" type="INT"/>
+    <field number="85" name="NoDlvyInst" type="INT"/>
+    <field number="86" name="DlvyInst" type="STRING"/>
+    <field number="87" name="AllocStatus" type="INT">
+      <value enum="0" description="ACCEPTED"/>
+      <value enum="1" description="REJECTED"/>
+      <value enum="2" description="PARTIAL_ACCEPT"/>
+      <value enum="3" description="RECEIVED"/>
+    </field>
+    <field number="88" name="AllocRejCode" type="INT">
+      <value enum="0" description="UNKNOWN_ACCOUNT"/>
+      <value enum="1" description="INCORRECT_QUANTITY"/>
+      <value enum="2" description="INCORRECT_AVERAGE_PRICE"/>
+      <value enum="3" description="UNKNOWN_EXECUTING_BROKER_MNEMONIC"/>
+      <value enum="4" description="COMMISSION_DIFFERENCE"/>
+      <value enum="5" description="UNKNOWN_ORDERID"/>
+      <value enum="6" description="UNKNOWN_LISTID"/>
+      <value enum="7" description="OTHER"/>
+    </field>
+    <field number="89" name="Signature" type="DATA"/>
+    <field number="90" name="SecureDataLen" type="INT"/>
+    <field number="91" name="SecureData" type="DATA"/>
+    <field number="92" name="BrokerOfCredit" type="STRING"/>
+    <field number="93" name="SignatureLength" type="INT"/>
+    <field number="94" name="EmailType" type="STRING">
+      <value enum="0" description="NEW"/>
+      <value enum="1" description="REPLY"/>
+      <value enum="2" description="ADMIN_REPLY"/>
+    </field>
+    <field number="95" name="RawDataLength" type="INT"/>
+    <field number="96" name="RawData" type="DATA"/>
+    <field number="97" name="PossResend" type="STRING"/>
+    <field number="98" name="EncryptMethod" type="INT">
+      <value enum="0" description="NONE_OTHER"/>
+      <value enum="1" description="PKCS"/>
+      <value enum="2" description="DES"/>
+      <value enum="3" description="PKCSDES"/>
+      <value enum="4" description="PGPDES"/>
+      <value enum="5" description="PGPDESMD5"/>
+      <value enum="6" description="PEMDESMD5"/>
+    </field>
+    <field number="99" name="StopPx" type="FLOAT"/>
+    <field number="100" name="ExDestination" type="STRING">
+      <value enum="0" description="NONE"/>
+      <value enum="4" description="POSIT"/>
+    </field>
+    <field number="102" name="CxlRejReason" type="INT">
+      <value enum="0" description="TOO_LATE_TO_CANCEL"/>
+      <value enum="1" description="UNKNOWN_ORDER"/>
+    </field>
+    <field number="103" name="OrdRejReason" type="INT">
+      <value enum="0" description="BROKER_OPTION"/>
+      <value enum="1" description="UNKNOWN_SYMBOL"/>
+      <value enum="2" description="EXCHANGE_CLOSED"/>
+      <value enum="3" description="ORDER_EXCEEDS_LIMIT"/>
+      <value enum="4" description="TOO_LATE_TO_ENTER"/>
+    </field>
+    <field number="104" name="IOIQualifier" type="STRING">
+      <value enum="X" description="CROSSING_OPPORTUNITY"/>
+      <value enum="O" description="AT_THE_OPEN"/>
+      <value enum="M" description="MORE_BEHIND"/>
+      <value enum="P" description="TAKING_A_POSITION"/>
+      <value enum="V" description="VERSUS"/>
+      <value enum="Q" description="CURRENT_QUOTE"/>
+      <value enum="C" description="AT_THE_CLOSE"/>
+      <value enum="S" description="PORTFOLIO_SHOWN"/>
+      <value enum="I" description="IN_TOUCH_WITH"/>
+      <value enum="W" description="INDICATION_WORKING_AWAY"/>
+      <value enum="A" description="ALL_OR_NONE"/>
+      <value enum="L" description="LIMIT"/>
+      <value enum="T" description="THROUGH_THE_DAY"/>
+    </field>
+    <field number="105" name="WaveNo" type="STRING"/>
+    <field number="106" name="Issuer" type="STRING"/>
+    <field number="107" name="SecurityDesc" type="STRING"/>
+    <field number="108" name="HeartBtInt" type="INT"/>
+    <field number="109" name="ClientID" type="STRING"/>
+    <field number="110" name="MinQty" type="INT"/>
+    <field number="111" name="MaxFloor" type="INT"/>
+    <field number="112" name="TestReqID" type="STRING"/>
+    <field number="113" name="ReportToExch" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="114" name="LocateReqd" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="115" name="OnBehalfOfCompID" type="STRING"/>
+    <field number="116" name="OnBehalfOfSubID" type="STRING"/>
+    <field number="117" name="QuoteID" type="STRING"/>
+    <field number="118" name="NetMoney" type="FLOAT"/>
+    <field number="119" name="SettlCurrAmt" type="FLOAT"/>
+    <field number="120" name="SettlCurrency" type="STRING"/>
+    <field number="121" name="ForexReq" type="STRING">
+      <value enum="Y" description="YES"/>
+      <value enum="N" description="NO"/>
+    </field>
+    <field number="122" name="OrigSendingTime" type="TIME"/>
+    <field number="123" name="GapFillFlag" type="STRING">
+      <value enum="Y" description="GAP_FILL_MESSAGE_MSGSEQNUM_FIELD_VALID"/>
+      <value enum="N" description="SEQUENCE_RESET_IGNORE_MSGSEQNUM"/>
+    </field>
+    <field number="124" name="NoExecs" type="INT"/>
+    <field number="125" name="CxlType" type="STRING">
+      <value enum="P" description="PARTIAL_CANCEL"/>
+      <value enum="F" description="FULL_REMAINING_QUANTITY"/>
+    </field>
+    <field number="126" name="ExpireTime" type="TIME"/>
+    <field number="127" name="DKReason" type="STRING">
+      <value enum="A" description="UNKNOWN_SYMBOL"/>
+      <value enum="B" description="WRONG_SIDE"/>
+      <value enum="C" description="QUANTITY_EXCEEDS_ORDER"/>
+      <value enum="D" description="NO_MATCHING_ORDER"/>
+      <value enum="E" description="PRICE_EXCEEDS_LIMIT"/>
+      <value enum="Z" description="OTHER" />
+    </field>
+    <field number="128" name="DeliverToCompID" type="STRING"/>
+    <field number="129" name="DeliverToSubID" type="STRING"/>
+    <field number="130" name="IOINaturalFlag" type="STRING">
+      <value enum="Y" description="NATURAL"/>
+      <value enum="N" description="NOT_NATURAL"/>
+    </field>
+    <field number="131" name="QuoteReqID" type="STRING"/>
+    <field number="132" name="BidPx" type="FLOAT"/>
+    <field number="133" name="OfferPx" type="FLOAT"/>
+    <field number="134" name="BidSize" type="INT"/>
+    <field number="135" name="OfferSize" type="INT"/>
+    <field number="136" name="NoMiscFees" type="INT"/>
+    <field number="137" name="MiscFeeAmt" type="FLOAT"/>
+    <field number="138" name="MiscFeeCurr" type="STRING"/>
+    <field number="139" name="MiscFeeType" type="STRING">
+      <value enum="1" description="REGULATORY"/>
+      <value enum="2" description="TAX"/>
+      <value enum="3" description="LOCAL_COMMISSION"/>
+      <value enum="4" description="EXCHANGE_FEES"/>
+      <value enum="5" description="STAMP"/>
+      <value enum="6" description="LEVY"/>
+      <value enum="7" description="OTHER"/>
+    </field>
+    <field number="140" name="PrevClosePx" type="FLOAT"/>
+  </fields>
+</fix>
@@ -0,0 +1,313 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<fix type='FIXT' major='1' minor='1' servicepack='0'>
+  <header>
+    <field name='BeginString' required='Y'/>
+    <field name='BodyLength' required='Y'/>
+    <field name='MsgType' required='Y'/>
+    <field name='SenderCompID' required='Y'/>
+    <field name='TargetCompID' required='Y'/>
+    <field name='OnBehalfOfCompID' required='N'/>
+    <field name='DeliverToCompID' required='N'/>
+    <field name='SecureDataLen' required='N'/>
+    <field name='SecureData' required='N'/>
+    <field name='MsgSeqNum' required='Y'/>
+    <field name='SenderSubID' required='N'/>
+    <field name='SenderLocationID' required='N'/>
+    <field name='TargetSubID' required='N'/>
+    <field name='TargetLocationID' required='N'/>
+    <field name='OnBehalfOfSubID' required='N'/>
+    <field name='OnBehalfOfLocationID' required='N'/>
+    <field name='DeliverToSubID' required='N'/>
+    <field name='DeliverToLocationID' required='N'/>
+    <field name='PossDupFlag' required='N'/>
+    <field name='PossResend' required='N'/>
+    <field name='SendingTime' required='Y'/>
+    <field name='OrigSendingTime' required='N'/>
+    <field name='XmlDataLen' required='N'/>
+    <field name='XmlData' required='N'/>
+    <field name='MessageEncoding' required='N'/>
+    <field name='LastMsgSeqNumProcessed' required='N'/>
+    <component name='HopGrp' required='N'/>
+    <field name='ApplVerID' required='N'/>
+    <field name='CstmApplVerID' required='N'/>
+  </header>
+  <trailer>
+    <field name='SignatureLength' required='N'/>
+    <field name='Signature' required='N'/>
+    <field name='CheckSum' required='Y'/>
+  </trailer>
+  <messages>
+    <message msgcat='admin' msgtype='0' name='Heartbeat'>
+      <field name='TestReqID' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='1' name='TestRequest'>
+      <field name='TestReqID' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='2' name='ResendRequest'>
+      <field name='BeginSeqNo' required='Y'/>
+      <field name='EndSeqNo' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='3' name='Reject'>
+      <field name='RefSeqNum' required='Y'/>
+      <field name='RefTagID' required='N'/>
+      <field name='RefMsgType' required='N'/>
+      <field name='SessionRejectReason' required='N'/>
+      <field name='Text' required='N'/>
+      <field name='EncodedTextLen' required='N'/>
+      <field name='EncodedText' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='4' name='SequenceReset'>
+      <field name='GapFillFlag' required='N'/>
+      <field name='NewSeqNo' required='Y'/>
+    </message>
+    <message msgcat='admin' msgtype='5' name='Logout'>
+      <field name='Text' required='N'/>
+      <field name='EncodedTextLen' required='N'/>
+      <field name='EncodedText' required='N'/>
+    </message>
+    <message msgcat='admin' msgtype='A' name='Logon'>
+      <field name='EncryptMethod' required='Y'/>
+      <field name='HeartBtInt' required='Y'/>
+      <field name='RawDataLength' required='N'/>
+      <field name='RawData' required='N'/>
+      <field name='ResetSeqNumFlag' required='N'/>
+      <field name='NextExpectedMsgSeqNum' required='N'/>
+      <field name='MaxMessageSize' required='N'/>
+      <field name='TestMessageIndicator' required='N'/>
+      <field name='Username' required='N'/>
+      <field name='Password' required='N'/>
+      <field name='DefaultApplVerID' required='Y'/>
+      <component name='MsgTypeGrp' required='N'/>
+    </message>
+  </messages>
+  <components>
+    <component name='HopGrp'>
+      <group name='NoHops' required='N'>
+        <field name='HopCompID' required='N'/>
+        <field name='HopSendingTime' required='N'/>
+        <field name='HopRefID' required='N'/>
+      </group>
+    </component>
+    <component name='MsgTypeGrp'>
+      <group name='NoMsgTypes' required='N'>
+        <field name='RefMsgType' required='N'/>
+        <field name='MsgDirection' required='N'/>
+        <field name='RefApplVerID' required='N'/>
+        <field name='RefCstmApplVerID' required='N'/>
+      </group>
+    </component>
+  </components>
+  <fields>
+    <field name='BeginSeqNo' number='7' type='SEQNUM'/>
+    <field name='BeginString' number='8' type='STRING'/>
+    <field name='BodyLength' number='9' type='LENGTH'/>
+    <field name='CheckSum' number='10' type='STRING'/>
+    <field name='EndSeqNo' number='16' type='SEQNUM'/>
+    <field name='MsgSeqNum' number='34' type='SEQNUM'/>
+    <field number='35' name='MsgType' type='STRING'>
+      <value enum='0' description='HEARTBEAT'/>
+      <value enum='1' description='TEST_REQUEST'/>
+      <value enum='2' description='RESEND_REQUEST'/>
+      <value enum='3' description='REJECT'/>
+      <value enum='4' description='SEQUENCE_RESET'/>
+      <value enum='5' description='LOGOUT'/>
+      <value enum='6' description='INDICATION_OF_INTEREST'/>
+      <value enum='7' description='ADVERTISEMENT'/>
+      <value enum='8' description='EXECUTION_REPORT'/>
+      <value enum='9' description='ORDER_CANCEL_REJECT'/>
+      <value enum='A' description='LOGON'/>
+      <value enum='B' description='NEWS'/>
+      <value enum='C' description='EMAIL'/>
+      <value enum='D' description='ORDER_SINGLE'/>
+      <value enum='E' description='ORDER_LIST'/>
+      <value enum='F' description='ORDER_CANCEL_REQUEST'/>
+      <value enum='G' description='ORDER_CANCEL_REPLACE_REQUEST'/>
+      <value enum='H' description='ORDER_STATUS_REQUEST'/>
+      <value enum='J' description='ALLOCATION_INSTRUCTION'/>
+      <value enum='K' description='LIST_CANCEL_REQUEST'/>
+      <value enum='L' description='LIST_EXECUTE'/>
+      <value enum='M' description='LIST_STATUS_REQUEST'/>
+      <value enum='N' description='LIST_STATUS'/>
+      <value enum='P' description='ALLOCATION_INSTRUCTION_ACK'/>
+      <value enum='Q' description='DONT_KNOW_TRADE'/>
+      <value enum='R' description='QUOTE_REQUEST'/>
+      <value enum='S' description='QUOTE'/>
+      <value enum='T' description='SETTLEMENT_INSTRUCTIONS'/>
+      <value enum='V' description='MARKET_DATA_REQUEST'/>
+      <value enum='W' description='MARKET_DATA_SNAPSHOT_FULL_REFRESH'/>
+      <value enum='X' description='MARKET_DATA_INCREMENTAL_REFRESH'/>
+      <value enum='Y' description='MARKET_DATA_REQUEST_REJECT'/>
+      <value enum='Z' description='QUOTE_CANCEL'/>
+      <value enum='a' description='QUOTE_STATUS_REQUEST'/>
+      <value enum='b' description='MASS_QUOTE_ACKNOWLEDGEMENT'/>
+      <value enum='c' description='SECURITY_DEFINITION_REQUEST'/>
+      <value enum='d' description='SECURITY_DEFINITION'/>
+      <value enum='e' description='SECURITY_STATUS_REQUEST'/>
+      <value enum='f' description='SECURITY_STATUS'/>
+      <value enum='g' description='TRADING_SESSION_STATUS_REQUEST'/>
+      <value enum='h' description='TRADING_SESSION_STATUS'/>
+      <value enum='i' description='MASS_QUOTE'/>
+      <value enum='j' description='BUSINESS_MESSAGE_REJECT'/>
+      <value enum='k' description='BID_REQUEST'/>
+      <value enum='l' description='BID_RESPONSE'/>
+      <value enum='m' description='LIST_STRIKE_PRICE'/>
+      <value enum='n' description='XML_MESSAGE'/>
+      <value enum='o' description='REGISTRATION_INSTRUCTIONS'/>
+      <value enum='p' description='REGISTRATION_INSTRUCTIONS_RESPONSE'/>
+      <value enum='q' description='ORDER_MASS_CANCEL_REQUEST'/>
+      <value enum='r' description='ORDER_MASS_CANCEL_REPORT'/>
+      <value enum='s' description='NEW_ORDER_CROSS'/>
+      <value enum='t' description='CROSS_ORDER_CANCEL_REPLACE_REQUEST'/>
+      <value enum='u' description='CROSS_ORDER_CANCEL_REQUEST'/>
+      <value enum='v' description='SECURITY_TYPE_REQUEST'/>
+      <value enum='w' description='SECURITY_TYPES'/>
+      <value enum='x' description='SECURITY_LIST_REQUEST'/>
+      <value enum='y' description='SECURITY_LIST'/>
+      <value enum='z' description='DERIVATIVE_SECURITY_LIST_REQUEST'/>
+      <value enum='AA' description='DERIVATIVE_SECURITY_LIST'/>
+      <value enum='AB' description='NEW_ORDER_MULTILEG'/>
+      <value enum='AC' description='MULTILEG_ORDER_CANCEL_REPLACE'/>
+      <value enum='AD' description='TRADE_CAPTURE_REPORT_REQUEST'/>
+      <value enum='AE' description='TRADE_CAPTURE_REPORT'/>
+      <value enum='AF' description='ORDER_MASS_STATUS_REQUEST'/>
+      <value enum='AG' description='QUOTE_REQUEST_REJECT'/>
+      <value enum='AH' description='RFQ_REQUEST'/>
+      <value enum='AI' description='QUOTE_STATUS_REPORT'/>
+      <value enum='AJ' description='QUOTE_RESPONSE'/>
+      <value enum='AK' description='CONFIRMATION'/>
+      <value enum='AL' description='POSITION_MAINTENANCE_REQUEST'/>
+      <value enum='AM' description='POSITION_MAINTENANCE_REPORT'/>
+      <value enum='AN' description='REQUEST_FOR_POSITIONS'/>
+      <value enum='AO' description='REQUEST_FOR_POSITIONS_ACK'/>
+      <value enum='AP' description='POSITION_REPORT'/>
+      <value enum='AQ' description='TRADE_CAPTURE_REPORT_REQUEST_ACK'/>
+      <value enum='AR' description='TRADE_CAPTURE_REPORT_ACK'/>
+      <value enum='AS' description='ALLOCATION_REPORT'/>
+      <value enum='AT' description='ALLOCATION_REPORT_ACK'/>
+      <value enum='AU' description='CONFIRMATION_ACK'/>
+      <value enum='AV' description='SETTLEMENT_INSTRUCTION_REQUEST'/>
+      <value enum='AW' description='ASSIGNMENT_REPORT'/>
+      <value enum='AX' description='COLLATERAL_REQUEST'/>
+      <value enum='AY' description='COLLATERAL_ASSIGNMENT'/>
+      <value enum='AZ' description='COLLATERAL_RESPONSE'/>
+      <value enum='BA' description='COLLATERAL_REPORT'/>
+      <value enum='BB' description='COLLATERAL_INQUIRY'/>
+      <value enum='BC' description='NETWORK_STATUS_REQUEST'/>
+      <value enum='BD' description='NETWORK_STATUS_RESPONSE'/>
+      <value enum='BE' description='USER_REQUEST'/>
+      <value enum='BF' description='USER_RESPONSE'/>
+      <value enum='BG' description='COLLATERAL_INQUIRY_ACK'/>
+      <value enum='BH' description='CONFIRMATION_REQUEST'/>
+      <value enum='BI' description='TRADING_SESSION_LIST_REQUEST'/>
+      <value enum='BJ' description='TRADING_SESSION_LIST'/>
+      <value enum='BK' description='SECURITY_LIST_UPDATE_REPORT'/>
+      <value enum='BL' description='ADJUSTED_POSITION_REPORT'/>
+      <value enum='BM' description='ALLOCATION_INSTRUCTION_ALERT'/>
+      <value enum='BN' description='EXECUTION_ACKNOWLEDGEMENT'/>
+      <value enum='BO' description='CONTRARY_INTENTION_REPORT'/>
+      <value enum='BP' description='SECURITY_DEFINITION_UPDATE_REPORT'/>
+    </field>
+    <field name='NewSeqNo' number='36' type='SEQNUM'/>
+    <field name='PossDupFlag' number='43' type='BOOLEAN'/>
+    <field name='RefSeqNum' number='45' type='SEQNUM'/>
+    <field name='SenderCompID' number='49' type='STRING'/>
+    <field name='SenderSubID' number='50' type='STRING'/>
+    <field name='SendingTime' number='52' type='UTCTIMESTAMP'/>
+    <field name='TargetCompID' number='56' type='STRING'/>
+    <field name='TargetSubID' number='57' type='STRING'/>
+    <field name='Text' number='58' type='STRING'/>
+    <field name='Signature' number='89' type='DATA'/>
+    <field name='SecureDataLen' number='90' type='LENGTH'/>
+    <field name='SecureData' number='91' type='DATA'/>
+    <field name='SignatureLength' number='93' type='LENGTH'/>
+    <field name='RawDataLength' number='95' type='LENGTH'/>
+    <field name='RawData' number='96' type='DATA'/>
+    <field name='PossResend' number='97' type='BOOLEAN'/>
+    <field name='EncryptMethod' number='98' type='INT'>
+      <value description='NONE_OTHER' enum='0'/>
+      <value description='PKCS' enum='1'/>
+      <value description='DES' enum='2'/>
+      <value description='PKCS_DES' enum='3'/>
+      <value description='PGP_DES' enum='4'/>
+      <value description='PGP_DES_MD5' enum='5'/>
+      <value description='PEM_DES_MD5' enum='6'/>
+    </field>
+    <field name='HeartBtInt' number='108' type='INT'/>
+    <field name='TestReqID' number='112' type='STRING'/>
+    <field name='OnBehalfOfCompID' number='115' type='STRING'/>
+    <field name='OnBehalfOfSubID' number='116' type='STRING'/>
+    <field name='OrigSendingTime' number='122' type='UTCTIMESTAMP'/>
+    <field name='GapFillFlag' number='123' type='BOOLEAN'/>
+    <field name='DeliverToCompID' number='128' type='STRING'/>
+    <field name='DeliverToSubID' number='129' type='STRING'/>
+    <field name='ResetSeqNumFlag' number='141' type='BOOLEAN'/>
+    <field name='SenderLocationID' number='142' type='STRING'/>
+    <field name='TargetLocationID' number='143' type='STRING'/>
+    <field name='OnBehalfOfLocationID' number='144' type='STRING'/>
+    <field name='DeliverToLocationID' number='145' type='STRING'/>
+    <field name='XmlDataLen' number='212' type='LENGTH'/>
+    <field name='XmlData' number='213' type='DATA'/>
+    <field number='347' name='MessageEncoding' type='STRING'>
+      <value enum='ISO-2022-JP' description='ISO_2022_JP'/>
+      <value enum='EUC-JP' description='EUC_JP'/>
+      <value enum='SHIFT_JIS' description='SHIFT_JIS'/>
+      <value enum='UTF-8' description='UTF_8'/>
+    </field>
+    <field name='EncodedTextLen' number='354' type='LENGTH'/>
+    <field name='EncodedText' number='355' type='DATA'/>
+    <field name='LastMsgSeqNumProcessed' number='369' type='SEQNUM'/>
+    <field name='RefTagID' number='371' type='INT'/>
+    <field name='RefMsgType' number='372' type='STRING'/>
+    <field name='SessionRejectReason' number='373' type='INT'>
+      <value description='INVALID_TAG_NUMBER' enum='0'/>
+      <value description='REQUIRED_TAG_MISSING' enum='1'/>
+      <value description='SENDINGTIME_ACCURACY_PROBLEM' enum='10'/>
+      <value description='INVALID_MSGTYPE' enum='11'/>
+      <value description='XML_VALIDATION_ERROR' enum='12'/>
+      <value description='TAG_APPEARS_MORE_THAN_ONCE' enum='13'/>
+      <value description='TAG_SPECIFIED_OUT_OF_REQUIRED_ORDER' enum='14'/>
+      <value description='REPEATING_GROUP_FIELDS_OUT_OF_ORDER' enum='15'/>
+      <value description='INCORRECT_NUMINGROUP_COUNT_FOR_REPEATING_GROUP' enum='16'/>
+      <value description='NON_DATA_VALUE_INCLUDES_FIELD_DELIMITER' enum='17'/>
+      <value description='TAG_NOT_DEFINED_FOR_THIS_MESSAGE_TYPE' enum='2'/>
+      <value description='UNDEFINED_TAG' enum='3'/>
+      <value description='TAG_SPECIFIED_WITHOUT_A_VALUE' enum='4'/>
+      <value description='VALUE_IS_INCORRECT' enum='5'/>
+      <value description='INCORRECT_DATA_FORMAT_FOR_VALUE' enum='6'/>
+      <value description='DECRYPTION_PROBLEM' enum='7'/>
+      <value description='SIGNATURE_PROBLEM' enum='8'/>
+      <value description='COMPID_PROBLEM' enum='9'/>
+      <value description='OTHER' enum='99'/>
+    </field>
+    <field name='MaxMessageSize' number='383' type='LENGTH'/>
+    <field name='NoMsgTypes' number='384' type='NUMINGROUP'/>
+    <field name='MsgDirection' number='385' type='CHAR'>
+      <value description='RECEIVE' enum='R'/>
+      <value description='SEND' enum='S'/>
+    </field>
+    <field name='TestMessageIndicator' number='464' type='BOOLEAN'/>
+    <field name='Username' number='553' type='STRING'/>
+    <field name='Password' number='554' type='STRING'/>
+    <field name='NoHops' number='627' type='NUMINGROUP'/>
+    <field name='HopCompID' number='628' type='STRING'/>
+    <field name='HopSendingTime' number='629' type='UTCTIMESTAMP'/>
+    <field name='HopRefID' number='630' type='SEQNUM'/>
+    <field name='NextExpectedMsgSeqNum' number='789' type='SEQNUM'/>
+    <field name='ApplVerID' number='1128' type='STRING'>
+      <value description='FIX27' enum='0'/>
+      <value description='FIX30' enum='1'/>
+      <value description='FIX40' enum='2'/>
+      <value description='FIX41' enum='3'/>
+      <value description='FIX42' enum='4'/>
+      <value description='FIX43' enum='5'/>
+      <value description='FIX44' enum='6'/>
+      <value description='FIX50' enum='7'/>
+      <value description='FIX50SP1' enum='8'/>
+      <value description='FIX50SP2' enum='9'/>
+    </field>
+    <field name='CstmApplVerID' number='1129' type='STRING'/>
+    <field name='RefApplVerID' number='1130' type='STRING'/>
+    <field name='RefCstmApplVerID' number='1131' type='STRING'/>
+    <field name='DefaultApplVerID' number='1137' type='STRING'/>
+  </fields>
+</fix>
@@ -0,0 +1,165 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+
+using bts.co2;
+
+namespace bts.history
+{
+   public interface IAlgo 
+   {
+      bool is_completed();
+      void set_param(string name, string val);
+
+      void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage curr_book,
+                        MarketDataIncrementalRefresh packet);
+      void process_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                          MarketDataIncrementalRefresh packet);
+   }
+   
+   abstract public partial class AlgoBase : IAlgo
+   {
+      public readonly Guid guid = Guid.NewGuid();
+      public ulong order_key { get; private set;}
+      
+      public string[] security_list { get; private set; }
+      public Action<FillEntry> on_fill;
+      public Action<OrderStatus> on_status;
+      
+      protected IOrderSink _sink;
+      public IOrderSink sink => _sink;
+
+      readonly protected SortedDictionary< string, Func<MDEntry, OrderIDEntry, bool> > _filter = new SortedDictionary<string, Func<MDEntry, OrderIDEntry, bool>>();      
+
+      protected AlgoBase(IOrderSink sink, string[] security_list) 
+      {
+         on_fill = new Action<FillEntry>(f => {});
+         on_status = new Action<OrderStatus>(s => {});
+         this._sink = sink;
+         this.security_list = security_list;
+         foreach (var sec in security_list)
+            _filter[sec] = (MDEntry md, OrderIDEntry id) => md.Symbol == sec;
+      }            
+                  
+      readonly Dictionary<string, string> _algo_params = new Dictionary<string, string>();
+      public IReadOnlyDictionary<string, string> algo_params => _algo_params;
+      public virtual void set_param(string name, string val)
+      {
+         _algo_params[name] = val;
+         var pi = this.GetType().GetProperty(name);
+         if(pi != null)
+         {
+            if(val.Contains(','))
+            {
+               var elem = pi.PropertyType.GetElementType();
+               var ary = val.Split(',').Select(x => Convert.ChangeType(x, elem)).ToArray();
+               var tary = Array.CreateInstance(elem, ary.Length);
+               Array.Copy(ary, tary, ary.Length);
+               pi.SetValue(this, tary);
+            }
+            else
+               pi.SetValue(this, Convert.ChangeType(val, pi.PropertyType));
+         }
+      }         
+
+      public virtual void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage curr_book,
+                                       MarketDataIncrementalRefresh packet)
+      {
+         if(!_filter.ContainsKey(security_desc))
+            return;
+         on_book(security_desc, security_key, current_book, curr_book, packet);
+      }
+
+      public virtual void process_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                                         MarketDataIncrementalRefresh packet)      
+      {
+         if (!_filter.ContainsKey(security_desc))
+            return;
+         on_trades(security_desc, security_key, trades, packet);
+      }      
+
+      public abstract object report();
+      public abstract bool is_completed();
+
+      public virtual Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
+                                        Action<Handle, FillEntry> more_on_fill = null,
+                                        Action<Handle, OrderStatus> more_on_status = null)
+         where C : AlgoBase
+      {
+         var order_key = IDGenerator.next_order_id();
+         var child = AlgoBase.algo_factory(typeof(C), _sink, security_list);
+         Handle h = hookup_child(child, security_list, side, price, size, asof, more_on_fill, more_on_status, order_key);
+         return h;
+      }
+
+      protected Handle hookup_child(AlgoBase child, string[] security_list, Side side, double price, uint size, Timestamp asof, 
+                                    Action<Handle, FillEntry> more_on_fill, Action<Handle, OrderStatus> more_on_status, uint order_key) 
+      {
+         child.order_key = order_key;
+         AlgoBase.set_algo_params(child, this.algo_params);
+         child.on_fill += f => this.on_fill(f);
+         child.on_status += s => this.on_status(s);
+         var h = new Handle(child);
+         if (more_on_fill != null)
+            child.on_fill += (f => more_on_fill(h, f));
+         if (more_on_status != null)
+            child.on_status += (s => more_on_status(h, s));
+
+         child.do_launch(security_list, side, price, size, asof, order_key);
+         return h;
+      }
+
+      protected virtual void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                     MarketDataIncrementalRefresh packet) { }
+      protected virtual void on_trades(string security_desc, uint security_key, IEnumerable<TradeMessage> trades, 
+                                       MarketDataIncrementalRefresh packet) { }
+                                       
+      protected abstract void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key);
+      
+      protected abstract void do_update(Timestamp asof, double price, uint size = uint.MinValue);
+      protected virtual void do_update(Timestamp asof, Dictionary<string, string> algo_params, double price, uint size = uint.MinValue)
+      {
+         foreach (var s in algo_params)
+            this.set_param(s.Key, s.Value);
+         this.do_update(asof, price, size);
+      }
+      protected abstract void do_cancel(Timestamp asof);
+
+      public class Handle : IEquatable<Handle>
+      {
+         readonly AlgoBase _algo;
+         public Handle(AlgoBase algo)
+         {
+            _algo = algo;
+         }
+         
+         public AlgoBase algo => _algo;
+         public ulong order_key => _algo.order_key;
+         public void update(Timestamp asof, double price, uint size = uint.MinValue)
+         {
+            _algo.do_update(asof, price, size);
+         }
+         public void update(Timestamp asof, Dictionary<string, string> algo_params, double price, uint size = uint.MinValue)
+         {
+            foreach (var s in algo_params)
+               _algo.set_param(s.Key, s.Value);
+            update(asof, price, size);
+         }
+         public void cancel(Timestamp asof)
+         {
+            _algo.do_cancel(asof);
+         }
+         public override int GetHashCode()
+         {
+            return this.algo.guid.GetHashCode();
+         }
+         public bool Equals(Handle other)
+         {
+            return this.algo.guid == other.algo.guid;
+         }
+      }
+   }
+}
@@ -0,0 +1,70 @@
+using System;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+using System.Collections.Generic;
+using System.Reflection;
+
+namespace bts.history
+{
+   public class AlgoFactory
+   {
+      static public IAlgo algo_factory(System.Type t, IOrderSink sink,
+                                       string[] security_list, string algo_params = null)
+      {
+         return AlgoBase.algo_factory(t, sink, security_list, algo_params);
+      }
+      static public void set_algo_params(IAlgo inst, string algo_params)
+      {
+         AlgoBase.set_algo_params((AlgoBase) inst, algo_params);
+      }
+      static public void set_algo_params(IAlgo inst, IEnumerable<string> param)
+      {
+         AlgoBase.set_algo_params((AlgoBase)inst, param);
+      }
+   }   
+   abstract partial class AlgoBase
+   {
+      static public AlgoBase algo_factory(System.Type t, IOrderSink sink, 
+                                          string[] security_list, string algo_params = null)
+      {
+         AlgoBase inst = inst = (AlgoBase)Activator.CreateInstance(t, sink, security_list);
+
+         if(algo_params != null)
+            set_algo_params(inst, algo_params);
+         return inst;
+      }      
+      static public void set_algo_params(AlgoBase inst, string algo_params)
+      {
+         string[] param = algo_params.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+         set_algo_params(inst, param);
+      }
+      static public void set_algo_params(AlgoBase inst, IEnumerable<string> param)
+      {
+         foreach (var par in param)
+         {
+            string[] tup = par.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
+            Log.assert(tup.Length == 2);
+            inst.set_param(tup[0], tup[1]);
+         }
+      }
+      static public void set_algo_params(AlgoBase inst, IReadOnlyDictionary<string, string> param)
+      {
+         foreach (var par in param)
+            inst.set_param(par.Key, par.Value);
+      }
+      public static Handle activate_self(AlgoBase algo, api.Side side, double price, uint size, Timestamp asof,
+                                         Action<Handle, FillEntry> more_on_fill = null,
+                                         Action<Handle, OrderStatus> more_on_status = null)
+      {
+         var order_key = IDGenerator.next_order_id();
+         //I am activating myself at the top
+         algo.do_launch(algo.security_list, side, price, size, asof, order_key);         
+         var h = new Handle(algo);
+         if(more_on_fill != null)
+            algo.on_fill += f => more_on_fill(h, f);
+         if(more_on_status != null)
+            algo.on_status += s => more_on_status(h, s);
+         return h;
+      }
+   }
+}
@@ -0,0 +1,235 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public abstract class CompositeBase<Child, Hedge> : AlgoBase
+      where Hedge : AlgoBase
+   {
+      protected List<Handle> _children = new List<Handle>();
+      protected Handle _child 
+      {
+         get 
+         {
+            Log.assert(_children.Count <= 1);
+            return _children.FirstOrDefault(); 
+         } 
+         set 
+         { 
+            Log.assert(_children.Count <= 1);
+            if(value == null)
+            {
+               if(_children.Count == 1)            
+                  _children.RemoveAt(0);
+            }
+            else
+            {
+               if(_children.Count == 0)
+                  _children.Add(value);
+               else
+                  _children[0] = value;
+            }
+         }
+      }
+      public CompositeBase(IOrderSink sink, string[] security_list) : base(sink, security_list) 
+      {
+      }
+
+      public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof, 
+                                         Action<Handle, FillEntry> more_on_fill = null, 
+                                         Action<Handle, OrderStatus> more_on_status = null)     
+      {         
+         if(typeof(C) == this.GetType())
+         {
+            return AlgoBase.activate_self(this, side, price, size, asof, more_on_fill, more_on_status);
+         }         
+         else
+         {
+            Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => on_child_fill(h, f));
+            if (more_on_fill != null)
+               of += more_on_fill;
+            Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) =>
+            {
+               on_child_status(h, s);
+               if (s.is_terminal())
+                  _children.Remove(h);
+            });
+            if (more_on_status != null)
+               os += more_on_status;
+
+            var handle = base_activate<C>(security_list, side, price, size, asof, of, os);
+            _children.Add(handle);
+            return handle;
+         }
+      }
+
+      protected Handle base_activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof, 
+                                        Action<Handle, FillEntry> of, Action<Handle, OrderStatus> os)
+         where C : AlgoBase
+      {
+         return base.activate<C>(security_list, side, price, size, asof, of, os);
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         foreach(var child in _children)
+            child?.update(asof, price, size);
+      }
+            
+
+      protected override void do_cancel(Timestamp asof)
+      {
+         var kids = new List<Handle>(_children);
+         foreach (var child in kids)
+            child?.cancel(asof);
+      }      
+      protected virtual void on_child_fill(Handle child, FillEntry fill) { }
+      protected virtual void on_child_status(Handle child, OrderStatus stat) { }
+   }
+
+   public abstract class CompositeAlgo<Child> : CompositeBase<Child, NullAlgo>
+   {
+      public CompositeAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+
+      public override bool is_completed()
+      {
+         bool comp = true;
+         foreach (var child in _children)
+            comp &= child.algo.is_completed();
+         return comp;
+      }
+      
+      public override object report()
+      {
+         string c = "";
+         foreach (var child in _children)
+            c+= child.algo.report();
+         return $"{c}";
+      }
+
+      public override void set_param(string name, string val)
+      {
+         base.set_param(name, val);
+         foreach (var child in _children)
+            child?.algo.set_param(name, val);
+      }
+
+      public override void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                        MarketDataIncrementalRefresh packet)
+      {
+         base.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var child in _children)
+            child?.algo.process_book(security_desc, security_key, current_book, working_book, packet);
+      }
+   }
+
+   public abstract class CompositeHedgeAlgo<Child, Hedge> : CompositeBase<Child, Hedge>
+      where Hedge : AlgoBase
+   {
+      protected int _payupcount;
+      protected Dictionary<Handle, Hedge> _hedges = new Dictionary<Handle, Hedge>();
+      public CompositeHedgeAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) {}
+
+      public override bool is_completed() 
+      {
+         bool comp = true;
+         foreach (var child in _children)
+            comp &= child.algo.is_completed();
+         foreach (var sub in _hedges.Values)
+         {
+            var hc = sub.is_completed();
+            if(!hc)
+               Log.assert(true);
+            comp &= hc;
+         }
+         return comp;
+      }
+
+      public override object report() 
+      {
+         string c = "";
+         foreach (var child in _children)
+            c+= child.algo.report();
+         
+         int ih = 0;
+         string sh = string.Empty;
+         foreach (var sub in _hedges.Values)
+         {
+            if(sub.report() is string sr)
+               sh += sr;
+            if(sub.report() is int ir)
+               ih += ir;
+         }
+         return $"{c} Hedge Payups {sh} {ih+_payupcount} Incomplete {_hedges.Values.Sum( x => x.is_completed() ? 0u : 1u) } ";
+      }
+
+      public override void set_param(string name, string val)
+      {
+         base.set_param(name, val);
+         foreach (var child in _children)
+            child?.algo.set_param(name, val);
+         foreach (var sub in _hedges.Values)
+            sub.set_param(name, val);
+      }
+
+      public override void process_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                                MarketDataIncrementalRefresh packet)
+      {
+         base.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var child in _children)
+            child?.algo.process_book(security_desc, security_key, current_book, working_book, packet);
+         foreach (var sub in _hedges.Values)
+            sub.process_book(security_desc, security_key, current_book, working_book, packet);
+      }
+
+      public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof, 
+                                         Action<Handle, FillEntry> more_on_fill = null, 
+                                         Action<Handle, OrderStatus> more_on_status = null)
+      {
+         if (typeof(Hedge) == typeof(C))
+         {
+            Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => {} );
+            if (more_on_fill != null)
+               of += more_on_fill;
+            Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) => {} );
+            if (more_on_status != null)
+               os += more_on_status;
+
+            var handle = base_activate<Hedge>(security_list, side, price, size, asof, of, os);                                         
+            _hedges[handle] = (Hedge)handle.algo;
+            return handle;
+         }
+         else
+            return base.activate<C>(security_list, side, price, size, asof, more_on_fill, more_on_status);
+      }
+
+      protected void send_hedge(string security_desc, api.Side side, double price, uint size, Timestamp asof)
+      {
+         Action<Handle, FillEntry> hedge_on_fill = (hand, f) =>
+         {
+            if (f.fully_filled)
+            {
+               if (_hedges.TryGetValue(hand, out Hedge h))
+               {
+                  if (h.report() is int pay)
+                     _payupcount += pay;
+                  _hedges.Remove(hand);
+               }
+            }
+         };
+         Action<Handle, OrderStatus> hedge_on_status = (hand,s) =>
+         {
+            if (s.is_terminal())
+               _hedges.Remove(hand);
+         };
+
+         var handle = this.activate<Hedge>(new[]{security_desc}, side, price, size, asof, 
+                                           hedge_on_fill, hedge_on_status);
+
+      }
+   }
+   
+}
@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using mbts.co2;
+
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public class CoverAlgo<Child, Hedge> : CompositeHedgeAlgo<Child, Hedge>
+      where Child : AlgoBase
+      where Hedge : AlgoBase
+   {
+      public CoverAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      
+      //algo state      
+      private MarketDataIncrementalRefresh _packet;
+      string[] _security_list;
+
+      //algo parameters
+      public double profit_ticks { get; set; } = 0.0;
+      public double hedge_factor { get; set; } = 1.0;
+
+      protected override void do_launch(string[] security_list, Side side, double price, uint size, Timestamp asof, ulong order_key) 
+      {
+         _security_list = security_list;
+         _child = this.activate<Child>(security_list, side, price, size, asof);
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue) 
+      {
+         _child.update(asof, price, size);
+      }
+
+      protected override void do_cancel(Timestamp asof) 
+      {
+         _child.cancel(asof);
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet) 
+      {
+         _packet = packet;
+      }
+
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+         base.on_child_fill(child, fill);
+                  
+         var side = Utils.opposite_side(fill.order.side);
+         uint size = (uint)(fill.size*this.hedge_factor);
+         double price = (side == Side.Buy) ?
+            fill.price - this.profit_ticks :
+            fill.price + this.profit_ticks;
+
+         send_hedge(fill.order.security_desc, side, price, size, fill.TransactTime);
+      }
+   }
+}
@@ -0,0 +1,13 @@
+
+
+namespace bts.history
+{
+   static class IDGenerator
+   {
+      static uint __myid = 999_999;
+      static public uint next_order_id()
+      {
+         return (++__myid);
+      }
+   }
+}
@@ -0,0 +1,153 @@
+
+using System;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   public class JoinAlgo<Child> : CompositeAlgo<Child>
+      where Child : AlgoBase
+   {
+      enum AlgoState
+      {
+         Begin, WaitingForJoin, Joined, Done
+      }
+
+      //algo state
+      AlgoState _state;
+      OrderStatus _status;
+      api.Side _myside;
+      uint _mysize;
+      Timestamp _myasof;
+      
+      co2.BookMessage _snapshot, _currentbook;
+      int _turncount, _pilecount;      
+
+      public JoinAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      //algo parameters      
+      public bool initial_pile { get; set; } = false;
+      public bool should_chase { get; set; } = true;
+      public double join_factor { get; set; } = 1.0;
+      public uint join_quantity{ get; set; } = 0;
+
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _state = AlgoState.Begin;
+         _myside = side;
+         _mysize = size;
+         _myasof = asof;
+         _child = null;
+      }
+
+      public override bool is_completed()
+      {
+         return _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         var b = base.report();
+         return $"{b} Turns,Piles,Join(Size,Price,piq)={_turncount},{_pilecount},({_status?.state?.Quantity},{_status?.state?.Price})";
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _status = stat;
+         if(stat.is_terminal())
+            _state = AlgoState.Done;
+      }
+      
+      bool has_turned()
+      {
+         bool turn = ((_myside == api.Side.Buy &&
+                       _currentbook.side_at(_myside, 0) > _snapshot.side_at(_myside, 0)) || //new top level bid price greater than old, market turn
+                      (_myside == api.Side.Sell &&
+                       _currentbook.side_at(_myside, 0) < _snapshot.side_at(_myside, 0))); //new top level ask price less than old, market turn            
+         return turn;
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         if(_snapshot == null)
+            _snapshot = current_book.Clone();            
+         _currentbook = current_book;
+         switch (_state)
+         {
+            case AlgoState.Begin:
+               {                  
+                  if(initial_pile)
+                  {
+                     var thresh = (this.join_quantity > 0 ? this.join_quantity : this.join_factor * _mysize);
+                     if(_currentbook.sidesize_at(_myside, 0) >= thresh)
+                     {
+                        _pilecount++;
+                        deploy_join_function(packet, 
+                                       security_desc, _currentbook.side_at(_myside, 0));
+                        _state = AlgoState.Joined;
+                        _snapshot = _currentbook.Clone();
+                     }
+                  }
+                  else
+                     _state = AlgoState.WaitingForJoin;
+               }
+               break;
+            case AlgoState.WaitingForJoin:
+               {
+                  bool turn = has_turned();                     
+                  var thresh = (this.join_quantity > 0 ? this.join_quantity : this.join_factor * _mysize);
+                  if(turn &&_currentbook.sidesize_at(_myside, 0) >= thresh)
+                  {
+                     _turncount++;                     
+                     deploy_join_function(packet,  
+                                    security_desc, _currentbook.side_at(_myside, 0));
+                     _state = AlgoState.Joined;
+                  }
+                  _snapshot = _currentbook.Clone();
+               }               
+               break;
+            case AlgoState.Joined:
+               {
+                  if (this.should_chase)
+                  {
+                     bool turn = has_turned();
+                     var thresh = (this.join_quantity > 0 ? this.join_quantity : this.join_factor * _mysize);
+                     if (turn && _currentbook.sidesize_at(_myside, 0) >= thresh)
+                     {
+                        _turncount++;
+                        deploy_join_function(packet,  
+                                       security_desc, _currentbook.side_at(_myside, 0));
+                        _snapshot = _currentbook.Clone();
+                     }                  
+                  }
+               }
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+
+
+      void deploy_join_function(MarketDataIncrementalRefresh packet, string security_desc, double joinprice)
+      {
+         if(_state == AlgoState.Joined || _state == AlgoState.WaitingForJoin)
+         {
+            Log.Testing.info("deploy_join_function ");
+
+            if(_child == null)
+            {
+               _child = this.activate<Child>(new[]{security_desc}, _myside, joinprice, _mysize, packet.timestamp());
+            }
+            else
+            {
+               _child.update(packet.timestamp(), joinprice);               
+            }
+         }
+      }
+   }
+}
@@ -0,0 +1,122 @@
+
+using System;
+using System.Collections.Generic;
+using mbts.co2;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public class LeanAlgo<Child> : CompositeAlgo<Child>
+      where Child : AlgoBase
+   {
+      enum AlgoState
+      {
+         LeanWatch, Payup, Done
+      }
+
+      //algo state
+      AlgoState _state;
+      OrderStatus _status;
+      api.Side _myside;
+      double _myprice;
+      uint _mysize;      
+            
+      co2.BookMessage _snapshot, _currentbook;
+      int _payupcount;
+
+      public LeanAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      //algo parameters
+      public double lean_factor { get; set; } = 3.0;
+      public uint lean_quantity { get; set; } = 0;
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _state = AlgoState.LeanWatch;
+         _myside = side;
+         _mysize = size;
+         _myprice = price;         
+         _child = this.activate<Child>(security_list, side, price, size, asof);
+      }
+      
+      public override bool is_completed()
+      {
+         return _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         return _payupcount;
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _status = stat;
+         if (stat.is_terminal())
+            _state = AlgoState.Done;
+      } 
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         if (_snapshot == null)
+            _snapshot = current_book.Clone();
+         _currentbook = current_book;
+
+         switch (_state)
+         {
+            case AlgoState.LeanWatch:
+               {
+                  bool payupprice = false, payupsize = false;
+                  var leansize = (this.lean_quantity == 0) ? this.lean_factor * this._mysize : this.lean_quantity;
+                  if (_myside == api.Side.Buy) 
+                  {
+                     double best_ask = _snapshot.ask_at(0);
+                     var best_ask_size = _snapshot.asksize_at(0);
+                     payupprice = _currentbook.ask_at(0) > best_ask;
+                     payupsize = best_ask_size < leansize;
+                  }
+                  else
+                  {
+                     double best_bid = _snapshot.bid_at(0);
+                     var best_bid_size = _snapshot.bidsize_at(0);
+                     payupprice = _currentbook.bid_at(0) < best_bid;
+                     payupsize = best_bid_size < leansize;
+                  }
+                  if (payupprice || payupsize)
+                  {
+                     _payupcount++;
+                     _state = AlgoState.Payup;
+                     payup_function(packet, security_desc);
+                  }
+               }
+               break;
+            case AlgoState.Payup:
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+
+      void payup_function(MarketDataIncrementalRefresh packet, string security_desc)
+      {
+         if(_state == AlgoState.Payup)
+         {
+            double newprice = 0.0;
+            //plow through
+            if (_myside == api.Side.Sell) 
+               newprice = _currentbook.all_bids().Min();
+            else
+               newprice = _currentbook.all_asks().Max();
+            //this should precipitate a fill
+            _child.update(packet.timestamp(), newprice);     
+            
+            Log.Testing.info($"payup_function  ");
+         }
+      }      
+   }
+}
@@ -0,0 +1,53 @@
+using System;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public class LimitOrder : AlgoBase
+   {
+      public LimitOrder(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+
+      protected OrderEntry _myorder;
+      OrderStatus _status;
+      uint _mysize;
+
+      public override object report() { return string.Empty; }
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)      
+      {
+         _mysize = size;
+         Log.assert(security_list.Length == 1);
+         var security_desc = security_list[0];
+         Action<FillEntry> fill_action = f =>
+         {
+            if (this.on_fill != null)
+               this.on_fill(f);
+         };
+         Action<OrderStatus> status_action = s =>
+         {
+            _status = s;
+            if (this.on_status != null)
+               this.on_status(s.clone());
+         };
+         Log.assert(_myorder == null);
+         _myorder = new OrderEntry(asof, order_key, security_desc, side, price, size, fill_action, status_action);
+         _sink.send(_myorder);
+      }
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         if(size == _mysize)
+            size = uint.MinValue;
+         _sink.update(_myorder.security_desc, asof, _myorder.order_id, price, size);
+      }
+      protected override void do_cancel(Timestamp asof)
+      {
+         if(_myorder == null)
+            return;
+         _sink.cancel(_myorder.security_desc, asof, _myorder.order_id);
+      }
+
+      public override bool is_completed() => _status?.is_terminal() ?? false;
+   }
+}
@@ -0,0 +1,18 @@
+using System;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public class MarketOrder : LimitOrder
+   {
+      public MarketOrder(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)      
+      {
+         base.do_launch(security_list, side, price, size, asof, order_key);
+         _myorder.market_order = true;
+      }
+   }
+}
@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace bts.history
+{
+   static class AlgoTestNativeMethods
+   {
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_launch_callback(byte* OrderDefinition, ulong length, uint order_key, double price, uint size);
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_update_callback(byte* Update, ulong length, uint order_key);
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_cancel_callback(uint order_key);
+
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_status_callback(byte* OrderState, ulong length, uint order_key);
+      [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
+      internal unsafe delegate void child_fill_callback(byte* Fill, ulong length, uint order_key);
+
+      [DllImport("libspark")]
+      internal static extern IntPtr try_create_algo(api.orderman.CustomOrderDefinition.CustomOneofCase algo_type);
+
+      [DllImport("libspark")]
+      internal static extern IntPtr create_algo_driver(byte[] Security, int seclength,
+                                                       byte[] OrderDefinition, int deflength, 
+                                                       child_launch_callback launch_cb,
+                                                       child_update_callback update_cb,
+                                                       child_cancel_callback cancel_cb,
+                                                       
+                                                       child_status_callback stat_cb,
+                                                       child_fill_callback fill_cb);
+      [DllImport("libspark")]
+      internal static extern void destroy_algo_driver(IntPtr algo_driver);
+      [DllImport("libspark")]
+      internal static extern void destroy_algo(IntPtr algo_driver);
+
+      [DllImport("libspark")]
+      internal static extern void launch_algo(IntPtr algo_driver, uint order_key, double price, uint size);
+
+      [DllImport("libspark")]
+      internal static extern void update_price(IntPtr algo_driver, double price);
+      [DllImport("libspark")]
+      internal static extern void update_size(IntPtr algo_driver, uint size);
+      [DllImport("libspark")]
+      internal static extern void cancel(IntPtr algo_driver);
+
+
+      [DllImport("libspark")]
+      internal static extern void advance_time(IntPtr algo_driver, ulong microsecond, uint steps);
+      [DllImport("libspark")]
+      internal static extern void update_book(IntPtr algo_driver, uint security_id, byte[] BookMessage, int length);
+      [DllImport("libspark")]
+      internal static extern void update_ticker(IntPtr algo_driver, uint security_id, byte[] TradeMessage, int length);
+
+      [DllImport("libspark")]
+      internal static extern void fill(IntPtr algo_driver, uint order_key, double price, uint quantity);
+
+
+      //see AlgoTestDriver.h
+      internal enum AlgoDriverState
+      {
+         CREATED,
+         // before_Launch() ->
+         READY,
+         // launch() ->
+         RUNNING,
+         CANCELED,
+         TERMINATED,
+         COMPLETED
+      };
+      
+      [DllImport("libspark")]
+      internal static extern AlgoDriverState algo_driver_state(IntPtr algo_driver);
+      
+      internal static bool state_is_terminal(AlgoDriverState st)
+      {
+         return st == AlgoDriverState.CANCELED || st == AlgoDriverState.TERMINATED || st == AlgoDriverState.COMPLETED;
+      }
+   }
+}
@@ -0,0 +1,314 @@
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using Google.Protobuf;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   public class NativeAlgo : CompositeAlgo<AlgoBase>
+   {
+      readonly Dictionary<ulong, api.orderman.OrderState> _order_key_vs_state = new Dictionary<ulong, api.orderman.OrderState>();
+      api.Side _myside;
+      double _myprice;
+      uint _mysize;
+      Timestamp _myasof;
+
+      api.Security _sec;
+      readonly api.orderman.OrderDefinition _def;      
+      IntPtr _proxy;
+
+      readonly AlgoTestNativeMethods.child_launch_callback _launchcb;
+      readonly AlgoTestNativeMethods.child_update_callback _updatecb;
+      readonly AlgoTestNativeMethods.child_cancel_callback _cancelcb;
+      
+      readonly AlgoTestNativeMethods.child_status_callback _statcb;
+      readonly AlgoTestNativeMethods.child_fill_callback _fillcb;
+
+      MessageParser<api.orderman.OrderDefinition> _defparser = new MessageParser<api.orderman.OrderDefinition>(() => new api.orderman.OrderDefinition());
+      MessageParser<api.orderman.Update> _updateparser = new MessageParser<api.orderman.Update>(() => new api.orderman.Update());
+      MessageParser<api.orderman.OrderState> _stateparser = new MessageParser<api.orderman.OrderState>(() => new api.orderman.OrderState());
+      MessageParser<api.orderman.Fill> _fillparser = new MessageParser<api.orderman.Fill>(() => new api.orderman.Fill());
+      OrderEntry _fake_order_for_status;
+
+
+      public NativeAlgo(IOrderSink sink, string[] security_list, 
+                        api.orderman.OrderDefinition def) : base(sink, security_list) 
+      {
+         _def = def;
+         unsafe
+         {
+            _launchcb = 
+               (byte * OrderDefinition, ulong length, uint order_key, double price, uint size) => { on_native_launch(OrderDefinition, length, order_key, price, size); } ;
+            _updatecb =
+               (byte* Update, ulong length, uint order_key) => { on_native_update(Update, length, order_key); };
+            _cancelcb =
+               (uint order_key) => { on_native_cancel(order_key); };
+
+            _statcb = 
+               (byte * OrderState, ulong length, uint order_key) => { on_native_status(OrderState, length, order_key); } ;
+            _fillcb = 
+               (byte * Fill, ulong length, uint order_key) => { on_native_fill(Fill, length, order_key); } ;
+         }
+         init_proxy(security_list);
+      }
+      
+      void init_proxy(string[] security_list)
+      {
+         Log.assert(security_list != null && security_list.Length == 1);
+         _sec = new api.Security { Id = _def.SecurityKey, Symbol = security_list[0] };
+         if (_sec.Symbol.Contains(':') || _sec.Symbol.Contains('-'))
+            _sec.Spread = new api.SpreadContract() { Normalized = true };
+         else
+            _sec.Future = new api.FutureContract();
+         reset();
+      }
+      
+      void reset()
+      {
+         unsafe
+         {
+            if(_proxy != IntPtr.Zero)
+            {
+               AlgoTestNativeMethods.destroy_algo_driver(_proxy);
+               _order_key_vs_state.Clear();
+            }
+            
+            var secser = _sec.ToByteArray();
+            var defser = _def.ToByteArray();
+            _proxy = AlgoTestNativeMethods.create_algo_driver(secser, secser.Length, defser, defser.Length,
+                                                              _launchcb, _updatecb, _cancelcb,
+                                                              _statcb, _fillcb);
+            Log.assert(_proxy != IntPtr.Zero);
+         }
+      }
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         Log.assert(security_list != null && security_list.Length == 1);
+         Log.assert(side == _def.Side);         
+         _myside = side;         
+         _myprice = price;
+         _mysize = size;
+         _myasof = asof;
+         AlgoTestNativeMethods.launch_algo(_proxy, (uint)order_key, price, size);
+         _fake_order_for_status = new OrderEntry(asof, order_key, security_list[0], side, price, size, null, null, "Native");
+      }
+
+      Handle hookup_child(AlgoBase child, string[] security_list, api.Side side, 
+                          double price, uint size, Timestamp asof, uint order_key) 
+      {
+         Action<Handle, FillEntry> of = new Action<Handle, FillEntry>((h, f) => 
+            on_child_fill(h, f));
+         Action<Handle, OrderStatus> os = new Action<Handle, OrderStatus>((h, s) =>
+         {
+            on_child_status(h, s);
+            if (s.is_terminal())
+               _children.Remove(h);
+         });
+         var handle = this.hookup_child(child, security_list, side, price, size, asof, of, os, order_key);
+         _children.Add(handle);
+         return handle;
+      }
+
+      // callback when this child launches other guys
+      unsafe void on_native_launch(byte* OrderDefinition, ulong length, uint order_key, double price, uint size)
+      {
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)OrderDefinition, arr, 0, (int)length);
+
+         api.orderman.OrderDefinition def = _defparser.ParseFrom(arr);         
+         switch(def.CustomDef.CustomCase)
+         {            
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Market:
+               {
+                  var child = AlgoBase.algo_factory(typeof(MarketOrder), _sink, this.security_list);
+                  hookup_child(child, this.security_list, def.Side, price, size, _myasof, order_key);
+               }
+               break;
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Limit:
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.GlbxIceberg:
+               {
+                  var child = AlgoBase.algo_factory(typeof(LimitOrder), _sink, this.security_list);
+                  hookup_child(child, this.security_list, def.Side, price, size, _myasof, order_key);
+               }
+               break;
+
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Stop:
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.StopLimit:
+               Log.assert(false);
+               break;
+
+            case api.orderman.CustomOrderDefinition.CustomOneofCase.Dynamic:
+               Log.assert(false);
+               break;
+            default:
+               {
+                  IntPtr native = AlgoTestNativeMethods.try_create_algo(def.CustomDef.CustomCase);
+                  if(native != IntPtr.Zero)
+                  {
+                     AlgoTestNativeMethods.destroy_algo(native); //TODO, boolean
+                     var child = new NativeAlgo(_sink, this.security_list, def);
+                     hookup_child(child, this.security_list, def.Side, price, size, _myasof, order_key);
+                     // var handle = this.hookup_child(child, security_list, def.Side, price, size, _myasof, null, null, order_key);
+                     // _children.Add(handle);
+                  }               
+               }
+            break;
+         }
+      }
+
+      unsafe void on_native_update(byte* Update, ulong length, uint order_key)
+      {
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)Update, arr, 0, (int)length);
+         api.orderman.Update upd = _updateparser.ParseFrom(arr);
+
+         var f = _children.FirstOrDefault(h => h.order_key == order_key);
+         if(f != null)
+            f.update(_lastmd.timestamp(), upd.Price, upd.Quantity);            
+      }
+
+      unsafe void on_native_cancel(uint order_key)
+      {
+         var f = _children.FirstOrDefault(h => h.order_key == order_key);
+         if(f != null)
+            f.cancel(_lastmd.timestamp());
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _order_key_vs_state[child.order_key] = stat.state;
+         process_status(stat.state);
+      }
+
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+         AlgoTestNativeMethods.fill(_proxy, (uint)fill.order.order_id, fill.price, (uint)fill.size);
+      }
+
+      unsafe void on_native_status(byte* OrderState, ulong length, uint order_key)
+      {
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)OrderState, arr, 0, (int)length);
+
+         api.orderman.OrderState curr = _stateparser.ParseFrom(arr);
+         _order_key_vs_state[order_key] = curr;
+        process_status(curr);
+      }
+      
+
+      private void process_status(api.orderman.OrderState curr)
+      {
+         bool done = is_completed();
+         api.orderman.OrderState aggstate = new api.orderman.OrderState
+         {
+            Status = done ? api.orderman.OrderStatus.Completed : curr.Status,
+            Price = curr.Price,
+            Quantity = curr.Quantity,
+            AvgPrice = _order_key_vs_state.Values.Average(x => x.Price),
+            FilledQuantity = _order_key_vs_state.Values.Sum(x => x.FilledQuantity),
+            LastChange = curr.LastChange,
+            LastMsg = curr.LastMsg,
+            LastPrice = curr.LastPrice
+         };
+         OrderStatus agg = new OrderStatus(_fake_order_for_status, aggstate);
+         agg.TransactTime = curr.LastChange;
+         agg.SendingTime = curr.LastChange;
+         if (this.on_status != null)
+            this.on_status(agg);
+      }
+
+      unsafe void on_native_fill(byte* Fill, ulong length, uint order_key)
+      {         
+         byte[] arr = new byte[length];
+         Marshal.Copy((IntPtr)Fill, arr, 0, (int)length);
+
+         api.orderman.Fill fill = _fillparser.ParseFrom(arr);
+         //this method is called as a result of us sending a fill down in AlgoTestNativeMethods.fill so we do nothing
+                  
+      }
+      
+      ~NativeAlgo()
+      {
+         AlgoTestNativeMethods.destroy_algo_driver(_proxy);
+      }
+
+      public override bool is_completed()
+      {
+         bool done = false;
+         if(_order_key_vs_state.Count > 0)
+         {
+            var astate = AlgoTestNativeMethods.algo_driver_state(_proxy);
+            done = AlgoTestNativeMethods.state_is_terminal(astate);
+            foreach(var st in _order_key_vs_state.Values)
+               done &= OrderStatus.is_terminal(st);
+         }
+         return done;
+      }
+
+      public override object report()
+      {         
+         return $"Native";
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         base.do_update(asof, price, size);
+         AlgoTestNativeMethods.update_price(_proxy, price);
+         if(size != uint.MinValue)
+         {
+            AlgoTestNativeMethods.update_size(_proxy, size);
+         }
+      }
+      
+      protected override void do_cancel(Timestamp asof)
+      {
+         base.do_cancel(asof);
+         AlgoTestNativeMethods.cancel(_proxy);         
+      }
+
+      MarketDataIncrementalRefresh _lastmd;
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                      MarketDataIncrementalRefresh packet)
+      {
+         security_key = _def.SecurityKey; //to fake this up for the AlgoTestDriver
+         var book = current_book.ToByteArray();
+         AlgoTestNativeMethods.update_book(_proxy, security_key, book, book.Length);
+
+         advance_time(packet);
+      }
+
+      protected override void on_trades(string security_desc, uint security_key, IEnumerable<co2.TradeMessage> trades,
+                                       MarketDataIncrementalRefresh packet)
+      {
+         security_key = _def.SecurityKey;
+         foreach(var trade in trades)
+         {
+            var tr = trade.ToByteArray();
+            AlgoTestNativeMethods.update_ticker(_proxy, security_key, tr, tr.Length);
+         }
+
+         advance_time(packet);            
+      }
+
+      void advance_time(MarketDataIncrementalRefresh packet)
+      {
+         if (_lastmd != null)
+         {
+            var dt = packet.timestamp() - _lastmd.timestamp();
+            if (dt.Seconds >= 0 && dt.Nanos > 0)
+            {
+               ulong us = (ulong)dt.Seconds * 1000_000ul + ((ulong)dt.Nanos / 1000ul);
+               AlgoTestNativeMethods.advance_time(_proxy, us, 1);
+            }
+         }
+         _lastmd = packet;
+      }            
+   }
+}
@@ -0,0 +1,22 @@
+
+using System;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public class NullAlgo : AlgoBase
+   {
+      public NullAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      public override bool is_completed() => true;
+
+      public override object report() { return string.Empty; }
+
+      protected override void do_launch(string[] security_list, Side side, double price, uint size, Timestamp asof, ulong order_key) { }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue) { }
+
+      protected override void do_cancel(Timestamp asof) { }
+   }
+}
@@ -0,0 +1,246 @@
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   public class SpreaderAlgo<Child, Hedge> : CompositeHedgeAlgo<Child, Hedge>
+      where Child : AlgoBase
+      where Hedge : AlgoBase
+   {
+      enum AlgoState
+      {
+         Begin, Quoting, Done
+      }
+
+      //algo state
+      string[] _security_list;
+      AlgoState _state;
+      double _myprice;
+      uint _mysize;
+      bool _sizeupdate;
+      api.Side _myside;
+      
+      SortedList<string, co2.BookMessage> _current_book = new SortedList<string, co2.BookMessage>();
+      SortedList<string, co2.BookMessage> _working_book = new SortedList<string, co2.BookMessage>();
+      OrderStatus _buystatus, _sellstatus;
+      Handle _buychild;
+      Handle _sellchild;
+      
+      private MarketDataIncrementalRefresh _packet;
+
+      public SpreaderAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      //algo parameters      
+      public double profit_ticks { get; set; } = 0.0;
+      public double[] tick_size { get; set; }
+      public double[] hedge_factors { get; set; }
+
+      SortedList<string, double> _tick_size = new SortedList<string, double>();
+      SortedList<string, double> _hedge_factors = new SortedList<string, double>();
+      string _quote_leg;
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _security_list = security_list;    
+         for(int i = 0; i < _security_list.Length; i++)
+         {
+            _current_book[_security_list[i]] = null;
+            _working_book[_security_list[i]] = null;
+            _tick_size[_security_list[i]] = tick_size[i];
+            _hedge_factors[_security_list[i]] = hedge_factors[i];
+            _quote_leg = _security_list[i]; //last
+         }     
+         _state = AlgoState.Begin;
+         _myprice = price;
+         _mysize = size;
+         _myside = side;
+         _buychild = null;
+         _sellchild = null;
+      }
+
+      protected override void do_update(Timestamp asof, double price, uint size = uint.MinValue)
+      {
+         _myprice = price;
+         if(size > uint.MinValue)
+         {
+            _mysize = size;
+            _sizeupdate = true;
+         }
+      }      
+
+      public override bool is_completed()
+      {
+         return base.is_completed() && _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         var b = base.report();
+         string qtleg = _security_list.Last();
+         // var buypiq = _sink.get_my_piq(qtleg, _buychild?.order_key ?? 0);
+         // var sellpiq = _sink.get_my_piq(qtleg, _sellchild?.order_key ?? 0);
+         // return $"{b} Quoting {_quote_leg} (bsz,bpx,bpiq ssz,spx,spiq) = ( {_buystatus?.state?.Quantity},{_buystatus?.state?.Price},{buypiq} at {_sellstatus?.state?.Quantity},{_sellstatus?.state?.Price},{sellpiq} )";
+         return b;
+      }
+
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         if(stat.side == api.Side.Buy)
+         {
+            _buystatus = stat;
+            if(stat.is_terminal())
+               _sellchild?.cancel(_buystatus.SendingTime);
+         }
+         else 
+         {
+            _sellstatus = stat;
+            if(stat.is_terminal())
+               _buychild?.cancel(_sellstatus.SendingTime);
+         }
+         //hedges are looked after elsewhere         
+         if( (_buystatus?.is_terminal() ?? true) && (_sellstatus?.is_terminal() ?? true) )
+            _state = AlgoState.Done;
+      }
+      
+      protected override void on_child_fill(Handle child, FillEntry fill)
+      {
+         var ff = factor_side(this._hedge_factors[_quote_leg]);
+         bool invert = ff != fill.order.side;
+
+         foreach(var sec in _security_list)
+         {
+            if(sec == _quote_leg)
+               continue;
+                     
+            var hedgeside = factor_side(this._hedge_factors[sec], invert);
+               
+            uint size = (uint)(fill.size * Math.Abs(this._hedge_factors[sec]));
+            double price = (hedgeside == api.Side.Buy) ? _current_book[sec].ask_at(0) : _current_book[sec].bid_at(0);
+            send_hedge(sec, hedgeside, price, size, fill.TransactTime);
+         }
+      }
+
+      api.Side factor_side(double mult, bool invert = false)
+      {
+         api.Side sd = api.Side.Undefined;
+         if(mult >= 0.0)
+            sd = api.Side.Buy;
+         else
+            sd = api.Side.Sell;
+         if(invert)
+            sd = Utils.opposite_side(sd);
+         return sd;
+      }
+
+      static double round_tick_up(double input, double tick)
+      {
+         return Math.Ceiling(input / tick) * tick;
+      }
+      static double round_tick_dn(double input, double tick)
+      {
+         return Math.Floor(input / tick) * tick;
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         _packet = packet;
+         _current_book[security_desc] = current_book;
+         _working_book[security_desc] = working_book;
+         var look_book = _current_book;
+         switch (_state)
+         {
+            case AlgoState.Begin:
+               {                  
+                  if(!_current_book.Values.Any(x => x == null) && !_working_book.Values.Any(x => x == null))
+                     _state = AlgoState.Quoting;
+               }
+               break;
+            case AlgoState.Quoting:
+               {
+                  uint sz = 0;
+                  
+                  List<bool> sides = new List<bool>();
+                  if(_myside == api.Side.Buy)
+                     sides.Add(false);
+                  else if (_myside == api.Side.Sell)
+                     sides.Add(true);
+                  else
+                  {
+                     sides.Add(false);
+                     sides.Add(true);
+                  }
+                  foreach(var invert in sides)
+                  {
+                     double qt = invert ? -_myprice : _myprice;  
+                     api.Side quoteside = factor_side(this._hedge_factors[_quote_leg], invert);
+                     foreach(var sec in _security_list)
+                     {
+                        var book = look_book[sec];                        
+                        if (sec != _quote_leg)
+                        {                
+                           var hedge_side = factor_side(this._hedge_factors[sec], invert); 
+                           var cross_side = Utils.opposite_side(hedge_side);
+                           qt += this._hedge_factors[sec] * book.side_at(cross_side, 0);
+                           sz = Math.Min(_mysize, (uint) (book.sidesize_at(cross_side, 0) / Math.Abs(this._hedge_factors[sec])) ); 
+                        }
+                        else
+                        {
+                           double tick = _tick_size[sec];
+                           if(quoteside == api.Side.Sell)
+                           {
+                              qt += profit_ticks;
+                              qt = round_tick_up(qt,tick);
+                              //qt = Math.Max(qt, current_book.ask_at(0));                              
+                           }
+                           else
+                           {
+                              qt -= profit_ticks;
+                              qt = round_tick_dn(qt,tick);
+                              //qt = Math.Min(qt, current_book.bid_at(0));
+                           }
+                        }
+                     }
+                     
+                     if (!double.IsNaN(qt) && sz > 0 && security_desc == _quote_leg)
+                     {
+                        if(quoteside == api.Side.Buy && (_buystatus?.state?.Price != qt || _sizeupdate) && (!_buystatus?.is_terminal() ?? true))
+                           quote_function(packet, _quote_leg, quoteside, qt, sz);
+                        else if(quoteside == api.Side.Sell && (_sellstatus?.state?.Price != qt || _sizeupdate) && (!_sellstatus?.is_terminal() ?? true))
+                           quote_function(packet, _quote_leg, quoteside, qt, sz);
+                     }
+                  }
+               }               
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }      
+      void quote_function(MarketDataIncrementalRefresh packet, string qtleg, 
+         api.Side quoteside, double qt, uint sz)
+      {
+         if (quoteside == api.Side.Buy)
+         {
+            if (_buychild == null)
+               _buychild = this.activate<Child>(new[] { qtleg }, quoteside, qt, sz, packet.timestamp());
+            else 
+               _buychild.update(packet.timestamp(), qt, sz);
+         }         
+         else 
+         {
+            if (_sellchild == null)
+               _sellchild = this.activate<Child>(new[] { qtleg }, quoteside, qt, sz, packet.timestamp());
+            else 
+               _sellchild.update(packet.timestamp(), qt, sz);            
+         }
+         _sizeupdate = false;
+      }
+   }
+}
@@ -0,0 +1,95 @@
+
+using System;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   public class TopAlgo<Child> : CompositeAlgo<Child>
+      where Child : AlgoBase
+   {
+      enum AlgoState
+      {
+         WaitingToCancel, CancelRequested, Done
+      }
+
+      //algo state
+      AlgoState _state;
+      OrderStatus _status;
+      api.Side _myside;
+      uint _mysize;
+      
+      co2.BookMessage _currentbook;
+      int _topcount;
+
+      public TopAlgo(IOrderSink sink, string[] security_list) : base(sink, security_list) { }
+      //algo parameters      
+      public double top_factor { get; set; } = 1.0;
+      public uint top_quantity { get; set; } = 0;
+
+      protected override void do_launch(string[] security_list, api.Side side, double price, uint size, Timestamp asof, ulong order_key)
+      {
+         _state = AlgoState.WaitingToCancel;
+         _myside = side;
+         _mysize = size;
+         _child = this.activate<Child>(security_list, side, price, size, asof);
+      }      
+
+      public override bool is_completed()
+      {
+         return _state == AlgoState.Done;
+      }
+
+      public override object report()
+      {
+         var b = base.report();
+         return $"{b} Cancels { _topcount }";
+      }
+      
+      protected override void on_child_status(Handle child, OrderStatus stat)
+      {
+         _status = stat;
+         if (stat.is_terminal())
+            _state = AlgoState.Done;
+      }
+
+      protected override void on_book(string security_desc, uint security_key, co2.BookMessage current_book, co2.BookMessage working_book,
+                                    MarketDataIncrementalRefresh packet)
+      {
+         _currentbook = current_book;
+         switch (_state)
+         {
+            case AlgoState.WaitingToCancel: 
+               {
+                  bool top = _currentbook.side_at(_myside, 0) == _status?.state?.Price;
+                  var thresh = (this.top_quantity > 0 ? this.top_quantity : this.top_factor * _mysize);
+                  if(top && _currentbook.sidesize_at(_myside, 0) < thresh)
+                  {
+                     cancel_top_function(packet, 
+                                    security_desc, _currentbook.side_at(_myside, 0));                     
+                     _state = AlgoState.CancelRequested;
+                     _topcount++;
+                  }
+               }               
+               break;
+            case AlgoState.CancelRequested:
+               break;
+            case AlgoState.Done:
+               break;
+            default:
+               Log.assert(false, "invalid algo state");
+               break;
+         }
+      }
+
+      void cancel_top_function(MarketDataIncrementalRefresh packet, string security_desc, object param)
+      {
+         Log.assert(_state == AlgoState.CancelRequested || _state == AlgoState.Done);
+         Log.Testing.info("cancel_top_function ");
+
+         _child.cancel(packet.timestamp());
+         _state = AlgoState.Done;
+      }
+   }
+}
@@ -0,0 +1,73 @@
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+
+   public class JoinCoverLean : CoverAlgo<JoinAlgo<TopAlgo<LimitOrder>>, LeanAlgo<LimitOrder>>
+   {
+      public JoinCoverLean(IOrderSink sink, string[] security_list) 
+         : base(sink, security_list) 
+      {
+      }
+   }
+
+   public class PegCoverLean : CoverAlgo<JoinAlgo<TopAlgo<LimitOrder>>, LeanAlgo<LimitOrder>>
+   {
+      public PegCoverLean(IOrderSink sink, string[] security_list) 
+         : base(sink, security_list) 
+      {
+         this.set_param(nameof(JoinAlgo<LimitOrder>.initial_pile), true.ToString());
+      }
+   }
+   public class Spreader : SpreaderAlgo<LimitOrder, LeanAlgo<LimitOrder>>
+   {
+      public Spreader(IOrderSink sink, string[] security_list) 
+         : base(sink, security_list) 
+      {          
+      }
+   }
+
+   public class TWAP : NativeAlgo
+   {
+      public TWAP(IOrderSink sink, string[] security_list) 
+         : base(sink, security_list, build_def())
+      {
+      }      
+      static api.orderman.OrderDefinition build_def()
+      {
+         api.orderman.OrderDefinition def = new api.orderman.OrderDefinition();
+         def.SecurityKey = 51880u;
+         def.Side = api.Side.Buy;
+         def.CustomDef = new api.orderman.CustomOrderDefinition { Twap = new api.orderman.TWAPOrderDefinition() };
+         def.CustomDef.Twap.Duration = new api.orderman.TimeDuration() { DurationInMinutes = 6, IntervalInSeconds = 10 } ;
+         def.CustomDef.Twap.Aggression = new api.orderman.OrderAggression() { AllowedScheduleSlippage = 1 };
+         
+         def.CustomDef.Twap.TacticalChild = new api.orderman.CustomOrderDefinition
+         {
+            Peg = new api.orderman.PegOrderDefinition()
+            {
+               Child = new api.orderman.CustomOrderDefinition { Limit = new api.orderman.LimitOrderDefinition() } ,
+               TickOffset = 0,
+            }
+         };      
+         
+         def.CustomDef.Twap.AggressiveChild = new api.orderman.CustomOrderDefinition
+         {
+            Lean = new api.orderman.LeanOrderDefinition()
+            {
+               Child = new api.orderman.CustomOrderDefinition { Limit = new api.orderman.LimitOrderDefinition() },
+               CrossThreshold = new spark.QuantityMeasure { Ratio = 3.0 } ,
+               TradeTowardBehavior = api.orderman.LeanOrderDefinition.Types.OnToward.NoChange,
+               MaxChaseTicks = 10,
+            }
+         };
+         return def;
+      }
+   }
+}
@@ -0,0 +1,33 @@
+using System;
+
+namespace bts.history
+{
+   internal class TradeStats
+   {
+      public DateTime security_timestamp { get; set; }
+      public string security_desc { get; set; }
+      public double our_average_buy_piq;
+      public double our_average_buy_q_len;
+      public double our_average_buy_price { get; set; }
+
+      public double our_average_sell_piq;
+      public double our_average_sell_q_len;
+      public double our_average_sell_price { get; set; }
+
+      public int our_total_buys { get; set; }
+      public int our_total_sells { get; set; }
+      public int our_total_volume { get; set; }
+      public double latest_pnl { get; set; }
+
+      public uint total_exchange_bought { get; set; }
+      public uint total_exchange_sold { get; set; }
+      public uint total_exchange_volume { get; set; }
+
+      public uint best_bid_size { get; set; }
+      public double best_bid { get; set; }
+      public double best_ask { get; set; }
+      public uint best_ask_size { get; set; }
+      public double last_trade_price { get; set; }
+      public uint last_trade_size { get; set; }
+   }
+}
@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System.Text;
+using System.IO;
+using mbts.config;
+using bts.config;
+
+namespace bts.history
+{
+   class BookAnalyzer : IAnalyzer
+   {
+      class SecData{
+         public uint n_updates = 0;
+         public System.IO.FileStream str;
+
+         public SecData(string full_path,
+                        string contract){
+            var contract_fr = contract.Replace(":",".");
+            string bk_file_name = String.Format ("{0}{1}",full_path,contract_fr);
+            str = new FileStream(bk_file_name, FileMode.OpenOrCreate);
+         }
+         public SecData (System.IO.FileStream str_a){
+            str = str_a;
+         }
+         public void inc_count(){
+            n_updates++;
+         }
+      }
+      private readonly Dictionary<string, co2.BookMessage> _books = 
+                     new Dictionary<string, co2.BookMessage>();
+      private readonly string _book_path;
+      IDictionary<string, SecData> _book_stream_map = 
+                              new Dictionary<string, SecData>();
+      private System.IO.FileStream _order_str, _trade_str;
+      
+      public BookAnalyzer(string symbol,
+                          HistoricalDataOptions options) {
+         _book_path = DataIOHelper.get_data_path("/books");
+         System.IO.Directory.CreateDirectory(_book_path);
+         _order_str = new FileStream(
+            String.Format ("{0}orders_{1}",_book_path,symbol),
+            FileMode.OpenOrCreate);
+         _trade_str = new FileStream(
+            String.Format ("{0}trades_{1}",_book_path,symbol),
+            FileMode.OpenOrCreate);
+      }
+
+      public void pre_analyze(MarketDataIncrementalRefresh md, 
+                              string debug_fix = null){
+         _write_orders(md);
+      }
+
+      public void analyze(MarketDataIncrementalRefresh md,
+                          string debug_fix = null)
+      {
+         foreach(var entry in md.MdEntries)
+         {
+            if(entry.MdEntryType == MDEntryType.Bid || 
+               entry.MdEntryType == MDEntryType.Offer)
+            {
+               if(!_books.TryGetValue(entry.Symbol, out co2.BookMessage book))
+               {
+                  book = new co2.BookMessage();
+                  Utils.init_book(book);
+                  _books[entry.Symbol] = book;
+               }
+               Utils.keep_book(book, entry);
+               if (Utils.quote_event_done(md))
+                  on_new_book(book, entry, md);
+            }
+            else if (entry.MdEntryType == MDEntryType.Trade)
+               on_new_trade(entry,md);
+         }
+         _write_orders(md);
+      }
+      private int _bn = 0;
+      private void on_new_book(co2.BookMessage book,
+                               MDEntry entry,
+                               MarketDataIncrementalRefresh md_inc){
+         ProcessedBook pb = new ProcessedBook();
+         pb.MsgSeqNum = md_inc.MsgSeqNum;
+         pb.SendingTime = md_inc.SendingTime;
+         pb.TransactTime = md_inc.TransactTime;
+         pb.Security = entry.Symbol;
+         pb.Book = book;        
+         SecData s;
+         if(!_book_stream_map.TryGetValue(pb.Security, out s)){
+            s = new SecData (_book_path,pb.Security);
+            _book_stream_map.Add(pb.Security,s);
+         }
+         pb.BookSeqNum = s.n_updates;
+         s.inc_count();
+         Google.Protobuf.MessageExtensions.WriteDelimitedTo(pb,s.str);
+         if(_bn++%10000==0)
+            Console.WriteLine("n_bks:{0}",_bn);
+      }
+
+      private int _n_ords = 0;
+      private HashSet<UInt64> _ord_map = new HashSet<UInt64>();
+      private void _write_orders(MarketDataIncrementalRefresh md){
+         foreach(var oid in md.OrderIdEntries)
+            if(oid.OrderUpdateAction==MDUpdateAction.New){          
+            //if(!_ord_map.Contains(oid.OrderId)){
+               NewOrderEntry new_ord = new NewOrderEntry();
+               new_ord.OrderId = oid.OrderId;
+               new_ord.SendingTime = md.SendingTime;
+               new_ord.TransactTime = md.TransactTime;
+               _ord_map.Add(oid.OrderId);
+               Google.Protobuf.MessageExtensions.WriteDelimitedTo(new_ord,_order_str);
+               if(_n_ords++%10000==0)
+                  Console.WriteLine("n_ords:{0}",_n_ords);
+            }
+      }
+
+      private void on_new_trade(MDEntry md_entry,
+                                MarketDataIncrementalRefresh md_inc){
+         string contract = md_entry.Symbol;                             
+         TradeEntry tr = new TradeEntry();
+         tr.Contract = contract;
+         tr.Price = md_entry.MdEntryPx;
+         tr.Qty = md_entry.MdEntrySize;
+         tr.TradeId = md_entry.MdTradeEntryId;
+         tr.OrderId = md_entry.OrderId;
+         tr.SendingTime = md_inc.SendingTime;
+         tr.TransactTime = md_inc.TransactTime;
+         Google.Protobuf.MessageExtensions.WriteDelimitedTo(tr,_trade_str);
+      }
+   }
+}
@@ -0,0 +1,136 @@
+using System;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   interface IAnalyzer 
+   {
+      //this data comes in before analysis time, but is useful to create complete books etc
+      void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null);
+      void analyze(MarketDataIncrementalRefresh md, string debug_fix = null);
+   }
+   
+   class QueueAnalyzer : IAnalyzer
+   {
+      const double _max_nanos = 200_000.0;      
+      readonly DistributionEstimator _turndist = new DistributionEstimator();
+      readonly DistributionEstimator _stackdist = new DistributionEstimator();
+      readonly MathNet.Numerics.Statistics.Histogram _turnhist = new MathNet.Numerics.Statistics.Histogram();
+      readonly MathNet.Numerics.Statistics.Histogram _stackhist = new MathNet.Numerics.Statistics.Histogram();
+
+      MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh();
+      Google.Protobuf.WellKnownTypes.Timestamp _laststacktime = new Google.Protobuf.WellKnownTypes.Timestamp();
+      Google.Protobuf.WellKnownTypes.Timestamp _lastturntime = new Google.Protobuf.WellKnownTypes.Timestamp();      
+      MDEntry _lastturn = new MDEntry();
+      MDEntry _lastentry = new MDEntry();
+      long _samples = 0;
+      long _stacksamples = 0;
+
+
+      readonly string _security_desc;
+      readonly MDEntryType _bid_or_ask;
+      public QueueAnalyzer(string security_desc) : this(security_desc, MDEntryType.Bid) {}
+      public QueueAnalyzer(string security_desc, MDEntryType bid_or_ask = MDEntryType.Bid)
+      {
+         _security_desc = security_desc;
+         _bid_or_ask = bid_or_ask;
+         foreach (var hist in new[] { _turnhist, _stackhist } )
+         {
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(0, 1000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(1000, 2000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(2000, 3000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(3000, 4000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(4000, 5000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(5000, 6000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(6000, 7000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(7000, 8000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(8000, 9000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(9000, 10000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(10000, 20000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(20000, 30000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(30000, 40000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(40000, 50000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(50000, 60000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(60000, 70000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(70000, 80000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(80000, 90000.0.Decrement()));
+            hist.AddBucket(new MathNet.Numerics.Statistics.Bucket(90000,  _max_nanos.Decrement()));
+            hist.LazySort();
+         }
+      }
+
+      public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
+      {
+      }
+
+      public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null)      
+      {            
+         var entries = md.MdEntries.Where(x => x.Symbol == _security_desc &&
+                                         (x.MdEntryType == _bid_or_ask) &&
+                                         (x.MdPriceLevel == 1) 
+                                    );
+         
+         foreach(var x in entries)
+         {
+            if( (_bid_or_ask == MDEntryType.Bid && x.MdEntryPx > _lastentry.MdEntryPx)  || //new top level bid price greater than old, market turn
+                (_bid_or_ask == MDEntryType.Offer && x.MdEntryPx < _lastentry.MdEntryPx) ) //new top level ask price less than old, market turn
+            {           
+               record_data(md, x, ref _lastturntime, _lastpacket.timestamp(), ref _samples, _turndist, _turnhist, "TURN");
+               _lastturn = x;
+            }
+            else if (x.MdEntryPx.AlmostEqual(_lastentry.MdEntryPx))
+            {
+               if (x.NumberOfOrders > _lastentry.NumberOfOrders) //price is same, norders increase
+               {
+                  record_data(md, x, ref _laststacktime, _lastturntime, ref _stacksamples, _stackdist, _stackhist, "STACK");
+               }
+            }
+            _lastentry = x;
+            _lastpacket = md; //inside the loop so we only get filtered packets
+         }                  
+      }
+
+      static void record_data(MarketDataIncrementalRefresh md, MDEntry entry,
+                              ref Google.Protobuf.WellKnownTypes.Timestamp last_time,
+                              Google.Protobuf.WellKnownTypes.Timestamp ref_time, ref long samples,
+                              DistributionEstimator dist, Histogram hist, string desc)
+      {
+         if(ref_time == null) 
+            return;
+         const double sumthresh = 10_000;
+         const double sumthresh2 = 25_000;
+         last_time = md.timestamp();
+         var delta = (md.timestamp() - ref_time);
+         double nanos = delta.Seconds * 1e9 + delta.Nanos;
+         if (delta.Seconds == 0 && delta.Nanos > 0 && delta.Nanos <= _max_nanos)
+         {
+            if (nanos > 100_000.0 || nanos < 1000)
+               nanos++; //for breakpoint                  
+            dist.add(nanos);
+            hist.AddData(nanos);
+            if (samples++ % 10 == 0)
+            {
+               Console.WriteLine($"{entry.Symbol} { entry.MdEntryType }  Trade date  {md.TradeDate.ToDateTime().ToShortDateString()}");
+               Console.WriteLine($"{desc} Current Sample {nanos}, Max filter { _max_nanos } { Environment.NewLine }");
+               Console.WriteLine($"{desc} Distribution {dist.ToString()}");
+               Console.WriteLine($"{desc} Histogram {Environment.NewLine + hist.ToString() }");
+               Console.WriteLine($"% < {sumthresh} {Environment.NewLine + (sumif(sumthresh, hist) / hist.DataCount).ToString("0.0%") }");
+               Console.WriteLine($"% < {sumthresh2} {Environment.NewLine + (sumif(sumthresh2, hist) / hist.DataCount).ToString("0.0%") }");
+               Console.WriteLine($"----------------------------------------------------------------------------- {Environment.NewLine}");
+            }
+         }
+      }
+
+      static double sumif(double sumthresh, Histogram hist)
+      {
+         double sum = 0.0;
+         for(int i = 0; i < hist.BucketCount; i++)
+         {
+            if(hist[i].UpperBound <= sumthresh)
+               sum += hist[i].Count;
+         }
+         return sum;
+      }
+   }
+}
@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+using Entry = System.Tuple<bts.api.SecurityType, bts.history.MarketDataIncrementalRefresh, string>;
+
+namespace bts.history
+{
+   class BackTester
+   {
+      readonly DataStreamer _stream;
+      MarketDataIncrementalRefresh _last = new MarketDataIncrementalRefresh() { SendingTime = new Timestamp(), TransactTime = new Timestamp() } ;
+      public BackTester()
+      {
+         _stream = new DataStreamer();
+      }
+
+      async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
+                               HistoricalDataOptions o)
+      {
+         start = start.ToUniversalTime();
+         end = end.ToUniversalTime();
+         var query = new MDQuery
+         {
+            Exchange = Venue.Glbx,
+            Symbol = symbol,
+            SecurityType = st,
+            Begin = Timestamp.FromDateTime(start),
+            End = Timestamp.FromDateTime(end),
+         };
+         if(DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
+            query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
+            
+         List<IAnalyzer> anal = new List<IAnalyzer>();
+         string algo_security = o.algo_security ?? string.Empty;
+         var t = System.Type.GetType(o.algo_type);
+         if (t == null)
+            t = System.Type.GetType("bts.history." + o.algo_type);
+         if (t != null)
+         {
+            anal.Add((IAnalyzer)Activator.CreateInstance(t, new[] { algo_security }));
+         }
+         else
+            anal.Add(new BookAnalyzer(algo_security, o));
+
+         int count = 0;
+         var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y) 
+            => x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime()));
+         await _stream.process_md_stream<MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
+        {           
+           if (md.timestamp().ToDateTime() < end)
+           {                            
+              bool preanal = (md.timestamp().ToDateTime() < start);
+              if(o.skip_preanalysis)
+                 preanal = false;
+              if(preanal && count++ % 50000 == 1)
+                 Console.WriteLine("Pre analysis " + md.timestamp().ToDateTime());
+               do_anal(md, anal, preanal, line);
+               Log.assert(md.timestamp().greater_or_equal( _last.timestamp() ));
+               _last = md;
+           }
+           else
+              throw new System.IO.EndOfStreamException();
+        }, comparer);
+      }
+      
+
+      void do_anal(MarketDataIncrementalRefresh md, IEnumerable<IAnalyzer> anal, bool is_before_analysis_time, string debug_fix = null)
+      {
+         if (is_before_analysis_time)
+         {
+            foreach (var d in anal)
+               d.pre_analyze(md, debug_fix);
+         }
+         else
+         {
+            foreach (var d in anal)
+               d.analyze(md, debug_fix);
+         }
+      }
+   }
+}
@@ -0,0 +1,241 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Threading.Tasks;
+using Amazon;
+using Amazon.S3;
+using Amazon.S3.Model;
+using bts.api;
+
+namespace bts.history
+{
+   class DataDownloader
+   {
+      private static AmazonS3Client _client;
+      //private const string _access_key = "AKIAJ4XUDWPQ7M42352A";
+      //private const string _secret_key = "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";
+      private const string _access_key = "AKIAIFIGOOWGBLEMO3PQ";
+      private const string _secret_key = "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";
+      static DataDownloader()
+      {
+         AmazonS3Config config = new AmazonS3Config();
+         //config.ServiceURL = "http://cme-bts-datamine.s3.us-east-1.amazonaws.com/";
+         config.RegionEndpoint = Amazon.RegionEndpoint.USEast1;
+         _client = new AmazonS3Client(_access_key, _secret_key, config);
+      }
+
+      private string _path;
+      private string _bucket_name; //http://cme-bts-datamine.s3.us-east-1.amazonaws.com/
+      public DataDownloader(string bucket_name = "cme-bts-datamine")
+      {
+         _path = mbts.config.Configuration.get<bts.config.System>().data_path;
+         _bucket_name = bucket_name;
+      }
+      bool is_in_cache(string key)
+      {
+         return File.Exists(_path + "/" + key);
+      }
+      Stream cached(string key)
+      {
+         if (is_in_cache(key))
+            return new FileStream(_path + "/" + key, FileMode.Open, FileAccess.Read, FileShare.Read);
+         else
+            return null;
+      }
+      async Task<Stream> download_s3(string key)
+      {
+         bts.Log.assert(!is_in_cache(key), $"key {key} already exists in cache!");
+         bts.Log.Gen.info($"downloading {key} from s3");
+         try
+         {
+            // var checkresp = await _client.ListObjectsAsync(new ListObjectsRequest { BucketName = _bucket_name} );
+            // foreach (S3Object o in checkresp.S3Objects)
+            // {
+            //    Console.WriteLine("{0}\t{1}\t{2}", o.Key, o.Size, o.LastModified);
+            // }
+
+            GetObjectRequest request = new GetObjectRequest { BucketName = _bucket_name, Key = key };
+            using (GetObjectResponse response = await _client.GetObjectAsync(request))
+            {
+               string title = response.Metadata["x-amz-meta-title"]; // Assume you have "title" as medata added to the object.
+               string contentType = response.Headers["Content-Type"];
+               bts.Log.Gen.info($"Object metadata, Title: {title}");
+               Console.WriteLine($"Downloading Object metadata, Title: {title} Key { key }");
+               Console.WriteLine($"Content type: {contentType}");
+               using (Stream response_stream = response.ResponseStream)
+               {
+                  var fullpath = _path + "/" + key;
+                  new FileInfo(fullpath).Directory.Create();                     
+                  FileStream writer = new FileStream(fullpath, FileMode.CreateNew);
+                  response_stream.CopyTo(writer);
+                  writer.Seek(0, SeekOrigin.Begin);
+                  return writer;
+               }
+            }
+         }
+         catch (AmazonS3Exception e)
+         {
+            bts.Log.Gen.error($"S3 Error encountered ***. Message:'{e.Message}' when writing an object {key}");            
+            throw;
+         }
+         catch (Exception e)
+         {
+            bts.Log.Gen.error($"Unknown encountered on server. Message:'{e.Message}' when writing an object {key}");
+            throw;
+         }
+      }
+      string map_mic(api.CMEMics mic)
+      {
+         switch (mic)
+         {
+            case CMEMics.Cbt:
+               return "xcbt";
+            case CMEMics.Cme:
+               return "xcme";
+            case CMEMics.Comex:
+               return "xcec";
+            case CMEMics.Dme:
+               return "dme";
+            case CMEMics.Nymex:
+               return "xnym";
+            default:
+               bts.Log.assert(false, "unknown mic map DataDownloader");
+               return string.Empty;
+         }
+      }
+      public string make_key(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
+      {
+         string micstring = map_mic(mic);
+         string date = dt.ToString("yyyyMMdd");
+         string key = date + "-MBO_" + micstring.ToLower() + "_" + symbol.ToLower();
+         if (st == api.SecurityType.Future)
+            key += "_fut_0-eth.gz";
+         else if (st == api.SecurityType.Spread)
+            key += "_fut_1-eth.gz";
+         string prefix = $"mbo/mbo/{date}/";
+         return prefix + key;
+      }
+      public static string data_path(DateTime dt)
+      {
+         string date = dt.ToString("yyyyMMdd");
+         string prefix = $"mbo/mbo/{date}/";
+         string p = mbts.config.Configuration.get<bts.config.System>().data_path + "/" + prefix;
+         return p;
+      }
+      public async Task<Stream> download(DateTime dt, string symbol, api.SecurityType st)
+      {
+         var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
+         var micmap = micconf.cme_mic;
+         if(micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic))
+         {
+            return await download(dt, themic, symbol, st);
+         }
+         throw new NotSupportedException("Can't find mic for symbol " + symbol);
+      }
+      public async Task<Stream> download(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
+      {         
+         string key = make_key(dt, mic, symbol, st);
+         return await download(key);
+      }
+
+      async Task<Stream> download_s3_uncompress(string key)
+      {                  
+         var gz = await download(key);
+         var fullpath = _path + "/" + key.Replace(".gz", "");
+         FileStream writer = new FileStream(fullpath, FileMode.CreateNew);
+         using ( GZipStream ungz = new GZipStream(gz, CompressionMode.Decompress))
+         {
+            ungz.CopyTo(writer);
+            writer.Seek(0, SeekOrigin.Begin);
+            return writer;
+         }
+      }
+
+      public async Task<Stream> download(string key)
+      {
+         if (!is_in_cache(key))
+            return await download_s3(key);
+         return cached(key);
+      }
+
+      public async Task<Stream> download_uncompressed(string key)
+      {
+         var unkey = key.Replace(".gz", "");
+         if (!is_in_cache(unkey))
+            return await download_s3_uncompress(key);
+         return cached(unkey);
+      }
+
+      public async Task<Stream> download_uncompressed(DateTime dt, string symbol, api.SecurityType st)
+      {
+         var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
+         var micmap = micconf.cme_mic;
+         if (micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic))
+         {
+            return await download_uncompressed(dt, themic, symbol, st);
+         }
+         throw new NotSupportedException("Can't find mic for symbol " + symbol);
+      }
+
+      public async Task<Stream> download_uncompressed(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
+      {
+         string key = make_key(dt, mic, symbol, st);
+         return await download_uncompressed(key);
+      }
+
+      public void download_compressed(DateTime dt,
+                                      string symbol,
+                                      api.SecurityType st,
+                                      string output_file_path){
+         if(!File.Exists(output_file_path)){
+            var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
+            var micmap = micconf.cme_mic;
+            if (micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic)){
+               string key = make_key(dt, themic, symbol, st);
+               var task = _download_s3(key, output_file_path);
+               while (!task.IsCompleted)
+                  System.Threading.Thread.Sleep(25);
+            }
+            else
+               throw new ApplicationException(
+                     String.Format("Could not find mic for {0}",symbol));
+        }
+     }
+
+     async Task<Stream> _download_s3(string s3_key,
+                                     string output_file_path){         
+        try{
+           GetObjectRequest request = new GetObjectRequest {
+                                             BucketName = _bucket_name, 
+                                             Key = s3_key 
+                                      };
+           using (GetObjectResponse response = await _client.GetObjectAsync(request)){
+              string title = response.Metadata["x-amz-meta-title"];
+              string contentType = response.Headers["Content-Type"];
+              bts.Log.Gen.info($"Object metadata, Title: {title}");
+              Console.WriteLine($"Downloading Object metadata, Title: {title} Key { s3_key }");
+              Console.WriteLine($"Content type: {contentType}");
+              using (Stream response_stream = response.ResponseStream){
+                 new FileInfo(output_file_path).Directory.Create();
+                 FileStream writer = new FileStream(output_file_path,
+                                                    FileMode.CreateNew);
+                 response_stream.CopyTo(writer);
+                 writer.Seek(0, SeekOrigin.Begin);
+                 return writer;
+              }
+           }
+        }
+        catch (AmazonS3Exception e){
+           bts.Log.Gen.error($"S3 Error encountered ***. Message:'{e.Message}' when writing an object {s3_key}");            
+           throw;
+        }
+        catch (Exception e){
+           bts.Log.Gen.error($"Unknown encountered on server. Message:'{e.Message}' when writing an object {s3_key}");
+           throw;
+        }
+      }
+
+   }
+
+}
@@ -0,0 +1,171 @@
+using System;
+using System.Collections.Generic;
+using bts.api.core;
+using System.Linq;
+using System.Threading.Tasks;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.co2;
+
+namespace bts.history
+{
+   class DataSender
+   {
+      readonly DataStreamer _stream;
+      readonly bts.co2.MarketDataReceiverService _mdservice;
+
+      Google.Protobuf.WellKnownTypes.Timestamp _time;
+      Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
+
+      public DataSender(ServiceContext context)
+      {
+         _stream = new DataStreamer();
+         _mdservice = context.locate<bts.co2.MarketDataReceiverService>();
+      }
+
+      bts.api.ResultMessage send()
+      {
+         co2.MDReply md = new co2.MDReply();
+         md.Channels.AddRange(_channels.Values);
+         var x = _mdservice.push_md(md);
+         return x;
+      }
+
+      co2.ChannelUpdate create(Product prod, MDEntry md)
+      {
+         var chanup = new co2.ChannelUpdate()
+         {
+            Key = md.SecurityId, //TODO join with the secdefs that I don't have
+            Security = new Security()
+            {
+               Id = md.SecurityId,
+               Symbol = prod.Symbol,
+               Exchange = Venue.Glbx,
+               ContractSpec = new ContractSpecification()
+               {
+                  MainFraction = 32,
+                  SubFraction = 2,
+                  PriceBreak = 0,
+                  LowTick = 0.015625,
+                  HighTick = 0.015625,
+                  CashPerPoint = 1000.0,
+                  PriceDisplayFormat = 3,
+                  PriceMultiplier = 1.0,
+               },
+               Glbx = new GLBXExchangeData()
+               {
+                  SecurityDesc = md.Symbol,
+                  ExchangeId = md.SecurityId,
+                  CmeSymbol = prod.Symbol,
+               }
+            }
+         };
+         if (prod.Type == SecurityType.Future)
+         {
+            chanup.Security.Future = new FutureContract()
+            {
+               ExpirationDay = 1,
+               ExpirationMonth = 9,
+               ExpirationYear = 2019
+            };
+         }
+         else if (prod.Type == SecurityType.Spread)
+         {
+            chanup.Security.Spread = new SpreadContract()
+            {
+               Normalized = true
+            };
+         }
+         else
+            System.Diagnostics.Trace.Assert(false, "unknown contract type");
+         return chanup;
+      }
+
+      void process(Product prod, MarketDataIncrementalRefresh mdr, string fix)
+      {
+         if (!_channels.TryGetValue(prod, out co2.ChannelReply chan))
+         {
+            chan = new co2.ChannelReply()
+            {
+               Platform = co2.Platform.Mdp3Onixs,
+               Channel = new co2.SecurityChannel()
+               {
+                  Entitled = true,
+                  Exchange = prod.Exchange,
+                  Type = prod.Type,
+                  Symbol = prod.Symbol,
+                  Flags = (uint)co2.MDRequest.Types.Flags.Direct
+               },
+            };
+            _channels.Add(prod, chan);
+         }
+         int count = 0;
+
+         foreach (var md in mdr.MdEntries)
+         {
+            var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId);
+            if (upd == null)
+            {
+               upd = create(prod, md);
+               upd.Direct = new BookMessage();
+               Utils.init_book(upd.Direct);
+               chan.Updates.Add(upd);
+            }
+            Utils.keep_book(upd.Direct, md);
+            count++;
+            if (Utils.quote_event_done( mdr ))
+               Utils.validate_book(upd.Direct, fix);
+         }
+      }
+
+      async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
+                              HistoricalDataOptions o)
+      {         
+         int us = 20;//o.dt_us;
+         var query = new MDQuery
+         {
+            Exchange = Venue.Glbx,
+            Symbol = symbol,
+            SecurityType = st,
+            Begin = Timestamp.FromDateTime(DateTime.SpecifyKind(start, DateTimeKind.Utc)),
+            End = Timestamp.FromDateTime(DateTime.SpecifyKind(end, DateTimeKind.Utc))
+         };
+         api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol };
+         int count = 0;
+         await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
+        {
+           if (md.timestamp().ToDateTime() < end)
+           {
+               process(prod, md, line);
+               maybe_send(md, us);
+               count++;
+           }
+        });
+      }
+      
+      void maybe_send(history.MarketDataIncrementalRefresh md, int us)
+      {
+         if (_time == null)
+         {
+            if (Utils.quote_event_done(md))
+            {
+               send(); //send the initial books
+               _time = md.timestamp();
+            }
+            else
+               return;
+         }       
+         else
+         {
+            var ts = (md.timestamp() - _time).ToTimeSpan();
+            var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
+            if (diff > us && Utils.quote_event_done(md))
+            {
+               send(); //send the next book
+               _time = md.timestamp();
+            }            
+         }  
+      }
+
+   }
+}
@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using bts.api;
+
+namespace bts.history
+{
+   class DataStreamer 
+   {
+      readonly DataDownloader _dl = new DataDownloader();
+      public DataStreamer()
+      {
+      }
+
+
+      #pragma warning disable CS0183
+      async public Task process_md_stream<T>(MDQuery request, Action<SecurityType, T, string> action, IComparer<T> comp = null) 
+         where T : Google.Protobuf.IMessage, new()
+      {
+         DateTime d = request.Begin.ToDateTime();
+         while (d < request.End.ToDateTime())
+         {
+            try
+            {
+               if(request.SecurityType == SecurityType.Synthetic)
+               {
+                  var s3f = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Future);
+                  var s3s = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Spread);
+                  using (MBOParser fparser = new MBOParser(s3f))
+                  {
+                     using(MBOParser sparser = new MBOParser(s3s))
+                     {
+                        var fut = fparser.next<T>(null, null, request.Skip).GetEnumerator();
+                        var spr = sparser.next<T>(null, null, request.Skip).GetEnumerator();                        
+                        bool futgo = fut.MoveNext(), sprgo = spr.MoveNext();  
+                        while(futgo || sprgo)
+                        {
+                           if(comp.Compare(fut.Current.Item1, spr.Current.Item1) < 0)
+                           {
+                              action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2.Replace('\u0001', '|'));
+                              if(futgo)
+                                 futgo = fut.MoveNext();                              
+                           }
+                           else if(comp.Compare(fut.Current.Item1, spr.Current.Item1) > 0)
+                           {
+                              action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2.Replace('\u0001', '|'));
+                              if (sprgo)
+                                 sprgo = spr.MoveNext();
+                           }
+                           else
+                           {
+                              action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2.Replace('\u0001', '|'));
+                              if (futgo)
+                                 futgo = fut.MoveNext();
+                              action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2.Replace('\u0001', '|'));
+                              if (sprgo)
+                                 sprgo = spr.MoveNext();
+                           }
+                        }                
+                     }
+                  }
+               }
+               else
+               {
+                  var s3 = await _dl.download_uncompressed(d, request.Symbol, request.SecurityType);
+                  using (MBOParser parser = new MBOParser(s3))
+                  {
+                     foreach (var (mbo,line) in parser.next<T>(null, null, request.Skip))
+                     {
+                        action(request.SecurityType, mbo, line.Replace('\u0001', '|'));
+                     }
+                  }
+               }
+            }
+            catch (Amazon.S3.AmazonS3Exception) { }
+            catch (System.IO.EndOfStreamException) { }
+            d = d.AddDays(1.0).Date;
+         }
+      }
+
+
+      async public Task process_md_stream_by_line(MDQuery request, Action<SecurityType, string, string> action, int peek_field = 0)
+      {
+         DateTime d = request.Begin.ToDateTime();
+         while (d < request.End.ToDateTime())
+         {
+            try
+            {
+               if (request.SecurityType == SecurityType.Synthetic)
+               {
+                  var s3f = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Future);
+                  var s3s = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Spread);
+                  using (MBOParser fparser = new MBOParser(s3f))
+                  {
+                     using (MBOParser sparser = new MBOParser(s3s))
+                     {
+                        var fut = fparser.next_line(peek_field).GetEnumerator();
+                        var spr = sparser.next_line(peek_field).GetEnumerator();
+                        bool futgo = fut.MoveNext(), sprgo = spr.MoveNext();
+                        while (futgo || sprgo)
+                        {
+                           action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2);
+                           if (futgo)
+                              futgo = fut.MoveNext();
+                              
+                           action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2);
+                           if (sprgo)
+                              sprgo = spr.MoveNext();
+                        }
+                     }
+                  }
+               }
+               else
+               {
+                  var s3 = await _dl.download_uncompressed(d, request.Symbol, request.SecurityType);
+                  using (MBOParser parser = new MBOParser(s3))
+                  {
+                     foreach (var (peek, line) in parser.next_line(peek_field))
+                        action(request.SecurityType, peek, line);
+                  }
+               }
+            }
+            catch (Amazon.S3.AmazonS3Exception) { }
+            catch (System.IO.EndOfStreamException) { }
+            d = d.AddDays(1.0).Date;
+         }
+      }
+   }   
+}
@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Linq;
+
+namespace bts.history
+{
+   class DistributionEstimator
+   {
+      List<double> _vals = new List<double>();
+      NormalEstimator _norm = new NormalEstimator();
+      public DistributionEstimator() 
+      {
+         
+      }
+      public void add(double x)
+      {
+         _vals.Add(x);
+         _norm.add(x);
+      }
+      public void delete(double x)
+      {
+         _vals.Remove(x);
+         _norm.delete(x);
+      }
+      public long num_samples => _vals.Count;
+      public double mean() => _norm.mean();
+      // {
+      //    if (num_samples == 0)
+      //       return 0.0;
+      //    double sum = _vals.Sum();
+      //    return sum / num_samples;
+      // }
+      public double sum_sq()
+      {
+         double m = mean();
+         double sq = 0.0;
+         for (int i = 0; i < num_samples; i++)
+            sq += (_vals[i] - m) * (_vals[i] - m);
+         return sq;
+      }
+      public double variance() => _norm.variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / num_samples;
+      // }
+      public double unbiased_variance() => _norm.unbiased_variance();
+      // {
+      //    if (num_samples <= 1)
+      //       return 0.0;
+      //    return sum_sq() / (num_samples - 1);
+      // }
+      public double std_deviation() => _norm.std_deviation();
+      // {
+      //    return Math.Sqrt(variance());
+      // }
+      public double unbiased_std_deviation() => _norm.unbiased_std_deviation();
+      // {
+      //    return Math.Sqrt(unbiased_variance());
+      // }
+      public double percentile(int p)
+      {
+         _vals.Sort();
+         double percentile = p / 100.0;
+         double n = (_vals.Count - 1) * percentile + 1;
+         if (n <= 1.0)
+            return _vals[0];
+         else if (n >= _vals.Count)
+            return _vals[_vals.Count - 1];
+         else
+         {
+            int k = (int)n;
+            double d = n - k;
+            return _vals[k - 1] + d * (_vals[k] - _vals[k - 1]);
+         }
+      }
+      public override string ToString()
+      {
+         var f = percentile(5);
+         var t = percentile(10);
+         var tw = percentile(25);
+         var h = percentile(50);
+         var s = percentile(75);
+         var nn = percentile(99);
+         var sum50 = f + t + tw + h;
+         return $@"
+   Normal( mean {mean()}, std_dev {std_deviation()} ) 
+   5th { f }, 
+   10th { t } 
+   25th { tw } 
+   50th { h } 
+   75th { s } 
+   99th { nn } 
+   [numSamples={ num_samples } ]
+";
+      }
+   }
+}
@@ -0,0 +1,210 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using bts.co2;
+using System.Threading.Tasks;
+using bts.history;
+using mbts.co2;
+using bts.api;
+
+namespace bts.history
+{
+   public class FIFOFillModel : FillModelBase, IFillModel
+   {
+      readonly Dictionary<ulong, List<MBOMessage>> _exchangeorders = 
+         new Dictionary<ulong, List<MBOMessage>>();
+      public IEnumerable<MBOMessage> exchange_orders_raw => _exchangeorders.SelectValues(_exchangeorders.Keys).SelectMany(x => x);
+      readonly protected SortedList<Timestamp, OrderStatus> _myorders =
+         new SortedList<Timestamp, OrderStatus>(HistoricalExtensions.timestamp_comparer);
+      public IEnumerable<OrderStatus> my_orders => _myorders.Values;
+
+      public FIFOFillModel(Bucket outgoing_latency) : base(outgoing_latency)
+      {
+      }
+      
+      public override void reset()
+      {
+         base.reset();
+         _myorders.Clear();
+         _exchangeorders.Clear();
+      }
+
+      public virtual void record_order_status(OrderStatus stat)
+      {
+         if (_disqual.ContainsKey(stat.order.order_id))
+         {
+            Log.Testing.error($"new status for disqualified error, not recording");
+         }
+         else if (!check_cross(stat))
+         {
+            int tries = 0; bool done = false;
+            while (tries++ < 99) //100 nano resolution from CME data makes this sorta ok
+            {
+               try
+               {
+                  _myorders.Add(stat.TransactTime, stat);
+                  done = true;
+                  break;
+               }
+               catch (ArgumentException)
+               {
+                  stat.TransactTime.Nanos += 1;
+               }
+            }
+            if (!done)
+               throw new ArgumentOutOfRangeException("can't place 100th dupe timestamp");
+         }
+      }
+
+      public void process_order(Timestamp transact_time, MBOMessage mbo)
+      {
+         get_mbo(mbo.OrderId).Add(mbo);
+      }
+
+      public void process_trade(Timestamp transact_time, string security_desc, co2.TradeMessage trade_summary,
+                                IEnumerable<ITradeDetail> orders)
+      {
+         foreach(var order in orders)
+         {
+            if(!_exchangeorders.TryGetValue(order.OrderId, out List<MBOMessage> operations))
+               continue;
+            if(!operations.Any()) //can't find the order, must have been here forever, they get filled not us
+               continue;
+            if(_myorders.Count == 0)
+               continue;
+            var market_operations = operations.OrderBy(x => x.TransactTime, HistoricalExtensions.timestamp_comparer).ToList();
+            var market_operation_index = find_first_greater_or_equal(market_operations, transact_time, x=>x.TransactTime, 
+                                                                     HistoricalExtensions.timestamp_comparer);
+            if(market_operation_index == 0) //transact_time is before first market operation we saw, on this order, they get filled not us
+               continue;
+            //this is the latest market operation for the filled order at the time of fill
+            var curr_market_operation = market_operations[market_operation_index-1];
+            //now we slide backwards to find the first operation that kept priority at the time of fill
+            for (int i = market_operation_index - 2; i >= 0; i--)
+            {
+               if(market_operations[i].MdOrderPriority == curr_market_operation.MdOrderPriority)
+               {
+                  curr_market_operation = market_operations[i];
+               }
+            }
+         
+            Log.assert(curr_market_operation.OrderUpdateAction != MDUpdateAction.Delete); //a deleted order should not get filled         
+            
+            //now we find the first operation for our order greater or equal to the transact time
+            int our_operation_index = find_first_greater_or_equal(_myorders, transact_time, HistoricalExtensions.timestamp_comparer); 
+            if(our_operation_index == 0) //transact_time before I'm in the market, they get filled not us
+               continue;
+         
+            string market_secdesc = curr_market_operation.SecurityDesc;
+            api.Side market_side = Utils.same_side(curr_market_operation.MdEntryType);
+            List<FillEntry> fills = new List<FillEntry>(); 
+            uint fill_qty = order.LastQty;
+            Dictionary<ulong, OrderStatus> latest_state = new Dictionary<ulong, OrderStatus>();
+            for(int i = our_operation_index - 1; i >= 0; i--)
+            {
+               //the closest operation to this trade time
+               OrderStatus curr_operation = _myorders.Values[i];            
+               if(curr_operation.order.security_desc != market_secdesc)
+                  continue;
+               if(curr_operation.order.side != market_side)
+                  continue;
+                  
+               //this order is completely spent, disqualify it from future fills in case a the algo behind it tries to keep updating, 
+               //thus creating new states ahead of this termnal state
+               if(_disqual.ContainsKey(curr_operation.order.order_id))
+                  continue;
+               if (curr_operation.is_terminal())
+               {
+                  _disqual[curr_operation.order.order_id] = 0; 
+                  continue;
+               }
+               
+               //when we begin searching, we only want to consider the latest valid, non terminal state of our order 
+               //FOR THIS ONE SEARCH, this is to prevent the case where an old state at a previous price is erronenously filled
+               if(latest_state.ContainsKey(curr_operation.order.order_id)) 
+                  continue;
+               latest_state[curr_operation.order.order_id] = curr_operation;
+               
+               //If our operation did not occur before the market's we are not filled
+               if (!curr_operation.TransactTime.less_or_equal(curr_market_operation.TransactTime))
+                  continue;
+               //here we have the latest status of an order on the right side of the market for the right security and the right price               
+               if (( market_side == api.Side.Buy && curr_operation.state.Price >= trade_summary.price || //fill us if we are better?
+                     market_side == api.Side.Sell && curr_operation.state.Price <= trade_summary.price) ) 
+               //this has to be be trade price, not mdentryprx because cme will report orders down in the book that suddenly aggress
+               {               
+                  uint size = Math.Min(fill_qty, curr_operation.state.Quantity);
+                  if (size > 0)
+                  {
+                     FillEntry f = generate_fill(transact_time, curr_operation, curr_operation.state.Price, size);                  
+                     f.fill_id = order.OrderId;
+                     if(market_side == Side.Buy)
+                        f.bid = trade_summary.price;
+                     else
+                        f.ask = trade_summary.price;
+                     f.total_traded = trade_summary.Size;
+                     f.num_orders = curr_market_operation.NumberOfOrders;
+                     bool same = curr_operation.order.side == Utils.same_side(trade_summary.AggressorSide);
+                     if(same)
+                        f.tag = $"agg side same {curr_operation.order.side}";
+                     if(f.fully_filled)
+                     {
+                        if (string.IsNullOrEmpty(f.tag))
+                           f.tag = $"{f.order.order_id} FULLFILL " + (same ? $"agg side same {curr_operation.order.side}" : "");
+                     }
+                     fills.Add(f);
+                     fill_qty -= size;                  
+                  }                     
+                  if (fill_qty <= 0)
+                  {
+                     Log.assert(fill_qty == 0);
+                     break;
+                  }
+               }
+            }
+         }
+      }
+
+      static int binary_search<T,U>(IList<U> list, T value, Func<U, T> selector, Comparer<T> comparer = null)
+      {
+         if (list == null)
+            throw new ArgumentNullException("list");
+         var comp = Comparer<T>.Default;
+         if (comparer != null)
+            comp = comparer;
+         if(selector == null)
+            selector = (x => (T)(object)x);
+         int lo = 0, hi = list.Count - 1;
+         while (lo < hi)
+         {
+            int m = lo + (hi - lo) / 2;
+            if (comp.Compare( selector(list[m]), value) < 0) lo = m + 1;
+            else hi = m - 1;
+         }
+         if (comp.Compare( selector(list[lo]), value) < 0) lo++;
+         return lo;
+      }
+
+      static int find_first_greater_or_equal<T,U>(IList<U> sorted_list, T key, Func<U, T> selector, Comparer<T> comp)
+      {
+         return binary_search(sorted_list, key, selector, comp);
+      }      
+
+      static int find_first_greater_or_equal<T, U>(SortedList<T, U> sorted_list, T key, Comparer<T> comp)
+      {
+         return binary_search(sorted_list.Keys, key, x => x,  comp);
+      }
+
+      List<MBOMessage> get_mbo(ulong id)
+      {
+         if (!_exchangeorders.TryGetValue(id, out List<MBOMessage> list))
+         {
+            _exchangeorders[id] = list = new List<MBOMessage>();
+         }
+         return list;
+      }
+
+   }
+}
@@ -0,0 +1,41 @@
+using System;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public class FillEntry : IHasTimestamp
+   {
+      static uint _fillid = 0;
+      public FillEntry(Timestamp ts, OrderEntry order, api.Side side, double price, ulong size)
+      {
+         this.TransactTime = ts;
+         this.SendingTime = ts;
+         this.order = order;
+         this.side = side;
+         this.price = price;
+         this.size = size;
+         this.fill_id = ++_fillid;
+      }
+      public Timestamp TransactTime { get; set;}
+      public Timestamp SendingTime { get; set;}
+      public OrderEntry order { get; }
+      public api.Side side { get; }
+      public double price { get; }
+      public ulong size { get; }
+      public bool fully_filled { get; set; }
+      public uint total_traded { get; set; }
+      public uint num_orders { get; set; }
+      public ulong fill_id { get; set; }
+
+      public uint bidsize { get; set; }
+      public double bid { get; set; }
+      public uint asksize { get; set; }
+      public double ask { get; set; }
+      public string tag { get; set; }
+
+      public override string ToString()
+      {
+         return $"{TransactTime.ToDateTime().ToLongDateString()} {TransactTime.ToDateTime().ToString("H:mm:ss.ffffff")} {order.tag ?? "Our   "} {order.security_desc} {side} {price} {size}    --    {bidsize} {bid} at {ask} { asksize }  {tag}";
+      }
+   }
+}
@@ -0,0 +1,141 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using bts.co2;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+
+namespace bts.history
+{
+   public class FillModelBase
+   {
+      readonly protected Dictionary<string, co2.BookMessage> _books = new Dictionary<string, co2.BookMessage>();
+      readonly protected Dictionary<ulong, uint> _disqual = new Dictionary<ulong, uint>();
+
+      public event Action<FillEntry> on_fill;
+      public event Action<OrderStatus> on_status_due_to_fill;
+      public Bucket outgoing_latency { get; }
+      
+      public FillModelBase(Bucket outgoing_latency)
+      {
+         this.outgoing_latency = outgoing_latency;
+      }
+
+      public virtual void reset()
+      {
+         _books.Clear();
+         _disqual.Clear();
+         on_fill = null;
+         on_status_due_to_fill = null;
+      }
+    
+
+      public virtual void process_book(Timestamp transact_time, string security_desc, BookMessage book)
+      {
+         _books[security_desc] = book;
+      }
+
+      protected co2.BookMessage get_book(string security_desc)
+      {
+         if (!_books.TryGetValue(security_desc, out co2.BookMessage book))
+         {
+            _books[security_desc] = book = new co2.BookMessage();
+         }
+         return book;
+      }
+
+      protected bool check_cross(OrderStatus status)
+      {
+         var fills = check_my_cross(status);
+         if (fills.Count > 0)
+            return fills.Any(x => x.fully_filled);
+         return false;
+      }
+
+      List<FillEntry> check_my_cross(OrderStatus status)
+      {
+         List<FillEntry> fills = new List<FillEntry>();
+         var book = get_book(status.order.security_desc);
+         if (status.order.side == Side.Buy)
+         {
+            int level = 0; int nasks = Math.Min(book.Asks.Count, 10);
+            while (status.state.Quantity > 0 && level < nasks)
+            {
+               var currask = book.ask_at(level);
+               if (status.state.Price >= currask || status.order.market_order)
+               {
+                  var qty = Math.Min(status.state.Quantity, book.asksize_at(level));
+                  if (qty > 0)
+                     fills.Add(generate_fill(status.TransactTime, status, currask, (uint)qty, book, "CROSS"));
+               }
+               else
+                  break;
+               level++;
+            }
+         }
+         else
+         {
+            int level = 0; int nbids = Math.Min(book.Bids.Count, 10);
+            while (status.state.Quantity > 0 && level < nbids)
+            {
+               var currbid = book.bid_at(level);
+               if (status.state.Price <= currbid || status.order.market_order)
+               {
+                  var qty = Math.Min(status.state.Quantity, book.bidsize_at(level));
+                  if (qty > 0)
+                     fills.Add(generate_fill(status.TransactTime, status, currbid, (uint)qty, book, "CROSS"));
+               }
+               else
+                  break;
+               level++;
+            }
+         }
+         return fills;
+      }
+
+      protected FillEntry generate_fill(Timestamp asof, OrderStatus status, double price, uint qty, co2.BookMessage book = null, string tag = null)
+      {
+         var f = new FillEntry(asof, status.order, status.order.side, price, qty);
+         f.SendingTime = f.TransactTime.add_nanos((int)outgoing_latency.LowerBound);
+         // f.piq = piq;
+         // f.queue_len = qlen;
+         f.bidsize = book?.bidsize_at(0) ?? 0;
+         f.bid = book?.bid_at(0) ?? 0;
+         f.ask = book?.ask_at(0) ?? 0;
+         f.asksize = book?.asksize_at(0) ?? 0;
+         f.tag = tag;
+         if (qty > status.state.Quantity)
+            throw new InvalidOperationException($"too much fill {qty} working quantity {status.state.Quantity}");
+         status.state.Quantity -= qty;
+         if (status.state.Quantity == 0)
+         {
+            Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? ""));
+            f.fully_filled = true;
+         }
+         else
+         {
+            Log.Testing.info("Partial FILL " + Stringizer.to_string(f));
+         }
+         var clone = status.clone();
+         clone.TransactTime = asof;
+         clone.SendingTime = clone.TransactTime.add_nanos((int)outgoing_latency.LowerBound);
+         clone.state.LastPrice = f.price;
+         clone.state.AvgPrice = status.state.AvgPrice * status.state.FilledQuantity + f.price * f.size;
+         clone.state.FilledQuantity += (uint)f.size;
+         clone.state.AvgPrice /= status.state.FilledQuantity;
+         if (f.fully_filled)
+         {
+            clone.state.Status = api.orderman.OrderStatus.Completed;
+            _disqual[status.order.order_id] = int.MaxValue;
+         }
+         if (this.on_status_due_to_fill != null)
+            this.on_status_due_to_fill(clone);
+         if (this.on_fill != null)
+            this.on_fill(f);
+         return f;
+      }
+
+   }
+}
@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using bts.co2;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history
+{
+   public interface IFillModel
+   {
+      IEnumerable<OrderStatus> my_orders { get; }
+      IEnumerable<MBOMessage> exchange_orders_raw { get; }
+
+      void reset();
+      void record_order_status(OrderStatus stat);
+      void process_order(Timestamp transact_time, MBOMessage mbo);
+      void process_trade(Timestamp transact_time, string security_desc, TradeMessage trade_summary, IEnumerable<ITradeDetail> orders);
+      void process_book(Timestamp transact_time, string security_desc, BookMessage book);
+      
+      event Action<FillEntry> on_fill;
+      event Action<OrderStatus> on_status_due_to_fill;
+   }
+}
@@ -0,0 +1,17 @@
+using System;
+using Google.Protobuf.WellKnownTypes;
+using MathNet.Numerics.Statistics;
+
+namespace bts.history
+{
+   public interface IOrderSink
+   {
+      void reset();
+      void send(OrderEntry order);
+      void update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = uint.MinValue);
+      void cancel(string security_desc, Timestamp asof, ulong order_id);
+      void record_order_status(OrderStatus s);
+      
+      event Action<OrderStatus> on_status;
+   }
+}
@@ -0,0 +1,369 @@
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.api;
+using Google.Protobuf.WellKnownTypes;
+using bts.co2;
+using mbts.co2;
+using MathNet.Numerics.Statistics;
+
+//priority -> order
+using QueueType = System.Collections.Generic.SortedList<ulong, bts.history.OrderEntry>;
+
+namespace bts.history
+{
+   public class KFillModel : FillModelBase, IFillModel
+   {
+      List<MBOMessage> _exchange_orders_raw = new List<MBOMessage>();
+      //price -> queue
+      readonly Dictionary<double, QueueType> _mbobuy = new Dictionary<double, QueueType>();
+      readonly Dictionary<double, QueueType> _mbosell = new Dictionary<double, QueueType>();
+      //orderid -> { side, price, priority }
+      readonly SortedDictionary<ulong, Tuple<api.Side, double, ulong>> _mbokey = new SortedDictionary<ulong, Tuple<api.Side, double, ulong>>();
+
+      readonly Dictionary<ulong, OrderStatus> _mystatus = new Dictionary<ulong, OrderStatus>();
+      public IEnumerable<OrderStatus> my_orders => _mystatus.Values;
+      public IEnumerable<MBOMessage> exchange_orders_raw => _exchange_orders_raw;
+
+      double bid_at(int level) => _mbobuy.Keys.OrderByDescending(x => x).Skip(level).FirstOrDefault(x => true, double.NaN);
+      double ask_at(int level) => _mbosell.Keys.OrderBy(x => x).Skip(level).FirstOrDefault(x => true, double.NaN);
+      // public uint bidsize_at(int level)
+      // {
+      //    //return _book.bidsize_at(level);
+      //    double bid = bid_at(level);
+      //    int sz = _mbobuy[bid].Values.Sum( x => (int)x.size );
+      //    return (uint)sz;
+      // }
+      // public uint asksize_at(int level)
+      // {
+      //    //return _book.asksize_at(level);
+      //    double ask = ask_at(level);
+      //    int sz = _mbosell[ask].Values.Sum(x => (int)x.size);
+      //    return (uint)sz;
+      // }
+
+      readonly bool _use_top;
+      readonly double _fifo_percent;
+      readonly uint _min_prorata;
+      readonly bool _use_leveling;
+      public KFillModel(Bucket outgoing_latency, bool use_top, double fifo_percent, uint min_prorata, bool use_leveling) 
+         : base(outgoing_latency)
+      {
+         _use_top = use_top;
+         _fifo_percent = fifo_percent;
+         _min_prorata = min_prorata;
+         _use_leveling = use_leveling;
+      }
+
+      public override void reset()
+      {
+         base.reset();
+         _mbobuy.Clear();
+         _mbosell.Clear();
+         _mbokey.Clear();
+         _mystatus.Clear();
+      }
+
+      public void record_order_status(OrderStatus stat)
+      {
+         Log.assert(stat.order.order_id != 0);
+         QueueType queue = get_mbo_side(stat.side).GetOrAdd(stat.state.Price, x =>
+         {
+            Log.Testing.error("FIRST in Q at " + stat.state.Price);
+            return new QueueType();
+         });
+         ulong prior = ulong.MinValue; ulong max = ulong.MinValue;
+         foreach (var q in queue)
+         {
+            if (q.Value.last_change.less_or_equal(stat.TransactTime))
+            {
+               if (q.Key > prior)
+                  prior = q.Key;
+            }
+            if (q.Key > max)
+               max = q.Key;
+         }
+         if (prior == ulong.MaxValue)
+            prior = max;
+
+         if (_mystatus.TryGetValue(stat.order.order_id, out OrderStatus existing_stat))
+         {
+            if (stat.is_terminal()) //remove
+            {
+               Log.assert(queue.Remove(stat.order_priority));
+               stat.order_priority = ulong.MaxValue;
+               _mystatus[stat.order.order_id] = stat;
+               _mbokey.Remove(stat.order.order_id);
+            }
+            else //change
+            {
+               if (stat.state.Quantity > existing_stat.state.Quantity || stat.state.Price != existing_stat.state.Price)
+               {
+                  Log.assert(queue.Remove(stat.order_priority));
+                  stat.order_priority = prior;
+                  stat.order.order_priority = prior;
+                  stat.order.tag = "our order";
+                  queue.Add(prior, stat.order);
+                  _mbokey[stat.order.order_id] = Tuple.Create(stat.order.side, stat.state.Price, prior);
+               }
+               _mystatus[stat.order.order_id] = stat; //order is already changed here if priority didn't change               
+            }
+         }
+         else //new
+         {
+            stat.order_priority = prior;
+            stat.order.order_priority = prior;
+            stat.order.tag = "our order";
+            queue.Add(prior, stat.order);
+            _mbokey[stat.order.order_id] = Tuple.Create(stat.order.side, stat.state.Price, prior);
+            _mystatus[stat.order.order_id] = stat;
+         }
+         Log.Testing.info($"insert myorder {stat.order.order_id} at price { stat.state.Price } prior {stat.order_priority}  piq { queue.Count } ");
+      }
+
+      public override void process_book(Timestamp transact_time, string security_desc, BookMessage book)
+      {
+         base.process_book(transact_time, security_desc, book);
+         if(book.bid_at(0) != bid_at(0))
+            Log.Testing.error("bid mismatch");
+         if(book.ask_at(0) != ask_at(0))
+            Log.Testing.error("ask mismatch");
+      }
+
+      public void process_order(Timestamp transact_time, MBOMessage mbo)
+      {
+         process_mbo(transact_time, mbo);
+         _exchange_orders_raw.Add(mbo);
+      }
+
+      public void process_trade(Timestamp transact_time, string security_desc, co2.TradeMessage trade_summary,
+                                IEnumerable<ITradeDetail> orders)
+      {
+         if(!orders.Any())
+            return;
+         var resting_side = Utils.opposide_side(trade_summary.AggressorSide);
+         var mbo = get_mbo_side(resting_side);
+         var queue = mbo[trade_summary.price];
+         List<ITradeDetail> matches = new List<ITradeDetail>();
+         uint aggressor_qty = trade_summary.Size; //this could be greater than the aggressor size in some cases where implieds trade
+         if(aggressor_qty != orders.First().LastQty)
+            Log.Testing.warn("trade had some implied");
+            
+         if (_use_top)
+            match_top(queue, trade_summary, ref aggressor_qty, matches);
+         uint fifo_lots = (uint)Math.Truncate(_fifo_percent * aggressor_qty);
+         if(fifo_lots > 0)
+         {            
+            if(aggressor_qty == 1 || aggressor_qty == 2)
+               fifo_lots = 1;
+            else if(aggressor_qty == 3)
+               fifo_lots = 2;
+            aggressor_qty -= fifo_lots;
+            match_fifo(queue, trade_summary, ref fifo_lots, matches);
+         }
+         if(aggressor_qty > 0)
+            match_prorata(queue, trade_summary, ref aggressor_qty, matches);
+         if(aggressor_qty > 0)
+            match_fifo(queue, trade_summary, ref aggressor_qty, matches);
+         validate_matches(orders, matches);
+      }
+      //https://www.cmegroup.com/confluence/display/EPICSANDBOX/CME+Globex+Matching+Algorithm+Steps#CMEGlobexMatchingAlgorithmSteps-Split
+      uint make_match(uint qty, ulong oid, List<ITradeDetail> matches)
+      {
+         var match = new OrderIDEntry { LastQty = qty, OrderId = oid };
+         matches.Add(match);
+         if (_mystatus.TryGetValue(match.OrderId, out OrderStatus existing_stat))
+         {
+            existing_stat.state.Quantity -= match.LastQty;
+            existing_stat.order.size -= match.LastQty;
+         }
+         return match.LastQty;
+      }
+
+      void match_top(QueueType queue, co2.TradeMessage trade_summary, ref uint aggressor_qty, List<ITradeDetail> matches)
+      {
+         var first = queue.FirstOrDefault();
+         if (first.Value != null && first.Value.is_top_eligible)
+         {
+            var qty = Math.Min(aggressor_qty, (uint)first.Value.size);
+            aggressor_qty -= make_match(qty, first.Value.order_id, matches);
+         }
+      }
+
+      void match_fifo(QueueType queue, TradeMessage trade_summary, ref uint fifo_lots, List<ITradeDetail> matches)
+      {
+         foreach (var lot in queue)
+         {
+            var td = Math.Min(fifo_lots, (uint)lot.Value.size);
+            if (td > 0)
+            {
+               fifo_lots -= make_match(td, lot.Value.order_id, matches);
+               if(fifo_lots <= 0)
+               {
+                  Log.assert(fifo_lots == 0);
+                  break;
+               }
+            }
+         }
+      }
+
+      void match_prorata(QueueType queue, TradeMessage trade_summary, ref uint aggressor_qty, List<ITradeDetail> matches)
+      {
+         uint orig_aggressor_qty = aggressor_qty;
+         uint total = queue.Sum(x => (uint)x.Value.size);
+         var noqueue = new QueueType(); //this is the list of orders that received no prorata, and are therefore eligible for leveling
+         foreach (var lot in queue)
+         {
+            uint alloc = (uint)Math.Truncate((double)lot.Value.size / total * aggressor_qty);
+            if(alloc >= _min_prorata)
+            {
+               var q = make_match(alloc, lot.Value.order_id, matches);
+               q = Math.Min(q, aggressor_qty);
+               aggressor_qty -= q;
+               if (aggressor_qty <= 0)
+               {
+                  Log.assert(aggressor_qty == 0);
+                  break;
+               }               
+            }
+            else
+               noqueue.Add(lot.Key, lot.Value);
+         }
+         if(_use_leveling && aggressor_qty > 0 && noqueue.Count > 0)
+         {
+            var l = noqueue.Select( x => x.Value ).OrderBy( xx => xx.size ).ThenBy( yy => yy.last_change.to_datetime() ).ToArray();
+            for(int i = 0; i < l.Length && aggressor_qty > 0; i++)
+            {
+               var q = make_match(1, l[i].order_id, matches);
+               aggressor_qty--;
+            }
+         }
+      }
+
+      void validate_matches(IEnumerable<ITradeDetail> orders, List<ITradeDetail> matches)
+      {
+         if(!matches.Any( x => _mystatus.ContainsKey( x.OrderId ) ))
+         {
+            var dict = matches.ToDictionary( x => x.OrderId, y => y.LastQty);
+            foreach(var order in orders)
+            {
+               var oid = order.OrderId;
+               var qty = dict[oid];
+               if(qty != order.LastQty)
+                  throw new InvalidOperationException($"invalid match, {qty} vs {order.LastQty} oid = {oid}");
+            }
+         }
+      }
+
+      public (int, int) get_my_piq(ulong orderid)
+      {
+         int piq = 0, count = 0;
+         if (_mystatus.TryGetValue(orderid, out OrderStatus myorder))
+         {
+            var mbo = get_mbo_side(myorder.side);
+            if (mbo.TryGetValue(myorder.state.Price, out QueueType q))
+            {
+               foreach (var ent in q.Values)
+               {
+                  if (ent.order_priority < myorder.order_priority)
+                     piq++;
+                  count++;
+               }
+               return (piq, count);
+            }
+         }
+         return (piq, count);
+      }
+
+      Dictionary<double, QueueType> get_mbo_side(api.Side side)
+      {
+         if (side == Side.Buy)
+            return _mbobuy;
+         else
+            return _mbosell;
+      }
+
+      OrderEntry find_order(ulong order_id)
+      {
+         OrderEntry order = null;
+         find_queue(order_id, out Side _, out double _, out ulong prior)?.TryGetValue(prior, out order);
+         return order;
+      }
+
+      QueueType find_queue(ulong order_id, out api.Side side, out double price, out ulong priority_key)
+      {
+         QueueType q = null; side = Side.Undefined; price = double.NaN; priority_key = 0;
+         if (_mbokey.TryGetValue(order_id, out Tuple<api.Side, double, ulong> tup))
+         {
+            var mbo = get_mbo_side(tup.Item1);
+            mbo.TryGetValue(tup.Item2, out q);
+            side = tup.Item1;
+            price = tup.Item2;
+            priority_key = tup.Item3;
+         }
+         return q;
+      }
+
+      void process_mbo(Timestamp ts, IMBOMessage order)
+      {
+         if (order.MdEntryType != MDEntryType.Bid && order.MdEntryType != MDEntryType.Offer)
+            return;
+         var mbo = get_mbo_side(Utils.same_side(order.MdEntryType));
+         if (order.OrderUpdateAction == MDUpdateAction.New)
+         {
+            bool added = false;
+            var q = mbo.GetOrAdd(order.MdEntryPx, x => { added = true; return new QueueType(); });
+            var side = Utils.same_side(order.MdEntryType);
+            var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, order.ReferenceId,
+                                          order.SecurityDesc, side, order.MdEntryPx, order.MdDisplayQty);
+            if ((added || q.Count == 1) && order.MdPriceLevel == 1)
+               neworder.is_top_eligible = true;
+            q.Add(order.MdOrderPriority, neworder);
+            _mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority);
+         }
+         else if (order.OrderUpdateAction == MDUpdateAction.Change)
+         {
+            var currq = find_queue(order.OrderId, out Side _, out double _, out ulong priority_key);
+            if (currq != null)
+               currq.Remove(priority_key); //this could have been previously deleted
+            var newq = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
+            newq.AddOrUpdate(order.MdOrderPriority,
+               x =>
+               {
+                  var side = Utils.same_side(order.MdEntryType); var refid = 0u;
+                  var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, refid,
+                                                order.SecurityDesc, side, order.MdEntryPx, order.MdDisplayQty);
+                  if (newq.Count == 0 && order.MdPriceLevel == 1)
+                     neworder.is_top_eligible = true;
+                  Log.assert(neworder.order_priority == order.MdOrderPriority, "bad priority");
+                  _mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority);
+                  return neworder;
+               },
+               (id, x) =>
+               {
+                  x.last_change = ts;
+                  if (x.order_priority != order.MdOrderPriority)
+                     Log.assert(false, "kd dec 12 2019, this is keyed on priority so this doens't make sense");
+                  x.price = order.MdEntryPx;
+                  x.size = order.MdDisplayQty;
+                  x.is_top_eligible = false;
+                  Log.assert(x.order_id == order.OrderId, "orderid invariant");
+                  _mbokey[x.order_id] = Tuple.Create(Utils.same_side(order.MdEntryType), x.price, x.order_priority);
+                  return x;
+               });
+         }
+         else if (order.OrderUpdateAction == MDUpdateAction.Delete)
+         {
+            var currq = find_queue(order.OrderId, out Side _, out double _, out ulong _);
+            if (currq == null)
+               Log.assert(true);
+            if (currq != null && currq.TryGetValue(order.MdOrderPriority, out OrderEntry removed_order))
+            {
+               removed_order.last_change = ts;
+               currq.Remove(order.MdOrderPriority);
+            }
+         }
+      }
+   }
+}
@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using Google.Protobuf.WellKnownTypes;
+using System.Linq;
+
+using Update = System.ValueTuple<Google.Protobuf.WellKnownTypes.Timestamp, ulong, double, uint>;
+using Cancel = System.ValueTuple<Google.Protobuf.WellKnownTypes.Timestamp, ulong>;
+using MathNet.Numerics.Statistics;
+using mbts.co2;
+using bts.api;
+using System.Threading.Tasks;
+
+namespace bts.history
+{
+   public class LatencyOrderSink : IOrderSink
+   {
+      Dictionary<ulong, OrderEntry> _orders = new Dictionary<ulong, OrderEntry>();
+      Dictionary<ulong, OrderStatus> _status = new Dictionary<ulong, OrderStatus>();
+
+      public Bucket incoming_latency { get; }
+      public Bucket outgoing_latency { get; }
+      public event Action<OrderStatus> on_status;
+      public LatencyOrderSink(Bucket incoming_latency, Bucket outgoing_latency)
+      {
+         this.incoming_latency = incoming_latency;
+         this.outgoing_latency = outgoing_latency;
+      }
+            
+      public void reset()
+      {
+         _orders.Clear();
+         _status.Clear();
+         on_status = null;
+      }
+
+      public void send(OrderEntry order)
+      {
+         var state = new api.orderman.OrderState
+         {
+            Status = api.orderman.OrderStatus.Working,
+            Quantity = (uint)order.size,
+            Price = order.price
+         };
+         var status = new OrderStatus(order, state);
+         order.last_change = status.TransactTime = order.create_time.add_nanos((int)this.incoming_latency.LowerBound);
+         status.SendingTime = status.TransactTime.add_nanos((int)this.outgoing_latency.LowerBound);
+         _orders[order.order_id] = order;
+         _status[order.order_id] = status;
+         send_status(status.clone());
+      }
+      
+      //from an external source, such as a fill model, probably not necessary other than to enforce not waking an order back up once it's terminal
+      public void record_order_status(OrderStatus s)
+      {
+         _status[s.order.order_id] = s;
+      }
+
+      public void update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = 0)
+      {
+         Log.assert(_orders.TryGetValue(order_id, out OrderEntry order));
+         Log.assert(_status.TryGetValue(order_id, out OrderStatus status));
+         if (!status.is_terminal())
+         {
+            order.price = price;
+            if (size > uint.MinValue)
+               order.size = size;
+            order.last_change = status.TransactTime = asof.add_nanos((int)this.incoming_latency.LowerBound);
+            status.SendingTime = status.TransactTime.add_nanos((int)this.outgoing_latency.LowerBound);
+            status.state.Quantity = (uint)order.size;
+            status.state.Price = order.price;
+            send_status(status.clone());
+         }
+      }
+
+      public void cancel(string security_desc, Timestamp asof, ulong order_id)
+      {
+         Log.assert(_orders.TryGetValue(order_id, out OrderEntry order));
+
+         Log.assert(_status.TryGetValue(order_id, out OrderStatus status));
+         if (!status.is_terminal())
+         {
+            order.last_change = asof;
+            order.last_change = status.TransactTime = asof.add_nanos((int)this.incoming_latency.LowerBound);
+            status.SendingTime = status.TransactTime.add_nanos((int)this.outgoing_latency.LowerBound);
+            status.state.Status = api.orderman.OrderStatus.Canceled;
+            send_status(status.clone());
+         }
+      }
+
+      void send_status(OrderStatus stat)
+      {
+         if (this.on_status != null)
+            this.on_status(stat);
+      }
+   }
+}