Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis

changed the keys
change aws keys
2020-04-06 16:00:24 -04:00 · 2020-04-06 16:00:12 -04:00 · 2020-03-31 11:00:26 -05:00 · 2020-03-03 17:09:56 -05:00 · 2020-03-02 18:32:19 -05:00 · 2020-03-02 09:46:10 -05:00
48 changed files with 3313 additions and 367 deletions
@@ -1,5 +1,7 @@
 nohup.out
 build/
+build
+cme_data/
 cmake-build-debug/
 .idea/
 .dir-locals.el
@@ -34,3 +36,5 @@ trace_data/
 /doc/user/public/*
 .ionide/*
 /.ignore
+/build
+/analysis_data
@@ -165,6 +165,12 @@
            "externalConsole": true,
            "requireExactSource": false
        },
+        {
+            "name": "Client Attach",
+            "type": "clr",
+            "request": "attach",
+            "processId": "${command:pickProcess}"
+        },
        {
            "name": ".NET Framework Attach",
            "type": "clr",
@@ -0,0 +1,123 @@
+drop table if exists msgs ;
+create table msgs (
+       MsgID Int4,
+       TradeDate varchar,
+       SeqNum Int4,
+       SendingTime varchar,
+       TransactTime varchar,
+       MatchEventIndicator varchar,
+       NoMDEntries Int2,
+       NoOrderIDEntries Int2,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+drop table if exists books ;
+create table books (
+       MsgID Int4,
+       SendingTime varchar,
+       TransactTime varchar,
+       SeqNum Int4,
+       bid_p_0 Real,
+       bid_sz_0 Int4,
+       bid_p_1 Real,
+       bid_sz_1 Int4,
+       bid_p_2 Real,
+       bid_sz_2 Int4,
+       bid_p_3 Real,
+       bid_sz_3 Int4,
+       bid_p_4 Real,
+       bid_sz_4 Int4,
+       bid_p_5 Real,
+       bid_sz_5 Int4,
+       bid_p_6 Real,
+       bid_sz_6 Int4,
+       bid_p_7 Real,
+       bid_sz_7 Int4,
+       bid_p_8 Real,
+       bid_sz_8 Int4,
+       bid_p_9 Real,
+       bid_sz_9 Int4,
+       Symbol Varchar,
+       ask_p_0 Real,
+       ask_sz_0 Int4,
+       ask_p_1 Real,
+       ask_sz_1 Int4,
+       ask_p_2 Real,
+       ask_sz_2 Int4,
+       ask_p_3 Real,
+       ask_sz_3 Int4,
+       ask_p_4 Real,
+       ask_sz_4 Int4,
+       ask_p_5 Real,
+       ask_sz_5 Int4,
+       ask_p_6 Real,
+       ask_sz_6 Int4,
+       ask_p_7 Real,
+       ask_sz_7 Int4,
+       ask_p_8 Real,
+       ask_sz_8 Int4,
+       ask_p_9 Real,
+       ask_sz_9 Int4,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+drop table if exists trades;
+create table trades (
+       MsgID Int4,
+       Symbol varchar,
+       SendingTime varchar,
+       TransactTime varchar,
+       TradeDate varchar,
+       MDEntryPx Real,
+       MDnetrySize Int4,
+       NumberOfOrders Int4,
+       AgressorSide Int4,
+       MDTradeEntryID Int4,
+       AgressorOID varchar,
+       RestingOID varchar,
+       PartialQty Int4,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+drop table if exists orders;
+create table orders (
+       MsgID Int4,
+       Symbol varchar,
+       SendingTime varchar,
+       TransactTime varchar,
+       MDEntryPx Real,
+       OrderID varchar,
+       MDDisplayQty Int4,
+       MDOrderPriority Int8,
+       OrderUpdateAction Int4,
+       Side Int4,
+       SendingTime_UnixTS Int8,
+       TransactTime_UnixTS Int8
+);
+
+-- create unique index msg_msg_id on msgs msgid;
+
+-- create view ranges as ( 
+--   select
+--      sendingtime,
+--      transacttime,
+--      bid_p_0 price,
+--      case when lag(bid_p_0) over (order by SendingTime) = bid_p_0 then null else 1 end seq 
+--   from
+--      u0v0_books);
+     
+-- create view groups as (
+--   select
+--      sendingtime,
+--      transacttime,
+--      price,
+--      sum(seq) over (order by SendingTime) grp
+--   from
+--      ranges);
+     
+     
+       
+   
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+DB=$1
+TABLE=$2
+FILE=$3
+
+zcat $FILE | psql -U postgres -d $DB -c "copy $TABLE from STDIN with delimiter as ',' CSV HEADER" 
@@ -0,0 +1,4 @@
+#!/bin/bash
+docker exec -it postgres psql -U postgres "$@"
+
+
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+#docker run --net host -e PGADMIN_DEFAULT_EMAIL=bts -e PGADMIN_DEFAULT_PASSWORD=BT5u53r dpage/pgadmin4
+
+docker run \
+       --rm \
+       -d \
+       -p 8000:80 \
+       -e PGADMIN_DEFAULT_EMAIL=bts \
+       -e PGADMIN_DEFAULT_PASSWORD=BT5u53r \
+       --name pgadmin \
+       dpage/pgadmin4 
@@ -0,0 +1,24 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SRC_ROOT=$DIR/../../..
+DATA_DIR=`readlink -f $SRC_ROOT/analysis_data`
+DB_DIR=$DATA_DIR/db
+
+mkdir -p $DB_DIR
+
+docker run \
+       --rm \
+       -d \
+       --name postgres \
+       --net host \
+       -v $DB_DIR:/var/lib/postgresql/data \
+       -v /home/$USER:/home/$USER \
+       -v $DATA_DIR:$DATA_DIR \
+       -v /etc/group:/etc/group \
+       -v /run/user/:/run/user/ \
+       -u $UID \
+       -t postgres \
+       postgres 
+
+
+
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SRC_ROOT=$DIR/../../
+DATE=$1
+DB=md_$DATE
+
+$DIR/db/psql.sh -c "drop database if exists $DB"
+$DIR/db/psql.sh -c "create database $DB"
+$DIR/db/psql.sh -d md_$DATE -f $DIR/db/create_db.sql
+
+PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE
+
+ALL_COMMANDS=""
+for TABLE in trades orders msgs books 
+do
+    for ITEM in $PROCESSED_DATA/*.$TABLE.gz
+    do
+	ALL_COMMANDS+="docker exec -t postgres /bin/bash -c \""
+	ALL_COMMANDS+="$DIR/db/import_csv.sh $DB $TABLE $ITEM"
+	ALL_COMMANDS+="\"\n"
+    done
+done
+
+echo -e $ALL_COMMANDS | parallel --will-cite
@@ -0,0 +1,29 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DATE=$1
+
+SRC_ROOT=$DIR/../../
+SPARK_BIN_ROOT=`readlink -f $SRC_ROOT/build`/ninja_release/server
+RAW_DATA=$SRC_ROOT/analysis_data/raw/$DATE
+PROCESSED_DATA=$SRC_ROOT/analysis_data/processed/$DATE
+
+mkdir -p $PROCESSED_DATA
+rm $PROCESSED_DATA/* >/dev/null
+
+ALL_COMMANDS=""
+for IN_PATH in $RAW_DATA/*.gz
+do
+    BASE_NAME=`basename $IN_PATH`
+    BASE_NAME_STEM=${BASE_NAME%.*}
+    for CMD in trades orders msgs books 
+    do
+	OUT_PATH=$PROCESSED_DATA/$BASE_NAME_STEM.$CMD.gz
+	ALL_COMMANDS+="(cd $SPARK_BIN_ROOT && zcat $IN_PATH | ./sparkd parsehistoricalfix $CMD | gzip >$OUT_PATH)\n"
+    done
+done
+echo -e $ALL_COMMANDS | parallel --will-cite
+
+
+	 
+
+    
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+d=$1
+end=${2-`date +%Y%m%d`}
+until [[ $d > $end ]]; do 
+    echo "$d"
+    d=$(date -d "$d + 1 day" +%Y%m%d)
+done
@@ -0,0 +1,8 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DATE=${1-`date -d "yesterday" +%Y%m%d`}
+
+$DIR/sync.sh $DATE &&
+$DIR/parse.sh $DATE &&
+$DIR/load.sh $DATE 
+    
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DATE=$1
+SRC_ROOT=$DIR/../../
+RAW_DATA_ROOT=$SRC_ROOT/analysis_data/raw/$DATE
+mkdir -p $RAW_DATA_ROOT
+AWS_ACCESS_KEY_ID=AKIAIFIGOOWGBLEMO3PQ AWS_SECRET_ACCESS_KEY=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 AWS_REGION=us-east-1 aws s3 sync s3://cme-bts-datamine/mbo/mbo/$DATE/ $RAW_DATA_ROOT
@@ -14,6 +14,13 @@ namespace bts.history
      class SecData{
         public uint n_updates = 0;
         public System.IO.FileStream str;
+
+         public SecData(string full_path,
+                        string contract){
+            var contract_fr = contract.Replace(":",".");
+            string bk_file_name = String.Format ("{0}{1}",full_path,contract_fr);
+            str = new FileStream(bk_file_name, FileMode.OpenOrCreate);
+         }
         public SecData (System.IO.FileStream str_a){
            str = str_a;
         }
@@ -26,16 +33,24 @@ namespace bts.history
      private readonly string _book_path;
      IDictionary<string, SecData> _book_stream_map = 
                              new Dictionary<string, SecData>();
-
+      private System.IO.FileStream _order_str, _trade_str;
+      
      public BookAnalyzer(string symbol,
                          HistoricalDataOptions options) {
-         _book_path = get_book_path(symbol, DateTime.Parse(options.replaystart));
+         _book_path = DataIOHelper.get_data_path("/books");
         System.IO.Directory.CreateDirectory(_book_path);
+         _order_str = new FileStream(
+            String.Format ("{0}orders_{1}",_book_path,symbol),
+            FileMode.OpenOrCreate);
+         _trade_str = new FileStream(
+            String.Format ("{0}trades_{1}",_book_path,symbol),
+            FileMode.OpenOrCreate);
      }

      public void pre_analyze(MarketDataIncrementalRefresh md, 
-                              string debug_fix = null)
-      {}
+                              string debug_fix = null){
+         _write_orders(md);
+      }

      public void analyze(MarketDataIncrementalRefresh md,
                          string debug_fix = null)
@@ -55,59 +70,62 @@ namespace bts.history
               if (Utils.quote_event_done(md))
                  on_new_book(book, entry, md);
            }
-            // else if (entry.MdEntryType == MDEntryType.Trade)
-            //    handle trades here
+            else if (entry.MdEntryType == MDEntryType.Trade)
+               on_new_trade(entry,md);
         }
+         _write_orders(md);
      }
-
-      public static string get_book_path (string symbol,
-                                          DateTime date){
-         return String.Format(@"{0}{1}/{2}/{3}/{4}/{5}/",
-                           Configuration.get<bts.config.System>().data_path,
-                           "trade_analysis",
-                           "market_data",
-                           date.ToShortDateString().Replace("/","_"),
-                           "books",
-                           symbol);
-      }
-
-      public static string get_book_path (string symbol,
-                                          string sec_name,
-                                          DateTime date){         
-         return String.Format(@"{0}{1}",
-                              get_book_path(symbol, date),
-                              sec_name);
-      }
-      
-      public static string get_algo_input_path (string algo_name){
-         return String.Format(@"{0}{1}/{2}/{3}/",
-                           Configuration.get<bts.config.System>().data_path,
-                           "trade_analysis",
-                           "algos",
-                           "algo_name");
-      }
-
+      private int _bn = 0;
      private void on_new_book(co2.BookMessage book,
                               MDEntry entry,
                               MarketDataIncrementalRefresh md_inc){
         ProcessedBook pb = new ProcessedBook();
         pb.MsgSeqNum = md_inc.MsgSeqNum;
         pb.SendingTime = md_inc.SendingTime;
-         pb.TransactTime = md_inc.TransactTime;         
+         pb.TransactTime = md_inc.TransactTime;
         pb.Security = entry.Symbol;
         pb.Book = book;        
         SecData s;
         if(!_book_stream_map.TryGetValue(pb.Security, out s)){
-            string file_name = pb.Security;
-            file_name = file_name.Replace(":",".");
-            file_name = String.Format ("{0}{1}",_book_path,file_name);
-            var str = new FileStream(file_name, FileMode.OpenOrCreate);
-            s = new SecData (str);
+            s = new SecData (_book_path,pb.Security);
            _book_stream_map.Add(pb.Security,s);
-         }         
+         }
         pb.BookSeqNum = s.n_updates;
         s.inc_count();
         Google.Protobuf.MessageExtensions.WriteDelimitedTo(pb,s.str);
+         if(_bn++%10000==0)
+            Console.WriteLine("n_bks:{0}",_bn);
+      }
+
+      private int _n_ords = 0;
+      private HashSet<UInt64> _ord_map = new HashSet<UInt64>();
+      private void _write_orders(MarketDataIncrementalRefresh md){
+         foreach(var oid in md.OrderIdEntries)
+            if(oid.OrderUpdateAction==MDUpdateAction.New){          
+            //if(!_ord_map.Contains(oid.OrderId)){
+               NewOrderEntry new_ord = new NewOrderEntry();
+               new_ord.OrderId = oid.OrderId;
+               new_ord.SendingTime = md.SendingTime;
+               new_ord.TransactTime = md.TransactTime;
+               _ord_map.Add(oid.OrderId);
+               Google.Protobuf.MessageExtensions.WriteDelimitedTo(new_ord,_order_str);
+               if(_n_ords++%10000==0)
+                  Console.WriteLine("n_ords:{0}",_n_ords);
+            }
+      }
+
+      private void on_new_trade(MDEntry md_entry,
+                                MarketDataIncrementalRefresh md_inc){
+         string contract = md_entry.Symbol;                             
+         TradeEntry tr = new TradeEntry();
+         tr.Contract = contract;
+         tr.Price = md_entry.MdEntryPx;
+         tr.Qty = md_entry.MdEntrySize;
+         tr.TradeId = md_entry.MdTradeEntryId;
+         tr.OrderId = md_entry.OrderId;
+         tr.SendingTime = md_inc.SendingTime;
+         tr.TransactTime = md_inc.TransactTime;
+         Google.Protobuf.MessageExtensions.WriteDelimitedTo(tr,_trade_str);
      }
   }
 }
@@ -13,8 +13,10 @@ namespace bts.history
   class DataDownloader
   {
      private static AmazonS3Client _client;
-      private const string _access_key = "AKIAJ4XUDWPQ7M42352A";
-      private const string _secret_key = "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";
+      //private const string _access_key = "AKIAJ4XUDWPQ7M42352A";
+      //private const string _secret_key = "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";
+      private const string _access_key = "AKIAIFIGOOWGBLEMO3PQ";
+      private const string _secret_key = "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";
      static DataDownloader()
      {
         AmazonS3Config config = new AmazonS3Config();
@@ -182,6 +184,58 @@ namespace bts.history
         return await download_uncompressed(key);
      }

+      public void download_compressed(DateTime dt,
+                                      string symbol,
+                                      api.SecurityType st,
+                                      string output_file_path){
+         if(!File.Exists(output_file_path)){
+            var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
+            var micmap = micconf.cme_mic;
+            if (micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic)){
+               string key = make_key(dt, themic, symbol, st);
+               var task = _download_s3(key, output_file_path);
+               while (!task.IsCompleted)
+                  System.Threading.Thread.Sleep(25);
+            }
+            else
+               throw new ApplicationException(
+                     String.Format("Could not find mic for {0}",symbol));
+        }
+     }
+
+     async Task<Stream> _download_s3(string s3_key,
+                                     string output_file_path){         
+        try{
+           GetObjectRequest request = new GetObjectRequest {
+                                             BucketName = _bucket_name, 
+                                             Key = s3_key 
+                                      };
+           using (GetObjectResponse response = await _client.GetObjectAsync(request)){
+              string title = response.Metadata["x-amz-meta-title"];
+              string contentType = response.Headers["Content-Type"];
+              bts.Log.Gen.info($"Object metadata, Title: {title}");
+              Console.WriteLine($"Downloading Object metadata, Title: {title} Key { s3_key }");
+              Console.WriteLine($"Content type: {contentType}");
+              using (Stream response_stream = response.ResponseStream){
+                 new FileInfo(output_file_path).Directory.Create();
+                 FileStream writer = new FileStream(output_file_path,
+                                                    FileMode.CreateNew);
+                 response_stream.CopyTo(writer);
+                 writer.Seek(0, SeekOrigin.Begin);
+                 return writer;
+              }
+           }
+        }
+        catch (AmazonS3Exception e){
+           bts.Log.Gen.error($"S3 Error encountered ***. Message:'{e.Message}' when writing an object {s3_key}");            
+           throw;
+        }
+        catch (Exception e){
+           bts.Log.Gen.error($"Unknown encountered on server. Message:'{e.Message}' when writing an object {s3_key}");
+           throw;
+        }
+      }
+
   }

 }
@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace bts.history{
-   class BookEntry{
-      // conveniant wrapper around ProcessedBook and its lifetime
-      // (timerange telling its birth and death)
-      private readonly ProcessedBook _pbook;
-      public readonly Timerange time_range;
-      public BookEntry (ProcessedBook pbook_a,
-                        Timerange time_range_a){
-         _pbook = pbook_a;
-         time_range = time_range_a;
-      }
-   }
-}
@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Google.Protobuf.WellKnownTypes;
+using System.IO;
+
+namespace bts.history{
+   // all book states for a security, for the duration of a day
+   class BookSeries{
+      // the prices we are getting from BookAnalzyer dont have
+      // the exponent populated   
+      private readonly IEnumerable<ProcessedBook> _books;
+
+      // this will change the ProcessedBook
+      public BookSeries(IEnumerable<ProcessedBook> pbooks,
+                        bool merge_duplicates_tops=false){
+         _books = _to_book_entries(pbooks, merge_duplicates_tops);
+      }
+
+      static IEnumerable<ProcessedBook> _to_book_entries(
+                                              IEnumerable<ProcessedBook> pbooks,
+                                              bool merge_duplicates_tops){
+         IEnumerable<ProcessedBook> rv = pbooks;
+         if(merge_duplicates_tops)
+            rv = MarketQueriesHelper.merge_duplicates_tops(pbooks);
+         MarketQueriesHelper.fill_time_range(rv);
+         return rv;
+      }
+
+      public int count(){
+         return _books.Count();
+      }
+      public IEnumerable<ProcessedBook> book_entries(){
+         return _books;
+      }
+   }
+}
@@ -0,0 +1,139 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System.Text;
+using System.IO;
+using mbts.config;
+using bts.config;
+using bts.api.orderman;
+using bts.co2;
+using mbts.co2;
+using Google.Protobuf.WellKnownTypes;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+
+
+// date-range, product, path (or file), optional spread
+namespace bts.history{
+   class CircleAnalyzer{
+      public CircleAnalyzer(HistoricalDataOptions hd_options){
+         Console.WriteLine("Circle:Hit key to continue:"); Console.ReadKey();
+         // product, path, book, trades, orders
+         string[] args = hd_options.algo_params.Split(',');
+         Log.assert(args.Count()==5);
+         var symbol = args[0];
+         var path = DataIOHelper.get_data_path(args[1]) + "/";
+         var book_file = path+args[2];
+         var trades_file = path+args[3];
+         var orders_file = path+args[4];
+         // generate smaller files (temp)
+      //    Console.WriteLine("Generating smaller files");
+      //    HashSet<string> sec_map = new HashSet<string>(
+      //          new string[]{"CLG0-CLM0","CLM0-CLZ0","CLG0-CLZ0"});
+      //    DataIOHelper.filter_books_for_contracts(book_file,
+      //                                            sec_map,
+      //                                            path + "t_" + args[2]);
+      //    DataIOHelper.filter_orders_for_contracts(orders_file,
+      //                                             sec_map,
+      //                                             path + "t_" + args[4]);
+
+      //    Console.WriteLine("Truncated, Hit key to continue:"); Console.ReadKey();
+
+         var trade_map = DataIOHelper.load_trades(trades_file);
+         _debug_trades (trade_map);
+         var order_map = DataIOHelper.load_orders(orders_file);         
+         var spr_book_map = DataIOHelper.load_books(book_file,symbol,true);
+         var circ_spreads = MarketQueriesHelper.generate_3_legged_circulars(
+                                                            symbol,
+                                                            spr_book_map.Keys);
+         List<SpreaderFill> spr_fills = new List<SpreaderFill>();
+         foreach(var s in circ_spreads)
+            _analyze_circle(s,spr_book_map,trade_map,order_map,ref spr_fills);
+         string out_file = path + "circle.csv";
+         _write_to_file(spr_fills,out_file);
+      }
+
+      public void analyze(){
+      }
+
+      private void _debug_trades(IDictionary<string,List<TradeRow>> tr_map){
+         var trs = tr_map["CLM0-CLZ0"];
+         Timerange r = new Timerange(){
+            T1 = new Timestamp(){
+               Seconds =  1581609969,
+               Nanos = 401912164                       
+            },
+            T2 = new Timestamp{
+               Seconds = 1581609969,
+               Nanos = 406509563      
+            }
+         };
+         MarketQueries.find_trades_in_time_range(trs,r);
+      }
+
+      // parse the legs, change the q's and call _analyze_circle with       
+      private static void _analyze_circle (
+                                    string synth_txt,
+                                    IDictionary<string,BookSeries> md_map,
+                                    IDictionary<string,List<TradeRow>> trd_map,
+                                    IDictionary<UInt64,IEnumerable<OrderRow>> ord_map,
+                                    ref List<SpreaderFill> spr_fills){         
+         // alternates quoting per leg
+         var spr_ord_orig = MarketQueriesHelper.parse_synthetic(synth_txt);
+         for(uint i=0; i<spr_ord_orig.ExecutionStrategy.LegParamsMap.Count; i++){
+            var spr_ord = MarketQueriesHelper.parse_synthetic(synth_txt);
+            foreach (var kv in spr_ord.ExecutionStrategy.LegParamsMap){
+               if(kv.Key==i)
+                  kv.Value.IsActiveQuoting = true;
+               else
+                  kv.Value.IsActiveQuoting = false;
+            }
+            spr_fills.AddRange(_analyze_circle(i,spr_ord,md_map,trd_map,ord_map));
+         }
+      }
+
+      private static IEnumerable<SpreaderFill> _analyze_circle (
+                                    uint q_idx,
+                                    SpreaderOrderDefinition spr,
+                                    IDictionary<string, BookSeries> md_map,
+                                    IDictionary<string,List<TradeRow>> trd_map,
+                                    IDictionary<UInt64,IEnumerable<OrderRow>> ord_map){
+         var t1 = DateTime.Now;         
+         List<SpreaderFill> rv = new List<SpreaderFill>();
+         rv.AddRange(MarketQueries.find_spreader_fills(spr,
+                                                       md_map,
+                                                       trd_map,
+                                                       ord_map,
+                                                       api.Side.Buy,
+                                                       -0.001));
+         rv.AddRange(MarketQueries.find_spreader_fills(spr,
+                                                       md_map,
+                                                       trd_map,
+                                                       ord_map,
+                                                       api.Side.Sell,
+                                                       0.001));
+         Console.WriteLine("Spr:{0} q_leg:{1} count:{2} time:{3} secs",
+                  spr.Security.Synthetic.Name,
+                  q_idx,
+                  rv.Count(),
+                  (DateTime.Now-t1).TotalSeconds);
+         return rv;
+      }
+
+      private static void _write_to_file (IEnumerable<SpreaderFill> spr_fills,
+                                          string file){
+         using(StreamWriter sw = new StreamWriter(file)){
+            string header = String.Empty;
+            for (int i=0; i<2; i++)
+               header += String.Format("{0},",DataIOHelper.leg_book_header());
+            header += DataIOHelper.quote_fill_header();
+            sw.WriteLine(header);
+            foreach (var f in spr_fills){
+               var ln = DataIOHelper.spreader_fill_to_csv(f);
+               sw.WriteLine(ln);
+            }
+         }
+      }
+   }
+}
@@ -0,0 +1,498 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.spark;
+using bts.api;
+using bts.api.orderman;
+using bts.co2;
+using mbts.config;
+using bts.config;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+using System.IO;
+using mbts.co2;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history{
+   // secondary level helper functions (parsing, reading files etc)
+   class DataIOHelper{
+      public static string get_data_path (string relative_path){
+         return String.Format(@"{0}/{1}",
+                           Configuration.get<bts.config.System>().data_path,
+                           relative_path);
+      }
+      public static IEnumerable<string> get_files_recursively (string dir){
+         List<string> rv = new List<string>(Directory.GetFiles(dir));
+         foreach (string d in Directory.GetDirectories(dir))
+            rv.AddRange(get_files_recursively(d));
+         return rv;
+      }
+      public static IDictionary<string,BookSeries> load_books(string file_name,
+                                                               string symbol, 
+                                                               bool top_only){
+        DateTime t1 = DateTime.Now;
+        var rv = new Dictionary<string,List<ProcessedBook>>();
+        int ln_no = 0;
+        using (StreamReader sr = new StreamReader(file_name)) {
+           string line;
+           while ((line = sr.ReadLine()) != null){
+              if(ln_no++>0){
+                 var pbook = _csv_to_processedbook(line,true,top_only);
+                 if(pbook != null){
+                    List<ProcessedBook> contract_books = null;
+                    if(!rv.TryGetValue(pbook.Security,out contract_books)){
+                       contract_books = new List<ProcessedBook>();
+                       rv.Add(pbook.Security,contract_books);
+                    }
+                    if(MarketQueriesHelper.is_sparse_book(pbook.Book))
+                       Log.assert(contract_books.Count==0,
+                                  String.Format("Illegal sparse book {0}",
+                                                pbook.BookSeqNum));
+                    else
+                       contract_books.Add(pbook);
+                 }
+              }
+           }
+        }
+        Console.WriteLine ("Loaded {0} books in {1} secs",
+                           ln_no, (DateTime.Now - t1).TotalSeconds);
+        foreach (var books in rv.Values)
+           MarketQueriesHelper.validate_book_series(books);
+        var ret = rv.ToDictionary(kv => kv.Key, kv => new BookSeries(kv.Value,top_only));
+        return ret;
+      }
+
+      public static void filter_books_for_contracts(string in_file,
+                                                    HashSet<string> sec_map,
+                                                    string out_file){
+         _filter_for_contracts(in_file,sec_map,out_file,
+                               l => _book_csv_contract(l));
+      }
+
+      public static void filter_orders_for_contracts(string in_file,
+                                                    HashSet<string> sec_map,
+                                                    string out_file){
+         _filter_for_contracts(in_file,sec_map,out_file,
+                               l => l.Split(',')[1]);
+      }
+
+      private static void _filter_for_contracts(string in_file,
+                                              HashSet<string> sec_map,
+                                              string out_file,
+                                              Func<string,string> get_contract){
+         int ln_no = 0;
+         StreamWriter sw = new StreamWriter(out_file);
+         using (StreamReader sr = new StreamReader(in_file)) {
+            string line;
+            while ((line = sr.ReadLine()) != null){
+               if(ln_no++ == 0)
+                  sw.WriteLine(line);
+               else{
+                  var contract = get_contract(line);
+                  if(sec_map.Contains(contract))
+                     sw.WriteLine(line);
+               }
+            }
+         }
+         sw.Close();
+      }
+
+      public static IDictionary<string,List<TradeRow>> load_trades(
+                                                            string file_name){
+         var rv = new Dictionary<string,List<TradeRow>>();
+         int ln_no = 0;
+         using (StreamReader sr = new StreamReader(file_name)) {
+            string line;
+            while ((line = sr.ReadLine()) != null){
+               if(ln_no++>0){
+                  var trow = _csv_to_traderow(line);
+                  List<TradeRow> contract_trades = null;
+                  if(!rv.TryGetValue(trow.Contract,out contract_trades)){
+                     contract_trades = new List<TradeRow>();
+                     rv.Add(trow.Contract,contract_trades);
+                  }
+                  contract_trades.Add(trow);
+               }
+            }
+         }
+         foreach (var t in rv.Values)
+            t.Sort((a,b) => a.TransactTime.compare(b.TransactTime));
+         Console.WriteLine("Loaded {0} trades", ln_no);
+         return rv;
+      }
+
+      public static IDictionary<UInt64, IEnumerable<OrderRow>> load_orders(
+                                                            string file_name){
+         var t1 = DateTime.Now;
+         var rv = new Dictionary<UInt64, List<OrderRow>>();
+         int ln_no = 0;
+         using (StreamReader sr = new StreamReader(file_name)) {
+            string line;
+            while ((line = sr.ReadLine()) != null){
+               if(ln_no++>0){
+                  var orow = _csv_to_orderrow(line);
+                  List<OrderRow> ord_actions;
+                  if(!rv.TryGetValue(orow.OrderId, out ord_actions)){
+                     ord_actions = new List<OrderRow>();
+                     rv.Add(orow.OrderId, ord_actions);
+                  }
+                  ord_actions.Add(orow);
+                  if(ln_no % 100000 == 0)
+                     Console.WriteLine("Loaded {0} orders in {1} secs",
+                                       ln_no, (DateTime.Now-t1).TotalSeconds);
+               }
+            }
+         }
+         Console.WriteLine("Loaded {0} orders in {1} secs", 
+                            ln_no, (DateTime.Now-t1).TotalSeconds);
+         return rv.ToDictionary(kv => kv.Key, kv => 
+                                    kv.Value as IEnumerable<OrderRow>);
+      }
+
+      static Timestamp _parse_time_string(string ts){
+         int yy = Int32.Parse(ts.Substring(0,4)),
+             mm = Int32.Parse(ts.Substring(4,2)),
+             dd = Int32.Parse(ts.Substring(6,2)),
+             hh = Int32.Parse(ts.Substring(8,2)),
+             mins = Int32.Parse(ts.Substring(10,2));
+         // now we are left with the remainder seconds 
+         // expressed in ns
+         string secs_in_ns_str = ts.Substring(12,ts.Count()-12);
+         Int64 secs_in_ns = Int64.Parse(secs_in_ns_str);
+         Int64 fractional_ns,
+               quotient_secs =  Math.DivRem(secs_in_ns,
+                                            1000000000,
+                                            out fractional_ns);
+         Log.assert(fractional_ns < Int32.MaxValue);
+         Log.assert(quotient_secs < Int32.MaxValue);
+         Int64 tot_seconds_epoch = Convert.ToInt64(
+             (new DateTime(yy,mm,dd,hh,mins,Convert.ToInt32(quotient_secs)) -
+             new DateTime(1970,1,1)).TotalSeconds);
+         return new Timestamp(){
+             Seconds = tot_seconds_epoch,
+             Nanos = Convert.ToInt32(fractional_ns)
+         };
+      }  
+
+      private static string _book_csv_contract(string csv_row) {
+         uint n_levels = 10;
+         string[] toks = csv_row.Split(',');
+         return toks[4+2*n_levels]; 
+      }
+      private static ProcessedBook _csv_to_processedbook(
+                                                string csv_row,
+                                                bool only_cal_sprs,
+                                                bool top_only){
+         uint n_levels = 10;
+         uint i = 0;
+         string[] toks = csv_row.Split(',');
+         uint sec_idx = 4+(n_levels*2);
+         if (only_cal_sprs)
+            if(!MarketQueriesHelper.is_single_prod_cal_spr(toks[sec_idx]))
+               return null;
+         ProcessedBook rv = new ProcessedBook(){Book = new BookMessage()};
+         Log.assert(toks.Length == 5+4*n_levels || toks.Length == 7+4*n_levels);
+         rv.MsgSeqNum = UInt32.Parse(toks[i++]);
+         rv.SendingTime = _parse_time_string(toks[i++]);
+         rv.TransactTime = _parse_time_string(toks[i++]);
+         rv.BookSeqNum = UInt32.Parse(toks[i++]);
+         uint iter_levels = top_only ? 1 : n_levels;
+         for (uint n=0; n<iter_levels; n++){
+            double b = Double.Parse(toks[i++]);
+            Int32 bqty = Int32.Parse(toks[i++]);
+            if(bqty < 0)
+               break;
+            rv.Book.Bids.Add(MarketQueriesHelper.to_pricesize(
+                             b,Convert.ToUInt32(bqty)));
+         }
+         i = sec_idx;
+         rv.Security = toks[i++];
+         for (uint n=0; n<iter_levels; n++){
+            double a = Double.Parse(toks[i++]);
+            Int32 aqty = Int32.Parse(toks[i++]);
+            if(aqty < 0)
+               break;
+            rv.Book.Asks.Add(MarketQueriesHelper.to_pricesize(
+                             a,Convert.ToUInt32(aqty)));
+         }
+         if(rv.Book.Bids.Count == 0)
+            rv.Book.Bids.Add(MarketQueriesHelper.to_pricesize(0,0));
+         if(rv.Book.Asks.Count == 0)
+            rv.Book.Asks.Add(MarketQueriesHelper.to_pricesize(0,0)); 
+         return rv;
+      }
+
+      private static TradeRow _csv_to_traderow(string csv_row,
+                                               bool only_cal_sprs=true){
+         UInt32 i = 0;
+         string[] toks = csv_row.Split(',');
+         TradeRow rv = new TradeRow();
+         rv.MsgId = UInt32.Parse(toks[i++]);
+         rv.Contract = toks[i++];
+         rv.SendingTime = _parse_time_string(toks[i++]);
+         rv.TransactTime = _parse_time_string(toks[i++]);
+         i++;//TradeDate
+         rv.MdEntryPx = Double.Parse(toks[i++]);
+         rv.MdEntrySz = UInt32.Parse(toks[i++]);
+         rv.NoOfOrders = UInt32.Parse(toks[i++]);
+         rv.AggressorSide = UInt32.Parse(toks[i++]);
+         rv.MdTradeEntryId = UInt64.Parse(toks[i++]);
+         rv.AggressorOid = UInt64.Parse(toks[i++]);
+         rv.RestingOid = UInt64.Parse(toks[i++]);
+         rv.PartialQty = UInt32.Parse(toks[i++]);
+         return rv;         
+      }
+
+      private static OrderRow _csv_to_orderrow(string csv_row,
+                                               bool only_cal_sprs=true){
+         UInt32 i = 0;
+         string[] toks = csv_row.Split(',');
+         OrderRow rv = new OrderRow();
+         rv.MsgId = UInt32.Parse(toks[i++]);
+         rv.Contract = toks[i++];
+         rv.SendingTime = _parse_time_string(toks[i++]);
+         rv.TransactTime = _parse_time_string(toks[i++]);
+         rv.MdEntryPx = Double.Parse(toks[i++]);
+         rv.OrderId = UInt64.Parse(toks[i++]);
+         rv.MdEntryQty = UInt32.Parse(toks[i++]);
+         rv.MdOrderPriority = UInt64.Parse(toks[i++]);
+         rv.OrderUpdateAction = UInt32.Parse(toks[i++]);
+         rv.Side = UInt32.Parse(toks[i++]);
+         return rv;
+      }
+
+      // Writing out to csv ///////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////  
+      #region Writing out
+
+      private static string _format_synthetic_detail(SpreaderOrderDefinition spr,
+                                                     Side side){
+         Func<Side,string> sign = (leg_side) => {
+            return leg_side == Side.Buy ? "+" : "-";
+         };
+         string rv = side == Side.Buy ? "BUY  " : "SELL  ";
+         var i=0;
+         foreach (var l in spr.Security.Synthetic.Legs){
+            bool is_quoting = 
+                spr.ExecutionStrategy.LegParamsMap.ElementAt(i++).Value.IsActiveQuoting;
+            rv += String.Format("{0}{1}({2})",
+                                            sign(l.Side),
+                                            is_quoting ? "Q" : String.Empty,
+                                            l.Security.Glbx.SecurityDesc);
+         }
+         rv = String.Format("'{0}'",rv);
+         return rv;
+      }
+
+      private static string _format_timestamp(Timestamp t, bool show_date = true){
+        if(show_date)  
+           return t.ToString().Replace("T","  ");
+        else
+           return t.ToString().Split('T')[1];
+      }
+      // individual legs of the spreader  
+      public static string leg_book_header(){
+        return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
+                        "Contract",
+                        "MsgID",
+                        "SeqNum",
+                        "ProcTime", //SendTime-TransactTime (not included here)
+                        "SameTopBooks",
+                        "SendTime",
+                        "NextTrans",
+                        "ActivePeriod", // NextTrans-SendTime
+                        "b_0",
+                        "bq_0",
+                        "a_0",
+                        "aq_0");
+      }      
+      public static string leg_book_entry_to_csv(ProcessedBook b){
+        return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
+                        b.Security,
+                        b.MsgSeqNum,
+                        b.BookSeqNum,
+                        MarketQueriesHelper.book_processing_time(b),
+                        (b.AbsorbedBooks == null || b.AbsorbedBooks.Count == 0) ? String.Empty :
+                            String.Format("{0}:{1}-{2}",
+                            b.AbsorbedBooks.Count, 
+                            b.AbsorbedBooks.First(),
+                            b.AbsorbedBooks.Last()),
+                        _format_timestamp(b.TimeRange.T1),
+                        _format_timestamp(b.TimeRange.T2),
+                        MarketQueriesHelper.period_ns(b.TimeRange),
+                        MarketQueriesHelper.book_price(b,true,0),
+                        MarketQueriesHelper.book_qty(b,true,0),
+                        MarketQueriesHelper.book_price(b,false,0),
+                        MarketQueriesHelper.book_qty(b,false,0));
+      }      
+      public static string synthetic_book_header(){
+         return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8}",
+                        "Synth",
+                        "idx",
+                        "T1",   // create time for this synthetic state
+                        "T2",   // end time  (anologous to above)
+                        "ActivePeriod", //T2-T1
+                        "En:Ex", //Leg that created this state:Leg that deactivated this state
+                        "MaxProcTime",  //Max (Leg proc time)
+                        "p",
+                        "q");
+      }
+      public static string synthetic_book_entry_to_csv(SyntheticPrice p){
+         var b = p.synthetic_book;
+         var min_max = MarketQueriesHelper.intersection_range_idx(
+             p.leg_books.Select(lb=>lb.TimeRange).ToArray());
+         return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8}",
+                        b.Security,
+                        p.idx,
+                        _format_timestamp(b.TimeRange.T1),
+                        _format_timestamp(b.TimeRange.T2),                        
+                        MarketQueriesHelper.period_ns(b.TimeRange),
+                        String.Format("{0}({1}):{2}({3})",
+                                min_max.Item1,
+                                p.leg_books.ElementAt(min_max.Item1).Security,
+                                min_max.Item2,
+                                p.leg_books.ElementAt(min_max.Item2).Security),
+                        p.max_processing_time(),
+                        p.viable_side == Side.Buy ? MarketQueriesHelper.book_price(b,true,0) :
+                                                    MarketQueriesHelper.book_price(b,false,0),
+                        p.viable_side == Side.Buy ? MarketQueriesHelper.book_qty(b,true,0) :
+                                                    MarketQueriesHelper.book_qty(b,false,0));
+      }      
+
+      public static string synthetic_price_to_csv(SyntheticPrice p){
+         string rv = String.Empty;
+         foreach (var l in p.leg_books)
+             rv += String.Format("{0},",leg_book_entry_to_csv(l));
+         rv += synthetic_book_entry_to_csv(p);
+         return rv;
+      }
+      public static string quote_fill_header(){
+         return string.Format("{0},{1},{2},{3},{4},{5},{6},{7}," + 
+                              "{8},{9},{10},{11},{12},{13},{14}," +
+                              "{15},{16},{17},{18},{19},{20},{21}," + 
+                              "{22},{23},{24},{25},{26},{27},{28},{29}," +
+                              "{30}",
+                        // contract
+                        "SyntheticD",                   //0
+                        "SyntheticS",
+                        "QuotingLeg",
+                        // stats
+                        "IsSnipe",                      //3
+                        "HedgeBooksID",
+                        "HedgeBooksActivePeriod",
+                        "TimeLine",
+                        "RequoteReactionTime",
+                        "HedgeTimeWindow",
+                        // trade
+                        "TradeMsgID",                   //9
+                        "TradeID",
+                        "TradeSendTime",
+                        "TradeTransactTime",
+                        "TradePrice",
+                        "TotQty",
+                        "NOrders",
+                        "TradeQty",
+                        "TradeAggressorSide",
+                        "TradeAggressorOID",
+                        "TradeRestingID",
+                        // requote order
+                        "QOrdMsgID",                    //20
+                        "QOrdPrice",
+                        "QOrdQty",
+                        "QOrdTransactTime",
+                        "QOrdTimeDiff",
+                        "QOrdMD",
+                        // opp order
+                        "OppOrdMsgID",                  //26
+                        "OppOrdPrice",
+                        "OppOrdQty",
+                        "OppOrdTransactTime",
+                        "OppOrdMD"                      //31
+                        );
+      }
+      public static string quote_fill_to_csv(SpreaderFill f){
+         var r = f.quote_trade;
+         Func<Side,string> sign = (side) => {
+            return side == Side.Buy ? "+" : "-";
+         };
+         Func<OrderRow,string> ord_sign = (o) => o.Side == 0 ? "+" : "-";
+         Func<ProcessedBook,string> top_str = (b) =>
+            b == null ? String.Empty :         
+            String.Format("{0} {1} {2} {3}",
+                MarketQueriesHelper.book_price(b,true,0),
+                MarketQueriesHelper.book_qty(b,true,0),
+                MarketQueriesHelper.book_price(b,false,0),
+                MarketQueriesHelper.book_qty(b,false,0));
+         var q_leg = MarketQueriesHelper.get_quoting_leg(f.spr_ord);
+         var q_side = MarketQueriesHelper.get_quoting_side(f.spr_ord,
+                                                           f.spr_ord_side);
+         return string.Format("{0},{1},{2},{3},{4},{5},{6},{7}," + 
+                              "{8},{9},{10},{11},{12},{13},{14}," +
+                              "{15},{16},{17},{18},{19},{20},{21}," + 
+                              "{22},{23},{24},{25},{26},{27},{28},{29}," +
+                              "{30}",
+                        // contract
+                        _format_synthetic_detail(f.spr_ord,f.spr_ord_side),
+                        String.Format("{0}/{1}",
+                                    f.spr_ord.Security.Synthetic.Name,
+                                    q_leg.Item2),
+                        String.Format("'{0}({1})'", r.Contract,sign(q_side)),
+                        // stats
+                        f.is_sniper,
+                        f.hedge_books.id,
+                        MarketQueriesHelper.period_ns(f.hedge_books.timerange),
+                        String.Format("{0} {1} {2} {3}",
+                                _format_timestamp(f.hedge_books.timerange.T1,false),
+                                _format_timestamp(f.quote_order_update.TransactTime,false),
+                                _format_timestamp(f.quote_trade.SendingTime,false),
+                                _format_timestamp(f.hedge_books.timerange.T2,false)),
+                        f.requote_reaction_time,
+                        f.hedging_window,
+                        // trade
+                        r.MsgId,
+                        r.MdTradeEntryId,
+                        _format_timestamp(r.SendingTime),
+                        _format_timestamp(r.TransactTime),
+                        r.MdEntryPx,
+                        r.MdEntrySz,
+                        r.NoOfOrders,
+                        r.PartialQty,                        
+                        r.AggressorSide,
+                        r.AggressorOid,
+                        r.RestingOid,
+                        // requote order
+                        f.quote_order_update.MsgId,
+                        f.quote_order_update.MdEntryPx,
+                        String.Format("{0}{1}",
+                            ord_sign(f.quote_order_update),
+                            f.quote_order_update.MdEntryQty),
+                        _format_timestamp(f.quote_order_update.TransactTime,false),
+                        f.requote_reaction_time,
+                        top_str(f.book_at_quote_update_time),
+                        // opp order
+                        f.opp_order_update == null ? 0 : 
+                                                f.opp_order_update.MsgId,
+                        f.opp_order_update == null ? Double.NaN : 
+                                                f.opp_order_update.MdEntryPx,
+                        f.opp_order_update == null ? String.Empty :
+                                              String.Format("{0}{1}",
+                                                ord_sign(f.opp_order_update),
+                                                f.opp_order_update.MdEntryQty),
+                        f.opp_order_update == null ? String.Empty : 
+                            _format_timestamp(f.opp_order_update.TransactTime,false),
+                        f.opp_order_update == null ? String.Empty :
+                            top_str(f.book_at_opp_update_time)
+                        );
+      }
+      public static string spreader_fill_to_csv(SpreaderFill f){
+         string rv = String.Empty; 
+         foreach (var h in f.hedge_books.contract_books)
+            rv += String.Format("{0},",leg_book_entry_to_csv(h));
+         rv += quote_fill_to_csv(f);
+         return rv;
+      }
+
+      #endregion
+   }
+}
@@ -1,96 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using Google.Protobuf.WellKnownTypes;
-using System.IO;
-
-namespace bts.history{
-   // all book states for a security, for the duration of a day
-   class BookState{
-      private readonly List<BookEntry> _books = new List<BookEntry>();      
-      public BookState(string file_path){
-         var pbooks = new List<ProcessedBook>();
-         FileStream fs = new FileStream (file_path, FileMode.Open);
-         while (fs.Position < fs.Length){
-            ProcessedBook pb = new ProcessedBook();
-            Google.Protobuf.MessageExtensions.MergeDelimitedFrom(pb,fs);
-            pbooks.Add(pb);
-         }
-         for (int i=0; i<pbooks.Count(); i++){
-            ProcessedBook pb_i = pbooks.ElementAt(i),
-                          pb_i_next = i+1 >= pbooks.Count() ? null :
-                                                      pbooks.ElementAt(i+1);
-            Timerange r = new Timerange(timestamp(pb_i),timestamp(pb_i_next));
-            _books.Add(new BookEntry(pb_i,r));
-         }
-      }
-
-      public int count(){
-         return _books.Count;
-      }
-      // returns all books values that exist in the timerange t
-      // (note, these are not necassarily the updates in t 
-      // but the possible book values that existed  in timerange t)
-      public IEnumerable<BookEntry> find_books_in_time_range (Timerange t){
-         var rv = new List<BookEntry>();
-         foreach (var b in _books){
-            Timerange b_life = b.time_range;
-            TimerangeIntersection intersect = t.intersection(b_life);
-            switch(intersect){
-               case TimerangeIntersection.LEFT_NOINTERSECTION:
-                  break;
-               case TimerangeIntersection.INTERSECTION:
-                  rv.Add(b);
-                  break;
-               case TimerangeIntersection.RIGHT_NOINTERSECTION:
-                  // since _books are sorted
-                  return rv;
-            }
-         }
-         return rv;
-      }
-      // given a book, finds all co-existing books
-      // in another book
-      // source_book_ptr -> {t1,t2} -> books in {t1,t2}
-      public IEnumerable<BookEntry> find_coexisting_books (
-                              BookEntry book,
-                              BookState b2_s){
-         return b2_s.find_books_in_time_range(book.time_range);
-      }
-      public IEnumerable<Tuple<BookEntry,BookEntry>> find_coexisting_books (
-                              BookEntry book,
-                              BookState b2_s,
-                              BookState b3_s){
-         var rv = new List<Tuple<BookEntry,BookEntry>>();
-         var b2_coex = b2_s.find_books_in_time_range(book.time_range);
-         foreach (BookEntry b2_e in b2_coex){
-            Timerange b1_b2_tr = Timerange.intersection(book.time_range, 
-                                                        b2_e.time_range);
-            var b3_coex = b3_s.find_books_in_time_range(b1_b2_tr);
-            foreach (var b3_e in b3_coex)
-               rv.Add(Tuple.Create(b2_e, b3_e));
-         }
-         return rv;
-      }
-      public void align_books_2(BookState bs_2,
-                                Action<BookEntry, BookEntry> on_book){
-         foreach (var b in _books){
-            var bs_2_coex = find_coexisting_books(b,bs_2);
-            foreach(var bs2_e in bs_2_coex)
-               on_book(b,bs2_e);
-         }
-      }
-      public void align_books_3(BookState bs_2,
-                                BookState bs_3,
-                                Action<BookEntry, BookEntry, BookEntry> on_book){
-         foreach (var b in _books)
-            foreach (var t in find_coexisting_books(b,bs_2,bs_3))
-               on_book(b,t.Item1,t.Item2);
-      }
-      #region  static helpers
-      public static Timestamp timestamp (ProcessedBook b){
-         return b!=null?b.SendingTime:Timerange.max_timestamp();
-      }            
-      #endregion
-   }
-}
@@ -5,39 +5,569 @@ using bts.spark;
 using bts.api;
 using bts.api.orderman;
 using bts.co2;
+using mbts.co2;
+using Google.Protobuf.WellKnownTypes;
+using bts.history;
+using QuoteLegTrades = System.Collections.Generic.IEnumerable<bts.history.TradeRow>;

 namespace bts.history{
-   class MarketQueries{
-      static IEnumerable<BookMessage> get_synthetic_prices(
-                                                 string synthetic_text,
-                                                 DateTime date){
-         var synth = MarketQueriesHelper.parse_synthetic(synthetic_text);
-         IEnumerable<BookState> book_states = 
-            synth.Security.Synthetic.Legs.Select(
-               l => MarketQueriesHelper.get_book_state(l.Security.Symbol,
-                                                       l.Security.Glbx.SecurityDesc,
-                                                       date));
-         // assert >= 2 and <= 3
-         var synth_books = new List<BookMessage>();
-         book_states.ElementAt(0).align_books_3(
-            book_states.ElementAt(1),
-            book_states.ElementAt(2),
-            (b1,b2,b3) => {
-               var b = compute_synthetic_book(synth,
-                                              b1,b2,b3);
-                                              synth_books.Add(b);
-            });
-         return synth_books;
+
+   // this is a snapshot of books for multiple
+   // contracts lined up.
+   public class BooksSnapshot{
+      public readonly UInt64 id;   
+      public readonly IEnumerable<ProcessedBook> contract_books;
+      public BooksSnapshot(
+                        UInt64 id_a,      
+                        IEnumerable<ProcessedBook> contract_books_a){
+         id=id_a;
+         contract_books = contract_books_a;
      }
-      // given a spread def and leg books computes the synthetic price
-      static BookMessage compute_synthetic_book(SpreaderOrderDefinition order_def,
-                                                params BookEntry[] leg_books){
-         bts.co2.BookMessage rv = new bts.co2.BookMessage();
-         return rv;
-      }      
-      static BookState get_book_state (string sec_txt,DateTime date){
-         string book_path = BookAnalyzer.get_book_path(sec_txt,date);
-         return null;
+      public Timerange timerange{
+         get{
+            var ranges = contract_books.Select(p => p.TimeRange);   
+            return MarketQueriesHelper.intersection_range(ranges.ToArray());
+         }
+      }
+      public bool reactable{
+         get{
+            return MarketQueriesHelper.period_ns(timerange) > 0;
+         }
+      }
+      public ProcessedBook at(int h){
+         return contract_books.ElementAt(h);
      }
   }
+   
+   public class SpreaderFill{
+      public readonly SpreaderOrderDefinition spr_ord;
+      public readonly Side spr_ord_side;
+      public readonly BooksSnapshot hedge_books;
+      public readonly TradeRow quote_trade;
+      public readonly OrderRow quote_order_update;
+      public readonly OrderRow opp_order_update;
+      public readonly ProcessedBook book_at_quote_update_time;
+      public readonly ProcessedBook book_at_opp_update_time;
+      public readonly uint total_quote_leg_trades;
+      public SpreaderFill(SpreaderOrderDefinition spr_ord_a,
+                          Side spr_ord_side_a,
+                          BooksSnapshot hedge_books_a,
+                          TradeRow quote_trade_a,
+                          OrderRow quote_order_update_a,
+                          OrderRow opp_order_update_a,
+                          ProcessedBook book_at_quote_update_time_a,
+                          ProcessedBook book_at_opp_update_time_a,
+                          uint total_quote_leg_trades_a){
+         spr_ord = spr_ord_a;
+         spr_ord_side = spr_ord_side_a;
+         hedge_books = hedge_books_a;
+         quote_trade = quote_trade_a;
+         quote_order_update = quote_order_update_a;
+         opp_order_update = opp_order_update_a;
+         book_at_quote_update_time = book_at_quote_update_time_a;
+         book_at_opp_update_time = book_at_opp_update_time_a;
+         total_quote_leg_trades = total_quote_leg_trades_a;
+         Log.assert(quote_order_update != null);
+         Log.assert(opp_order_update == null ||
+                    (quote_order_update.Side != opp_order_update.Side));      
+      }
+      public Int64 requote_reaction_time{
+         get{
+            return MarketQueriesHelper.diff_ns(
+                  quote_order_update.TransactTime,
+                  hedge_books.timerange.T1);
+         }                  
+      }
+      public Int64 hedging_window{
+         get{
+            return MarketQueriesHelper.diff_ns(
+                  hedge_books.timerange.T2,
+                  quote_trade.SendingTime);
+         }
+      }
+      public bool is_sniper{
+         get{
+            return quote_trade.AggressorOid == quote_order_update.OrderId;
+         }             
+      }
+   }
+
+   public class SyntheticPrice{
+      public readonly IEnumerable<ProcessedBook> leg_books;
+      public readonly ProcessedBook synthetic_book;
+      public SyntheticPrice(IEnumerable<ProcessedBook> leg_book_a,
+                            ProcessedBook synthetic_book_a){
+         leg_books = leg_book_a.ToArray();
+         synthetic_book = synthetic_book_a;
+      }
+      public api.Side viable_side = api.Side.Undefined;
+      public UInt16 idx = 0;
+      public Int64 max_processing_time(){
+         var rv = leg_books.MaxElement(b => 
+               MarketQueriesHelper.book_processing_time(b));
+         return MarketQueriesHelper.book_processing_time(rv);
+      }
+   }
+
+   class MarketQueries{      
+      // Synthetic value comp /////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////
+      #region Synthetic Book Calc
+
+      public static void get_synthetic_prices(
+                            SpreaderOrderDefinition synth,
+                            IEnumerable<BookSeries> leg_book_series,
+                            Action<SyntheticPrice> on_price){
+         align_bookseries(
+               leg_book_series,
+               bks_snp => {
+                  var sbe = _compute_synthetic_top_book(
+                                                synth, 
+                                                bks_snp.contract_books.ToArray());
+                  on_price(new SyntheticPrice(bks_snp.contract_books,sbe));
+               });
+      }
+
+      // stop gap, till it is easier to call native function
+      private static ProcessedBook _compute_synthetic_top_book(
+                                             SpreaderOrderDefinition order_def,
+                                             params ProcessedBook[] leg_books){
+         double top_bid=0, top_ask=0;
+         UInt32 top_bid_q=UInt32.MaxValue, top_ask_q=UInt32.MaxValue;
+         for (int i=0; i<leg_books.Count();i++){
+            ProcessedBook lb = leg_books[i];
+            double b = MarketQueriesHelper.book_price(lb,true,0),
+                   a = MarketQueriesHelper.book_price(lb,false,0);
+            UInt32 bq = MarketQueriesHelper.book_qty(lb,true,0),
+                   aq = MarketQueriesHelper.book_qty(lb,false,0);
+            ContractLeg l = order_def.Security.Synthetic.Legs[i];
+            top_bid += l.PriceMultiplier *(l.Side == Side.Buy ? b : a);
+            top_ask += l.PriceMultiplier * (l.Side == Side.Buy ? a : b);
+            top_bid_q = Math.Min(top_bid_q, l.Side == Side.Buy ? bq : aq);
+            top_ask_q = Math.Min(top_ask_q, l.Side == Side.Buy ? aq : bq);
+         }
+         BookMessage sbm = new BookMessage();
+         sbm.Bids.Add(MarketQueriesHelper.to_pricesize(top_bid, top_bid_q));
+         sbm.Asks.Add(MarketQueriesHelper.to_pricesize(top_ask, top_ask_q));
+         return _synth_book_to_entry (order_def,sbm,0,leg_books);
+      }
+      private static ProcessedBook _synth_book_to_entry(
+                                                SpreaderOrderDefinition order_def,
+                                                BookMessage bm,
+                                                UInt32 seq_no,
+                                                params ProcessedBook[] leg_books){
+         Timerange r = MarketQueriesHelper.intersection_range(
+                              leg_books.Select(l=>l.TimeRange).ToArray());         
+         return new ProcessedBook {
+            Security = order_def.Security.Synthetic.Name,
+            Book = bm,
+            TimeRange = r
+         };
+      }
+
+      #endregion
+
+      // Alligning Books //////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////
+      #region Books
+
+      public static void align_bookseries(
+                              IEnumerable<BookSeries> leg_book_serieses,
+                              Action<IEnumerable<ProcessedBook>> on_book){
+         align_bookseries(leg_book_serieses.ToArray(),
+                          bsnp => on_book(bsnp.contract_books));
+      }
+
+      public static UInt64 _id_store = 0;
+      public static void align_bookseries(
+                              IEnumerable<BookSeries> leg_book_serieses,
+                              Action<BooksSnapshot> on_book){
+         // book series with all leg books         
+         var all = new List<Tuple<uint,ProcessedBook>>();
+         ProcessedBook[] curr_vals = 
+            leg_book_serieses.Select(i=>(ProcessedBook)null).ToArray();
+         Func<bool> is_ready = () => {
+            foreach(var l in curr_vals)
+               if(l==null)
+                  return false;
+            return true;
+         };
+         uint ord = 0;
+         foreach (var s in leg_book_serieses){
+            all.AddRange(s.book_entries().Select(be=>Tuple.Create(ord,be)));
+            ord++;
+         }
+         all.Sort((x,y)=>
+            MarketQueriesHelper.compare(x.Item2.TimeRange.T1,
+                                        y.Item2.TimeRange.T1));
+         foreach (var t in all){
+            curr_vals[t.Item1]=t.Item2;
+            if(is_ready())
+               on_book(new BooksSnapshot(_id_store++, curr_vals));
+         }
+      }
+
+      // finds the last book observed (send time) at at_time
+      public static ProcessedBook find_book_at (BookSeries bser,
+                                                Timestamp at_time){
+         var i = MarketQueriesHelper.bin_search_nearest(
+               bser.book_entries(),
+               at_time,
+               (b,t) => MarketQueriesHelper.diff_ns(b.TimeRange.T1,at_time));
+         // nearest can be on either side
+         var b_i = bser.book_entries().ElementAt(i);
+         var diff_i = MarketQueriesHelper.diff_ns(b_i.TimeRange.T1, at_time);
+         if(diff_i<=0)
+            return b_i;
+         else{
+            if(i>0){
+               var b_i1 = bser.book_entries().ElementAt(i-1);
+               var diff_i1 = MarketQueriesHelper.diff_ns(b_i1.TimeRange.T1, at_time);
+               Log.assert(diff_i1<=0);
+               return b_i1;
+            }
+            return null;
+         }
+      }
+      #endregion
+      
+      #region Orders and Trades
+
+      public static IEnumerable<TradeRow> find_trades_in_time_range_lin (
+                                                IEnumerable<TradeRow> trades,
+                                                Timerange r){
+         return trades.Where(t => MarketQueriesHelper.within(r,t.TransactTime));
+      }
+
+      public static IEnumerable<TradeRow> find_trades_in_time_range (
+                                                IEnumerable<TradeRow> trades,
+                                                Timerange r){
+         List<TradeRow> rv = new List<TradeRow>();
+         if(trades.Count()==0)
+            return rv;
+         int f = MarketQueriesHelper.bin_search_nearest(
+                  trades,
+                  r.T1, 
+                  (t,b) => MarketQueriesHelper.diff_ns(t.TransactTime,b));
+         // iterate and as soon as something is out
+         // and over, return
+         for(int i=f; i<trades.Count(); i++){
+            TradeRow t = trades.ElementAt(i);
+            switch(MarketQueriesHelper.ts_location(r,t.TransactTime)){
+               case TimerangeIntersection.INTERSECTION:
+                  rv.Add(t);
+                  break;
+               case TimerangeIntersection.LEFT_NOINTERSECTION:
+                  break;
+               case TimerangeIntersection.RIGHT_NOINTERSECTION:
+                  // return as soon as the trade surpasses the range
+                  return rv;
+            }
+         }
+      //    var alt_rv = find_trades_in_time_range_lin(trades,r);
+      //    if(rv.Count() != alt_rv.Count())
+      //       Log.assert(rv.Count() == alt_rv.Count(), 
+      //                   String.Format("{0}!={1}", rv.Count(),alt_rv.Count()));
+         return rv;
+      }
+
+      // For all order updates to a single order, finds
+      // the latest priority change (ie TransTime closest 
+      // to t1) bw t0 and t1
+      // Returns null if there is no priority change
+      // within the window
+      public static OrderRow find_last_priority_change (
+                                    IEnumerable<OrderRow> ord_updates,
+                                    Timestamp t0,
+                                    Timestamp t1){
+         for (int i=ord_updates.Count()-1; i>=0; i--){
+            OrderRow ord = ord_updates.ElementAt(i);
+            if(MarketQueriesHelper.within(t0,t1,ord.TransactTime)){
+               if(i==0 || ord.OrderUpdateAction==0)
+                  return ord;
+               else{
+                  OrderRow prev_r = ord_updates.ElementAt(i-1);
+                  if(ord.MdOrderPriority != prev_r.MdOrderPriority)
+                     return ord;
+               }
+            }
+         }
+         return null;
+      }
+
+      // just a conveniant wrapper around the above
+      public static OrderRow find_last_priority_change (
+                        IDictionary<UInt64,IEnumerable<OrderRow>> ord_map,
+                        UInt64 oid,
+                        Timestamp t0,
+                        Timestamp t1){
+         IEnumerable<OrderRow> ord_updates;
+         if(ord_map.TryGetValue(oid, out ord_updates))
+            return find_last_priority_change(ord_updates,t0,t1);
+         return null;
+      }
+
+      //(assumes the order side is never changed)
+      public static Nullable<Side> find_order_side (
+                        UInt64 oid,
+                        IDictionary<UInt64,IEnumerable<OrderRow>> ord_map){
+         IEnumerable<OrderRow> orows = null;
+         if(ord_map.TryGetValue(oid, out orows))
+            if(orows.Count() > 0)
+               return orows.ElementAt(0).Side == 0 ? Side.Buy : Side.Sell;
+         return null;
+      }
+
+      // returns false if the order changes sides
+      public static bool unch_side (
+                        ulong oid,
+                        IDictionary<UInt64,IEnumerable<OrderRow>> ord_map){
+         IEnumerable<OrderRow> o_upds;
+         if(ord_map.TryGetValue(oid, out o_upds))
+            for(int i=1; i<o_upds.Count(); i++)
+               if(o_upds.ElementAt(i).Side != o_upds.ElementAt(i-1).Side)
+                  return false;
+         return true;
+      }
+
+      public static bool unch_contract (
+                        ulong oid,
+                        IDictionary<UInt64,IEnumerable<OrderRow>> ord_map,
+                        string contract){
+         IEnumerable<OrderRow> o_upds;
+         if(ord_map.TryGetValue(oid, out o_upds))
+            foreach (var u in o_upds)
+               if(u.Contract != contract)
+                  return false;
+         return true;
+      }
+
+      public static bool trade_err_check (
+                        TradeRow r,
+                        IDictionary<UInt64,IEnumerable<OrderRow>> ord_map){
+         if (r.AggressorOid == r.RestingOid)
+            return false;
+         // side doesnt change on either order
+         // implied trades
+         if(r.AggressorSide == 0){
+            Console.WriteLine ("Ignoring, implied tid:{0}", r.MdTradeEntryId);
+            return false;
+         }
+         if(!unch_side(r.AggressorOid,ord_map)){
+            Console.WriteLine ("Ignoring, switched sides tid:{0} a:{1}", 
+                                r.MdTradeEntryId,r.AggressorOid);
+            return false;
+         }
+         if(!unch_side(r.RestingOid,ord_map)){
+            Console.WriteLine ("Ignoring, wwitched sides tid:{0} a:{1}",
+                                r.MdTradeEntryId,r.RestingOid);
+            return false;
+         }
+         //crossing orders has same side
+         Nullable<Side> a_side = find_order_side(r.AggressorOid,ord_map),
+                        p_side = find_order_side(r.RestingOid,ord_map);
+         if(a_side.HasValue && p_side.HasValue)
+            if(a_side.Value == p_side.Value){
+               Console.WriteLine ("Ignoring, same sides tid:{0} a:{1} p:{2}", 
+                                   r.MdTradeEntryId,r.AggressorOid,r.RestingOid);
+               return false;
+            }
+         // same contract on either order as the trade
+         if(!unch_contract(r.AggressorOid,ord_map,r.Contract)){
+            Console.WriteLine ("Ignoring, incorrect contract tid:{0} a:{1}", 
+                                r.MdTradeEntryId,r.AggressorOid);
+            return false;
+         }
+         if(!unch_contract(r.AggressorOid,ord_map,r.Contract)){
+            Console.WriteLine ("Ignoring, incorrect contract tid:{0} a:{1}", 
+                                r.MdTradeEntryId,r.AggressorOid);
+            return false;
+         }         
+         // header row         
+         return true;
+      }
+
+      // finds the last updates (or create) that lead to a priority change
+      // for the order on either side of the trade
+      public static Tuple<OrderRow,OrderRow> find_crossing_orders (
+                        TradeRow trade,
+                        IDictionary<UInt64,IEnumerable<OrderRow>> ord_map){
+         if(trade_err_check(trade,ord_map)){
+            Log.assert(trade.AggressorSide == 1 ||
+                       trade.AggressorSide == 2);
+            OrderRow r_ord = find_last_priority_change(ord_map,
+                                                trade.RestingOid,
+                                                MarketQueriesHelper.ref_min_dt,
+                                                trade.TransactTime);
+            OrderRow a_ord = find_last_priority_change(ord_map,
+                                                trade.AggressorOid,
+                                                MarketQueriesHelper.ref_min_dt,
+                                                trade.TransactTime);
+            return Tuple.Create(r_ord,a_ord);
+         }
+         return null;
+      }
+
+      // tuple.Item3 is true is if the quoter order is the aggressor
+      // (ie snipe)
+      public static Tuple<OrderRow,OrderRow,bool> find_possible_requote_2 (
+                              Timerange hedge_books_tr,
+                              TradeRow q_leg_trade,
+                              IDictionary<UInt64,IEnumerable<OrderRow>> ord_map,
+                              Side q_side){
+         Log.assert(MarketQueriesHelper.within(hedge_books_tr,
+                                               q_leg_trade.TransactTime));
+         // 0 = No aggressor
+         // 1 = Buy
+         // 2 = Sell
+         Func<OrderRow,Side> ord_side = o => {
+            var u = o.Side;
+            Log.assert(u==0 || u==1);
+            return u == 0 ? Side.Buy : Side.Sell;
+         };
+         var pa_tup = find_crossing_orders(q_leg_trade,ord_map);
+         if(pa_tup != null){
+            var p_ord = pa_tup.Item1;
+            var a_ord = pa_tup.Item2;
+            if(p_ord != null && q_side == ord_side(p_ord)){
+               // passive quoting order
+               if(MarketQueriesHelper.is_after(p_ord.TransactTime,
+                                               hedge_books_tr.T1))
+                  return Tuple.Create(p_ord,a_ord,false);
+            }
+            else if (a_ord != null && q_side == ord_side(a_ord)){
+               return Tuple.Create(p_ord,a_ord,true);
+            }
+         }
+         return null;
+      }
+
+      // finds all the qleg fills paired with hedge books snapshots
+      // (ie q leg trades that happened during the time the hedgebook
+      // snapshots were active) that would have given the desired
+      // spreader price and side
+      public static IEnumerable<Tuple<BooksSnapshot,QuoteLegTrades>> 
+                              find_qleg_trades_at_price(
+                                       SpreaderOrderDefinition spr,
+                                       IDictionary<string, BookSeries> md_map,
+                                       IDictionary<string,List<TradeRow>> trd_map,
+                                       api.Side spr_side,
+                                       double spr_price){
+         var rv = new List<Tuple<BooksSnapshot,QuoteLegTrades>>();
+         var q_legs = MarketQueriesHelper.find_legs(spr,true);
+         Log.assert(q_legs.Count()==1);
+         var q_leg = q_legs.ElementAt(0);         
+         var non_q_legs = MarketQueriesHelper.find_legs(spr,false);
+         Log.assert(non_q_legs.Count()>=1);
+         var non_q_books = non_q_legs.Select(
+               l => md_map[MarketQueriesHelper.sec_name(l)]);
+         List<TradeRow> qleg_trades = null;
+         if(trd_map.TryGetValue(MarketQueriesHelper.sec_name(q_leg),
+                                out qleg_trades)){
+            MarketQueries.align_bookseries(non_q_books, h_bks_snp => {
+               if (h_bks_snp.reactable){
+                  var qleg_trades_in_tr = find_trades_in_time_range(
+                                          qleg_trades,h_bks_snp.timerange);
+                  qleg_trades_in_tr = qleg_trades_in_tr.Where(tr =>
+                        conform_to_price(spr, spr_side, spr_price, h_bks_snp,tr));
+                  if(qleg_trades_in_tr.Count()>0)                  
+                     rv.Add(Tuple.Create(new BooksSnapshot(
+                              h_bks_snp.id, h_bks_snp.contract_books.ToArray()),
+                              qleg_trades_in_tr.ToArray() as QuoteLegTrades));
+               }
+            });
+         }
+         return rv;
+      }
+
+      public static IEnumerable<SpreaderFill> find_spreader_fills(
+                                    SpreaderOrderDefinition spr_ord,
+                                    IDictionary<string, BookSeries> md_map,
+                                    IDictionary<string,List<TradeRow>> trd_map,
+                                    IDictionary<UInt64,IEnumerable<OrderRow>> ord_map,
+                                    api.Side spr_ord_side,
+                                    double spr_ord_price){
+         Func<OrderRow,ProcessedBook> book_at_ord = (o) => {
+            if(o != null){
+               BookSeries bser;
+               if(md_map.TryGetValue(o.Contract, out bser))
+                   return find_book_at(bser,o.TransactTime);
+            }
+            return null;
+         };
+         var rv = new List<SpreaderFill>();
+         var q_side = MarketQueriesHelper.get_quoting_side(spr_ord,spr_ord_side);
+         var hedge_books_q_leg_trades = find_qleg_trades_at_price(
+                                                spr_ord,
+                                                md_map,
+                                                trd_map,
+                                                spr_ord_side,
+                                                spr_ord_price);
+         foreach (var tup in hedge_books_q_leg_trades){
+            BooksSnapshot hedge_books = tup.Item1;
+            IEnumerable<TradeRow> q_leg_trades = tup.Item2;
+            foreach (var qlt in q_leg_trades){
+               var tr_ord = find_possible_requote_2(hedge_books.timerange,
+                                                    qlt,
+                                                    ord_map,
+                                                    q_side);
+               var q_ord_row = tr_ord == null ? null : 
+                               tr_ord.Item3 ? tr_ord.Item2 : tr_ord.Item1;               
+               if(q_ord_row!=null){
+                  var opp_ord_row = tr_ord.Item3 ? tr_ord.Item1 : tr_ord.Item2;
+                  SpreaderFill spr_fill = new SpreaderFill(
+                                    spr_ord,
+                                    spr_ord_side,
+                                    hedge_books,
+                                    qlt,
+                                    q_ord_row,
+                                    opp_ord_row,
+                                    book_at_ord(q_ord_row),
+                                    book_at_ord(opp_ord_row),
+                                    (uint)hedge_books_q_leg_trades.Count());
+                  rv.Add(spr_fill);
+               }
+            }
+         }
+         return rv;
+      }
+      
+      // returns true if price or better      
+      public static bool conform_to_price(
+                                    SpreaderOrderDefinition spr,
+                                    api.Side spr_order_side,
+                                    double spr_price,
+                                    BooksSnapshot hedge_books,
+                                    TradeRow q_trade){
+         double comp_price = 0;
+         var h = 0;
+         for (var i=0; i<spr.Security.Synthetic.Legs.Count(); i++){
+            var leg = spr.Security.Synthetic.Legs.ElementAt(i);
+            var leg_params = spr.ExecutionStrategy.LegParamsMap.ElementAt(i).Value;
+            if (leg_params.IsActiveQuoting)
+               comp_price += leg.PriceMultiplier * q_trade.MdEntryPx;
+            else{
+               // --------------------------------------------------   
+               // leg_side | order_side |  eff_leg_side | hedge_side
+               // --------------------------------------------------
+               // buy          buy           buy           sell
+               // buy          sell          sell          buy
+               // sell         buy           sell          buy
+               // sell         sell          buy           sell      
+               // --------------------------------------------------
+               var is_hedge_buy = leg.Side != spr_order_side;
+               double hedge_price = MarketQueriesHelper.book_price(
+                                                      hedge_books.at(h),
+                                                      is_hedge_buy,
+                                                      0);
+               comp_price += leg.PriceMultiplier * hedge_price;
+               h++;
+            }
+         }
+         return spr_order_side==Side.Buy && comp_price <= spr_price ||
+                spr_order_side == Side.Sell && comp_price >= spr_price;
+      }     
+      
+      #endregion
+   }
 }
@@ -4,21 +4,224 @@ using System.Linq;
 using bts.spark;
 using bts.api;
 using bts.api.orderman;
+using bts.co2;
+using mbts.config;
+using bts.config;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+using System.IO;
+using mbts.co2;
+using Google.Protobuf.WellKnownTypes;

 namespace bts.history{
+   // in local time (eastern) 
+   class TradingTime{
+      public readonly uint start_m, stop_m;
+      public TradingTime(uint start_hrs, uint start_mins,
+                         uint stop_hrs, uint stop_mins){
+         start_m = start_hrs*60 + start_mins;
+         stop_m = stop_hrs*60 + stop_mins;
+      }
+      public bool within (Timestamp t){
+         DateTime t_utc = MarketQueriesHelper.to_utc_dt(t);
+         DateTime t_local = TimeZoneInfo.ConvertTime(t_utc,TimeZoneInfo.Local);
+         uint mins = Convert.ToUInt32(t_local.Hour * 60 + t_local.Minute);
+         return mins >= start_m && mins <= stop_m;
+      }
+      public static bool within (Timestamp t, IEnumerable<TradingTime> t_list){
+         foreach (var tt in t_list)
+            if(tt.within(t))
+               return true;
+         return false;
+      }
+   } 
   // secondary level helper functions (parsing, reading files etc)
-   class MarketQueriesHelper{      
+   class MarketQueriesHelper{
+      // time range helpers ///////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////
+      #region time range helpers
+      private readonly static UInt64 _ref_secs = Convert.ToUInt64(
+         (new DateTime(2019, 1,1) - 
+          new DateTime(1970, 1, 1)).TotalSeconds);
+      public readonly static DateTime epoch_dt = 
+        new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+      public readonly static Timestamp ref_min_dt = 
+         Timestamp.FromDateTime(new DateTime(2019,1,1,0,0,0,DateTimeKind.Utc));
+      public static int compare (Timestamp xt, Timestamp yt){
+        var rv = xt.Seconds.CompareTo(yt.Seconds);
+        return rv != 0 ? rv : xt.Nanos.CompareTo(yt.Nanos);
+      }  
+      public static Int64 period_ns(Timerange r){
+         return diff_ns(r.T2,r.T1);
+      }
+      public static DateTime to_utc_dt(Timestamp ts){
+         DateTime rv = epoch_dt.AddSeconds(ts.Seconds);
+         rv = rv.AddMilliseconds(ts.Nanos/1000000);
+         return rv;
+      }
+      // returns where x falls on y (from x's POV)
+      public static TimerangeIntersection intersection_type (Timerange x,
+                                                             Timerange y){
+         if (_is_before_or_same(x.T2,y.T1))
+            return TimerangeIntersection.LEFT_NOINTERSECTION;
+         if (_is_before_or_same(y.T2,x.T1))
+            return TimerangeIntersection.RIGHT_NOINTERSECTION;
+         return TimerangeIntersection.INTERSECTION;
+      }
+      // returns where y falls wrt x's range
+      public static TimerangeIntersection ts_location (Timerange x,
+                                                       Timestamp y){
+         if (_is_before(y,x.T1))
+            return TimerangeIntersection.LEFT_NOINTERSECTION;
+         if (is_after(y,x.T2))
+            return TimerangeIntersection.RIGHT_NOINTERSECTION;
+         return TimerangeIntersection.INTERSECTION;
+      }
+      // conveniance access to the above function
+      public static bool intersects(Timerange x, Timerange y){
+          return intersection_type(x,y) == TimerangeIntersection.INTERSECTION;
+      }
+      // finds the overlap between n ranges
+      public static Timerange intersection_range(params Timerange[] ranges){
+         Timerange max_t1 = ranges.MaxElement(r => _from_ref_ns(r.T1)),
+                   min_t2 = ranges.MinElement(r => _from_ref_ns(r.T2));
+         return new Timerange {T1=max_t1.T1,T2=min_t2.T2};
+      }
+      public static Tuple<UInt16,UInt16> intersection_range_idx(
+                                                    params Timerange[] ranges){
+         if(ranges.Length == 0)
+           return null;                
+         UInt16 entry=0, exit=0;
+         // greatest T1
+         for (UInt16 i=0; i<ranges.Length; i++)
+            if(_is_after_or_same(ranges[i].T1, ranges[entry].T1))
+               entry = i;
+         // least T2   
+         for (UInt16 i=0; i<ranges.Length; i++)
+            if(_is_before_or_same(ranges[i].T2, ranges[exit].T2))
+               exit = i;
+          return Tuple.Create(entry,exit);
+      }
+      private static bool _is_before (Timestamp x,
+                                      Timestamp y){
+         return diff_ns(x,y)<0;
+      }
+      private static bool _is_before_or_same (Timestamp x,
+                                              Timestamp y){
+         return diff_ns(x,y)<=0;
+      }
+      public static bool is_after (Timestamp x,
+                                   Timestamp y){
+         return diff_ns(x,y)>0;
+      }
+      private static bool _is_after_or_same (Timestamp x,
+                                              Timestamp y){
+         return !_is_before(x,y);
+      }
+      public static bool within (Timerange r, Timestamp t){
+         return _is_before_or_same(r.T1,t) &&
+                _is_after_or_same(r.T2,t);     
+      }
+      public static bool within (Timestamp t1, Timestamp t2, Timestamp t){
+         return within(new Timerange{T1=t1,T2=t2},t);
+      }
+      // x-y ns
+      public static Int64 diff_ns (Timestamp x,
+                                   Timestamp y){
+         var xns = Convert.ToInt64(_from_ref_ns(x));
+         var yns = Convert.ToInt64(_from_ref_ns(y));
+         return xns - yns;
+      }
+      private static UInt64 _from_ref_ns (Timestamp ts){
+         UInt64 secs = Convert.ToUInt64(ts.Seconds),
+                ns = Convert.ToUInt64(ts.Nanos);
+         return (secs - _ref_secs)*1000000000 + ns;
+      }
+      public static bool within_trading_time(SyntheticPrice p,
+                                             string symbol){
+         TradingTime[] close_times;
+         Log.assert(_close_times.TryGetValue(symbol,out close_times));
+         foreach (var be in p.leg_books)
+            if (TradingTime.within(be.TimeRange.T1,close_times))
+               return false;
+         return true;
+      }
+      #endregion
+            
+      // temp exponent filling ////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////
+      #region symbol constants
+      private static Dictionary<string, int> _exponent_map = new Dictionary<string, int>()
+      {
+         {"CL",-2},
+         {"HE",0},
+         {"ZC",0},
+         {"ZL",-2},
+         {"ZS",-2},
+         {"NG",-2},
+         {"GC",-2}
+      };
+      // these are in central time
+      private static Dictionary<string, TradingTime[]> _close_times = new
+      Dictionary<string, TradingTime[]> ()
+      {
+            // 5:00 - 6:00 ET close
+            {"CL",new TradingTime[]{new TradingTime(17,0,18,0)}},
+            {"NG",new TradingTime[]{new TradingTime(17,0,18,0)}},
+            // Monday - Friday: 9:30 a.m - 2:05 p.m. ET
+            {"HE",new TradingTime[]{new TradingTime(00,00,09,30),
+                                    new TradingTime(14,05,24,00)}},
+            // Sunday – Friday 7:00 p.m. – 7:45 a.m. CT and
+            // Monday – Friday 8:30 a.m. – 1:20 p.m. CT
+            // Closed : 8:45 a.m. - 9:30 a.m. ET
+            //          2:20 p.m. - 8:00 p.m. ET
+            {"ZL",new TradingTime[]{new TradingTime(08,45,09,30),
+                                    new TradingTime(14,20,20,00)}},
+            {"ZC",new TradingTime[]{new TradingTime(08,45,09,30),
+                                    new TradingTime(14,20,20,00)}},
+            {"ZS",new TradingTime[]{new TradingTime(08,45,09,30),
+                                    new TradingTime(14,20,20,00)}},
+      };      
+      #endregion
+                            
+      // synthetic parsing and generation /////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////
+      #region security parsing and generation
+
+        // single prod calendar spread
+      public static bool is_single_prod_cal_spr (string s){
+         if(!s.Contains(":")){
+            string[] toks = s.Split('-');
+            Log.assert(toks.Count() == 2);
+            string sym1 = toks[0].Substring(0,2),
+                   sym2 =  toks[1].Substring(0,2);
+            return sym1 == sym2;
+         }
+         return false;
+      }
+
+      // works for spreaders with 1 q leg only (asserts otherwise)
+      public static Side get_quoting_side (SpreaderOrderDefinition spr_ord,
+                                           Side ord_side){
+         var leg = get_quoting_leg(spr_ord);
+         return leg.Item1.Side == ord_side ? Side.Buy : Side.Sell;
+      }
+
      // name, tick_size, (symbol, leg, qty, price_mult, q/h)*
      public static SpreaderOrderDefinition parse_synthetic (string synthetic_text){         
         SpreaderOrderDefinition rv = new SpreaderOrderDefinition();
+         rv.ExecutionStrategy = new ExecutionStrategy();
         uint i=0;
         string[] toks = synthetic_text.Split(',');
         string synth_name = toks[i++];
         double tick_size = Double.Parse(toks[i++]);
-         rv.Security.Synthetic.Name = toks[i++];
+         rv.Security = new Security();
+         rv.Security.Synthetic = new SyntheticContract();
+         rv.Security.Synthetic.Name = synth_name;

-         // (symbol, leg, qty, price_mult, q/h)*
-         int n_legs = (toks.Length-2)/4;
+         // (symbol,leg,qty,price_mult,q/h)*
+         int n_legs = (toks.Length-2)/5;
         for (int l=0;l<n_legs;l++){
            // contract leg
            string symbol = toks[i++],
@@ -43,18 +246,342 @@ namespace bts.history{

            // LegSpreaderParams
            LegSpreaderParams leg_params = new LegSpreaderParams();
-            leg_params.IsActiveQuoting = toks[i++].ToLower().StartsWith("q");
+            leg_params.IsActiveQuoting = is_quoting;
            leg_params.EnableHedging = true;
            rv.ExecutionStrategy.LegParamsMap.Add(Convert.ToUInt32(l),leg_params);
            // sec_text list
         }
         return rv;
      }
-      public static BookState get_book_state (string symbol,
-                                             string sec_txt,
-                                             DateTime date){
-         string book_path = BookAnalyzer.get_book_path(symbol, sec_txt,date);
-         return new BookState(book_path);
+
+      public static Dictionary<string, int> mmap = new Dictionary<string, int>()
+      {
+         {"F",1},{"G",2},{"H",3},{"J",4},{"K",5},{"M",6},
+         {"N",7},{"Q",8},{"U",9},{"V",10},{"X",11},{"Z",12}
+      };
+      public static IEnumerable<string> generate_3_legged_circulars(
+                                  string symbol,
+                                  IEnumerable<string> cal_sprds){
+         int exp;
+         Log.assert(_exponent_map.TryGetValue(symbol,out exp),"Could not find exp");
+         double tick_size = Math.Pow(10,exp);
+         List<string> rv = new List<string>();
+         HashSet<string> smap = new HashSet<string>(cal_sprds);
+         var exp_map = new SortedDictionary<DateTime,string>();         
+         // parse out exp
+         Func<string,DateTime> get_exp = (t) => {
+            if(t.Count()==4){
+               int month = mmap[t.ElementAt(2).ToString()];
+               int yr = Int32.Parse(t.ElementAt(3).ToString());
+               yr = (yr == 9 ? -1 : yr) + 2020;            
+               return new DateTime(yr,month,1);
+            }
+            else{
+               Console.WriteLine("Could not parse {0}",t); 
+               return DateTime.MinValue; 
+            }
+         };
+         foreach (var s in smap)
+            if(!s.Contains(".")){
+               string[] toks = s.Split('-');
+               Log.assert(toks.Count() == 2);
+               foreach (string t in toks){
+                  // check for sym here
+                  string sym = t.Substring(0,2);
+                  if(sym == symbol){
+                     var e = get_exp(t);
+                     if(e != DateTime.MinValue)
+                        exp_map[e] = t;
+                  }
+               }
+            }
+         foreach (var kv1 in exp_map)
+            foreach (var kv2 in exp_map)
+               foreach (var kv3 in exp_map)
+                  if(kv2.Key>kv1.Key && kv3.Key>kv2.Key){
+                     string s1 = String.Format("{0}-{1}",kv1.Value,kv2.Value),
+                            s2 = String.Format("{0}-{1}",kv2.Value,kv3.Value),
+                            s3 = String.Format("{0}-{1}",kv1.Value,kv3.Value);
+                     if(smap.Contains(s1) && smap.Contains(s2) && smap.Contains(s3)){
+                        var spr_name = String.Format("{0}_{1}{2}{3}",
+                                                      symbol,
+                                                      kv1.Value.Substring(2),
+                                                      kv2.Value.Substring(2),
+                                                      kv3.Value.Substring(2));
+                        var spr =
+                         String.Format("{0},{1},"+
+                                       "{2},{3},{4},{5},{6}," +
+                                       "{7},{8},{9},{10},{11}," +
+                                       "{12},{13},{14},{15},{16}",
+                                       spr_name,tick_size,
+                                       symbol,s1,1,1,"Q",
+                                       symbol,s2,1,1,"H",
+                                       symbol,s3,-1,-1,"H");
+                        rv.Add(spr);
+                     }
+                  }
+         return rv;
      }
+
+      public static IEnumerable<ContractLeg> find_legs(
+                                                SpreaderOrderDefinition spr,
+                                                bool is_quoting){
+         List<ContractLeg> rv = new List<ContractLeg>();
+         for (int i=0; i< spr.Security.Synthetic.Legs.Count; i++)
+            if(spr.ExecutionStrategy.LegParamsMap.ElementAt(i).Value.IsActiveQuoting ==
+               is_quoting)
+               rv.Add(spr.Security.Synthetic.Legs.ElementAt(i));
+         return rv;
+      }
+
+      public static string sec_name (ContractLeg l){
+         return l.Security.Glbx.SecurityDesc;
+      }
+
+      public static Tuple<ContractLeg,UInt16> 
+            get_quoting_leg(SpreaderOrderDefinition spr){
+         UInt16 i=0;
+         foreach (var kv in spr.ExecutionStrategy.LegParamsMap){
+            if(kv.Value.IsActiveQuoting)
+               return Tuple.Create(spr.Security.Synthetic.Legs[i],i);
+            i++;  
+         }
+         return null;
+      }
+
+      #endregion
+
+      #region book accessors     
+
+      public static void fill_time_range (IEnumerable<ProcessedBook> pbooks){
+         for (int i=0; i<pbooks.Count(); i++){
+            ProcessedBook pb_i = pbooks.ElementAt(i),
+                          pb_i_next = i+1 >= pbooks.Count() ? null :
+                                             pbooks.ElementAt(i+1);
+            pb_i.TimeRange = new Timerange{
+                  T1 = pb_i.SendingTime,
+                  T2 = (pb_i_next == null ? pb_i.SendingTime : 
+                                            pb_i_next.TransactTime)
+            };
+         }   
+      }
+
+      public static IEnumerable<ProcessedBook> merge_duplicates_tops(
+                                        IEnumerable<ProcessedBook> bser){
+         List<ProcessedBook> rv = new List<ProcessedBook>();
+         if (bser.Count()>0)
+            rv.Add(bser.ElementAt(0));
+         for(int i=1;i<bser.Count();i++){
+            var last_v = rv.Last();
+            var curr_v = bser.ElementAt(i);
+            if(!compare_book_tops(last_v.Book,curr_v.Book,false))
+               rv.Add(curr_v);
+            else{
+               last_v.Book.Bids[0].Size = Math.Max(last_v.Book.Bids[0].Size,
+                                                   curr_v.Book.Bids[0].Size);
+               last_v.Book.Asks[0].Size = Math.Max(last_v.Book.Asks[0].Size,
+                                                   curr_v.Book.Asks[0].Size);
+               last_v.AbsorbedBooks.Add(curr_v.BookSeqNum);
+               last_v.SameTops.Add(curr_v);
+            }
+         }
+         return rv;
+      }
+    
+      private static bool _compare_book_messages(PriceSizeMessage p1,
+                                                 PriceSizeMessage p2,
+                                                 bool compare_qtys = true){
+         return (!compare_qtys ? true : p1.Size == p2.Size) && 
+                p1.Exponent == p2.Exponent &&
+                p1.Mantissa == p2.Mantissa;
+      }
+
+      public static bool compare_books_sides(IEnumerable<PriceSizeMessage> ps1,
+                                             IEnumerable<PriceSizeMessage> ps2){
+         if(ps1.Count() != ps2.Count())
+            return false;
+          for(int i=0;i<ps1.Count();i++)
+             if(!_compare_book_messages(ps1.ElementAt(i),ps2.ElementAt(i)))
+                return false;
+          return true;
+      }
+
+      public static bool compare_books(BookMessage bm1, BookMessage bm2){
+         return compare_books_sides(bm1.Bids,bm2.Bids) && 
+                compare_books_sides(bm1.Asks,bm2.Asks);
+      }
+
+      private static PriceSizeMessage _empty_pricesize = new PriceSizeMessage();
+      public static bool compare_book_tops(BookMessage bm1, 
+                                           BookMessage bm2,
+                                           bool compare_qtys = true){
+         var emp = _empty_pricesize;
+         Func<BookMessage,bool,PriceSizeMessage> top = (bm,is_buy) => {
+            var pss = is_buy ? bm.Bids : bm.Asks; 
+            return pss.Count>0 ? pss.ElementAt(0) : _empty_pricesize;
+         };
+         return _compare_book_messages(top(bm1,true),
+                                       top(bm2,true),
+                                       compare_qtys) &&
+                _compare_book_messages(top(bm1,false),
+                                       top(bm2,false),
+                                       compare_qtys);
+      }
+
+      public static PriceSizeMessage to_pricesize (double price, UInt32 qty){
+         IntPrice int_price = mbts.co2.IntPrice.from_double(price);
+         PriceSizeMessage rv = new PriceSizeMessage();
+         rv.Mantissa = int_price.mantissa;
+         rv.Exponent = int_price.exponent;
+         rv.Size = qty;
+         return rv;
+      }
+
+      public static bool reactable (ProcessedBook be){
+          return diff_ns(be.TimeRange.T2,be.TimeRange.T1) > 0;
+      }
+
+      public static double book_price(ProcessedBook b, bool is_bid, UInt16 level){
+         var vals = is_bid ? b.Book.Bids : b.Book.Asks;
+         var i = vals[level];
+         if (i.Size == 0)
+            return Double.NaN;
+         else{ 
+            var iprice = new mbts.co2.IntPrice(i.Mantissa, Convert.ToSByte(i.Exponent));
+            return iprice.price();
+         }
+      }
+
+      public static UInt32 book_qty(ProcessedBook b, bool is_bid, UInt16 level){
+         var vals = is_bid ? b.Book.Bids : b.Book.Asks;
+         var i = vals[level];
+         return i.Size;
+      }
+
+      public static Int64 book_processing_time (ProcessedBook pb){
+         return diff_ns(pb.SendingTime,pb.TransactTime);
+      }   
+      public static void validate_book_series_order (
+                                    IEnumerable<ProcessedBook> books){
+         for(int i=1;i< books.Count();i++){
+            var prev = books.ElementAt(i-1);
+            var curr = books.ElementAt(i);
+            Log.assert(MarketQueriesHelper.compare(curr.SendingTime,
+                                                    prev.SendingTime) >= 0);
+            Log.assert(curr.MsgSeqNum >= prev.MsgSeqNum);
+             // assert that msg seq no and time are in increasing order             
+         }
+      }
+      public static bool is_sparse(IEnumerable<PriceSizeMessage> psms){
+         UInt32 last_qty = 1;
+         foreach (var p in psms){
+            if(p.Size != 0 && last_qty == 0)
+               return true;
+            last_qty = p.Size;
+         }
+         return false;
+      }
+      public static bool is_sparse_book (BookMessage m){
+         return is_sparse(m.Bids) || is_sparse(m.Asks);        
+      }
+      public static void validate_book_entry(ProcessedBook pb){                    
+          var b = pb.Book;
+          Int64 last_price = Int64.MinValue;
+          Action<PriceSizeMessage,bool> test_entry = (psm, is_bid) => {
+             if(psm.Size>0){
+                // TODO (check for market close time) 
+                // Log.assert(psm.Mantissa > last_price,
+                //             String.Format("{0}: {1} {2}>{3}", 
+                //                            pb.MsgSeqNum, 
+                //                            (is_bid ? "Bid" : "Ask"),
+                //                            psm.Mantissa, last_price));
+                last_price = psm.Mantissa;
+             }
+          };
+          for(int i = b.Bids.Count()-1; i>=0; i--)
+             test_entry(b.Bids[i],true);
+          for(int i = 0; i < b.Asks.Count(); i++)
+            test_entry(b.Asks[i],false);
+      }
+      public static void validate_book_series (IEnumerable<ProcessedBook> books){
+         validate_book_series_order(books);
+         foreach(var b in books)
+            validate_book_entry(b);
+      }
+      public static IEnumerable<BookSeries> get_non_quoting_books (
+                                            SpreaderOrderDefinition spr,
+                                            IDictionary<string,BookSeries> md_map){
+         var rv = new List<BookSeries>();
+         uint i=0;
+         foreach (var l in spr.Security.Synthetic.Legs)
+            if(!spr.ExecutionStrategy.LegParamsMap[i++].IsActiveQuoting)
+               rv.Add(md_map[l.Security.SecurityName()]);
+         return rv;    
+      }
+
+      #endregion
+
+      #region Misc
+      public static IEnumerable<IEnumerable<T>> break_into_multiples<T>(
+                                                      IEnumerable<T> full,
+                                                      UInt16 chunk_sz){
+         List<List<T>> rv = new List<List<T>>();
+         for (UInt16 i=0; i < full.Count(); i++){
+            if (i % chunk_sz == 0)
+               rv.Add(new List<T>());
+            rv.Last().Add(full.ElementAt(i));
+         }
+         UInt16 c=0;
+         rv.ForEach(k => k.ForEach(j => c++));
+         Log.assert(c == full.Count());
+         return rv;
+      }      
+
+      public static int bin_search_nearest<S,T> (IEnumerable<S> sorted_items,
+                                                  T to_find,
+                                                  Func<S,T,Int64> diff,
+                                                  Tuple<int,int> range = null){         
+         Func<int,Int64> diff_i = (idx) => 
+            diff(sorted_items.ElementAt(idx),to_find);
+         range = range == null ?
+            Tuple.Create(0, sorted_items.Count()-1) : range;
+         Log.assert(range.Item2>=range.Item1);
+         if (range.Item2 == range.Item1)
+            return range.Item1;// closest item
+         else{
+            int mid_idx = (range.Item2 - range.Item1)/2 + range.Item1;
+            var d = diff_i(mid_idx);
+            if(d == 0)
+               return mid_idx;
+            else if (d < 0){ 
+               // mid is smaller, go right
+               int new_left = mid_idx+1;
+               if(new_left >= sorted_items.Count())
+                  return mid_idx;
+               var d1 = diff_i(new_left);
+               if(Math.Abs(d) < Math.Abs(d1) || d == -1*d1) 
+               // if more toward mid or in the middle of mid and new_left
+                  return mid_idx;
+               return bin_search_nearest(
+                        sorted_items,
+                        to_find,diff,
+                        Tuple.Create(new_left,range.Item2));
+            }
+            else{ 
+               // mid is greater, go left
+               int new_right = mid_idx-1;
+               if (new_right < 0)
+                  return mid_idx;
+               var d1 = diff_i(new_right);   
+               if(Math.Abs(d) < Math.Abs(d1) || d == -1*d1) 
+                  return mid_idx;
+               return bin_search_nearest(
+                        sorted_items,
+                        to_find,diff,
+                        Tuple.Create(range.Item1,new_right));
+            }
+         }
+      }
+      #endregion
   }
 }
@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MathNet.Numerics.Statistics;
+using System.Text;
+using System.IO;
+using mbts.config;
+using bts.config;
+using bts.api.orderman;
+using Google.Protobuf.WellKnownTypes;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+
+namespace bts.history{   
+   class Microscope{
+      enum Actions{
+         Book,
+         Trade,
+         Order
+      }
+      // looks deeper into the data for
+      // 1. book updates for <contract,date>* bw msg seq no's
+      // 2. book updates for <contract,date>* bw t1 and t2
+      // 3. book turn stats
+      // 4. trades
+      // 5. orders
+      //(writes output to csv that can be plotted in some cases)
+      public Microscope(HistoricalDataOptions hd_options){
+         Console.WriteLine("Miscroscope:Hit key to continue:"); Console.ReadKey();
+         //book,symbol,date,contract,m1,m2
+         string[] args = hd_options.algo_params.Split(',');
+         if (Actions.Book.ToString().ToLower()==args[0].ToLower()){
+            string symbol = args[1];
+            DateTime dt = DateTime.Parse(args[2]);
+            string contract = args[3];
+            UInt32 m1 = UInt32.Parse(args[4]),
+                   m2 = UInt32.Parse(args[5]);
+            //_find_unreactables(symbol,contract,dt);
+            // BookSeries bser = MarketQueriesHelper.get_book_series(symbol,
+            //                                                      contract,
+            //                                                      dt);
+            // var books = MarketQueries.find_in_range(bser,m1,m2);
+            // foreach (BookEntry b in books)
+            //    Console.WriteLine(MarketQueriesHelper.book_entry_to_csv(b));
+         }
+      }
+
+      public void analyze(){
+      }
+   }
+}
@@ -0,0 +1,254 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using bts.spark;
+using bts.api;
+using bts.api.orderman;
+using bts.co2;
+using Google.Protobuf.WellKnownTypes;
+
+namespace bts.history{
+   class SanityTests{
+//       public static void run_tests(){
+//          Console.Write ("Tests started...");
+//          test_books_in_range();
+//          test_random_intersection();
+//          Console.Write ("Tests completed");
+//       }
+      public static void test_bin_search(){
+         Random rnd = new Random(DateTime.Now.Millisecond);
+         int sz = 1000;
+         int n_runs = 1000;
+         for(int t=0; t<n_runs; t++){
+            List<int> rnd_nos = new List<int>();
+            int n=0;
+            for(int i=0; i<sz; i++){
+               n += rnd.Next(10,100);
+               rnd_nos.Add(n);
+            }
+            // exact match
+            for (int i=0 ;i<sz; i++){               
+               int f = MarketQueriesHelper.bin_search_nearest(rnd_nos,rnd_nos[i],(x,y)=>x-y);
+               Log.assert(rnd_nos[i]==rnd_nos[f],String.Format("{0}!={1}",i,f));
+            }
+            // nearest match
+            for (int i=0 ;i<sz-1; i++){
+               int d0 = rnd_nos.ElementAt(i) + 3,
+                   d1 = rnd_nos.ElementAt(i+1) - 3;
+               int f0 = MarketQueriesHelper.bin_search_nearest(rnd_nos,d0,(x,y)=>x-y),
+                   f1 = MarketQueriesHelper.bin_search_nearest(rnd_nos,d1,(x,y)=>x-y);
+               Log.assert(f0==i,String.Format("{0}!={1}",f0,i));
+               Log.assert(f1==i+1,String.Format("{0}!={1}",f1,i+1));
+            }
+         }
+      }
+//       public static void test_books_in_range(){
+//       //    DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc);
+//       //    List<ProcessedBook> bseries_list = new List<ProcessedBook>();         
+
+//       //    Action<UInt32> add_b = (UInt32 usecs) => {
+//       //       ProcessedBook b = new ProcessedBook();
+//       //       b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count());
+//       //       b.SendingTime = b.TransactTime = from_datetime(base_time, usecs);
+//       //       bseries_list.Add(b);
+//       //    };
+
+//       //    add_b(1000);
+//       //    add_b(1020);
+//       //    add_b(1025);
+//       //    add_b(1035);
+//       //    add_b(1050);
+//       //    add_b(1055);
+//       //    add_b(1060);
+//       //    add_b(1065);
+//       //    add_b(1080);
+//       //    add_b(1110);
+
+//       //    BookSeries bser_1 = new BookSeries (bseries_list);
+//       //    var calc_bes = bser_1.book_entries();
+//       //    Log.assert(calc_bes.Count()==10);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(0).TimeRange) == 20000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(1).TimeRange) == 5000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(2).TimeRange) == 10000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(3).TimeRange) == 15000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(4).TimeRange) == 5000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(5).TimeRange) == 5000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(6).TimeRange) == 5000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(7).TimeRange) == 15000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(8).TimeRange) == 30000);
+//       //    Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(9).TimeRange) > 1000000000);
+
+//       //    // find in range /////////////////////////////////////////////////////
+//       //    // test find_books_in_time_range
+//       //    // 1005-1053
+//       //    var bitems = bser_1.find_books_in_time_range(
+//       //                                  new Timerange{
+//       //                                     T1=from_datetime(base_time, 1005),
+//       //                                     T2=from_datetime(base_time, 1053)});
+//       //    Log.assert(bitems.Count()==5);
+//       //    for (int i=0; i<bitems.Count();i++){
+//       //       BookEntry be = bitems.ElementAt(i);
+//       //       Log.assert(be.PBook.MsgSeqNum==Convert.ToUInt32(i));
+//       //    }
+//       //    // test find_books_in_time_range
+//       //    // 1030-2000
+//       //    bitems = bser_1.find_books_in_time_range(
+//       //                                  new Timerange{
+//       //                                     T1=from_datetime(base_time, 1030),
+//       //                                     T2=from_datetime(base_time, 2000)});
+//       //    Log.assert(bitems.Count()==8);
+//       //    for (int i=0; i<bitems.Count();i++){
+//       //       BookEntry be = bitems.ElementAt(i);
+//       //       Log.assert(be.PBook.MsgSeqNum==Convert.ToUInt32(i+2));
+//       //    }
+
+//          // find coexisting 1 ////////////////////////////////////////////////////
+//       //    bseries_list.Clear();
+//       //    add_b(900);
+//       //    add_b(990);
+//       //    add_b(1010);
+//       //    add_b(1015);   //3
+//       //    add_b(1030);
+//       //    BookSeries bser_2 = new BookSeries (bseries_list);
+//       //    var coex = MarketQueries.find_coexisting_books(
+//       //                      bser_1,
+//       //                      bser_2,
+//       //                      new Timerange{T1=from_datetime(base_time, 999),
+//       //                                    T2=from_datetime(base_time, 1051)});
+//       //    _compare_seq_no (coex.ElementAt(0),0,1);  //1000
+//       //    _compare_seq_no (coex.ElementAt(1),0,2);
+//       //    _compare_seq_no (coex.ElementAt(2),0,3);
+//       //    _compare_seq_no (coex.ElementAt(3),1,3);  //1020
+//       //    _compare_seq_no (coex.ElementAt(4),2,3);  //1025
+//       //    _compare_seq_no (coex.ElementAt(5),2,4);  //1025
+//       //    _compare_seq_no (coex.ElementAt(6),3,4);  //1035
+//       //    _compare_seq_no (coex.ElementAt(7),4,4);  //1050
+
+//       //    MarketQueries.align_bookseries(
+//       //       l => {
+//       //          var b1 = l.ElementAt(0);
+//       //          var b2 = l.ElementAt(1);
+//       //          var r = MarketQueriesHelper.intersection_type(b1.TimeRange, b2.TimeRange);
+//       //          Log.assert(r == TimerangeIntersection.INTERSECTION);
+//       //       },
+//       //       bser_1,
+//       //       bser_2);
+//       }
+
+//       public static void test_random_intersection(){
+//          // 3 lists         
+//          Func<int,int,int,int?,BookSeries> cons_series = (n_items,min_i,max_i,r) => {
+//             List<ProcessedBook> book_ser_list = new List<ProcessedBook>();
+//             Random rnd = new Random(DateTime.Now.Millisecond);
+//             int moving_usecs = 0;
+//             for(int i = 0; i < n_items; i++){
+//                moving_usecs += r.HasValue ? r.Value : rnd.Next(min_i,max_i);
+//                _add_to_ser(ref book_ser_list,
+//                            Convert.ToUInt32(moving_usecs));
+//             }
+//             return new BookSeries(book_ser_list);
+//          };
+//          // non ranodm data
+//          BookSeries bser_1 = cons_series(100,1,20,20),
+//                     bser_2 = cons_series(200,2,15,10),
+//                     bser_3 = cons_series(400,2,15,5);
+//          var res = new List<Tuple<BookEntry,BookEntry,BookEntry>>();
+//          MarketQueries.align_bookseries(
+//             l => res.Add(Tuple.Create(l.ElementAt(0),
+//                                       l.ElementAt(1),
+//                                       l.ElementAt(2))),
+//             bser_1,
+//             bser_2,
+//             bser_3);
+//          _verify_intersection(bser_1,bser_2,bser_3,res);
+//          // random data
+//          int n_rnd_tests = 10;
+//          for (int i=0;i<n_rnd_tests;i++){
+//             bser_1 = cons_series(100,1,19,null);
+//             bser_2 = cons_series(100,3,23,null);
+//             bser_3 = cons_series(100,5,31,null);
+//             res = new List<Tuple<BookEntry,BookEntry,BookEntry>>();
+//             MarketQueries.align_bookseries(
+//                l => res.Add(Tuple.Create(l.ElementAt(0),
+//                                          l.ElementAt(1),
+//                                          l.ElementAt(2))),    
+//                bser_1,
+//                bser_2,
+//                bser_3);
+//             _verify_intersection(bser_1,bser_2,bser_3,res);
+//          }
+//       }
+
+//       public static Timestamp from_datetime(DateTime dt, UInt32 usecs){         
+//          if (dt.Millisecond != 0){
+//             usecs += Convert.ToUInt32(dt.Millisecond) * 1000;
+//             dt = dt.AddMilliseconds(dt.Millisecond*-1.0);
+//          }
+//          Timestamp rv = new Timestamp();
+//          rv.Seconds = Convert.ToInt64((dt - new DateTime(1970,1,1)).TotalSeconds);
+//          rv.Nanos = Convert.ToInt32(usecs) * 1000;
+//          return rv;
+//       }      
+      
+//       private static void _compare_seq_no (Tuple<BookEntry,BookEntry> b,
+//                                            UInt32 s1, UInt32 s2){
+//          Log.assert(b.Item1.PBook.BookSeqNum == s1);
+//          Log.assert(b.Item2.PBook.BookSeqNum == s2);
+//       }
+
+//       private static void _add_to_ser(ref List<ProcessedBook> bseries_list,
+//                                       UInt32 usecs){
+//          DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc);
+//          ProcessedBook b = new ProcessedBook();
+//          b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count());
+//          b.SendingTime = b.TransactTime = from_datetime(base_time, usecs);
+//          bseries_list.Add(b);
+//       }
+
+//       private static void _verify_intersection (
+//                   BookSeries bser_1,
+//                   BookSeries bser_2,
+//                   BookSeries bser_3,
+//                   IEnumerable<Tuple<BookEntry,BookEntry,BookEntry>> i_set){
+//          Func<Tuple<BookEntry,BookEntry,BookEntry>, int> count = (t_a) => {
+//             int rv = 0;
+//             foreach (var t in i_set)
+//                if(t.Item1.PBook.BookSeqNum == t_a.Item1.PBook.BookSeqNum &&
+//                   t.Item2.PBook.BookSeqNum == t_a.Item2.PBook.BookSeqNum &&
+//                   t.Item3.PBook.BookSeqNum == t_a.Item3.PBook.BookSeqNum)
+//                   rv++;
+//             return rv;
+//          };
+//          // 1. uniqueness of data
+//          // each tuple should be present only once
+//          foreach (var t in i_set)
+//             Log.assert(count(t)==1);
+//          // 2. make sure each tuple in I has overlap
+//          foreach (var t in i_set){
+//             Timerange r = MarketQueriesHelper.intersection_range(
+//                                                        t.Item1.TimeRange,
+//                                                        t.Item2.TimeRange,
+//                                                        t.Item3.TimeRange);
+//             Log.assert(MarketQueriesHelper.period_ns(r)>0);
+//          }
+//          // 3. find overlaps throgh exhaustive search and verify
+//          // they are found in I
+//          foreach (BookEntry b1 in bser_1.book_entries())
+//             foreach (BookEntry b2 in bser_2.book_entries())
+//                foreach (BookEntry b3 in bser_3.book_entries()){
+//                   var intr = MarketQueriesHelper.intersection_range(
+//                                                                 b1.TimeRange,
+//                                                                 b2.TimeRange,
+//                                                                 b3.TimeRange);
+//                   if(MarketQueriesHelper.period_ns(intr)>0)
+//                      Log.assert(count(Tuple.Create(b1,b2,b3))==1);
+//                }
+//       }
+
+//       // test find_books_in_time_range
+//       // find_coexisting_books 1
+//       // find_coexisting_books 2
+//       // align_2_bookseries
+//       // align_3_bookseries
+   }
+}
@@ -6,69 +6,139 @@ using System.Text;
 using System.IO;
 using mbts.config;
 using bts.config;
+using bts.api.orderman;
+using bts.co2;
+using mbts.co2;
 using Google.Protobuf.WellKnownTypes;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;

-namespace bts.history{   
+
+// date-range, product, path (or file), optional spread
+namespace bts.history{
   class SpreaderTickSniperAnalyzer{
-      // product, date, result file, spreaders
      public SpreaderTickSniperAnalyzer(HistoricalDataOptions hd_options){
-         string input_file_name = hd_options.algo_params;
-         string path = String.Format(@"{0}/{1}",BookAnalyzer.get_algo_input_path("sniper"));
-         input_file_name = String.Format(@"{0}/{1}",path, input_file_name);  
-         FileStream fs = new FileStream (input_file_name, FileMode.Open);
-         StreamReader sr = new StreamReader(fs);
-         string product = sr.ReadLine();
-         DateTime day = DateTime.Parse (sr.ReadLine());
-         string output_file = sr.ReadLine ();
-         string spread;
-         while (fs.Position < fs.Length){
-            // read spreads
-            spread = sr.ReadLine();
+         Console.WriteLine("SpreaderTickSniperAnalyzer:Hit key to continue:"); Console.ReadKey();         
+         // product, input_path, output_path
+         string[] args = hd_options.algo_params.Split(',');
+         Log.assert(args.Count()==3);         
+         var symbol = args[0];
+         var path =  DataIOHelper.get_data_path(args[1]);
+         var output_path = DataIOHelper.get_data_path(args[2]);         
+         var files = DataIOHelper.get_files_recursively(path);
+         files = files.Where(f => f.Contains("raw_books.csv"));
+         var price_map = files.ToDictionary(f => f, f => 
+                                    new List<SyntheticPrice>());
+         ushort prl_files = 1;
+         var file_groups = MarketQueriesHelper.break_into_multiples(files,prl_files);
+         
+         foreach (var fg in file_groups){
+            Parallel.ForEach(fg, f =>{                  
+               Console.WriteLine("Processing {0} ...", f);
+               var f_opps = price_map[f];
+               var spr_book_map = DataIOHelper.load_books(f,symbol,true);
+               Console.WriteLine("Downloaded {0} {1} books",f,spr_book_map.Count());
+               _execute_single(spr_book_map,symbol,ref f_opps);
+            });
         }
-         _temp_test();
-      }
-
-
-      // given a spread and date, return all possible book values
-      // for that spread
-      private static IEnumerable<ProcessedBook> _get_spread_prices (string spread,
-                                                                    DateTime day){
-         return null;
-      }
-
-      public void _temp_test()
-      {
-         System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
-         sw.Start();
-         List<long> time_diff = new List<long>();
-         string file = @"C:\Users\taha.BLUETRADESYS\AppData\Roaming\bts\data\preprocessed_md\CL\10_29_2019\CLZ9-BZZ9";
-         FileStream fs = new FileStream (file, FileMode.Open);
-         while (fs.Position < fs.Length){
-            try{
-               ProcessedBook pb = new ProcessedBook();
-               Google.Protobuf.MessageExtensions.MergeDelimitedFrom(pb,fs);
-               Timerange tr = new Timerange (pb.TransactTime,pb.SendingTime);
-               time_diff.Add(tr.period_ns());
-               if(time_diff.Count%1000==0)
-                  Console.WriteLine(tr.period_ns());
-            }
-            catch (Exception ex){
-               Console.WriteLine(ex);
-            }
+         List<SyntheticPrice> opps = new List<SyntheticPrice>();
+         foreach (var kv in price_map)
+            opps.AddRange(kv.Value);
+         opps.Sort((a,b) => a.synthetic_book.TimeRange.T1.compare(
+               b.synthetic_book.TimeRange.T1));
+         for(UInt16 i=0;i<opps.Count;i++)
+            opps[i].idx=i;
+         // write to file
+         string output_file = String.Format("{0}/sniper.csv",
+                                 output_path);
+         (new FileInfo(output_file)).Directory.Create();
+         using (StreamWriter tw = new StreamWriter(output_file)){
+            _write_header(tw);
+            foreach (var o in opps)
+               tw.WriteLine(DataIOHelper.synthetic_price_to_csv(o));
         }
-         sw.Stop();
-         time_diff.Sort();
-         var median = time_diff[time_diff.Count/2];
-         Console.WriteLine ("count:{0} time_ms:{1} min:{2} max:{3} mean:{4} median:{5}", 
-                           time_diff.Count, 
-                           sw.ElapsedMilliseconds,
-                           time_diff.Min(),
-                           time_diff.Max(),
-                           time_diff.Average(),
-                           time_diff[time_diff.Count/2]);
-      } 
+      }

      public void analyze(){
      }
+
+      private static void _write_header(StreamWriter tw){
+         string header = String.Empty;
+         for (int i=0; i<3; i++)
+            header += String.Format("{0},", DataIOHelper.leg_book_header());
+         header += DataIOHelper.synthetic_book_header();
+         tw.WriteLine(header);      
+      }
+
+      // generates all 3 legged circular synthetics for the spreads in
+      // spr_book_map and evaluates their prices
+      private void _execute_single (IDictionary<string,BookSeries> spr_book_map,
+                                    string symbol,
+                                    ref List<SyntheticPrice> opps){
+         Console.WriteLine("Generating circular spreads");
+         var circ_spreads = MarketQueriesHelper.generate_3_legged_circulars(
+                                                            symbol,
+                                                            spr_book_map.Keys);
+         Console.WriteLine("Generated {0} circular spreads", circ_spreads.Count());
+         var spread_map = circ_spreads.ToDictionary(
+                              k=>k,
+                              k=>MarketQueriesHelper.parse_synthetic(k));
+         foreach (var kv in spread_map)
+            _evaluate_synthetic(symbol,spr_book_map,kv.Value,ref opps);
+      }
+
+      private static void _evaluate_synthetic(
+                              string symbol,
+                              IDictionary<string, BookSeries> leg_book_map,
+                              SpreaderOrderDefinition s,
+                              ref List<SyntheticPrice> opps){
+         var n = s.Security.Synthetic.Name;
+         List<BookSeries> leg_books = new List<BookSeries>();
+         foreach (var l in s.Security.Synthetic.Legs){
+            BookSeries bser = null;
+            var leg_name = l.Security.Glbx.SecurityDesc;
+            Log.assert(leg_book_map.TryGetValue(leg_name, out bser),
+                       String.Format("Could not find book for {0}", leg_name));
+            leg_books.Add(bser);
+         }
+         System.Diagnostics.Stopwatch w = new System.Diagnostics.Stopwatch();
+         w.Restart();
+         int spr_count = 0;
+         List<SyntheticPrice> l_opps = new List<SyntheticPrice>();
+         MarketQueries.get_synthetic_prices(s,leg_books,spr=> {
+            if(MarketQueriesHelper.within_trading_time(spr,symbol) && 
+               _detect_opp(n,spr))
+               l_opps.Add(spr);
+            spr_count++;
+         });
+         w.Stop();
+         opps.AddRange(l_opps);
+         Console.WriteLine("{0}:{1} {2} msecs", n,spr_count,w.ElapsedMilliseconds);
+      }
+
+      private static bool _detect_opp (
+                                string n,
+                                SyntheticPrice spr){
+         ProcessedBook p = spr.synthetic_book;
+         double b = MarketQueriesHelper.book_price(p,true,0),
+                a = MarketQueriesHelper.book_price(p,false,0);
+         UInt32 bq = MarketQueriesHelper.book_qty(p,true,0),
+                aq = MarketQueriesHelper.book_qty(p,false,0);
+         if(bq>0 && b>0){
+            Console.WriteLine("SELL:{0} t:{1} active_for:{2} ns {3}:{4}",
+                        n, p.TimeRange.T1,
+                        MarketQueriesHelper.period_ns(p.TimeRange),b,bq);
+            spr.viable_side = api.Side.Buy;
+            return true;
+         }
+         if(aq>0 && a<0){
+            Console.WriteLine("BUY:{0} t:{1} active_for:{2} ns {3}:{4}",
+                     n, p.TimeRange.T1, MarketQueriesHelper.period_ns(p.TimeRange),
+                     a,aq);
+            spr.viable_side = api.Side.Sell;
+            return true;
+         }
+         return false;
+      }
   }
 }
@@ -2,65 +2,10 @@ using System;
 using System.Collections.Generic;
 using Google.Protobuf.WellKnownTypes;

-namespace bts.history{
-   enum TimeLineRegion{
-      BEFORE,
-      MIDDLE,
-      AFTER
-   }
+namespace bts.history{   
   enum TimerangeIntersection{
      LEFT_NOINTERSECTION,
      INTERSECTION,    // for later
      RIGHT_NOINTERSECTION,
   }
-   // Wrapper for a start/stop (with ns precision) and helper functions
-   class Timerange{
-      public readonly Timestamp t1, t2;
-      public static UInt64 ref_secs = Convert.ToUInt64(
-         (new DateTime(DateTime.Now.Year, 1,1) - 
-          new DateTime(1970, 1, 1)).TotalSeconds);
-      public Timerange(Timestamp t1_a, Timestamp t2_a){
-         t1=t1_a;
-         t2=t2_a;
-      }
-      public Int64 period_ns(){
-         return Convert.ToInt64(_from_ref_ns(t2)-_from_ref_ns(t1));
-      }
-      public bool illformed(){
-         return period_ns()<0;
-      }
-      public TimerangeIntersection intersection (Timerange tr){
-         if (is_before(tr.t2,t1))
-            return TimerangeIntersection.LEFT_NOINTERSECTION;
-         if (is_before(t2,tr.t1))
-            return TimerangeIntersection.RIGHT_NOINTERSECTION;
-         return TimerangeIntersection.INTERSECTION;
-      }
-      public static Timerange intersection(params Timerange[] ranges){
-         Timerange max_t1 = ranges.MaxElement(r => Timerange._from_ref_ns(r.t1)),
-                   min_t2 = ranges.MinElement(r => Timerange._from_ref_ns(r.t2));
-         return new Timerange(max_t1.t1,min_t2.t2);
-      }
-      public static Timestamp max_timestamp(){
-         Timestamp ts = new Timestamp();
-         ts.Seconds = long.MaxValue;
-         ts.Nanos = 0;
-         return ts;
-      }
-      public static bool is_before (Timestamp t1,
-                                    Timestamp t2){
-         return diff_ns(t1,t2)<0;
-      }
-      // t1-t2 ns
-      public static Int64 diff_ns (Timestamp t1,
-                                   Timestamp t2){
-         return Convert.ToInt64(_from_ref_ns(t1))-
-                Convert.ToInt64(_from_ref_ns(t2));
-      }
-      private static UInt64 _from_ref_ns (Timestamp ts){
-         UInt64 secs = Convert.ToUInt64(ts.Seconds),
-                ns = Convert.ToUInt64(ts.Nanos);
-         return (secs - ref_secs)*1000000000 + ns;
-      } 
-   }
 }
@@ -19,7 +19,7 @@ namespace bts.history
      static ManualResetEvent _quit = new ManualResetEvent(false);
      public static int Run(HistoricalDataOptions o)
      {
-         Console.WriteLine(o.ToString());  
+         Console.WriteLine(o.ToString());
         if ("sniper".Equals (o.algo_type))
         {
             Console.WriteLine("Analyzing sniper");
@@ -27,7 +27,55 @@ namespace bts.history
             sa.analyze();
             return 0;
         }
-
+         if ("circle".Equals (o.algo_type))
+         {
+             Console.WriteLine("Analyzing circle");
+             CircleAnalyzer ca = new CircleAnalyzer(o);
+             ca.analyze();
+             return 0;
+         }
+         if ("circle".Equals (o.algo_type))
+         {
+             Log.assert(false);
+             return 0;
+         }
+         if ("microscope".Equals (o.algo_type))             
+         {
+             Console.WriteLine("Starting miscroscope");
+             Microscope m = new Microscope(o);
+             m.analyze();
+             return 0;
+         }
+         if ("download".Equals (o.algo_type))             
+         {
+            Console.WriteLine("Starting download");
+            //date1-date2,sym1 ... ,symn,location(relative to data path)
+            string[] toks = o.algo_params.Split(',');
+            string[] dt_beg_end = toks[0].Split('-');
+            Log.assert(dt_beg_end.Length==2);
+            DateTime beg = DateTime.Parse(dt_beg_end[0]),
+                     end = DateTime.Parse(dt_beg_end[1]);
+            string dir = DataIOHelper.get_data_path(toks[2]);
+            var d = new DataDownloader();
+            for (DateTime i = beg; i <= end; i = i.AddDays(1))
+               for (int j = 1; j<toks.Length-1; j++){
+                  var symbol = toks[j]; 
+                  var full_path = String.Format("{0}/{1}_{2}_raw.gz",
+                                                dir, 
+                                                i.ToString("yyyy/MM/dd").Replace("/",String.Empty),
+                                                symbol);
+                  Console.WriteLine ("Downloading {0} {1} ...",
+                                      i.ToShortDateString(), symbol);
+                  try{                    
+                     d.download_compressed(i,symbol,SecurityType.Spread, full_path);
+                  }
+                  catch (Exception ex){
+                     String.Format("Could not download {0} {1} Ex:{2}",
+                                    symbol, i.ToShortDateString(), ex.Message);
+                  }
+               }
+            return 0;
+         }
         if (!string.IsNullOrEmpty(o.replayend) && !string.IsNullOrEmpty(o.replaystart) && 
             !string.IsNullOrEmpty(o.replaysymbol) )
         {
@@ -1 +1 @@
-set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
+set(sources files/cert_secure_logon.txt;files/fix/spark_fix.cfg;files/fix/spec/fix/FIX40.xml;files/fix/spec/fix/FIX41.xml;files/fix/spec/fix/FIX42.xml;files/fix/spec/fix/FIX43.xml;files/fix/spec/fix/FIX44.xml;files/fix/spec/fix/FIX50.xml;files/fix/spec/fix/FIX50SP1.xml;files/fix/spec/fix/FIX50SP2.xml;files/fix/spec/fix/FIXT11.xml;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orLine truncated
+set(sources api/Apicore.proto;api/Apigateway.proto;api/Apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/HistoricalDataService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/UI.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/fix.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/fix.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/Co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrLine truncated
@@ -24,7 +24,7 @@ enum MDEntryType
   TRADE = 50;
   OPENING_PRICE = 52;
   SETTLEMENT_PRICE = 54;
-}   
+}
 message MDEntry
 {
   MDUpdateAction md_update_action = 279; //delimiter, this member must be first
@@ -152,6 +152,32 @@ message ProcessedBook
   google.protobuf.Timestamp transact_time = 30;
   string security = 40;
   co2.BookMessage book = 50;
+   repeated uint32 absorbed_books = 60;
+   repeated ProcessedBook same_tops = 65;
+   // this is the book's sending time (T1) till
+   // the next book's transaction time
+   Timerange time_range = 70;
+}
+
+message Timerange{
+   google.protobuf.Timestamp t1 = 10;
+   google.protobuf.Timestamp t2 = 20;
+}
+
+message TradeEntry{
+   string contract = 1;
+   double price = 10;
+   uint32 qty = 20;
+   string trade_id = 30;
+   uint64 order_id = 40;
+   google.protobuf.Timestamp transact_time = 50;
+   google.protobuf.Timestamp sending_time = 60;
+}
+
+message NewOrderEntry{
+   uint64 order_id = 10;
+   google.protobuf.Timestamp transact_time = 20;
+   google.protobuf.Timestamp sending_time = 30;
 }

 //this is a generated class to help accessing the mbo fields, look at the partial class extension
@@ -192,3 +218,31 @@ service HistoricalDataService
   rpc get_md_stream (MDQuery) returns (MDStream);
 }

+// Symbol,SendingTime,TransactTime,MDEntryPx,MDEntrySize,NumberOfOrders,AggressorSide,MDTradeEntryID,AgressorOID,RestingOID,PartialQty
+message TradeRow{
+   uint32 msg_id = 10;
+   string contract = 20;
+   google.protobuf.Timestamp sending_time = 30;
+   google.protobuf.Timestamp transact_time = 40;
+   double md_entry_px = 50;
+   uint32 md_entry_sz = 60;
+   uint32 no_of_orders = 70;
+   uint32 aggressor_side = 80;
+   uint64 md_trade_entry_id = 90;
+   uint64 aggressor_oid = 100;
+   uint64 resting_oid = 110;
+   uint32 partial_qty = 120;
+}
+
+message OrderRow{
+   uint32 msg_id = 10;
+   string contract = 20;
+   google.protobuf.Timestamp sending_time = 30;
+   google.protobuf.Timestamp transact_time = 40;
+   uint64 order_id = 50;
+   double md_entry_px = 60;
+   uint32 md_entry_qty = 70;
+   uint64 md_order_priority = 80;
+   uint32 order_update_action = 90;
+   uint32 side = 100;
+}
@@ -49,6 +49,7 @@ enum class MessageTypes : uint32_t
   TestRequest                 = '1',
   TradingSessionStatus        = 'h',
   TradingSessionStatusRequest = 'g',
+   MDIncrementalUpdate         = 'X',
   ICETraderLogin              = ('U' << 16) + ('C' << 8) + 'G',
   ICETraderLoginResponse      = ('U' << 16) + ('C' << 8) + 'I',   
   ICETraderLogout             = ('U' << 16) + ('C' << 8) + 'H',   
@@ -26,8 +26,8 @@ security_key default_key(const MDConsumerHandle& h)

 std::string desc_key(const MDConsumerHandle& h) 
 { 
-   auto&& sec = h.get_normalized_security();
-   auto&& desc = securities::descriptor(sec);
+   auto sec = h.get_normalized_security();
+   auto desc = securities::descriptor(sec);
   return desc;
 }

@@ -85,12 +85,12 @@ private:
         {
            if (_consumer->id() == id) 
            {
-               handle_index.lock([this, handle](auto& index) 
+               handle_index.lock([handle](auto& index) 
               {
                  auto key = Key_Func(handle);
                  index[ key ] = handle;
               });
-               security_index.lock([this, handle, &sec](auto& index) 
+               security_index.lock([handle, &sec](auto& index) 
               {
                  auto key = Key_Func(handle);
                  index[ key ] = sec;
@@ -107,4 +107,4 @@ typedef HandleIndex<desc_key> DescHandleIndex;
 }}


-#endif
+#endif
@@ -6,7 +6,7 @@ namespace bts {
 namespace orderman {

 class ChildTemplate;
-class LegData;
+struct LegData;

 class PyramidAlgo : public IAlgoOrder {
 public:
@@ -6,7 +6,7 @@ namespace bts {
 namespace orderman {

 class ChildTemplate;
-class LegData;
+struct LegData;

 class TASAlgo : public IAlgoOrder {
 public:
@@ -61,7 +61,7 @@ CAPI AlgoTestDriverBase* create_algo_driver(const char* security, long seclength
         return nullptr;
      auto handle = new AlgoTestDriverBase(*algoptr, def, Price(0.0), Quantity(0), ATDFlags::NONE, sec);      
      
-      handle->on_child_launching([launch_cb, handle] (const ChildOrder& child) 
+      handle->on_child_launching([launch_cb] (const ChildOrder& child) 
      {
         size_t sz = child.def->ByteSizeLong();
         char* buffer = new char[sz];
@@ -71,7 +71,7 @@ CAPI AlgoTestDriverBase* create_algo_driver(const char* security, long seclength
         launch_cb(buffer, sz, child.key.value(), price, quantity);
      });
      
-      handle->on_child_update([update_cb, handle] (const OrderKey key, const api::orderman::Update& upd)
+      handle->on_child_update([update_cb] (const OrderKey key, const api::orderman::Update& upd)
      {
         size_t sz = upd.ByteSizeLong();
         char* buffer = new char[sz];
@@ -79,12 +79,12 @@ CAPI AlgoTestDriverBase* create_algo_driver(const char* security, long seclength
         update_cb(buffer, sz, key.value());         
      });
      
-      handle->on_child_cancel([cancel_cb, handle] (const OrderKey key)
+      handle->on_child_cancel([cancel_cb] (const OrderKey key)
      {
         cancel_cb(key.value());         
      });      
      
-      handle->on_child_changed([stat_cb, handle] (const ChildOrder& , const ChildOrder& curr) 
+      handle->on_child_changed([stat_cb] (const ChildOrder& , const ChildOrder& curr) 
      {
         api::orderman::OrderState orderstate;
         orderstate.set_status( (api::orderman::OrderStatus) curr.cstate.status );
@@ -101,7 +101,7 @@ CAPI AlgoTestDriverBase* create_algo_driver(const char* security, long seclength
         stat_cb(buffer, sz, curr.key.value());
      });   
      
-      handle->on_child_changed([fill_cb, handle] (const ChildOrder& old, const ChildOrder& curr) 
+      handle->on_child_changed([fill_cb] (const ChildOrder& old, const ChildOrder& curr) 
      {
         auto currfill = curr.cstate.total_fills.value();
         auto oldfill = old.cstate.total_fills.value();
@@ -302,4 +302,4 @@ CAPI IAlgoOrder* try_create_algo(CustomOrderDefinition::CustomCase algo_type)



-}}}
+}}}
@@ -4188,11 +4188,9 @@ TEST_CASE ("scratch_pad", "[spreader]"){
   REQUIRE(rv_1.i==0);
   REQUIRE(rvuser_1.rv.i==100);   
 }
-TEST_CASE ("scratch_pad_2", "[spreader]"){
-   time_t rawtime;   
-   time (&rawtime);
-   struct tm* ptm = gmtime (&rawtime);
-   LOG(OrderMan, Info, "hrs:{} mins:{} sec:{}", ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
+TEST_CASE ("scratch_pad_2", "[spreader]"){   
+   auto lambda = []() {std::cout << "yes its me";};
+   lambda();
 }

 }}
@@ -6,21 +6,15 @@
 namespace bts {
 namespace utils {

-static log::LoggingConfigPoller *poller=nullptr;
-
 void init_spark_lib()
 {
   threading::configure_current_thread("main");
   log::start_default_logging_thread();
-   poller = new log::LoggingConfigPoller ();
 }

 void finit_spark_lib()
 {
   LOG(Gen, Info, "finit_spark_lib");
-   if (poller) {
-      delete poller;
-   }
 }

 }}
@@ -1 +1 @@
-set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunFix.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;fix/AdminModelWrapper.cpp;fix/DropCopySession.cpp;fix/FixInit.cpp;fix/FixRouter.cpp;fix/OrderSession.cpp;fix/SecurityIndex.cpp;fix/SessionBase.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/MarketDataReceiverServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
@@ -0,0 +1,551 @@
+#include "ParseHistoricalFIX.h"
+#include "lib/jetfix/codec/FIXMsg.h"
+#include "lib/jetfix/codec/FIXParser.h"
+#include "lib/jetfix/codec/Fields.h"
+#include "lib/jetfix/codec/Group.h"
+#include "lib/jetfix/codec/MessageTypes.h"
+#include "lib/utils/bts_chrono.h"
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/path.hpp>
+
+#include <fstream>
+#include <unordered_map>
+
+namespace bts {
+namespace cmd {
+
+using namespace jetfix;
+
+typedef StringField<75> TradeDate;
+typedef IntField<34> SeqNum;
+typedef StringField<52> SendingTime;
+typedef StringField<60> TransactTime;
+typedef StringField<5799> MatchEventIndicator;
+
+typedef IntField<268> NoMDEntries;
+typedef StringField<279> MDUpateAction;
+typedef StringField<269> MDEntryType;
+typedef StringField<48> SecurityID;
+typedef StringField<55> Symbol;
+typedef StringField<83> ReptSeq;
+typedef FloatField<270> MDEntryPx;
+typedef IntField<271> MDEntrySize;
+typedef IntField<346> NumberOfOrders;
+typedef IntField<1023> MDPriceLevel;
+typedef StringField<286> OpenCloseSettlFlag;
+typedef StringField<731> SettlPriceType;
+typedef StringField<5797> AggressorSide;
+typedef StringField<37711> MDTradeEntryID;
+typedef StringField<5796> TradingReferenceDate;
+typedef StringField<1149> HighLimitPrice;
+typedef StringField<1148> LowLimitPrice;
+typedef StringField<1143> MaxPriceVariation;
+typedef StringField<1180> ApplID;
+typedef IntField<9633> ReferenceID;
+typedef IntField<32> LastQty;
+
+typedef IntField<37705> NoOrderIDEntries;
+typedef StringField<37> OrderID;
+typedef IntField<37707> MDOrderPriority;
+typedef IntField<37706> MDDisplayQty;
+typedef StringField<37708> OrderUpdateAction;
+
+typedef Group<NoOrderIDEntries, OrderID, MDDisplayQty, MDOrderPriority,
+              OrderUpdateAction, ReferenceID, LastQty>
+    OrderIDEntries;
+
+typedef Group<NoMDEntries, MDUpateAction, MDEntryType, SecurityID, Symbol,
+              ReptSeq, MDEntryPx, MDEntrySize, NumberOfOrders, MDPriceLevel,
+              OpenCloseSettlFlag, SettlPriceType, AggressorSide, MDTradeEntryID,
+              TradingReferenceDate, HighLimitPrice, LowLimitPrice,
+              MaxPriceVariation, ApplID, ReferenceID, LastQty, OrderID,
+              MDOrderPriority, MDDisplayQty>
+    MDEntries;
+
+typedef FIXMsg<MessageTypes::MDIncrementalUpdate, TradeDate, SeqNum,
+               SendingTime, TransactTime, MatchEventIndicator, MDEntries,
+               OrderIDEntries>
+    MDIncrementalUpdate;
+
+std::string to_str(std::string const &s) { return s; }
+
+template <class V> std::string to_str(V const &v) { return std::to_string(v); }
+
+template <class F, class G> std::string get_if_valid(G const &update) {
+   if (update.template has<F>())
+      return to_str(update.template get<F>());
+   else
+      return "";
+}
+
+void print_msg_headers(std::ostream &out) {
+   out << "MsgID,"
+       << "TradeDate,"
+       << "SeqNum,"
+       << "SendingTime,"
+       << "TransactTime,"
+       << "MatchEventIndicator,"
+       << "NoMDEntries,"
+       << "NoOrderIDEntries,"
+       << "SendingTime_UnixTS,"
+       << "TransactTime_UnixTS\n";
+}
+
+uint64_t date_str_to_unix_ts(std::string const &data_str) {
+   static boost::posix_time::ptime unix_epoch(
+       boost::gregorian::date(1970, 1, 1));
+
+   uint64_t year = std::stoi(data_str.substr(0, 4));
+   uint64_t month = std::stoi(data_str.substr(4, 2));
+   uint64_t day = std::stoi(data_str.substr(6, 2));
+   uint64_t hours = std::stoi(data_str.substr(8, 2));
+   uint64_t minutes = std::stoi(data_str.substr(10, 2));
+   uint64_t seconds = std::stoi(data_str.substr(12, 2));
+   uint64_t nanos = std::stoi(data_str.substr(14));
+
+   boost::posix_time::ptime t(
+       boost::gregorian::date(year, month, day),
+       boost::posix_time::time_duration(hours, minutes, seconds));
+
+   uint64_t total_secs = (t - unix_epoch).total_seconds();
+   auto result = total_secs * 1000000000 + nanos;
+   return result;
+}
+
+void print_msgs(size_t msg_idx, MDIncrementalUpdate const &line,
+                std::ostream &out) {
+   line.get<SeqNum>();
+   out << msg_idx << "," << get_if_valid<TradeDate>(line) << ","
+       << get_if_valid<SeqNum>(line) << "," << get_if_valid<SendingTime>(line)
+       << "," << get_if_valid<TransactTime>(line) << ","
+       << get_if_valid<MatchEventIndicator>(line) << ","
+       << (line.has<MDEntries>() ? line.get<MDEntries>().size() : 0) << ","
+       << (line.has<OrderIDEntries>() ? line.get<OrderIDEntries>().size() : 0)
+       << "," << date_str_to_unix_ts(get_if_valid<SendingTime>(line)) << ","
+       << date_str_to_unix_ts(get_if_valid<TransactTime>(line)) << "\n";
+}
+
+struct PriceSize {
+   double price = 0;
+   int32_t size = -1;
+};
+
+struct Book {
+   Book() {
+      book[0].resize(10);
+      book[1].resize(10);
+   }
+   std::string symbol;
+   std::string transact_time;
+   std::string send_time;
+   uint32_t seqnum;
+   uint32_t msg_id;
+   std::array<std::vector<PriceSize>, 2> book;
+};
+
+size_t n_printed_books = 10;
+
+void print_book_side(std::vector<PriceSize> const &side,
+                     std::ostream &book_stream) {
+   for (size_t i = 0; i < n_printed_books; i++)
+      if (i < side.size())
+         book_stream << "," << side[i].price << "," << side[i].size;
+      else
+         book_stream << "," << 0 << "," << 0;
+}
+
+void print_book_header(std::ostream &book_stream) {
+   book_stream << "MsgID,SendingTime,TransactTime, SeqnNum";
+   for (size_t i = 0; i < n_printed_books; i++)
+      book_stream << ",bid_p_" << i << ",bid_sz_" << i;
+   book_stream << ",Symbol";
+   for (size_t i = 0; i < n_printed_books; i++)
+      book_stream << ",ask_p_" << i << ",ask_sz_" << i;
+   book_stream << ",SendingTime_UnixTS,"
+               << "TransactTime_UnixTS\n";
+}
+
+void print_book(Book &b, std::ostream &book_stream) {
+   book_stream << b.msg_id << "," << b.send_time << "," << b.transact_time
+               << "," << b.seqnum;
+   print_book_side(b.book[0], book_stream);
+   book_stream << "," << b.symbol;
+   print_book_side(b.book[1], book_stream);
+   book_stream << "," << date_str_to_unix_ts(b.send_time) << ","
+               << date_str_to_unix_ts(b.transact_time);
+   book_stream << "\n";
+}
+
+void update_and_maybe_print_book(uint32_t msg_id,
+                                 MDIncrementalUpdate const &update,
+                                 std::unordered_map<std::string, Book> &books,
+                                 std::ostream &book_stream) {
+   std::set<Book *> to_update;
+   if (update.has<MDEntries>()) {
+      auto &md_entries = update.get<MDEntries>();
+      for (size_t i = 0; i < md_entries.size(); i++) {
+         auto &entry = md_entries[i];
+         if (entry.has<MDPriceLevel>()) {
+            if (entry.get<MDEntryType>() == "0" ||
+                entry.get<MDEntryType>() == "1") {
+               auto side = entry.get<MDEntryType>()[0] - '0';
+               auto &sym = entry.get<Symbol>();
+               auto &book = books[sym];
+               to_update.insert(&book);
+               auto &book_side = book.book[side];
+               book.symbol = sym;
+               book.transact_time = update.get<TransactTime>();
+               book.send_time = update.get<SendingTime>();
+               book.transact_time = update.get<TransactTime>();
+               book.seqnum = update.get<SeqNum>();
+               book.msg_id = msg_id;
+               auto level = entry.get<MDPriceLevel>() - 1;
+
+               if (entry.get<MDUpateAction>() == "0") { // insert new level
+                  for (int i = 8; i >= level; i--)
+                     book_side[i + 1] = book_side[i];
+                  book_side[level] = {entry.get<MDEntryPx>(),
+                                      entry.get<MDEntrySize>()};
+               } else if (entry.get<MDUpateAction>() ==
+                          "1") // update existing level
+                  book_side[level] = {entry.get<MDEntryPx>(),
+                                      entry.get<MDEntrySize>()};
+               else {
+                  // delete level
+                  for (int i = level; i < 9; i++)
+                     book_side[i] = book_side[i + 1];
+                  book_side[9] = {0, 0};
+               }
+            }
+         }
+      }
+      auto indicator = update.get<MatchEventIndicator>();
+      if (indicator[5] == '1')
+         for (auto b : to_update)
+            print_book(*b, book_stream);
+   }
+}
+void print_trades_header(std::ostream &trade_stream) {
+   trade_stream
+       << "MsgID, "
+          "Symbol,SendingTime,TransactTime,TradeDate,MDEntryPx,MDEntrySize,"
+          "NumberOfOrders,AggressorSide,MDTradeEntryID,AgressorOID,"
+          "RestingOID,PartialQty,"
+          "SendingTime_UnixTS,"
+          "TransactTime_UnixTS\n";
+}
+
+struct TradeSummaryItem {
+   uint32_t msg_id;
+   std::string symbol;
+   std::string sending_time;
+   std::string transact_time;
+   std::string trade_date;
+   double md_entry_px;
+   uint32_t md_entry_size;
+   uint32_t n_orders;
+   std::string agressor_side;
+   std::string md_trade_entry_id;
+};
+
+struct TradeOrder {
+   std::string order_id;
+   uint32_t last_qty;
+};
+
+struct TradeSummaryEvent {
+   std::vector<TradeSummaryItem> md_entries;
+   std::vector<TradeOrder> orderid_entries;
+};
+
+void print_trades(TradeSummaryEvent &event, std::ostream &trade_stream) {
+   size_t current_order = 0;
+
+   size_t total_orders = 0;
+   for (auto &trade : event.md_entries)
+      total_orders += trade.n_orders;
+   if (total_orders != event.orderid_entries.size())
+      return;
+
+   for (auto &trade : event.md_entries) {
+      for (auto i = current_order; i < current_order + trade.n_orders; i++) {
+         trade_stream << trade.msg_id << "," << trade.symbol << ","
+                      << trade.sending_time << "," << trade.transact_time << ","
+                      << trade.trade_date << "," << trade.md_entry_px << ","
+                      << trade.md_entry_size << "," << trade.n_orders << ","
+                      << trade.agressor_side << "," << trade.md_trade_entry_id
+                      << "," << event.orderid_entries[current_order].order_id
+                      << "," << event.orderid_entries[i].order_id << ","
+                      << event.orderid_entries[i].last_qty << ","
+                      << date_str_to_unix_ts(trade.transact_time) << ","
+                      << date_str_to_unix_ts(trade.sending_time) << "\n";
+      }
+      current_order += trade.n_orders;
+   }
+}
+
+void update_and_maybe_print_trades(uint32_t msg_id,
+                                   MDIncrementalUpdate const &update,
+                                   TradeSummaryEvent &event,
+                                   std::ostream &trade_stream) {
+   if (update.has<MDEntries>()) {
+      auto &md_entries = update.get<MDEntries>();
+      bool is_trade_summary = false;
+      for (size_t i = 0; i < md_entries.size(); i++) {
+
+         auto &entry = md_entries[i];
+         if (entry.get<MDEntryType>() == "2") {
+            is_trade_summary = true;
+            TradeSummaryItem item;
+            item.msg_id = msg_id;
+            item.symbol = entry.get<Symbol>();
+            item.sending_time = update.get<SendingTime>();
+            item.transact_time = update.get<TransactTime>();
+            item.trade_date = update.get<TradeDate>();
+            item.md_entry_px = entry.get<MDEntryPx>();
+            item.md_entry_size = entry.get<MDEntrySize>();
+            item.n_orders = entry.get<NumberOfOrders>();
+            item.agressor_side = entry.get<AggressorSide>();
+            item.md_trade_entry_id = entry.get<MDTradeEntryID>();
+            event.md_entries.emplace_back(std::move(item));
+         }
+
+         if (is_trade_summary) {
+            if (update.has<OrderIDEntries>()) {
+               auto &oid_entries = update.get<OrderIDEntries>();
+               for (size_t i = 0; i < oid_entries.size(); i++) {
+                  auto &oid = oid_entries[i];
+                  TradeOrder order_id_entry;
+                  order_id_entry.order_id = oid.get<OrderID>();
+                  order_id_entry.last_qty = oid.get<LastQty>();
+                  event.orderid_entries.emplace_back(std::move(order_id_entry));
+               }
+            }
+            auto indicator = update.get<MatchEventIndicator>();
+            if (indicator[7] == '1') {
+               print_trades(event, trade_stream);
+               event.md_entries.clear();
+               event.orderid_entries.clear();
+            }
+         }
+      }
+   }
+}
+
+void print_orders_header(std::ostream &orders_stream) {
+   orders_stream << "MsgID,"
+                 << "Symbol,"
+                 << "SendingTime,"
+                 << "TransactTime,"
+                 << "MDEntryPx,"
+                 << "OrderID,"
+                 << "MDDisplayQty,"
+                 << "MDOrderPriority,"
+                 << "OrderUpdateAction,"
+                 << "Side,"
+                 << "SendingTime_UnixTS,"
+                 << "TransactTime_UnixTS\n";
+}
+
+struct Order {
+   uint32_t msg_id;
+   std::string sending_time;
+   std::string transact_time;
+   std::string side;
+
+   std::string order_id;
+   std::string order_update_action;
+   uint32_t md_order_priority;
+   uint32_t display_qty;
+
+   std::string symbol;
+   double md_entry_px;
+};
+
+void print_order(Order const &order, std::ostream &orders_stream) {
+   orders_stream << order.msg_id << "," << order.symbol << ","
+                 << order.sending_time << "," << order.transact_time << ","
+                 << order.md_entry_px << "," << order.order_id << ","
+                 << order.display_qty << "," << order.md_order_priority << ","
+                 << order.order_update_action << "," << order.side << ","
+                 << date_str_to_unix_ts(order.transact_time) << ","
+                 << date_str_to_unix_ts(order.sending_time) << "\n";
+}
+
+void print_orders(uint32_t msg_id, MDIncrementalUpdate const &update,
+                  std::ostream &orders_stream) {
+   if (update.has<MDEntries>()) {
+      auto &md_entries = update.get<MDEntries>();
+      auto &first_entry = md_entries[0];
+
+      if (first_entry.get<MDEntryType>() == "0" ||
+          first_entry.get<MDEntryType>() == "1") {
+
+         Order order;
+         order.msg_id = msg_id;
+         order.sending_time = update.get<SendingTime>();
+         order.transact_time = update.get<TransactTime>();
+         order.side = first_entry.get<MDEntryType>();
+
+         if (first_entry.has<MDPriceLevel>()) {
+            // combined template
+            // https://www.cmegroup.com/confluence/display/EPICSANDBOX/Incremental+Refresh+SBE+Template+Book+Processing
+            if (update.has<OrderIDEntries>()) {
+               auto &oid_entries = update.get<OrderIDEntries>();
+               for (size_t i = 0; i < oid_entries.size(); i++) {
+
+                  auto &oid = oid_entries[i];
+                  order.order_id = oid.get<OrderID>();
+                  order.order_update_action = oid.get<OrderUpdateAction>();
+                  order.md_order_priority = oid.get<MDOrderPriority>();
+                  order.display_qty = oid.get<MDDisplayQty>();
+
+                  auto &md_entry =
+                      update.get<MDEntries>()[oid.get<ReferenceID>() - 1];
+                  order.symbol = md_entry.get<Symbol>();
+                  order.md_entry_px = md_entry.get<MDEntryPx>();
+                  print_order(order, orders_stream);
+               }
+            }
+         } else {
+            // MBO only template
+            for (size_t i = 0; i < md_entries.size(); i++) {
+               auto &entry = md_entries[i];
+               order.order_id = entry.get<OrderID>();
+               order.order_update_action = entry.get<MDUpateAction>();
+               order.md_order_priority = entry.get<MDOrderPriority>();
+               order.display_qty = entry.get<MDDisplayQty>();
+               order.symbol = entry.get<Symbol>();
+               order.md_entry_px = entry.get<MDEntryPx>();
+               print_order(order, orders_stream);
+            }
+         }
+      }
+   }
+}
+
+void get_files(const std::string &dir, std::vector<std::string> &out_files) {
+   if (boost::filesystem::is_directory(dir)) {
+      boost::filesystem::path p(dir);
+      for (auto &entry : boost::make_iterator_range(
+               boost::filesystem::directory_iterator(p), {})) {
+         if (boost::filesystem::is_regular_file(entry)) {
+            auto file_name = entry.path().string();
+            if (file_name.find("_raw") != std::string::npos &&
+                file_name.find(".csv") == std::string::npos)
+               out_files.push_back(file_name);
+         }
+      }
+   }
+}
+
+int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
+
+   if (args[0] == "orders" || args[0] == "trades" || args[0] == "books" ||
+       args[0] == "msgs") {
+
+      std::unordered_map<std::string, Book> books;
+      TradeSummaryEvent current_trade_summary;
+      MDIncrementalUpdate update;
+      FIXParser<MDIncrementalUpdate> parser;
+      auto &input = std::cin;
+
+      if (args[0] == "orders")
+         print_orders_header(std::cout);
+      if (args[0] == "trades")
+         print_trades_header(std::cout);
+      if (args[0] == "msgs")
+         print_msg_headers(std::cout);
+      if (args[0] == "books")
+         print_book_header(std::cout);
+
+      std::string line;
+      size_t c = 0;
+      while (std::getline(input, line)) {
+         c++;
+         auto parsed_ptr = parser.parse(InMsgBuffer(
+             line.data(), line.size(), std::shared_ptr<std::string>{}));
+         if (parsed_ptr) {
+            auto msg = std::dynamic_pointer_cast<const MDIncrementalUpdate>(
+                parsed_ptr);
+            if (args[0] == "orders")
+               print_orders(c, *msg, std::cout);
+            if (args[0] == "trades")
+               update_and_maybe_print_trades(c, *msg, current_trade_summary,
+                                             std::cout);
+            if (args[0] == "msgs")
+               print_msgs(c, *msg, std::cout);
+            if (args[0] == "books")
+               update_and_maybe_print_book(c, *msg, books, std::cout);
+         }
+      }
+      return 0;
+   }
+
+   // first arg is the path to where the files are
+   // second arg is optional to parse orders/trades or not
+   auto dir = args[0];
+   auto process_orders = args.size() == 1 ? false : (args[1] == "orders");
+   std::vector<std::string> file_names;
+   get_files(dir, file_names);
+   for (const auto &f : file_names)
+      _execute_single(f, process_orders);
+   return 0;
+}
+
+void ParseHistoricalFIX::_execute_single(const std::string &file,
+                                         bool process_orders) {
+   std::cout << "Processing file:" << file << std::endl;
+   std::ifstream input(file);
+   std::unique_ptr<std::ofstream> books_stream, msgs_stream, trades_stream,
+       orders_stream;
+   books_stream = std::make_unique<std::ofstream>(file + "_books.csv");
+   print_book_header(*books_stream);
+   if (process_orders) {
+      msgs_stream = std::make_unique<std::ofstream>(file + "_msgs.csv");
+      trades_stream = std::make_unique<std::ofstream>(file + "_trades.csv");
+      orders_stream = std::make_unique<std::ofstream>(file + "_orders.csv");
+      print_msg_headers(*msgs_stream);
+      print_trades_header(*trades_stream);
+      print_orders_header(*orders_stream);
+   }
+   std::unordered_map<std::string, Book> books;
+   TradeSummaryEvent current_trade_summary;
+
+   MDIncrementalUpdate update;
+   FIXParser<MDIncrementalUpdate> parser;
+
+   std::string line;
+   size_t c = 0;
+   auto ok = 0;
+   auto start = Timer::now();
+   while (std::getline(input, line)) {
+      c++;
+      auto parsed_ptr = parser.parse(InMsgBuffer(
+          line.data(), line.size(), std::shared_ptr<std::string>{}));
+      if (parsed_ptr) {
+         auto msg =
+             std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
+         update_and_maybe_print_book(c, *msg, books, *books_stream);
+         if (process_orders) {
+            print_msgs(c, *msg, *msgs_stream);
+            update_and_maybe_print_trades(c, *msg, current_trade_summary,
+                                          *trades_stream);
+            print_orders(c, *msg, *orders_stream);
+         }
+      }
+      ok += (bool)(parsed_ptr);
+   }
+   auto sec_duration = ns_diff(Timer::now(), start) / 1e12;
+   std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
+             << " out of " << c << " time: " << std::fixed
+             << std::setprecision(3) << sec_duration << " secs\n";
+}
+
+std::string ParseHistoricalFIX::help() {
+   return "ParseHistoricalFIX <input> will produce 2 files, "
+          "<input>_mdentries.csv and <input>order_entries.csv";
+}
+
+} // namespace cmd
+} // namespace bts
@@ -0,0 +1,20 @@
+#pragma once
+
+#include "ICmd.h"
+
+namespace bts {
+namespace cmd {
+
+class ParseHistoricalFIX : public ICmd 
+{
+private:
+   void _execute_single(const std::string& full_file_name,
+                        bool process_orders);
+public:
+   int execute(std::vector<std::string> const &args);
+   std::string help ();
+};
+
+}}
+
+
@@ -94,7 +94,7 @@ void DropCopySession::onMessage(const FIX44::SecurityDefinitionRequest& req, con
 {
   if (req.getField(FIX::FIELD::SecurityRequestType) == std::to_string(FIX::SecurityRequestType_ALL_SECURITIES))
   {
-      auto&& secs = _router.get_securities();
+      // auto&& secs = _router.get_securities();
      // if (req.isSetField(FIX::FIELD::Symbol))
      //    secs = secs.Where(x => x.Symbol.ToLower() == req.Symbol.getValue().ToLower());
      // if (req.IsSetSecurityType())
@@ -33,15 +33,15 @@ public:
   
 public:
   void fromAdmin(const FIX::Message& message, const FIX::SessionID& sessionID) 
-                  throw(FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::RejectLogon);
+                  throw(FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::RejectLogon) override;
   void fromApp(const FIX::Message &message,const FIX::SessionID &sessionID) 
-                throw(FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::UnsupportedMessageType);
-   void toAdmin(FIX::Message&, const FIX::SessionID&);
-   void toApp(FIX::Message&, const FIX::SessionID&) throw(FIX::DoNotSend);                
+                throw(FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::UnsupportedMessageType) override;
+   void toAdmin(FIX::Message&, const FIX::SessionID&) override;
+   void toApp(FIX::Message&, const FIX::SessionID&) throw(FIX::DoNotSend) override;                
                     
-   void onCreate(const FIX::SessionID&);
-   void onLogon(const FIX::SessionID&);
-   void onLogout(const FIX::SessionID&);
+   void onCreate(const FIX::SessionID&) override;
+   void onLogon(const FIX::SessionID&) override;
+   void onLogout(const FIX::SessionID&) override;

 private:
   IFixSession& session(const FIX::SessionID&);
@@ -58,4 +58,4 @@ private:
   std::unique_ptr<PImpl> me;
 };

-}}
+}}
@@ -12,6 +12,7 @@
 #include "server/sparkd/cmds/Help.h"
 #include "server/sparkd/cmds/KeyCode.h"
 #include "server/sparkd/cmds/Md.h"
+#include "server/sparkd/cmds/ParseHistoricalFIX.h"
 #include "server/sparkd/cmds/PrintHedgeOrders.h"
 #include "server/sparkd/cmds/QuickTest.h"
 #include "server/sparkd/cmds/Run.h"
@@ -43,6 +44,7 @@ int main(int argc, char **argv) {
   REGISTER(FIXSim);
   REGISTER(Span);
   REGISTER(Web);
+   REGISTER(ParseHistoricalFIX);
   REGISTER(PrintHedgeOrders);
   REGISTER(AdminReport);
   END_CMD;
@@ -1 +1 @@
-set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
+set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/Apicore.proto;../lib/api/Apigateway.proto;../lib/api/Apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/HistoricalDataService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/Co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
@@ -3,5 +3,5 @@ if [ -f /.dockerenv ]; then
    ninja
 else
    cid=`docker ps | grep -P "emacs_ide|bluetradingsystems/linux_docker" | awk '{print $1}'`
-    docker exec $cid /bin/bash -c "cd build/clang_debug/ && ninja"
+    docker exec $cid /bin/bash -c "cd /home/ppinto/temp/spark_build/clang_debug && ninja"
 fi
@@ -1,23 +1,21 @@
 #/bin/bash
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-mkdir -p $DIR/../build/
-cd $DIR/../build/
+SRC_ROOT=$DIR/../
+mkdir -p $SRC_ROOT/build/
+BUILD_ROOT=`readlink -f $SRC_ROOT/build`

-rm -rf clang_debug 
-mkdir clang_debug
-cd clang_debug
-cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=/usr/lib/ccache/clang++ ../..
-cd ..
+rm -rf $BUILD_ROOT/clang_debug 
+mkdir $BUILD_ROOT/clang_debug
+cd $BUILD_ROOT/clang_debug
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=/usr/lib/ccache/clang++ $SRC_ROOT 

-rm -rf ninja_debug
-mkdir ninja_debug
-cd ninja_debug
-cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=/usr/lib/ccache/g++ ../..
-cd ..
+rm -rf $BUILD_ROOT/ninja_debug
+mkdir $BUILD_ROOT/ninja_debug
+cd $BUILD_ROOT/ninja_debug
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=/usr/lib/ccache/g++ $SRC_ROOT

-rm -rf ninja_release
-mkdir ninja_release
-cd ninja_release
-cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=g++ ../..
-cd ..
+rm -rf $BUILD_ROOT/ninja_release
+mkdir $BUILD_ROOT/ninja_release
+cd $BUILD_ROOT/ninja_release
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=g++ $SRC_ROOT

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SRC_ROOT=$DIR/../
+mkdir -p $SRC_ROOT/build/
+BUILD_ROOT=`readlink -f $SRC_ROOT/build`
+
+mkdir -p $SRC_ROOT/cme_data
+
+DATE=${1-`date +%Y%m%d`}
+
+AWS_ACCESS_KEY_ID=AKIAJ4XUDWPQ7M42352A AWS_SECRET_ACCESS_KEY=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 AWS_REGION=us-east-1 aws s3 sync s3://cme-bts-datamine/mbo/mbo/$DATE/ $SRC_ROOT/cme_data
Author	SHA1	Message	Date
Taha Afzal	843f42c3e3	Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis	2020-04-06 16:00:24 -04:00
Taha Afzal	3afc3d64b6	changed the keys	2020-04-06 16:00:12 -04:00
pfisher	44f2cdd7c0	change aws keys	2020-03-31 11:00:26 -05:00
Taha Afzal	3f36f54781	Added TopBooks to ProcessedBook	2020-03-03 17:09:56 -05:00
Taha Afzal	59e66cf1ef	Sniper reads from multiple files recursively	2020-03-02 18:32:19 -05:00
Taha Afzal	dec178f133	ignoring all trades that are not making sense	2020-03-02 09:46:10 -05:00
Taha Afzal	e9c9a3bad6	Function to get orders for both sides of the trade	2020-03-02 09:46:10 -05:00
Taha Afzal	46a4aa99df	Added quote order marketat the time the quote order was updated	2020-03-02 09:46:10 -05:00
Taha Afzal	1097bbcd60	Q leg display	2020-03-02 09:46:10 -05:00
Taha Afzal	840ba7e0b3	Added total qty and no of orders	2020-03-02 09:46:10 -05:00
Taha Afzal	7e7e88cedc	Proximity search bug	2020-03-02 09:46:10 -05:00
Taha Afzal	727a3d5b97	Cleaning up some mess	2020-03-02 09:46:10 -05:00
Taha Afzal	3044d5d5ec	Bin searching for trades	2020-03-02 09:46:10 -05:00
Taha Afzal	506084242a	Refinement of orders/trades	2020-03-02 09:46:10 -05:00
Taha Afzal	067946127a	cleaning up	2020-03-02 09:46:10 -05:00
Taha Afzal	297848bcdb	Re-org of passive quote detection	2020-03-02 09:46:10 -05:00
Pedro Pinto	376d64cfbe	updated db script to incorporate timestamps	2020-02-06 10:07:14 -05:00
Pedro Pinto	bdf3ae7c92	Added unix timestamps to csvs	2020-02-06 09:22:22 -05:00
Pedro Pinto	97d0637d88	Updated tools	2020-02-04 09:27:19 -05:00
Pedro Pinto	b4d85cdadf	Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis	2020-02-04 09:24:40 -05:00
Pedro Pinto	2b0cb84069	Changed synchronization script to default to synching yesterday	2020-02-04 09:23:25 -05:00
Pedro Pinto	0412c45ed3	Removed build step	2020-02-03 15:48:20 -05:00
Pedro Pinto	b984613b8a	Non interactive use of postgres	2020-02-03 15:23:21 -05:00
Pedro Pinto	e712433d6f	More tweaks.	2020-02-03 15:21:27 -05:00
Pedro Pinto	4d8b20d084	Fixed path in postgres sql.	2020-02-03 15:14:08 -05:00
Pedro Pinto	df86187457	Tool tweaks	2020-02-03 14:56:01 -05:00
Pedro Pinto	620ea4b4af	Tweaked date tool	2020-02-03 13:56:21 -05:00
Pedro Pinto	e2951b4436	Added pgadmin tool	2020-02-03 10:12:32 -05:00
Pedro Pinto	0ebc3cf111	Fixed a few issue with analysis tools	2020-02-03 09:58:39 -05:00
Pedro Pinto	fd9b2990ae	Added easier to use tools for analysis	2020-01-31 16:28:18 -05:00
Pedro Pinto	15d24f96c2	Updated externals, private tools	2020-01-31 16:28:18 -05:00
Pedro Pinto	b1bbc6ab0d	Updated docker	2020-01-31 16:28:17 -05:00
Pedro Pinto	39be9e41c8	Rewound externals because tip is broken	2020-01-31 16:28:17 -05:00
Pedro Pinto	fd91b78742	sync/parse/load tools	2020-01-31 16:28:17 -05:00
Pedro Pinto	95a5dd0ceb	updated docker	2020-01-31 16:28:17 -05:00
Pedro Pinto	a9aa76deed	Updated gitignore and externals	2020-01-31 16:28:17 -05:00
Taha Afzal	a80a4f327e	Fixed theoutput data zero fills	2020-01-31 13:51:52 -05:00
Taha Afzal	7e9a853a50	filtering books	2020-01-31 11:38:11 -05:00
Taha Afzal	0e0d4412b8	Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis	2020-01-30 22:45:06 -05:00
Taha Afzal	b3a19eab05	Filtering quote fills for orders	2020-01-30 22:44:42 -05:00
Pedro Pinto	fdb1826e01	Added a tool to download cme historical data	2020-01-30 13:27:02 -05:00
Taha Afzal	e2d6bc1c45	Order parsing	2020-01-29 19:43:36 -05:00
Taha Afzal	d5a06cc3e7	circle filling in	2020-01-27 22:58:22 -05:00
Taha Afzal	3e7c2e3c18	circle analysis	2020-01-22 11:50:49 -05:00
Taha Afzal	ea6eaf6e1d	contracts show in entry:exit	2020-01-21 10:40:11 -05:00
Taha Afzal	dea2358c51	cleaned up some wrappers to simplify the code	2020-01-17 18:43:08 -05:00
Taha Afzal	491d0c0b7a	no exponent mapping	2020-01-17 11:50:06 -05:00
Taha Afzal	f593b457a5	Market close times factored in	2020-01-17 11:25:26 -05:00
Taha Afzal	10a3ba52f4	Added max processing time	2020-01-16 11:03:27 -05:00
Taha Afzal	56bc7f216a	Parallelized the sniper analysis across file (in batches)	2020-01-15 22:11:04 -05:00
Taha Afzal	a57b99b2e8	BEtter formatting of output	2020-01-14 14:49:40 -05:00
Taha Afzal	d3a0595934	Cleaning	2020-01-14 10:54:55 -05:00
Taha Afzal	a659fbc7cd	Sniper analysis works on multiple files now	2020-01-14 09:32:08 -05:00
Taha Afzal	542b8e946d	mass sniper analysis	2020-01-13 11:16:06 -05:00
Taha Afzal	d4d1f8f533	download args changed	2020-01-12 18:18:56 -05:00
Taha Afzal	624a432c44	ignoring ticks from when the market is closed	2020-01-12 13:13:05 -05:00
Pedro Pinto	68f09c2e27	Fixed crash when the timen take to parse was rounded to 0 secs.	2020-01-11 18:45:36 -05:00
Taha Afzal	3d22a58bac	ParseHistoricalFIX takes a path and processes all files with name _raw in it	2020-01-10 19:19:52 -05:00
Taha Afzal	33e46b2fb1	Batch download	2020-01-10 11:14:25 -05:00
Taha Afzal	cb357440ed	cleaned up some old code	2020-01-09 14:12:04 -05:00
Taha Afzal	a88670b95d	Added headers to the output	2020-01-09 13:22:39 -05:00
Taha Afzal	bb70db70b5	Better output generation	2020-01-09 11:28:41 -05:00
Taha Afzal	2c3e4b7d4d	loading and validating	2020-01-06 20:04:45 -05:00
Taha Afzal	843c4f447f	Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis	2020-01-03 17:07:26 -05:00
Taha Afzal	d41c76e6a3	validatio	2020-01-03 17:06:55 -05:00
Pedro Pinto	805722cdc7	Fixed the calculation of the book in the historical data parser.	2020-01-03 16:07:32 -05:00
Pedro Pinto	ef64d3c25e	Fixed weird auto&& in HandleIndex.h	2020-01-03 12:27:52 -05:00
Pedro Pinto	0b0cf744a6	Added more information to parser	2020-01-03 12:22:44 -05:00
Pedro Pinto	c3410db0aa	Fixed clang build.	2020-01-03 12:22:44 -05:00
Eric Harding	c5a89b0bb6	fix a couple clang warnings	2020-01-03 11:45:44 -05:00
Taha Afzal	89f5f94588	validation	2020-01-02 19:39:50 -05:00
Taha Afzal	c8ac56b0a1	Merge branch 'trade_analysis' of bitbucket.org:bluetradesys/spark into trade_analysis	2020-01-02 19:33:15 -05:00
Pedro Pinto	b152b2b853	Added a parser of historical MD to facilitate analysis.	2020-01-02 14:01:57 -05:00
Pedro Pinto	354390ed11	Fixed clang build.	2020-01-02 14:01:48 -05:00
Taha Afzal	08839aadb2	book validation code	2020-01-01 22:03:23 -05:00
Taha Afzal	b7cb17500c	Downloading new book data	2019-12-31 18:32:45 -05:00
Taha Afzal	01b50e1138	Added function skeleton to compure possible fills	2019-12-30 19:33:04 -05:00
Taha Afzal	8fdb7485ba	new order and trade protos	2019-12-30 19:22:40 -05:00
Taha Afzal	293d51038f	Circle spreads initial analysis	2019-12-27 18:55:54 -05:00
Taha Afzal	8db81fa248	countering for the trades and orders files	2019-12-27 10:27:18 -05:00
Taha Afzal	47628089ac	Recording trades and orders	2019-12-26 21:08:49 -05:00
Taha Afzal	8b2c5a60aa	Added NG exp	2019-12-24 12:41:35 -05:00
Taha Afzal	f1df673e61	spread parsing gix, Z29 showed up	2019-12-23 18:43:14 -05:00
Taha Afzal	9faee77360	merging vwap	2019-12-23 16:38:58 -05:00
Taha Afzal	17a1190e34	added exponents for HE,ZC	2019-12-13 18:16:13 -05:00
Taha Afzal	82834dc9e6	Parallelized the processing of file for sniper	2019-12-13 14:46:30 -05:00
Taha Afzal	2ff96e3528	more helper functions	2019-12-13 12:37:17 -05:00
Taha Afzal	74f1bf2480	functions to remove duplicates	2019-12-12 11:35:19 -05:00
Taha Afzal	4e48bcf6b5	refactoring	2019-12-11 13:33:07 -05:00
Taha Afzal	6fd829df49	Silly bug that was overwriting the lined up markets	2019-12-11 11:49:30 -05:00
Taha Afzal	0dcb3d3dfd	Added microscope for a closer look!	2019-12-08 21:53:39 -05:00
Taha Afzal	212f2a3d36	can run the sniper on one synthetic only, too	2019-12-08 10:39:04 -05:00
Taha Afzal	649225ef6c	unifying the spread calc function	2019-12-07 22:55:18 -05:00
Taha Afzal	35495dc7cf	better formatting	2019-12-06 20:17:14 -05:00
Taha Afzal	8ee78c83d2	writes result to a file:	2019-12-06 18:19:29 -05:00
Taha Afzal	b4beb7c0db	moved BookEntry and TimeRange into protos	2019-12-06 13:19:03 -05:00
Taha Afzal	f84517c15c	cleaning up some mes	2019-12-05 17:32:05 -05:00
Taha Afzal	075354a975	Sniper analysis works on generated spreaders	2019-12-05 11:08:45 -05:00
Taha Afzal	4450476a40	simplified lining up of books	2019-12-04 09:49:26 -05:00
Taha Afzal	3b0bbe6908	simplified	2019-12-02 13:09:11 -05:00
Taha Afzal	1147edcbb9	tests	2019-12-02 12:44:10 -05:00
Taha Afzal	c50133ba02	tests	2019-12-01 16:17:14 -05:00
Taha Afzal	f1dbacb989	forcing an exponent to fix the prices	2019-11-29 16:43:21 -05:00
Taha Afzal	0535eedbbf	merging	2019-11-29 15:34:44 -05:00
Taha Afzal	2740de913c	BookState is now called BookSeries	2019-11-29 09:47:46 -05:00
Taha Afzal	ba53543e43	Parsing the sniper input file correctly	2019-11-27 18:07:59 -05:00