From 47628089ac41d720dc6eba468e6fc802deb87589 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Thu, 26 Dec 2019 21:08:49 -0500 Subject: [PATCH] Recording trades and orders --- .../Analyzers/BookAnalyzer.cs | 75 +++++++++++++++---- .../api/services/HistoricalDataService.proto | 16 ++++ 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/src/client/spark_historical/Analyzers/BookAnalyzer.cs b/src/client/spark_historical/Analyzers/BookAnalyzer.cs index c1f4b193b..1f3a492c7 100644 --- a/src/client/spark_historical/Analyzers/BookAnalyzer.cs +++ b/src/client/spark_historical/Analyzers/BookAnalyzer.cs @@ -13,7 +13,17 @@ namespace bts.history { class SecData{ public uint n_updates = 0; - public System.IO.FileStream str; + public System.IO.FileStream str, trade_str; + + public SecData(string full_path, + string contract){ + var contract_fr = contract.Replace(":","."); + string bk_file_name = String.Format ("{0}{1}",full_path,contract_fr); + str = new FileStream(bk_file_name, FileMode.OpenOrCreate); + string tr_file_name = + String.Format ("{0}trades_{1}",full_path,contract_fr); + trade_str = new FileStream(tr_file_name,FileMode.OpenOrCreate); + } public SecData (System.IO.FileStream str_a){ str = str_a; } @@ -26,18 +36,23 @@ namespace bts.history private readonly string _book_path; IDictionary _book_stream_map = new Dictionary(); - + private System.IO.FileStream _order_str; + public BookAnalyzer(string symbol, HistoricalDataOptions options) { _book_path = MarketQueriesHelper.get_book_path( symbol, DateTime.Parse(options.replaystart)); System.IO.Directory.CreateDirectory(_book_path); + _order_str = new FileStream( + String.Format ("{0}orders_{1}",_book_path,symbol), + FileMode.OpenOrCreate); } public void pre_analyze(MarketDataIncrementalRefresh md, - string debug_fix = null) - {} + string debug_fix = null){ + _write_orders(md); + } public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null) @@ -57,10 +72,11 @@ namespace bts.history if (Utils.quote_event_done(md)) on_new_book(book, entry, md); } - // else if (entry.MdEntryType == MDEntryType.Trade) - // handle trades here + else if (entry.MdEntryType == MDEntryType.Trade) + on_new_trade(entry,md); } - } + _write_orders(md); + } private int _bn = 0; private void on_new_book(co2.BookMessage book, MDEntry entry, @@ -68,23 +84,52 @@ namespace bts.history ProcessedBook pb = new ProcessedBook(); pb.MsgSeqNum = md_inc.MsgSeqNum; pb.SendingTime = md_inc.SendingTime; - pb.TransactTime = md_inc.TransactTime; + pb.TransactTime = md_inc.TransactTime; pb.Security = entry.Symbol; pb.Book = book; SecData s; if(!_book_stream_map.TryGetValue(pb.Security, out s)){ - string file_name = pb.Security; - file_name = file_name.Replace(":","."); - file_name = String.Format ("{0}{1}",_book_path,file_name); - var str = new FileStream(file_name, FileMode.OpenOrCreate); - s = new SecData (str); + s = new SecData (_book_path,pb.Security); _book_stream_map.Add(pb.Security,s); - } + } pb.BookSeqNum = s.n_updates; s.inc_count(); Google.Protobuf.MessageExtensions.WriteDelimitedTo(pb,s.str); if(_bn++%10000==0) - Console.WriteLine("b:{0}",_bn); + Console.WriteLine("n_bks:{0}",_bn); + } + + private int _n_ords = 0; + private void _write_orders(MarketDataIncrementalRefresh md){ + foreach(var oid in md.OrderIdEntries) + if(oid.OrderUpdateAction == MDUpdateAction.New){ + // write it along with the time + NewOrderEntry new_ord = new NewOrderEntry(); + new_ord.OrderId = oid.OrderId; + new_ord.SendingTime = md.SendingTime; + new_ord.TransactTime = md.TransactTime; + Google.Protobuf.MessageExtensions.WriteDelimitedTo(new_ord,_order_str); + if(_n_ords++%10000==0) + Console.WriteLine("n_ords:{0}",_n_ords); + } + } + + private void on_new_trade(MDEntry md_entry, + MarketDataIncrementalRefresh md_inc){ + string contract = md_entry.Symbol; + TradeEntry tr = new TradeEntry(); + tr.Price = md_entry.MdEntryPx; + tr.Qty = md_entry.MdEntrySize; + tr.TradeId = md_entry.MdTradeEntryId; + tr.OrderId = md_entry.OrderId; + tr.SendingTime = md_inc.SendingTime; + tr.TransactTime = md_inc.TransactTime; + SecData s; + if(!_book_stream_map.TryGetValue(contract, out s)){ + s = new SecData (_book_path, contract); + _book_stream_map.Add(contract,s); + } + Google.Protobuf.MessageExtensions.WriteDelimitedTo(tr,s.trade_str); } } } \ No newline at end of file diff --git a/src/lib/api/services/HistoricalDataService.proto b/src/lib/api/services/HistoricalDataService.proto index d8b900f67..d338e275b 100644 --- a/src/lib/api/services/HistoricalDataService.proto +++ b/src/lib/api/services/HistoricalDataService.proto @@ -163,6 +163,22 @@ message BookEntry{ ProcessedBook p_book = 10; Timerange time_range = 20; } + +message TradeEntry{ + double price = 10; + uint32 qty = 20; + string trade_id = 30; + uint64 order_id = 40; + google.protobuf.Timestamp transact_time = 50; + google.protobuf.Timestamp sending_time = 60; +} + +message NewOrderEntry{ + uint64 order_id = 10; + google.protobuf.Timestamp transact_time = 20; + google.protobuf.Timestamp sending_time = 30; +} + //this is a generated class to help accessing the mbo fields, look at the partial class extension message MBOMessage {