diff --git a/src/client/spark_historical/Query/MongoGenerator.cs b/src/client/spark_historical/Query/MongoGenerator.cs index c2f40fa7a..0cabebc57 100644 --- a/src/client/spark_historical/Query/MongoGenerator.cs +++ b/src/client/spark_historical/Query/MongoGenerator.cs @@ -18,6 +18,7 @@ namespace bts.history { public MarketDataIncrementalRefresh md; public MDEntry book_or_trade_entry; + public IEnumerable mbo; } readonly DataStreamer _stream; @@ -123,18 +124,29 @@ namespace bts.history private void send(MarketDataIncrementalRefresh md) { + IDictionary> cache = Query.extract_mbo(md); + + foreach (var pair in cache) + { + if (!_mbo_operations.TryGetValue(pair.Key, out ConcurrentQueue q)) + { + _mbo_operations[pair.Key] = q = new ConcurrentQueue(); + start_mbo_processor(pair.Key).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ; + } + _mbo_operations[pair.Key].Enqueue(new Operation { md = md, mbo = pair.Value }); + } + foreach (var mdval in md.MdEntries) { if (!_mbo_operations.TryGetValue(mdval.security_desc(), out ConcurrentQueue q)) { _mbo_operations[mdval.security_desc()] = q = new ConcurrentQueue(); - start_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ; + start_mbo_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ; } q.Enqueue(new Operation { md = md, book_or_trade_entry = mdval }); } } - - async Task start_processor(string secdesc) + async Task start_mbo_processor(string secdesc) { var t = Task.Run(async () => { @@ -149,9 +161,16 @@ namespace bts.history session.StartTransaction(); while (operations.TryDequeue(out Operation oper)) { - Log.assert(oper.book_or_trade_entry != null); - write_book_or_trade_entry(oper.md, oper.book_or_trade_entry); - Log.assert(oper.book_or_trade_entry.security_desc() == secdesc); + if(oper.mbo != null) + { + write_mbo(secdesc, oper.md, oper.mbo); + } + else if(oper.book_or_trade_entry != null) + { + write_book_or_trade_entry(oper.md, oper.book_or_trade_entry); + Log.assert(oper.book_or_trade_entry.security_desc() == secdesc); + } + Log.assert(oper.mbo == null || oper.book_or_trade_entry == null); if (count++ > 50_000) break; } @@ -223,8 +242,8 @@ namespace bts.history MsgSeqNum = md.MsgSeqNum, SendingTime = md.SendingTime, TransactTime = md.TransactTime, - SecurityDesc = entry.security_desc(), - Md = md, + SecurityDesc = entry.security_desc(), + Md = md, }; if(entry.MdPriceLevel == 1) hist.TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] }; @@ -235,25 +254,26 @@ namespace bts.history table.InsertOne( hist ); } - // void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable mbo) - // { - // string tname = table_name(secdesc); - // var table = _db.GetCollection(tname); - // var orders = mbo.Select(x => - // { - // HistoryRow hist = new HistoryRow - // { - // TradeDate = md.TradeDate, - // MsgSeqNum = md.MsgSeqNum, - // SendingTime = md.SendingTime, - // TransactTime = md.TransactTime, - // SecurityDesc = x.SecurityDesc, - // Order = x, - // }; - // return hist; - // }); - // table.InsertMany(orders); - // } + void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable mbo) + { + string tname = table_name(secdesc); + var table = _db.GetCollection(tname); + var orders = mbo.Select(x => + { + HistoryRow hist = new HistoryRow + { + TradeDate = md.TradeDate, + MsgSeqNum = md.MsgSeqNum, + SendingTime = md.SendingTime, + TransactTime = md.TransactTime, + SecurityDesc = x.SecurityDesc, + Order = x, + Md = md, + }; + return hist; + }); + table.InsertMany(orders); + } public async void Dispose() { diff --git a/src/client/spark_historical/Query/Query.cs b/src/client/spark_historical/Query/Query.cs index 1fcaf92a8..968204860 100644 --- a/src/client/spark_historical/Query/Query.cs +++ b/src/client/spark_historical/Query/Query.cs @@ -189,7 +189,7 @@ namespace bts.history return null; } - public IDictionary> extract_mbo(MarketDataIncrementalRefresh md) + static public IDictionary> extract_mbo(MarketDataIncrementalRefresh md) { Dictionary> cache = new Dictionary>(); foreach (var order in md.OrderIdEntries) diff --git a/src/lib/api/services/HistoricalDataService.proto b/src/lib/api/services/HistoricalDataService.proto index f92f19590..7452a955c 100644 --- a/src/lib/api/services/HistoricalDataService.proto +++ b/src/lib/api/services/HistoricalDataService.proto @@ -178,7 +178,7 @@ message HistoryRow co2.BookMessage book = 50; co2.TopBookMessage top_book = 55; co2.TradeMessage trade = 60; - MBOMessage order = 70; //deprecated + MBOMessage order = 70; MarketDataIncrementalRefresh md = 80; }