From 69612eaee788ca837effd84c2de7374358bdffa2 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Wed, 4 Dec 2019 23:33:48 -0500 Subject: [PATCH] thought better of two tables, this makes simulation way easier --- .vscode/launch.json | 2 +- src/client/spark_api/history/HistoryEntry.cs | 36 +---- src/client/spark_historical/Algos/AlgoBase.cs | 3 +- .../spark_historical/HistoricalExtensions.cs | 4 - .../spark_historical/Query/MongoGenerator.cs | 129 +++++++----------- src/client/spark_historical/Query/Query.cs | 21 +-- .../api/services/HistoricalDataService.proto | 34 ++--- 7 files changed, 77 insertions(+), 152 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 75fa6159f..5667f1343 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -109,7 +109,7 @@ "--rend", "11/26/2019", "--rsymbol", "ZN", "--rsecuritytype", "Synthetic", - "--post_process_db", "roll", + "--post_process_db", "roll2", "--post_process_db_host", "sparkvm", "--post_process_db_port", "27017", "--post_process_db_user", "mongo", diff --git a/src/client/spark_api/history/HistoryEntry.cs b/src/client/spark_api/history/HistoryEntry.cs index c895a834e..f9a8ccd10 100644 --- a/src/client/spark_api/history/HistoryEntry.cs +++ b/src/client/spark_api/history/HistoryEntry.cs @@ -10,31 +10,9 @@ namespace bts.history DateTime transact_time { get ; } DateTime timestamp(); } - - public interface IMBP : ITimestamped - { - string SecurityDesc { get ; } - co2.BookMessage Book { get ; } - co2.TopBookMessage TopBook { get ; } - co2.TradeMessage Trade { get ; } - } + - public interface IMBO : ITimestamped - { - MDUpdateAction OrderUpdateAction { get ; } - ulong OrderId { get; } - uint SecurityId { get; } - string SecurityDesc { get; } - ulong MdOrderPriority { get; } - - double MdEntryPx { get; } - uint MdPriceLevel { get; } - ulong MdDisplayQty { get; } - } - - - - public sealed partial class HistoryRow : IMBP + public sealed partial class HistoryRow : ITimestamped { public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } } public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } } @@ -45,14 +23,4 @@ namespace bts.history } } - public sealed partial class HistoryOrderRow : IMBO - { - public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } } - public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } } - public DateTime transact_time { get { return this.TransactTime.ToDateTime(); } set { this.TransactTime = Timestamp.FromDateTime(value); } } - public DateTime timestamp() - { - return this.sending_time; - } - } } \ No newline at end of file diff --git a/src/client/spark_historical/Algos/AlgoBase.cs b/src/client/spark_historical/Algos/AlgoBase.cs index a905456a3..2db101e17 100644 --- a/src/client/spark_historical/Algos/AlgoBase.cs +++ b/src/client/spark_historical/Algos/AlgoBase.cs @@ -54,8 +54,9 @@ namespace bts.history } } + public interface IAlgo {} - abstract class AlgoBase + abstract class AlgoBase : IAlgo { public readonly Guid guid = Guid.NewGuid(); public ulong order_key { get; private set;} diff --git a/src/client/spark_historical/HistoricalExtensions.cs b/src/client/spark_historical/HistoricalExtensions.cs index b14861c58..6a1f60de9 100644 --- a/src/client/spark_historical/HistoricalExtensions.cs +++ b/src/client/spark_historical/HistoricalExtensions.cs @@ -16,10 +16,6 @@ namespace bts.history { return md.SecurityDesc; } - public static string security_desc(this HistoryOrderRow md) - { - return md.SecurityDesc; - } public static bool less(this Timestamp lhs, Timestamp rhs) { return lhs.Seconds * 1e9 + lhs.Nanos < rhs.Seconds * 1e9 + rhs.Nanos; diff --git a/src/client/spark_historical/Query/MongoGenerator.cs b/src/client/spark_historical/Query/MongoGenerator.cs index a17a7dfa9..3554afc24 100644 --- a/src/client/spark_historical/Query/MongoGenerator.cs +++ b/src/client/spark_historical/Query/MongoGenerator.cs @@ -18,7 +18,7 @@ namespace bts.history { public MarketDataIncrementalRefresh md; public MDEntry book_or_trade_entry; - public IEnumerable mbo; + public IEnumerable mbo; } readonly DataStreamer _stream; @@ -28,8 +28,6 @@ namespace bts.history readonly CancellationTokenSource _cts = new CancellationTokenSource(); bool _done_reading; - readonly ConcurrentDictionary> _book_operations = - new ConcurrentDictionary>(); readonly ConcurrentDictionary> _mbo_operations = new ConcurrentDictionary>(); @@ -37,7 +35,6 @@ namespace bts.history readonly MongoClient _client; readonly IMongoDatabase _db; - const bool SHOULD_PROCESS_ORDERS = true; public MongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port, string user = "mongo", string pass = "enter") { @@ -67,23 +64,18 @@ namespace bts.history int count = 0; Stopwatch watch = new Stopwatch(); #pragma warning disable CS4014 watch.Start(); - - //start_line_processor().ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); await _stream.process_md_stream_by_line(query, (sectype, _, line) => { try { var md = ProtoFix.parse_fix(line); - send_books_and_trades(md); - if(SHOULD_PROCESS_ORDERS) - send_orders(md); + send(md); } catch (QuickFix.MessageParseError ex) { Log.FIX.warn("Message parse error: " + ex.ToString()); } -// _lines.Enqueue(line); if (++count % 10_000 == 1) { var qlen = get_qlen(); @@ -111,57 +103,13 @@ namespace bts.history int get_qlen() { - var qlen = _book_operations.Values.Select(x => x.Count).Sum(); var blen = _mbo_operations.Values.Select(x => x.Count).Sum(); - return qlen + blen; + return blen; } - private void send_books_and_trades(MarketDataIncrementalRefresh md) + private void send(MarketDataIncrementalRefresh md) { - foreach (var entry in md.MdEntries) - { - if (!_book_operations.TryGetValue(entry.security_desc(), out ConcurrentQueue q)) - { - _book_operations[entry.security_desc()] = q = new ConcurrentQueue(); - start_book_processor(entry.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ; - } - q.Enqueue(new Operation { md = md, book_or_trade_entry = entry }); - } - } - async Task start_book_processor(string secdesc) - { - var t = Task.Run( async () => - { - using (var session = await _client.StartSessionAsync()) - { - while(!_cts.IsCancellationRequested) - { - await Task.Delay(1); - if(!_book_operations.TryGetValue(secdesc, out ConcurrentQueue operations)) - continue; - int count = 0; - session.StartTransaction(); - while(operations.TryDequeue(out Operation oper)) - { - Log.assert(oper.book_or_trade_entry != null); - write_book_or_trade_entry(oper.md, oper.book_or_trade_entry); - Log.assert(oper.book_or_trade_entry.security_desc() == secdesc); - if (count++ > 10_000) - break; - } - session.CommitTransactionAsync(); - if(_done_reading && operations.Count == 0) - break; - } - } - }, _cts.Token); - _qtask.Add( t ); - await t; - } - - private void send_orders(MarketDataIncrementalRefresh md) - { - Dictionary> cache = new Dictionary>(); + Dictionary> cache = new Dictionary>(); foreach (var order in md.OrderIdEntries) { if (order.ReferenceId == 0) @@ -172,7 +120,6 @@ namespace bts.history MDEntry mdval = md.MdEntries[mdidx]; - //now merge this order stuff into an MDEntry because we can't do templates....no I don't miss c++ mdval = mdval.Clone(); mdval.LastQty = order.LastQty; mdval.MdDisplayQty = order.MdDisplayQty; @@ -198,26 +145,32 @@ namespace bts.history _mbo_operations[pair.Key].Enqueue(new Operation { md = md, mbo = pair.Value }); } + foreach (var mdval in md.MdEntries) + { + if (!_mbo_operations.TryGetValue(mdval.security_desc(), out ConcurrentQueue q)) + { + _mbo_operations[mdval.security_desc()] = q = new ConcurrentQueue(); + start_mbo_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ; + } + q.Enqueue(new Operation { md = md, book_or_trade_entry = mdval }); + } + //each order in here can be for a different secdesc void cache_mbo(MDEntry order) { - HistoryOrderRow mbo = new HistoryOrderRow(); - mbo.TradeDate = md.TradeDate; - mbo.MsgSeqNum = md.MsgSeqNum; - mbo.SendingTime = md.SendingTime; - mbo.TransactTime = md.TransactTime; - + MBOMessage mbo = new MBOMessage(); mbo.OrderUpdateAction = order.MdUpdateAction; mbo.OrderId = order.OrderId; mbo.SecurityId = order.SecurityId; mbo.SecurityDesc = order.security_desc(); mbo.MdOrderPriority = order.MdOrderPriority; - + mbo.MdEntryPx = order.MdEntryPx; mbo.MdPriceLevel = order.MdPriceLevel; mbo.MdDisplayQty = order.MdDisplayQty; - if (!cache.TryGetValue(mbo.SecurityDesc, out List list)) - cache[mbo.SecurityDesc] = list = new List(); + + if (!cache.TryGetValue(order.security_desc(), out List list)) + cache[order.security_desc()] = list = new List(); list.Add(mbo); } } @@ -236,10 +189,18 @@ namespace bts.history session.StartTransaction(); while (operations.TryDequeue(out Operation oper)) { - Log.assert(oper.mbo != null); - write_mbo(secdesc, oper.md, oper.mbo); + if(oper.mbo != null) + { + write_mbo(secdesc, oper.md, oper.mbo); + } + else if(oper.book_or_trade_entry != null) + { + write_book_or_trade_entry(oper.md, oper.book_or_trade_entry); + Log.assert(oper.book_or_trade_entry.security_desc() == secdesc); + } + Log.assert(oper.mbo == null || oper.book_or_trade_entry == null); if (count++ > 50_000) - break; + break; } session.CommitTransactionAsync(); if(_done_reading && operations.Count == 0) @@ -307,9 +268,10 @@ namespace bts.history MsgSeqNum = md.MsgSeqNum, SendingTime = md.SendingTime, TransactTime = md.TransactTime, - SecurityDesc = entry.security_desc(), - TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] }, - }; + SecurityDesc = entry.security_desc(), + }; + if(entry.MdPriceLevel == 1) + hist.TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] }; if (OrderBook.quote_event_done(md)) hist.Book = book; @@ -317,11 +279,24 @@ namespace bts.history table.InsertOne( hist ); } - void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable mbo) + void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable mbo) { - string tname = table_name(secdesc) + "_MBO"; - var table = _db.GetCollection(tname); - table.InsertMany(mbo); + string tname = table_name(secdesc); + var table = _db.GetCollection(tname); + var orders = mbo.Select(x => + { + HistoryRow hist = new HistoryRow + { + TradeDate = md.TradeDate, + MsgSeqNum = md.MsgSeqNum, + SendingTime = md.SendingTime, + TransactTime = md.TransactTime, + SecurityDesc = x.SecurityDesc, + Order = x, + }; + return hist; + }); + table.InsertMany(orders); } public async void Dispose() diff --git a/src/client/spark_historical/Query/Query.cs b/src/client/spark_historical/Query/Query.cs index e22fa7e68..6e325ce28 100644 --- a/src/client/spark_historical/Query/Query.cs +++ b/src/client/spark_historical/Query/Query.cs @@ -50,18 +50,7 @@ namespace bts.history m.UnmapMember(x => x.SendingTime); m.UnmapMember(x => x.TransactTime); m.SetIgnoreExtraElements(true); - }); - BsonClassMap.RegisterClassMap(m => - { - m.AutoMap(); - m.MapMember(x => x.trade_date).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); - m.MapMember(x => x.sending_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); - m.MapMember(x => x.transact_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); - m.UnmapMember(x => x.TradeDate); - m.UnmapMember(x => x.SendingTime); - m.UnmapMember(x => x.TransactTime); - m.SetIgnoreExtraElements(true); - }); + }); //BsonSerializer.RegisterSerializationProvider(new SerializationProvider()); } @@ -94,13 +83,13 @@ namespace bts.history return res; } - public IEnumerable run_algo(AlgoBase algo, IFillModel fill_model, uint latency, - IEnumerable bookstream, IEnumerable orderstream) + public IEnumerable run_algo(IAlgo algo, IFillModel fill_model, uint latency, + IEnumerable datastream) { - + return null; } } - + public interface IFillModel {} public static class QueryExtensions { static public IMongoQueryable where(this IMongoQueryable q, string where) diff --git a/src/lib/api/services/HistoricalDataService.proto b/src/lib/api/services/HistoricalDataService.proto index e8b1a8b19..58bc8241a 100644 --- a/src/lib/api/services/HistoricalDataService.proto +++ b/src/lib/api/services/HistoricalDataService.proto @@ -154,26 +154,8 @@ message ProcessedBook co2.BookMessage book = 50; } -message HistoryRow +message MBOMessage { - google.protobuf.Timestamp trade_date = 5; - uint32 msg_seq_num = 10; - google.protobuf.Timestamp sending_time = 20; - google.protobuf.Timestamp transact_time = 30; - - string security_desc = 40; - co2.BookMessage book = 50; - co2.TopBookMessage top_book = 55; - co2.TradeMessage trade = 60; -} - -message HistoryOrderRow -{ - google.protobuf.Timestamp trade_date = 5; - uint32 msg_seq_num = 10; - google.protobuf.Timestamp sending_time = 20; - google.protobuf.Timestamp transact_time = 30; - MDUpdateAction order_update_action = 279; uint64 order_id = 37; uint32 security_id = 48; @@ -185,6 +167,20 @@ message HistoryOrderRow uint64 md_display_qty = 37706; } +message HistoryRow +{ + google.protobuf.Timestamp trade_date = 5; + uint32 msg_seq_num = 10; + google.protobuf.Timestamp sending_time = 20; + google.protobuf.Timestamp transact_time = 30; + + string security_desc = 40; + co2.BookMessage book = 50; + co2.TopBookMessage top_book = 55; + co2.TradeMessage trade = 60; + MBOMessage order = 70; +} + service HistoricalDataService { rpc get_trading_slices(SlicesQuery) returns (Slices);