From ab2e9fe6bdb5e5cf1f96efbe2f748d075ad37bb6 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Tue, 3 Dec 2019 22:34:04 -0500 Subject: [PATCH] I might have fixed the dates --- .../Algos/AlgoStatsWrapper.cs | 2 +- ...istoricalDataReplayer.cs => BackTester.cs} | 8 +- ...calDataDownloader.cs => DataDownloader.cs} | 8 +- ...{HistoricalDataSender.cs => DataSender.cs} | 8 +- ...toricalDataStreamer.cs => DataStreamer.cs} | 8 +- ...abaseGenerator.cs => DatabaseGenerator.cs} | 10 +- .../HistoricalDataServiceServer.cs | 2 +- src/client/spark_historical/MBOParser.cs | 6 +- ...calMongoGenerator.cs => MongoGenerator.cs} | 169 ++++++++---------- src/client/spark_historical/ProtoFix.cs | 19 +- src/client/spark_historical/Query.cs | 45 ++++- .../spark_historical/TimestampSerializer.cs | 77 ++++++++ .../sparkcmd/Cmd/HistoricalDataCommand.cs | 6 +- 13 files changed, 228 insertions(+), 140 deletions(-) rename src/client/spark_historical/{HistoricalDataReplayer.cs => BackTester.cs} (92%) rename src/client/spark_historical/{HistoricalDataDownloader.cs => DataDownloader.cs} (94%) rename src/client/spark_historical/{HistoricalDataSender.cs => DataSender.cs} (93%) rename src/client/spark_historical/{HistoricalDataStreamer.cs => DataStreamer.cs} (95%) rename src/client/spark_historical/{HistoricalDatabaseGenerator.cs => DatabaseGenerator.cs} (94%) rename src/client/spark_historical/{HistoricalMongoGenerator.cs => MongoGenerator.cs} (55%) create mode 100644 src/client/spark_historical/TimestampSerializer.cs diff --git a/src/client/spark_historical/Algos/AlgoStatsWrapper.cs b/src/client/spark_historical/Algos/AlgoStatsWrapper.cs index 609fe22ab..c10a7c35e 100644 --- a/src/client/spark_historical/Algos/AlgoStatsWrapper.cs +++ b/src/client/spark_historical/Algos/AlgoStatsWrapper.cs @@ -58,7 +58,7 @@ namespace bts.history _symbol = options.replaysymbol; _securitytype = (SecurityType)Enum.Parse(typeof(SecurityType), options.replaysecuritytype); _tradedate = DateTime.Parse(options.replaystart); - var path = HistoricalDataDownloader.data_path(_tradedate.Date); + var path = DataDownloader.data_path(_tradedate.Date); new FileInfo(path).Directory.Create(); _fs = new StreamWriter( File.Open(path + $"/{_symbol}_{_securitytype}_{algo.GetType().Name}_{algo.bucket.LowerBound.ToString("0") }_trades.txt", FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite) ); diff --git a/src/client/spark_historical/HistoricalDataReplayer.cs b/src/client/spark_historical/BackTester.cs similarity index 92% rename from src/client/spark_historical/HistoricalDataReplayer.cs rename to src/client/spark_historical/BackTester.cs index e722b2943..408b1b044 100644 --- a/src/client/spark_historical/HistoricalDataReplayer.cs +++ b/src/client/spark_historical/BackTester.cs @@ -10,13 +10,13 @@ using Entry = System.Tuple().data_path; _bucket_name = bucket_name; @@ -98,7 +98,7 @@ namespace bts.history case CMEMics.Nymex: return "xnym"; default: - bts.Log.assert(false, "unknown mic map HistoricalDataDownloader"); + bts.Log.assert(false, "unknown mic map DataDownloader"); return string.Empty; } } diff --git a/src/client/spark_historical/HistoricalDataSender.cs b/src/client/spark_historical/DataSender.cs similarity index 93% rename from src/client/spark_historical/HistoricalDataSender.cs rename to src/client/spark_historical/DataSender.cs index c591aa8d7..097796432 100644 --- a/src/client/spark_historical/HistoricalDataSender.cs +++ b/src/client/spark_historical/DataSender.cs @@ -9,17 +9,17 @@ using bts.co2; namespace bts.history { - class HistoricalDataSender + class DataSender { - readonly HistoricalDataStreamer _stream; + readonly DataStreamer _stream; readonly bts.co2.MarketDataReceiverService _mdservice; Google.Protobuf.WellKnownTypes.Timestamp _time; Dictionary _channels = new Dictionary(); - public HistoricalDataSender(ServiceContext context) + public DataSender(ServiceContext context) { - _stream = new HistoricalDataStreamer(); + _stream = new DataStreamer(); _mdservice = context.locate(); } diff --git a/src/client/spark_historical/HistoricalDataStreamer.cs b/src/client/spark_historical/DataStreamer.cs similarity index 95% rename from src/client/spark_historical/HistoricalDataStreamer.cs rename to src/client/spark_historical/DataStreamer.cs index 35b2cb25b..bf6f1a8b3 100644 --- a/src/client/spark_historical/HistoricalDataStreamer.cs +++ b/src/client/spark_historical/DataStreamer.cs @@ -5,10 +5,10 @@ using bts.api; namespace bts.history { - class HistoricalDataStreamer + class DataStreamer { - readonly HistoricalDataDownloader _dl = new HistoricalDataDownloader(); - public HistoricalDataStreamer() + readonly DataDownloader _dl = new DataDownloader(); + public DataStreamer() { } @@ -79,7 +79,7 @@ namespace bts.history } - async public Task process_md_stream_by_line(MDQuery request, Action action, int peek_field = 55) + async public Task process_md_stream_by_line(MDQuery request, Action action, int peek_field = 0) { DateTime d = request.Begin.ToDateTime(); while (d < request.End.ToDateTime()) diff --git a/src/client/spark_historical/HistoricalDatabaseGenerator.cs b/src/client/spark_historical/DatabaseGenerator.cs similarity index 94% rename from src/client/spark_historical/HistoricalDatabaseGenerator.cs rename to src/client/spark_historical/DatabaseGenerator.cs index 8125f0573..7e2442900 100644 --- a/src/client/spark_historical/HistoricalDatabaseGenerator.cs +++ b/src/client/spark_historical/DatabaseGenerator.cs @@ -13,7 +13,7 @@ using System.Net.Security; namespace bts.history { - class HistoricalDatabaseGenerator : IDisposable + class DatabaseGenerator : IDisposable { const string _certstring = "MIIEMDCCApigAwIBAgIDWCURMA0GCSqGSIb3DQEBDAUAMDoxODA2BgNVBAMMLzMzZTdhMTgzLTY0YmYtNGVjYi1hOWRiLTFjZjhjZmQyZmY3NSBQcm9qZWN0IENBMB4XDTE5MTEyOTE4MTQzNVoXDTI5MTEyNjE4MTQzNVowOjE4MDYGA1UEAwwvMzNlN2ExODMtNjRiZi00ZWNiLWE5ZGItMWNmOGNmZDJmZjc1IFByb2plY3QgQ0EwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDMZDOJWfSlSYd4Qim73HvT9MyxvOuDBA4HI9dI/4h7/cCFwdFOxygmpiB0naEyaUB/Xz3YK+Z/0sbTXemWXiKkQD/LTRSCGjzs+3iN3COf6QcFe7M/nyIKFW+BwEsF1i4r2gVlJDhC5QfEYH8Vj5VyY6sVavVoyxIiZfgo13JafVOP2D5YbeO1Q7lFcl4Jl+UJCoBVsrsvgG0nmPs3XDdJuQndlYqJGB5i8t6XybAT6PzeEG2GbKOfX3IIaeq0O4fkJMBO2psa7JsublBj6/RhSorU1j+QMC+MMZkr2UYOPdkycAHdIT3GpEJtKx/H3BSewsvEZkDBWlQ6YpdCJculFHuYZGntNxTtFDLnTMVLKH/g8GJZBxomlj1bvRDQW+9MqKldQq/Oe7Lo2IjbMuj/0a9CTEG2jOu6lL0NOFFtslO78PjKTfG6tzbPWKwlti78qG2jvhQjr+CKdqvKN2OehUMMaB0FTNpPAJAWHMexw4YCRIAeF2neVmgkeaunxxMCAwEAAaM/MD0wHQYDVR0OBBYEFMmBSdQd0LEmHanWuSwNlQmj6e9IMA8GA1UdEwQIMAYBAf8CAQAwCwYDVR0PBAQDAgEGMA0GCSqGSIb3DQEBDAUAA4IBgQBJtdspuw62weMuPiOTjm70vjrCn8GZB0BGJ6HK47zUJ2sefdKCjhdn/L359fmln08ykTq7OPuNUAo3YS+QnLKm8tazcF+g7AL1o8sr5inQO2d1kERaWCgYdTT0R3yYc+J/3yxSsxvkbnCK/prs2fzIfUfaybylsvUuiQJqFmpb8h4dO0sXXMMDKyXrTczrYsrNR6r8d1LDofv6CcRVb1I8Ei/51dJEjNahUlARWD44rBwgipwzxADeG8dLkaoq2CK8yvmQ5jtEhEkJKtRZEsAmm8NjXkWuwbXQ8pMVrZbZode39MC4yM1lIp/XW98/34BvxF0kkBbCiPRZrVYK1Ok/7/hDP4Ek9xb3vhQD37T58n5tA2ZROtmSJ/BFxc29lLcfslBPdCmJW4JdcJvsY9rxwi11cHa7Ya8K1M2WVtMX34N3CGpvtScbwiQjOhaqL3S1GO2uYeiNF/qP82zvb2g1r1d7KZC45rst0uHKMyRmw+3tifhjWxkdqFGu65Lg+3c="; @@ -24,7 +24,7 @@ namespace bts.history public bool commit_now; } - readonly HistoricalDataStreamer _stream; + readonly DataStreamer _stream; readonly Dictionary _books = new Dictionary(); readonly Dictionary _trades = new Dictionary(); @@ -36,12 +36,12 @@ namespace bts.history readonly Npgsql.NpgsqlConnection _sql; bool _use_timescale = true; - public HistoricalDatabaseGenerator(string post_process_db, string post_process_db_host, string post_process_db_port, - string user = "postgres", string pass = "enter") + public DatabaseGenerator(string post_process_db, string post_process_db_host, string post_process_db_port, + string user = "postgres", string pass = "enter") { if(_use_timescale && pass == "enter") pass = "password"; - _stream = new HistoricalDataStreamer(); + _stream = new DataStreamer(); var conn = $@"Host={post_process_db_host}; Port={post_process_db_port}; Username={user}; Password={pass}; sslmode=Prefer;"; var sql = new Npgsql.NpgsqlConnection(conn); sql.ProvideClientCertificatesCallback = provide_cert; diff --git a/src/client/spark_historical/HistoricalDataServiceServer.cs b/src/client/spark_historical/HistoricalDataServiceServer.cs index 8f14ab5a9..291a31f46 100644 --- a/src/client/spark_historical/HistoricalDataServiceServer.cs +++ b/src/client/spark_historical/HistoricalDataServiceServer.cs @@ -11,7 +11,7 @@ namespace bts.history { class HistoricalDataServiceServer : HistoricalDataServiceBase { - readonly HistoricalDataStreamer _stream = new HistoricalDataStreamer(); + readonly DataStreamer _stream = new DataStreamer(); public HistoricalDataServiceServer(ServiceContext context) : base(context, "") { } diff --git a/src/client/spark_historical/MBOParser.cs b/src/client/spark_historical/MBOParser.cs index 57b76f408..f2298c736 100644 --- a/src/client/spark_historical/MBOParser.cs +++ b/src/client/spark_historical/MBOParser.cs @@ -69,13 +69,15 @@ namespace bts.history } } - public IEnumerable> next_line(int peek_field = 55 /*sym*/) + public IEnumerable> next_line(int peek_field = 0) { while (_reader.Peek() != -1) { _line++; string line = _reader.ReadLine(); - string peek = ProtoFix.peek_field(line, peek_field); + string peek = string.Empty; + if(peek_field != 0) + peek = ProtoFix.peek_field(line, peek_field); yield return Tuple.Create(peek, line); } } diff --git a/src/client/spark_historical/HistoricalMongoGenerator.cs b/src/client/spark_historical/MongoGenerator.cs similarity index 55% rename from src/client/spark_historical/HistoricalMongoGenerator.cs rename to src/client/spark_historical/MongoGenerator.cs index 1915998bc..0ccfbec43 100644 --- a/src/client/spark_historical/HistoricalMongoGenerator.cs +++ b/src/client/spark_historical/MongoGenerator.cs @@ -22,15 +22,16 @@ using MongoDB.Bson.Serialization.Serializers; namespace bts.history { - class HistoricalMongoGenerator : IDisposable + class MongoGenerator : IDisposable { class Operation - { - public string security_desc; + { public string line; + public MarketDataIncrementalRefresh md; + public MDEntry entry; } - readonly HistoricalDataStreamer _stream; + readonly DataStreamer _stream; readonly ConcurrentDictionary _books = new ConcurrentDictionary(); readonly ConcurrentDictionary _trades = new ConcurrentDictionary(); @@ -42,11 +43,11 @@ namespace bts.history readonly MongoClient _client; readonly IMongoDatabase _db; - public HistoricalMongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port, + public MongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port, string user = "mongo", string pass = "enter") { Query.setup_bson(); - _stream = new HistoricalDataStreamer(); + _stream = new DataStreamer(); var conn = $@"mongodb://{user}:{pass}@{post_process_db_host}:{post_process_db_port}/admin"; _client = new MongoClient(conn); _db = _client.GetDatabase(post_process_db); @@ -71,15 +72,13 @@ namespace bts.history int count = 0; Stopwatch watch = new Stopwatch(); #pragma warning disable CS4014 watch.Start(); - await _stream.process_md_stream_by_line(query, (sectype, secdesc, line) => + + start_processor(string.Empty).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); + var queue = _operations[string.Empty] = new ConcurrentQueue(); + + await _stream.process_md_stream_by_line(query, (sectype, _, line) => { - if(!_operations.TryGetValue(secdesc, out ConcurrentQueue queue)) - { - queue = new ConcurrentQueue(); - _operations[secdesc] = queue; - start_processor(secdesc).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); - } - queue.Enqueue(new Operation { security_desc = secdesc, line = line } ); + queue.Enqueue(new Operation { line = line } ); if (++count % 10_000 == 1) { @@ -87,7 +86,8 @@ namespace bts.history Console.WriteLine($"preprocessed {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length"); if (qlen > 1_000_000) { - while (qlen > 500_000) + var reps = 0; + while (qlen > 500_000 && reps++ < 30) { System.Threading.Thread.Sleep(2000); qlen = _operations.Values.Select(x => x.Count).Sum(); @@ -113,22 +113,24 @@ namespace bts.history if(!_operations.TryGetValue(secdesc, out ConcurrentQueue operations)) continue; int count = 0; - session.StartTransaction(); - while(operations.TryDequeue(out Operation oper)) + //the empty secdesc queue parses fix and routes MDEntry via security desc + //preserving the order of per secdesc operations in their respective queues + if(secdesc == string.Empty) { - try + route_operation(operations); + } + else + { + session.StartTransaction(); + while(operations.TryDequeue(out Operation oper)) { - var md = ProtoFix.parse_fix(oper.line); - parse_packet(md); + Log.assert(secdesc == oper.entry?.security_desc()); + parse_entry(oper.md, oper.entry); + if(count++ > 10_000) + break; } - catch(QuickFix.MessageParseError ex) - { - Log.FIX.warn("Message parse error: " + ex.ToString()); - } - if(count++ > 10_000) - break; + session.CommitTransactionAsync(); } - session.CommitTransactionAsync(); } } }, _cts.Token); @@ -136,33 +138,56 @@ namespace bts.history await t; } - private void parse_packet(MarketDataIncrementalRefresh md) + private void route_operation(ConcurrentQueue operations) { - foreach (var entry in md.MdEntries) + while (operations.TryDequeue(out Operation oper)) { - string tname = table_name(entry.security_desc()); - if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer) + Log.assert(oper.line != null && oper.entry == null && oper.md == null); + try { - if (!_books.TryGetValue(entry.security_desc(), out co2.BookMessage book)) + var md = ProtoFix.parse_fix(oper.line); + foreach (var entry in md.MdEntries) { - book = new co2.BookMessage(); - OrderBook.init_book(book); - _books[entry.security_desc()] = book; - } - OrderBook.keep_book(book, entry); - write_book(tname, book, entry, md); + if(!_operations.TryGetValue(entry.security_desc(), out ConcurrentQueue q)) + { + q = new ConcurrentQueue(); + _operations[entry.security_desc()] = q; + start_processor(entry.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ; + } + q.Enqueue(new Operation { md = md, entry = entry }); + } } - else if (entry.MdEntryType == MDEntryType.Trade) + catch (QuickFix.MessageParseError ex) { - co2.TradeMessage tm = OrderBook.create_trade(md, entry); - if (!_trades.ContainsKey(entry.security_desc())) - _trades[entry.security_desc()] = tm; - - write_trade(tname, tm, entry, md); + Log.FIX.warn("Message parse error: " + ex.ToString()); } } } + private void parse_entry(MarketDataIncrementalRefresh md, MDEntry entry) + { + string tname = table_name(entry.security_desc()); + if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer) + { + if (!_books.TryGetValue(entry.security_desc(), out co2.BookMessage book)) + { + book = new co2.BookMessage(); + OrderBook.init_book(book); + _books[entry.security_desc()] = book; + } + OrderBook.keep_book(book, entry); + write_book(tname, book, entry, md); + } + else if (entry.MdEntryType == MDEntryType.Trade) + { + co2.TradeMessage tm = OrderBook.create_trade(md, entry); + if (!_trades.ContainsKey(entry.security_desc())) + _trades[entry.security_desc()] = tm; + + write_trade(tname, tm, entry, md); + } + } + string table_name(string security_desc) { var tname = $"{security_desc}"; @@ -171,7 +196,7 @@ namespace bts.history void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md) { - HistoryEntry row = new HistoryEntry + HistoryEntry hist = new HistoryEntry { TradeDate = md.TradeDate, MsgSeqNum = md.MsgSeqNum, @@ -181,14 +206,14 @@ namespace bts.history Trade = tm }; var table = _db.GetCollection(tname); - table.InsertOne(row); + table.InsertOne( hist ); } void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md) { if (book.Bids.FirstOrDefault()?.Size == 0 && book.Asks.FirstOrDefault()?.Size == 0) return; - HistoryEntry row = new HistoryEntry + HistoryEntry hist = new HistoryEntry { TradeDate = md.TradeDate, MsgSeqNum = md.MsgSeqNum, @@ -198,9 +223,9 @@ namespace bts.history WorkingBook = book, }; if (OrderBook.quote_event_done(md)) - row.Book = book; + hist.Book = book; var table = _db.GetCollection(tname); - table.InsertOne(row); + table.InsertOne( hist ); } public async void Dispose() @@ -210,50 +235,4 @@ namespace bts.history await qtask; } } - - class TimestampSerializer : IBsonSerializer - { - readonly DateTimeSerializer _dt = new DateTimeSerializer(DateTimeKind.Utc, BsonType.Document); - public System.Type ValueType => typeof(Timestamp); - public TimestampSerializer() - { - } - public object Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) - { - // Timestamp val = new Timestamp(); - // val.Seconds = context.Reader.ReadInt64(nameof(val.Seconds)); - // val.Nanos = context.Reader.ReadInt32(nameof(val.Nanos)); - // return val; - DateTime dt = _dt.Deserialize(context, args); - return Timestamp.FromDateTime(dt); - } - - public void Serialize(BsonSerializationContext context, BsonSerializationArgs args, object value) - { - // Timestamp val = value as Timestamp; - // if (val == null) - // context.Writer.WriteNull(); - // else - // { - // context.Writer.WriteStartDocument(); - // context.Writer.WriteInt64(nameof(val.Seconds), val.Seconds); - // context.Writer.WriteInt32(nameof(val.Nanos), val.Nanos); - // var dt = BsonUtils.ToMillisecondsSinceEpoch(val.ToDateTime().ToUniversalTime()); - // context.Writer.WriteDateTime("DateTime", dt); - // context.Writer.WriteEndDocument(); - // } - DateTime dt = ((Timestamp)value).ToDateTime(); - _dt.Serialize(context, args, dt); - } - } - - class SerializationProvider : IBsonSerializationProvider - { - public IBsonSerializer GetSerializer(System.Type type) - { - if (type == typeof(Timestamp)) - return new TimestampSerializer(); - return null; - } - } } \ No newline at end of file diff --git a/src/client/spark_historical/ProtoFix.cs b/src/client/spark_historical/ProtoFix.cs index d9aecd137..36852aa46 100644 --- a/src/client/spark_historical/ProtoFix.cs +++ b/src/client/spark_historical/ProtoFix.cs @@ -25,17 +25,14 @@ namespace bts static string get_between(string source, string start, string end) { - int Start, End; - if (source.Contains(start) && source.Contains(end)) - { - Start = source.IndexOf(start, 0) + start.Length; - End = source.IndexOf(end, Start); - return source.Substring(Start, End - Start); - } - else - { - return ""; - } + int spos = source.IndexOf(start, 0); + if(spos < 0) + return string.Empty; + spos += start.Length; + int epos = source.IndexOf(end, spos); + if(epos < 0) + return string.Empty; + return source.Substring(spos, epos - spos); } public static Timestamp peek_timestamp(string fixstring, int field = 60) diff --git a/src/client/spark_historical/Query.cs b/src/client/spark_historical/Query.cs index 660c6ca96..28d5ebd5a 100644 --- a/src/client/spark_historical/Query.cs +++ b/src/client/spark_historical/Query.cs @@ -7,14 +7,46 @@ using MongoDB.Driver.Linq; using bts.history; using Google.Protobuf.WellKnownTypes; using MongoDB.Bson.Serialization; +using bts.co2; +using MongoDB.Bson.Serialization.Serializers; namespace bts.history { + public class HistoryRow + { + public DateTime TradeDate { get; set; } + public uint MsgSeqNum { get; set; } + public DateTime SendingTime { get; set; } + public DateTime TransactTime { get; set; } + public string SecurityDesc { get; set; } + public BookMessage Book { get; set; } + public BookMessage WorkingBook { get; set; } + public TradeMessage Trade { get; set; } + public MBOOrderMessage Order { get; set; } + + public HistoryRow(HistoryEntry entry) + { + this.entry = entry; + this.TradeDate = entry.TradeDate.ToDateTime(); + this.MsgSeqNum = entry.MsgSeqNum; + this.SendingTime = entry.SendingTime.ToDateTime(); + this.TransactTime = entry.TransactTime.ToDateTime(); + this.SecurityDesc = entry.SecurityDesc; + this.Book = entry.Book; + this.WorkingBook = entry.WorkingBook; + this.Trade = entry.Trade; + this.Order = entry.Order; + } + public HistoryEntry entry { get; } + public DateTime timestamp() => entry.timestamp().ToDateTime(); + } + public class Query { readonly IMongoClient _client; readonly IMongoDatabase _db; - + + public static void setup_bson() { } static Query() { BsonClassMap.RegisterClassMap(m => @@ -22,15 +54,16 @@ namespace bts.history m.MapProperty(x => x.Bids); m.MapProperty(x => x.Asks); }); - // BsonClassMap.RegisterClassMap(m => + // BsonClassMap.RegisterClassMap(m => // { // m.AutoMap(); + // m.MapMember(x => x.TradeDate).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); + // m.MapMember(x => x.SendingTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); + // m.MapMember(x => x.TransactTime).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document)); // m.SetIgnoreExtraElements(true); // }); - //BsonSerializer.RegisterSerializationProvider(new SerializationProvider()); - } - - public static void setup_bson() {} + BsonSerializer.RegisterSerializationProvider(new SerializationProvider()); + } public Query(string database, string host = "sparkvm", int port = 27017, string user = "mongo", string pass = "enter") diff --git a/src/client/spark_historical/TimestampSerializer.cs b/src/client/spark_historical/TimestampSerializer.cs new file mode 100644 index 000000000..171ba40ee --- /dev/null +++ b/src/client/spark_historical/TimestampSerializer.cs @@ -0,0 +1,77 @@ + + +using System; +using Google.Protobuf.WellKnownTypes; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; + +namespace bts.history +{ + class TimestampSerializer : SerializerBase, IBsonDocumentSerializer + { + readonly DateTimeSerializer _dt = new DateTimeSerializer(DateTimeKind.Utc, BsonType.Document); + public TimestampSerializer() + { + } + public override Timestamp Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + Timestamp val = new Timestamp(); + val.Seconds = context.Reader.ReadInt64(nameof(val.Seconds)); + val.Nanos = context.Reader.ReadInt32(nameof(val.Nanos)); + return val; + // DateTime dt = _dt.Deserialize(context, args); + // return Timestamp.FromDateTime(dt); + } + + public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, Timestamp val) + { + if (val == null) + context.Writer.WriteNull(); + else + { + context.Writer.WriteStartDocument(); + context.Writer.WriteInt64(nameof(val.Seconds), val.Seconds); + context.Writer.WriteInt32(nameof(val.Nanos), val.Nanos); + var dt = BsonUtils.ToMillisecondsSinceEpoch(val.ToDateTime().ToUniversalTime()); + context.Writer.WriteDateTime("DateTime", dt); + context.Writer.WriteEndDocument(); + } + // DateTime dt = ((Timestamp)value).ToDateTime(); + // _dt.Serialize(context, args, dt); + } + + public bool TryGetMemberSerializationInfo(string memberName, out BsonSerializationInfo serializationInfo) + { + switch (memberName) + { + case "Id": + serializationInfo = new BsonSerializationInfo("_id", new ObjectIdSerializer(), typeof(ObjectId)); + return true; + case nameof(Timestamp.Seconds): + serializationInfo = new BsonSerializationInfo(nameof(Timestamp.Seconds), new Int64Serializer(), typeof(long)); + return true; + case nameof(Timestamp.Nanos): + serializationInfo = new BsonSerializationInfo(nameof(Timestamp.Nanos), new Int32Serializer(), typeof(int)); + return true; + case "DateTime": + serializationInfo = new BsonSerializationInfo("DateTime", new DateTimeSerializer(), typeof(DateTime)); + return true; + default: + serializationInfo = null; + return false; + } + } + } + + class SerializationProvider : IBsonSerializationProvider + { + public IBsonSerializer GetSerializer(System.Type type) + { + if (type == typeof(Timestamp)) + return new TimestampSerializer(); + return null; + } + } +} \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs b/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs index b2bb01cd0..ae447a7fb 100644 --- a/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs +++ b/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs @@ -38,13 +38,13 @@ namespace bts.history st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecuritytype, true); if(!string.IsNullOrEmpty(o.post_process_db)) { - var rep = new HistoricalMongoGenerator(o.post_process_db, o.post_process_db_host, o.post_process_db_port, + var rep = new MongoGenerator(o.post_process_db, o.post_process_db_host, o.post_process_db_port, o.post_process_db_user, o.post_process_db_pass); rep.generate(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult(); } else { - var rep = new HistoricalDataReplayer(); + var rep = new BackTester(); rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult(); } Console.ReadKey(); @@ -53,7 +53,7 @@ namespace bts.history if (!string.IsNullOrEmpty(o.s3key)) { - var dl = new HistoricalDataDownloader(); + var dl = new DataDownloader(); var str = dl.download(o.s3key).GetAwaiter().GetResult(); using (MBOParser parser = new MBOParser(str)) {