diff --git a/.vscode/launch.json b/.vscode/launch.json index fb5ac11c7..0dac8ebdd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -369,14 +369,11 @@ "program": "sparkcmd", "args": [ "historical", - "--rstart", "10/21/2019", - "--rend", "10/21/2019", + "--rstart", "11/20/2019", + "--rend", "11/20/2019", "--rsymbol", "CL", - "--rsecuritytype", "Future", - "--algo_security", "CLX9", - "--algo_type", "TWAP", - "--algo_params", "size=100;profit_ticks=1;tick_size=1,1,1;hedge_factors=1,1,-1", - "--skip_preanalysis", "true", + "--rsecuritytype", "Spread", + "--post_process_db", "cl_history", ], "env": {"LD_LIBRARY_PATH" : "."}, "stopAtEntry": false, diff --git a/src/client/sparkcmd/Cmd/Historical/Analyzers/BookAnalyzer.cs b/src/client/sparkcmd/Cmd/Historical/Analyzers/BookAnalyzer.cs index 182e22ae6..c4e8341a5 100644 --- a/src/client/sparkcmd/Cmd/Historical/Analyzers/BookAnalyzer.cs +++ b/src/client/sparkcmd/Cmd/Historical/Analyzers/BookAnalyzer.cs @@ -52,7 +52,8 @@ namespace bts.history _books[entry.Symbol] = book; } OrderBook.keep_book(book, entry); - on_new_book(book, entry, md); + if (OrderBook.quote_event_done(md)) + on_new_book(book, entry, md); } // else if (entry.MdEntryType == MDEntryType.Trade) // handle trades here diff --git a/src/client/sparkcmd/Cmd/Historical/HistoricalDatabaseGenerator.cs b/src/client/sparkcmd/Cmd/Historical/HistoricalDatabaseGenerator.cs index 5e80f0af4..af5952921 100644 --- a/src/client/sparkcmd/Cmd/Historical/HistoricalDatabaseGenerator.cs +++ b/src/client/sparkcmd/Cmd/Historical/HistoricalDatabaseGenerator.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using bts.api; using bts.co2; @@ -23,13 +24,21 @@ namespace bts.history var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;"; _sql = new Npgsql.NpgsqlConnection(conn); var create = new Npgsql.NpgsqlCommand( - $@"create database if not exists {_post_process_db} + $@"CREATE DATABASE {_post_process_db} with owner = postgres encoding = 'UTF8' connection limit = -1; ", _sql); _sql.Open(); - create.ExecuteNonQuery(); + try + { + create.ExecuteNonQuery(); + } + catch(Exception ex) + { + if(!ex.Message.Contains("already exists")) + throw ex; + } _sql.Close(); //TODO google says I have to do this _sql = null; } @@ -54,10 +63,15 @@ namespace bts.history if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue) query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime()); + int count = 0; var comparer = Comparer.Create((x, y) => x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime())); await _stream.process_md_stream(query, (sectype, md, line) => { + if(count++ % 50000 == 1) + Console.WriteLine($"preprocessed {count} lines"); + if(md.timestamp().ToDateTime() > end) + throw new System.IO.EndOfStreamException(); foreach (var entry in md.MdEntries) { if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer) @@ -84,7 +98,7 @@ namespace bts.history { var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;Database={_post_process_db}"; _sql = new Npgsql.NpgsqlConnection(conn); - var name = $"{symbol}_history"; + var name = $"{symbol}_{st.ToString()}_history"; var cmd = new Npgsql.NpgsqlCommand( $@"CREATE TABLE IF NOT EXISTS {name} (id serial NOT NULL PRIMARY KEY, seqnum INTEGER NOT NULL CHECK(seqnum > 0), transact_time TIMESTAMPTZ NOT NULL, sending_time TIMESTAMPTZ NOT NULL, @@ -98,22 +112,38 @@ namespace bts.history private void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md) { var json = Google.Protobuf.JsonFormatter.Default.Format(tm); - var cmd = new Npgsql.NpgsqlCommand( - $@"INSERT INTO {tname} (transact_time, sending_time, symbol, security_type, security_desc, data_type, data ) - VALUES ({md.MsgSeqNum}, {md.TransactTime.ToString()}, {md.SendingTime.ToString()}, - {entry.security_desc()}, {nameof(TradeMessage)}, { json } ) - ", _sql); - cmd.ExecuteNonQuery(); + var cmd = new Npgsql.NpgsqlCommand( + $"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " + + $"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(TradeMessage)}', '{ json }' ) ", + _sql); + try + { + cmd.ExecuteNonQuery(); + } + catch (Exception ex) + { + throw ex; + } } private void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md) { + if(!book.Bids.Any( x=>x.Size > 0) && !book.Asks.Any(x => x.Size > 0)) + return; + var json = Google.Protobuf.JsonFormatter.Default.Format(book); var cmd = new Npgsql.NpgsqlCommand( - $@"INSERT INTO {tname} (transact_time, sending_time, symbol, security_type, security_desc, data_type, data ) - VALUES ({md.MsgSeqNum}, {md.TransactTime.ToString()}, {md.SendingTime.ToString()}, - {entry.security_desc()}, {nameof(BookMessage)}, { json } ) - ", _sql); + $"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " + + $"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(BookMessage)}', '{ json }' ) ", + _sql); + try + { + cmd.ExecuteNonQuery(); + } + catch(Exception ex) + { + throw ex; + } } public void Dispose() diff --git a/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs b/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs index 88775b908..225b97df3 100644 --- a/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs +++ b/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs @@ -95,7 +95,7 @@ namespace bts.history [Option('5', "post_process_db", Required = false, Default = "", HelpText = "Name of the postgres db to generate from post processing")] public string post_process_db { get; set; } - [Option('6', "post_process_db_ip", Required = true, Default = "127.0.0.1", HelpText = "IP of the postgres db to generate from post processing")] + [Option('6', "post_process_db_ip", Required = false, Default = "127.0.0.1", HelpText = "IP of the postgres db to generate from post processing")] public string post_process_db_ip { get; set; } diff --git a/tools/simple_config.sh b/tools/simple_config.sh new file mode 100644 index 000000000..b932db5c2 --- /dev/null +++ b/tools/simple_config.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +(cd /sparkdata/conf; python -m SimpleHTTPServer) \ No newline at end of file