Files
spark/src/client/sparkcmd/Cmd/Historical/HistoricalDatabaseGenerator.cs
T

155 lines
6.3 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using bts.api;
using bts.co2;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
class HistoricalDatabaseGenerator : IDisposable
{
readonly HistoricalDataStreamer _stream;
MarketDataIncrementalRefresh _last = new MarketDataIncrementalRefresh() { SendingTime = new Timestamp(), TransactTime = new Timestamp() };
readonly string _post_process_db;
readonly string _post_process_db_ip;
Npgsql.NpgsqlConnection _sql;
public HistoricalDatabaseGenerator(string post_process_db, string post_process_db_ip)
{
_post_process_db = post_process_db;
_post_process_db_ip = post_process_db_ip;
_stream = new HistoricalDataStreamer();
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;";
_sql = new Npgsql.NpgsqlConnection(conn);
var create = new Npgsql.NpgsqlCommand(
$@"CREATE DATABASE {_post_process_db}
with owner = postgres
encoding = 'UTF8'
connection limit = -1;
", _sql);
_sql.Open();
try
{
create.ExecuteNonQuery();
}
catch(Exception ex)
{
if(!ex.Message.Contains("already exists"))
throw ex;
}
_sql.Close(); //TODO google says I have to do this
_sql = null;
}
readonly Dictionary<string, co2.BookMessage> _books = new Dictionary<string, co2.BookMessage>();
async public Task generate(DateTime start, DateTime end, string symbol, SecurityType st,
HistoricalDataOptions o)
{
var tname = maybe_create_table(start, end, symbol, st, o);
start = start.ToUniversalTime();
end = end.ToUniversalTime();
var query = new MDQuery
{
Exchange = Venue.Glbx,
Symbol = symbol,
SecurityType = st,
Begin = Timestamp.FromDateTime(start),
End = Timestamp.FromDateTime(end),
LineSkip = o.lineskip,
};
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
int count = 0;
var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y)
=> x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime()));
await _stream.process_md_stream<MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
{
if(count++ % 50000 == 1)
Console.WriteLine($"preprocessed {count} lines");
if(md.timestamp().ToDateTime() > end)
throw new System.IO.EndOfStreamException();
foreach (var entry in md.MdEntries)
{
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
{
if (!_books.TryGetValue(entry.Symbol, out co2.BookMessage book))
{
book = new co2.BookMessage();
OrderBook.init_book(book);
_books[entry.security_desc()] = book;
}
OrderBook.keep_book(book, entry);
write_book(tname, book, entry, md);
}
else if (entry.MdEntryType == MDEntryType.Trade)
{
var tm = OrderBook.create_trade(md, entry);
write_trade(tname, tm, entry, md);
}
}
}, comparer);
}
private string maybe_create_table(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o)
{
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;Database={_post_process_db}";
_sql = new Npgsql.NpgsqlConnection(conn);
var name = $"{symbol}_{st.ToString()}_history";
var cmd = new Npgsql.NpgsqlCommand(
$@"CREATE TABLE IF NOT EXISTS {name} (id serial NOT NULL PRIMARY KEY,
seqnum INTEGER NOT NULL CHECK(seqnum > 0), transact_time TIMESTAMPTZ NOT NULL, sending_time TIMESTAMPTZ NOT NULL,
security_desc TEXT NOT NULL, data_type TEXT NOT NULL, data jsonb NOT NULL)",
_sql);
_sql.Open();
cmd.ExecuteNonQuery();
return name;
}
private void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md)
{
var json = Google.Protobuf.JsonFormatter.Default.Format(tm);
var cmd = new Npgsql.NpgsqlCommand(
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(TradeMessage)}', '{ json }' ) ",
_sql);
try
{
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
throw ex;
}
}
private void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md)
{
if(!book.Bids.Any( x=>x.Size > 0) && !book.Asks.Any(x => x.Size > 0))
return;
var json = Google.Protobuf.JsonFormatter.Default.Format(book);
var cmd = new Npgsql.NpgsqlCommand(
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(BookMessage)}', '{ json }' ) ",
_sql);
try
{
cmd.ExecuteNonQuery();
}
catch(Exception ex)
{
throw ex;
}
}
public void Dispose()
{
if(_sql != null)
_sql.Close();
}
}
}