155 lines
6.3 KiB
C#
155 lines
6.3 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using bts.api;
|
|
using bts.co2;
|
|
using Google.Protobuf.WellKnownTypes;
|
|
|
|
namespace bts.history
|
|
{
|
|
class HistoricalDatabaseGenerator : IDisposable
|
|
{
|
|
readonly HistoricalDataStreamer _stream;
|
|
MarketDataIncrementalRefresh _last = new MarketDataIncrementalRefresh() { SendingTime = new Timestamp(), TransactTime = new Timestamp() };
|
|
readonly string _post_process_db;
|
|
readonly string _post_process_db_ip;
|
|
Npgsql.NpgsqlConnection _sql;
|
|
|
|
public HistoricalDatabaseGenerator(string post_process_db, string post_process_db_ip)
|
|
{
|
|
_post_process_db = post_process_db;
|
|
_post_process_db_ip = post_process_db_ip;
|
|
_stream = new HistoricalDataStreamer();
|
|
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;";
|
|
_sql = new Npgsql.NpgsqlConnection(conn);
|
|
var create = new Npgsql.NpgsqlCommand(
|
|
$@"CREATE DATABASE {_post_process_db}
|
|
with owner = postgres
|
|
encoding = 'UTF8'
|
|
connection limit = -1;
|
|
", _sql);
|
|
_sql.Open();
|
|
try
|
|
{
|
|
create.ExecuteNonQuery();
|
|
}
|
|
catch(Exception ex)
|
|
{
|
|
if(!ex.Message.Contains("already exists"))
|
|
throw ex;
|
|
}
|
|
_sql.Close(); //TODO google says I have to do this
|
|
_sql = null;
|
|
}
|
|
|
|
readonly Dictionary<string, co2.BookMessage> _books = new Dictionary<string, co2.BookMessage>();
|
|
|
|
async public Task generate(DateTime start, DateTime end, string symbol, SecurityType st,
|
|
HistoricalDataOptions o)
|
|
{
|
|
var tname = maybe_create_table(start, end, symbol, st, o);
|
|
start = start.ToUniversalTime();
|
|
end = end.ToUniversalTime();
|
|
var query = new MDQuery
|
|
{
|
|
Exchange = Venue.Glbx,
|
|
Symbol = symbol,
|
|
SecurityType = st,
|
|
Begin = Timestamp.FromDateTime(start),
|
|
End = Timestamp.FromDateTime(end),
|
|
LineSkip = o.lineskip,
|
|
};
|
|
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
|
|
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
|
|
|
|
int count = 0;
|
|
var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y)
|
|
=> x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime()));
|
|
await _stream.process_md_stream<MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
|
|
{
|
|
if(count++ % 50000 == 1)
|
|
Console.WriteLine($"preprocessed {count} lines");
|
|
if(md.timestamp().ToDateTime() > end)
|
|
throw new System.IO.EndOfStreamException();
|
|
foreach (var entry in md.MdEntries)
|
|
{
|
|
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
|
|
{
|
|
if (!_books.TryGetValue(entry.Symbol, out co2.BookMessage book))
|
|
{
|
|
book = new co2.BookMessage();
|
|
OrderBook.init_book(book);
|
|
_books[entry.security_desc()] = book;
|
|
}
|
|
OrderBook.keep_book(book, entry);
|
|
write_book(tname, book, entry, md);
|
|
}
|
|
else if (entry.MdEntryType == MDEntryType.Trade)
|
|
{
|
|
var tm = OrderBook.create_trade(md, entry);
|
|
write_trade(tname, tm, entry, md);
|
|
}
|
|
}
|
|
}, comparer);
|
|
}
|
|
|
|
private string maybe_create_table(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o)
|
|
{
|
|
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;Database={_post_process_db}";
|
|
_sql = new Npgsql.NpgsqlConnection(conn);
|
|
var name = $"{symbol}_{st.ToString()}_history";
|
|
var cmd = new Npgsql.NpgsqlCommand(
|
|
$@"CREATE TABLE IF NOT EXISTS {name} (id serial NOT NULL PRIMARY KEY,
|
|
seqnum INTEGER NOT NULL CHECK(seqnum > 0), transact_time TIMESTAMPTZ NOT NULL, sending_time TIMESTAMPTZ NOT NULL,
|
|
security_desc TEXT NOT NULL, data_type TEXT NOT NULL, data jsonb NOT NULL)",
|
|
_sql);
|
|
_sql.Open();
|
|
cmd.ExecuteNonQuery();
|
|
return name;
|
|
}
|
|
|
|
private void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md)
|
|
{
|
|
var json = Google.Protobuf.JsonFormatter.Default.Format(tm);
|
|
var cmd = new Npgsql.NpgsqlCommand(
|
|
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
|
|
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(TradeMessage)}', '{ json }' ) ",
|
|
_sql);
|
|
try
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
throw ex;
|
|
}
|
|
}
|
|
|
|
private void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md)
|
|
{
|
|
if(!book.Bids.Any( x=>x.Size > 0) && !book.Asks.Any(x => x.Size > 0))
|
|
return;
|
|
|
|
var json = Google.Protobuf.JsonFormatter.Default.Format(book);
|
|
var cmd = new Npgsql.NpgsqlCommand(
|
|
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
|
|
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(BookMessage)}', '{ json }' ) ",
|
|
_sql);
|
|
try
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
catch(Exception ex)
|
|
{
|
|
throw ex;
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
if(_sql != null)
|
|
_sql.Close();
|
|
}
|
|
}
|
|
} |