able to preprocess trades and books into a postgres

This commit is contained in:
Kevin Darby committed 2019-11-27 16:18:56 -05:00
1 parent 097edb2726
commit 8b9d689a5f
5 files changed
+53 -22

No files matched your search

+4 -7
View File
@@ -369,14 +369,11 @@
"program": "sparkcmd",
"args": [
"historical",
"--rstart", "10/21/2019",
"--rend", "10/21/2019",
"--rstart", "11/20/2019",
"--rend", "11/20/2019",
"--rsymbol", "CL",
"--rsecuritytype", "Future",
"--algo_security", "CLX9",
"--algo_type", "TWAP",
"--algo_params", "size=100;profit_ticks=1;tick_size=1,1,1;hedge_factors=1,1,-1",
"--skip_preanalysis", "true",
"--rsecuritytype", "Spread",
"--post_process_db", "cl_history",
],
"env": {"LD_LIBRARY_PATH" : "."},
"stopAtEntry": false,
@@ -52,7 +52,8 @@ namespace bts.history
_books[entry.Symbol] = book;
}
OrderBook.keep_book(book, entry);
on_new_book(book, entry, md);
if (OrderBook.quote_event_done(md))
on_new_book(book, entry, md);
}
// else if (entry.MdEntryType == MDEntryType.Trade)
// handle trades here
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using bts.api;
using bts.co2;
@@ -23,13 +24,21 @@ namespace bts.history
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;";
_sql = new Npgsql.NpgsqlConnection(conn);
var create = new Npgsql.NpgsqlCommand(
$@"create database if not exists {_post_process_db}
$@"CREATE DATABASE {_post_process_db}
with owner = postgres
encoding = 'UTF8'
connection limit = -1;
", _sql);
_sql.Open();
create.ExecuteNonQuery();
try
{
create.ExecuteNonQuery();
}
catch(Exception ex)
{
if(!ex.Message.Contains("already exists"))
throw ex;
}
_sql.Close(); //TODO google says I have to do this
_sql = null;
}
@@ -54,10 +63,15 @@ namespace bts.history
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
int count = 0;
var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y)
=> x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime()));
await _stream.process_md_stream<MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
{
if(count++ % 50000 == 1)
Console.WriteLine($"preprocessed {count} lines");
if(md.timestamp().ToDateTime() > end)
throw new System.IO.EndOfStreamException();
foreach (var entry in md.MdEntries)
{
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
@@ -84,7 +98,7 @@ namespace bts.history
{
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;Database={_post_process_db}";
_sql = new Npgsql.NpgsqlConnection(conn);
var name = $"{symbol}_history";
var name = $"{symbol}_{st.ToString()}_history";
var cmd = new Npgsql.NpgsqlCommand(
$@"CREATE TABLE IF NOT EXISTS {name} (id serial NOT NULL PRIMARY KEY,
seqnum INTEGER NOT NULL CHECK(seqnum > 0), transact_time TIMESTAMPTZ NOT NULL, sending_time TIMESTAMPTZ NOT NULL,
@@ -98,22 +112,38 @@ namespace bts.history
private void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md)
{
var json = Google.Protobuf.JsonFormatter.Default.Format(tm);
var cmd = new Npgsql.NpgsqlCommand(
$@"INSERT INTO {tname} (transact_time, sending_time, symbol, security_type, security_desc, data_type, data )
VALUES ({md.MsgSeqNum}, {md.TransactTime.ToString()}, {md.SendingTime.ToString()},
{entry.security_desc()}, {nameof(TradeMessage)}, { json } )
", _sql);
cmd.ExecuteNonQuery();
var cmd = new Npgsql.NpgsqlCommand(
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(TradeMessage)}', '{ json }' ) ",
_sql);
try
{
cmd.ExecuteNonQuery();
}
catch (Exception ex)
{
throw ex;
}
}
private void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md)
{
if(!book.Bids.Any( x=>x.Size > 0) && !book.Asks.Any(x => x.Size > 0))
return;
var json = Google.Protobuf.JsonFormatter.Default.Format(book);
var cmd = new Npgsql.NpgsqlCommand(
$@"INSERT INTO {tname} (transact_time, sending_time, symbol, security_type, security_desc, data_type, data )
VALUES ({md.MsgSeqNum}, {md.TransactTime.ToString()}, {md.SendingTime.ToString()},
{entry.security_desc()}, {nameof(BookMessage)}, { json } )
", _sql);
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(BookMessage)}', '{ json }' ) ",
_sql);
try
{
cmd.ExecuteNonQuery();
}
catch(Exception ex)
{
throw ex;
}
}
public void Dispose()
@@ -95,7 +95,7 @@ namespace bts.history
[Option('5', "post_process_db", Required = false, Default = "", HelpText = "Name of the postgres db to generate from post processing")]
public string post_process_db { get; set; }
[Option('6', "post_process_db_ip", Required = true, Default = "127.0.0.1", HelpText = "IP of the postgres db to generate from post processing")]
[Option('6', "post_process_db_ip", Required = false, Default = "127.0.0.1", HelpText = "IP of the postgres db to generate from post processing")]
public string post_process_db_ip { get; set; }
+3
View File
@@ -0,0 +1,3 @@
#!/bin/bash
(cd /sparkdata/conf; python -m SimpleHTTPServer)