able to preprocess trades and books into a postgres
This commit is contained in:
1 parent
097edb2726
commit
8b9d689a5f
5 files changed
+53
-22
No files matched your search
Vendored
+4
-7
@@ -369,14 +369,11 @@
|
||||
"program": "sparkcmd",
|
||||
"args": [
|
||||
"historical",
|
||||
"--rstart", "10/21/2019",
|
||||
"--rend", "10/21/2019",
|
||||
"--rstart", "11/20/2019",
|
||||
"--rend", "11/20/2019",
|
||||
"--rsymbol", "CL",
|
||||
"--rsecuritytype", "Future",
|
||||
"--algo_security", "CLX9",
|
||||
"--algo_type", "TWAP",
|
||||
"--algo_params", "size=100;profit_ticks=1;tick_size=1,1,1;hedge_factors=1,1,-1",
|
||||
"--skip_preanalysis", "true",
|
||||
"--rsecuritytype", "Spread",
|
||||
"--post_process_db", "cl_history",
|
||||
],
|
||||
"env": {"LD_LIBRARY_PATH" : "."},
|
||||
"stopAtEntry": false,
|
||||
|
||||
@@ -52,7 +52,8 @@ namespace bts.history
|
||||
_books[entry.Symbol] = book;
|
||||
}
|
||||
OrderBook.keep_book(book, entry);
|
||||
on_new_book(book, entry, md);
|
||||
if (OrderBook.quote_event_done(md))
|
||||
on_new_book(book, entry, md);
|
||||
}
|
||||
// else if (entry.MdEntryType == MDEntryType.Trade)
|
||||
// handle trades here
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using bts.api;
|
||||
using bts.co2;
|
||||
@@ -23,13 +24,21 @@ namespace bts.history
|
||||
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;";
|
||||
_sql = new Npgsql.NpgsqlConnection(conn);
|
||||
var create = new Npgsql.NpgsqlCommand(
|
||||
$@"create database if not exists {_post_process_db}
|
||||
$@"CREATE DATABASE {_post_process_db}
|
||||
with owner = postgres
|
||||
encoding = 'UTF8'
|
||||
connection limit = -1;
|
||||
", _sql);
|
||||
_sql.Open();
|
||||
create.ExecuteNonQuery();
|
||||
try
|
||||
{
|
||||
create.ExecuteNonQuery();
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
if(!ex.Message.Contains("already exists"))
|
||||
throw ex;
|
||||
}
|
||||
_sql.Close(); //TODO google says I have to do this
|
||||
_sql = null;
|
||||
}
|
||||
@@ -54,10 +63,15 @@ namespace bts.history
|
||||
if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue)
|
||||
query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime());
|
||||
|
||||
int count = 0;
|
||||
var comparer = Comparer<MarketDataIncrementalRefresh>.Create((x, y)
|
||||
=> x.timestamp().ToDateTime().CompareTo(y.timestamp().ToDateTime()));
|
||||
await _stream.process_md_stream<MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
|
||||
{
|
||||
if(count++ % 50000 == 1)
|
||||
Console.WriteLine($"preprocessed {count} lines");
|
||||
if(md.timestamp().ToDateTime() > end)
|
||||
throw new System.IO.EndOfStreamException();
|
||||
foreach (var entry in md.MdEntries)
|
||||
{
|
||||
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
|
||||
@@ -84,7 +98,7 @@ namespace bts.history
|
||||
{
|
||||
var conn = $@"Server={_post_process_db_ip};Port=5432; User Id=postgres;Password=enter;Database={_post_process_db}";
|
||||
_sql = new Npgsql.NpgsqlConnection(conn);
|
||||
var name = $"{symbol}_history";
|
||||
var name = $"{symbol}_{st.ToString()}_history";
|
||||
var cmd = new Npgsql.NpgsqlCommand(
|
||||
$@"CREATE TABLE IF NOT EXISTS {name} (id serial NOT NULL PRIMARY KEY,
|
||||
seqnum INTEGER NOT NULL CHECK(seqnum > 0), transact_time TIMESTAMPTZ NOT NULL, sending_time TIMESTAMPTZ NOT NULL,
|
||||
@@ -98,22 +112,38 @@ namespace bts.history
|
||||
private void write_trade(string tname, TradeMessage tm, MDEntry entry, MarketDataIncrementalRefresh md)
|
||||
{
|
||||
var json = Google.Protobuf.JsonFormatter.Default.Format(tm);
|
||||
var cmd = new Npgsql.NpgsqlCommand(
|
||||
$@"INSERT INTO {tname} (transact_time, sending_time, symbol, security_type, security_desc, data_type, data )
|
||||
VALUES ({md.MsgSeqNum}, {md.TransactTime.ToString()}, {md.SendingTime.ToString()},
|
||||
{entry.security_desc()}, {nameof(TradeMessage)}, { json } )
|
||||
", _sql);
|
||||
cmd.ExecuteNonQuery();
|
||||
var cmd = new Npgsql.NpgsqlCommand(
|
||||
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
|
||||
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(TradeMessage)}', '{ json }' ) ",
|
||||
_sql);
|
||||
try
|
||||
{
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
private void write_book(string tname, BookMessage book, MDEntry entry, MarketDataIncrementalRefresh md)
|
||||
{
|
||||
if(!book.Bids.Any( x=>x.Size > 0) && !book.Asks.Any(x => x.Size > 0))
|
||||
return;
|
||||
|
||||
var json = Google.Protobuf.JsonFormatter.Default.Format(book);
|
||||
var cmd = new Npgsql.NpgsqlCommand(
|
||||
$@"INSERT INTO {tname} (transact_time, sending_time, symbol, security_type, security_desc, data_type, data )
|
||||
VALUES ({md.MsgSeqNum}, {md.TransactTime.ToString()}, {md.SendingTime.ToString()},
|
||||
{entry.security_desc()}, {nameof(BookMessage)}, { json } )
|
||||
", _sql);
|
||||
$"INSERT INTO {tname} (seqnum, transact_time, sending_time, security_desc, data_type, data ) " +
|
||||
$"VALUES ({md.MsgSeqNum}, '{md.TransactTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{md.SendingTime.ToString().Replace('T', ' ').Replace("Z", "+00")}', '{entry.security_desc()}', '{nameof(BookMessage)}', '{ json }' ) ",
|
||||
_sql);
|
||||
try
|
||||
{
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
|
||||
@@ -95,7 +95,7 @@ namespace bts.history
|
||||
[Option('5', "post_process_db", Required = false, Default = "", HelpText = "Name of the postgres db to generate from post processing")]
|
||||
public string post_process_db { get; set; }
|
||||
|
||||
[Option('6', "post_process_db_ip", Required = true, Default = "127.0.0.1", HelpText = "IP of the postgres db to generate from post processing")]
|
||||
[Option('6', "post_process_db_ip", Required = false, Default = "127.0.0.1", HelpText = "IP of the postgres db to generate from post processing")]
|
||||
public string post_process_db_ip { get; set; }
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
|
||||
(cd /sparkdata/conf; python -m SimpleHTTPServer)
|
||||
Reference in new issue
Block a user