thought better of two tables, this makes simulation way easier

This commit is contained in:
Kevin Darby committed 2019-12-04 23:33:48 -05:00
1 parent b7ad777de4
commit 69612eaee7
7 files changed
+77 -152

No files matched your search

+1 -1
View File
@@ -109,7 +109,7 @@
"--rend", "11/26/2019",
"--rsymbol", "ZN",
"--rsecuritytype", "Synthetic",
"--post_process_db", "roll",
"--post_process_db", "roll2",
"--post_process_db_host", "sparkvm",
"--post_process_db_port", "27017",
"--post_process_db_user", "mongo",
+2 -34
View File
@@ -10,31 +10,9 @@ namespace bts.history
DateTime transact_time { get ; }
DateTime timestamp();
}
public interface IMBP : ITimestamped
{
string SecurityDesc { get ; }
co2.BookMessage Book { get ; }
co2.TopBookMessage TopBook { get ; }
co2.TradeMessage Trade { get ; }
}
public interface IMBO : ITimestamped
{
MDUpdateAction OrderUpdateAction { get ; }
ulong OrderId { get; }
uint SecurityId { get; }
string SecurityDesc { get; }
ulong MdOrderPriority { get; }
double MdEntryPx { get; }
uint MdPriceLevel { get; }
ulong MdDisplayQty { get; }
}
public sealed partial class HistoryRow : IMBP
public sealed partial class HistoryRow : ITimestamped
{
public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } }
public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } }
@@ -45,14 +23,4 @@ namespace bts.history
}
}
public sealed partial class HistoryOrderRow : IMBO
{
public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } }
public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } }
public DateTime transact_time { get { return this.TransactTime.ToDateTime(); } set { this.TransactTime = Timestamp.FromDateTime(value); } }
public DateTime timestamp()
{
return this.sending_time;
}
}
}
@@ -54,8 +54,9 @@ namespace bts.history
}
}
public interface IAlgo {}
abstract class AlgoBase
abstract class AlgoBase : IAlgo
{
public readonly Guid guid = Guid.NewGuid();
public ulong order_key { get; private set;}
@@ -16,10 +16,6 @@ namespace bts.history
{
return md.SecurityDesc;
}
public static string security_desc(this HistoryOrderRow md)
{
return md.SecurityDesc;
}
public static bool less(this Timestamp lhs, Timestamp rhs)
{
return lhs.Seconds * 1e9 + lhs.Nanos < rhs.Seconds * 1e9 + rhs.Nanos;
@@ -18,7 +18,7 @@ namespace bts.history
{
public MarketDataIncrementalRefresh md;
public MDEntry book_or_trade_entry;
public IEnumerable<HistoryOrderRow> mbo;
public IEnumerable<MBOMessage> mbo;
}
readonly DataStreamer _stream;
@@ -28,8 +28,6 @@ namespace bts.history
readonly CancellationTokenSource _cts = new CancellationTokenSource();
bool _done_reading;
readonly ConcurrentDictionary<string, ConcurrentQueue<Operation>> _book_operations =
new ConcurrentDictionary<string, ConcurrentQueue<Operation>>();
readonly ConcurrentDictionary<string, ConcurrentQueue<Operation>> _mbo_operations =
new ConcurrentDictionary<string, ConcurrentQueue<Operation>>();
@@ -37,7 +35,6 @@ namespace bts.history
readonly MongoClient _client;
readonly IMongoDatabase _db;
const bool SHOULD_PROCESS_ORDERS = true;
public MongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port,
string user = "mongo", string pass = "enter")
{
@@ -67,23 +64,18 @@ namespace bts.history
int count = 0; Stopwatch watch = new Stopwatch();
#pragma warning disable CS4014
watch.Start();
//start_line_processor().ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted);
await _stream.process_md_stream_by_line(query, (sectype, _, line) =>
{
try
{
var md = ProtoFix.parse_fix<MarketDataIncrementalRefresh>(line);
send_books_and_trades(md);
if(SHOULD_PROCESS_ORDERS)
send_orders(md);
send(md);
}
catch (QuickFix.MessageParseError ex)
{
Log.FIX.warn("Message parse error: " + ex.ToString());
}
// _lines.Enqueue(line);
if (++count % 10_000 == 1)
{
var qlen = get_qlen();
@@ -111,57 +103,13 @@ namespace bts.history
int get_qlen()
{
var qlen = _book_operations.Values.Select(x => x.Count).Sum();
var blen = _mbo_operations.Values.Select(x => x.Count).Sum();
return qlen + blen;
return blen;
}
private void send_books_and_trades(MarketDataIncrementalRefresh md)
private void send(MarketDataIncrementalRefresh md)
{
foreach (var entry in md.MdEntries)
{
if (!_book_operations.TryGetValue(entry.security_desc(), out ConcurrentQueue<Operation> q))
{
_book_operations[entry.security_desc()] = q = new ConcurrentQueue<Operation>();
start_book_processor(entry.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
}
q.Enqueue(new Operation { md = md, book_or_trade_entry = entry });
}
}
async Task start_book_processor(string secdesc)
{
var t = Task.Run( async () =>
{
using (var session = await _client.StartSessionAsync())
{
while(!_cts.IsCancellationRequested)
{
await Task.Delay(1);
if(!_book_operations.TryGetValue(secdesc, out ConcurrentQueue<Operation> operations))
continue;
int count = 0;
session.StartTransaction();
while(operations.TryDequeue(out Operation oper))
{
Log.assert(oper.book_or_trade_entry != null);
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
if (count++ > 10_000)
break;
}
session.CommitTransactionAsync();
if(_done_reading && operations.Count == 0)
break;
}
}
}, _cts.Token);
_qtask.Add( t );
await t;
}
private void send_orders(MarketDataIncrementalRefresh md)
{
Dictionary<string, List<HistoryOrderRow>> cache = new Dictionary<string, List<HistoryOrderRow>>();
Dictionary<string, List<MBOMessage>> cache = new Dictionary<string, List<MBOMessage>>();
foreach (var order in md.OrderIdEntries)
{
if (order.ReferenceId == 0)
@@ -172,7 +120,6 @@ namespace bts.history
MDEntry mdval = md.MdEntries[mdidx];
//now merge this order stuff into an MDEntry because we can't do templates....no I don't miss c++
mdval = mdval.Clone();
mdval.LastQty = order.LastQty;
mdval.MdDisplayQty = order.MdDisplayQty;
@@ -198,26 +145,32 @@ namespace bts.history
_mbo_operations[pair.Key].Enqueue(new Operation { md = md, mbo = pair.Value });
}
foreach (var mdval in md.MdEntries)
{
if (!_mbo_operations.TryGetValue(mdval.security_desc(), out ConcurrentQueue<Operation> q))
{
_mbo_operations[mdval.security_desc()] = q = new ConcurrentQueue<Operation>();
start_mbo_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
}
q.Enqueue(new Operation { md = md, book_or_trade_entry = mdval });
}
//each order in here can be for a different secdesc
void cache_mbo(MDEntry order)
{
HistoryOrderRow mbo = new HistoryOrderRow();
mbo.TradeDate = md.TradeDate;
mbo.MsgSeqNum = md.MsgSeqNum;
mbo.SendingTime = md.SendingTime;
mbo.TransactTime = md.TransactTime;
MBOMessage mbo = new MBOMessage();
mbo.OrderUpdateAction = order.MdUpdateAction;
mbo.OrderId = order.OrderId;
mbo.SecurityId = order.SecurityId;
mbo.SecurityDesc = order.security_desc();
mbo.MdOrderPriority = order.MdOrderPriority;
mbo.MdEntryPx = order.MdEntryPx;
mbo.MdPriceLevel = order.MdPriceLevel;
mbo.MdDisplayQty = order.MdDisplayQty;
if (!cache.TryGetValue(mbo.SecurityDesc, out List<HistoryOrderRow> list))
cache[mbo.SecurityDesc] = list = new List<HistoryOrderRow>();
if (!cache.TryGetValue(order.security_desc(), out List<MBOMessage> list))
cache[order.security_desc()] = list = new List<MBOMessage>();
list.Add(mbo);
}
}
@@ -236,10 +189,18 @@ namespace bts.history
session.StartTransaction();
while (operations.TryDequeue(out Operation oper))
{
Log.assert(oper.mbo != null);
write_mbo(secdesc, oper.md, oper.mbo);
if(oper.mbo != null)
{
write_mbo(secdesc, oper.md, oper.mbo);
}
else if(oper.book_or_trade_entry != null)
{
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
}
Log.assert(oper.mbo == null || oper.book_or_trade_entry == null);
if (count++ > 50_000)
break;
break;
}
session.CommitTransactionAsync();
if(_done_reading && operations.Count == 0)
@@ -307,9 +268,10 @@ namespace bts.history
MsgSeqNum = md.MsgSeqNum,
SendingTime = md.SendingTime,
TransactTime = md.TransactTime,
SecurityDesc = entry.security_desc(),
TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] },
};
SecurityDesc = entry.security_desc(),
};
if(entry.MdPriceLevel == 1)
hist.TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] };
if (OrderBook.quote_event_done(md))
hist.Book = book;
@@ -317,11 +279,24 @@ namespace bts.history
table.InsertOne( hist );
}
void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<HistoryOrderRow> mbo)
void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<MBOMessage> mbo)
{
string tname = table_name(secdesc) + "_MBO";
var table = _db.GetCollection<HistoryOrderRow>(tname);
table.InsertMany(mbo);
string tname = table_name(secdesc);
var table = _db.GetCollection<HistoryRow>(tname);
var orders = mbo.Select(x =>
{
HistoryRow hist = new HistoryRow
{
TradeDate = md.TradeDate,
MsgSeqNum = md.MsgSeqNum,
SendingTime = md.SendingTime,
TransactTime = md.TransactTime,
SecurityDesc = x.SecurityDesc,
Order = x,
};
return hist;
});
table.InsertMany(orders);
}
public async void Dispose()
+5 -16
View File
@@ -50,18 +50,7 @@ namespace bts.history
m.UnmapMember(x => x.SendingTime);
m.UnmapMember(x => x.TransactTime);
m.SetIgnoreExtraElements(true);
});
BsonClassMap.RegisterClassMap<HistoryOrderRow>(m =>
{
m.AutoMap();
m.MapMember(x => x.trade_date).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
m.MapMember(x => x.sending_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
m.MapMember(x => x.transact_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
m.UnmapMember(x => x.TradeDate);
m.UnmapMember(x => x.SendingTime);
m.UnmapMember(x => x.TransactTime);
m.SetIgnoreExtraElements(true);
});
});
//BsonSerializer.RegisterSerializationProvider(new SerializationProvider());
}
@@ -94,13 +83,13 @@ namespace bts.history
return res;
}
public IEnumerable<OrderOrFill> run_algo(AlgoBase algo, IFillModel fill_model, uint latency,
IEnumerable<IMBP> bookstream, IEnumerable<IMBO> orderstream)
public IEnumerable<OrderOrFill> run_algo(IAlgo algo, IFillModel fill_model, uint latency,
IEnumerable<HistoryRow> datastream)
{
return null;
}
}
public interface IFillModel {}
public static class QueryExtensions
{
static public IMongoQueryable<T> where<T>(this IMongoQueryable<T> q, string where)
@@ -154,26 +154,8 @@ message ProcessedBook
co2.BookMessage book = 50;
}
message HistoryRow
message MBOMessage
{
google.protobuf.Timestamp trade_date = 5;
uint32 msg_seq_num = 10;
google.protobuf.Timestamp sending_time = 20;
google.protobuf.Timestamp transact_time = 30;
string security_desc = 40;
co2.BookMessage book = 50;
co2.TopBookMessage top_book = 55;
co2.TradeMessage trade = 60;
}
message HistoryOrderRow
{
google.protobuf.Timestamp trade_date = 5;
uint32 msg_seq_num = 10;
google.protobuf.Timestamp sending_time = 20;
google.protobuf.Timestamp transact_time = 30;
MDUpdateAction order_update_action = 279;
uint64 order_id = 37;
uint32 security_id = 48;
@@ -185,6 +167,20 @@ message HistoryOrderRow
uint64 md_display_qty = 37706;
}
message HistoryRow
{
google.protobuf.Timestamp trade_date = 5;
uint32 msg_seq_num = 10;
google.protobuf.Timestamp sending_time = 20;
google.protobuf.Timestamp transact_time = 30;
string security_desc = 40;
co2.BookMessage book = 50;
co2.TopBookMessage top_book = 55;
co2.TradeMessage trade = 60;
MBOMessage order = 70;
}
service HistoricalDataService
{
rpc get_trading_slices(SlicesQuery) returns (Slices);