we actually do want one row per order for better query perf

This commit is contained in:
Kevin Darby committed 2019-12-10 21:14:57 -05:00
1 parent 445d77223f
commit deadbc6cf2
3 files changed
+49 -29

No files matched your search

@@ -18,6 +18,7 @@ namespace bts.history
{
public MarketDataIncrementalRefresh md;
public MDEntry book_or_trade_entry;
public IEnumerable<MBOMessage> mbo;
}
readonly DataStreamer _stream;
@@ -123,18 +124,29 @@ namespace bts.history
private void send(MarketDataIncrementalRefresh md)
{
IDictionary<string, List<MBOMessage>> cache = Query.extract_mbo(md);
foreach (var pair in cache)
{
if (!_mbo_operations.TryGetValue(pair.Key, out ConcurrentQueue<Operation> q))
{
_mbo_operations[pair.Key] = q = new ConcurrentQueue<Operation>();
start_mbo_processor(pair.Key).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
}
_mbo_operations[pair.Key].Enqueue(new Operation { md = md, mbo = pair.Value });
}
foreach (var mdval in md.MdEntries)
{
if (!_mbo_operations.TryGetValue(mdval.security_desc(), out ConcurrentQueue<Operation> q))
{
_mbo_operations[mdval.security_desc()] = q = new ConcurrentQueue<Operation>();
start_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
start_mbo_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
}
q.Enqueue(new Operation { md = md, book_or_trade_entry = mdval });
}
}
async Task start_processor(string secdesc)
async Task start_mbo_processor(string secdesc)
{
var t = Task.Run(async () =>
{
@@ -149,9 +161,16 @@ namespace bts.history
session.StartTransaction();
while (operations.TryDequeue(out Operation oper))
{
Log.assert(oper.book_or_trade_entry != null);
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
if(oper.mbo != null)
{
write_mbo(secdesc, oper.md, oper.mbo);
}
else if(oper.book_or_trade_entry != null)
{
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
}
Log.assert(oper.mbo == null || oper.book_or_trade_entry == null);
if (count++ > 50_000)
break;
}
@@ -223,8 +242,8 @@ namespace bts.history
MsgSeqNum = md.MsgSeqNum,
SendingTime = md.SendingTime,
TransactTime = md.TransactTime,
SecurityDesc = entry.security_desc(),
Md = md,
SecurityDesc = entry.security_desc(),
Md = md,
};
if(entry.MdPriceLevel == 1)
hist.TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] };
@@ -235,25 +254,26 @@ namespace bts.history
table.InsertOne( hist );
}
// void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<MBOMessage> mbo)
// {
// string tname = table_name(secdesc);
// var table = _db.GetCollection<HistoryRow>(tname);
// var orders = mbo.Select(x =>
// {
// HistoryRow hist = new HistoryRow
// {
// TradeDate = md.TradeDate,
// MsgSeqNum = md.MsgSeqNum,
// SendingTime = md.SendingTime,
// TransactTime = md.TransactTime,
// SecurityDesc = x.SecurityDesc,
// Order = x,
// };
// return hist;
// });
// table.InsertMany(orders);
// }
void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<MBOMessage> mbo)
{
string tname = table_name(secdesc);
var table = _db.GetCollection<HistoryRow>(tname);
var orders = mbo.Select(x =>
{
HistoryRow hist = new HistoryRow
{
TradeDate = md.TradeDate,
MsgSeqNum = md.MsgSeqNum,
SendingTime = md.SendingTime,
TransactTime = md.TransactTime,
SecurityDesc = x.SecurityDesc,
Order = x,
Md = md,
};
return hist;
});
table.InsertMany(orders);
}
public async void Dispose()
{
+1 -1
View File
@@ -189,7 +189,7 @@ namespace bts.history
return null;
}
public IDictionary<string, List<MBOMessage>> extract_mbo(MarketDataIncrementalRefresh md)
static public IDictionary<string, List<MBOMessage>> extract_mbo(MarketDataIncrementalRefresh md)
{
Dictionary<string, List<MBOMessage>> cache = new Dictionary<string, List<MBOMessage>>();
foreach (var order in md.OrderIdEntries)
@@ -178,7 +178,7 @@ message HistoryRow
co2.BookMessage book = 50;
co2.TopBookMessage top_book = 55;
co2.TradeMessage trade = 60;
MBOMessage order = 70; //deprecated
MBOMessage order = 70;
MarketDataIncrementalRefresh md = 80;
}