we actually do want one row per order for better query perf
This commit is contained in:
1 parent
445d77223f
commit
deadbc6cf2
3 files changed
+49
-29
No files matched your search
@@ -18,6 +18,7 @@ namespace bts.history
|
||||
{
|
||||
public MarketDataIncrementalRefresh md;
|
||||
public MDEntry book_or_trade_entry;
|
||||
public IEnumerable<MBOMessage> mbo;
|
||||
}
|
||||
|
||||
readonly DataStreamer _stream;
|
||||
@@ -123,18 +124,29 @@ namespace bts.history
|
||||
|
||||
private void send(MarketDataIncrementalRefresh md)
|
||||
{
|
||||
IDictionary<string, List<MBOMessage>> cache = Query.extract_mbo(md);
|
||||
|
||||
foreach (var pair in cache)
|
||||
{
|
||||
if (!_mbo_operations.TryGetValue(pair.Key, out ConcurrentQueue<Operation> q))
|
||||
{
|
||||
_mbo_operations[pair.Key] = q = new ConcurrentQueue<Operation>();
|
||||
start_mbo_processor(pair.Key).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
|
||||
}
|
||||
_mbo_operations[pair.Key].Enqueue(new Operation { md = md, mbo = pair.Value });
|
||||
}
|
||||
|
||||
foreach (var mdval in md.MdEntries)
|
||||
{
|
||||
if (!_mbo_operations.TryGetValue(mdval.security_desc(), out ConcurrentQueue<Operation> q))
|
||||
{
|
||||
_mbo_operations[mdval.security_desc()] = q = new ConcurrentQueue<Operation>();
|
||||
start_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
|
||||
start_mbo_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
|
||||
}
|
||||
q.Enqueue(new Operation { md = md, book_or_trade_entry = mdval });
|
||||
}
|
||||
}
|
||||
|
||||
async Task start_processor(string secdesc)
|
||||
async Task start_mbo_processor(string secdesc)
|
||||
{
|
||||
var t = Task.Run(async () =>
|
||||
{
|
||||
@@ -149,9 +161,16 @@ namespace bts.history
|
||||
session.StartTransaction();
|
||||
while (operations.TryDequeue(out Operation oper))
|
||||
{
|
||||
Log.assert(oper.book_or_trade_entry != null);
|
||||
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
|
||||
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
|
||||
if(oper.mbo != null)
|
||||
{
|
||||
write_mbo(secdesc, oper.md, oper.mbo);
|
||||
}
|
||||
else if(oper.book_or_trade_entry != null)
|
||||
{
|
||||
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
|
||||
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
|
||||
}
|
||||
Log.assert(oper.mbo == null || oper.book_or_trade_entry == null);
|
||||
if (count++ > 50_000)
|
||||
break;
|
||||
}
|
||||
@@ -223,8 +242,8 @@ namespace bts.history
|
||||
MsgSeqNum = md.MsgSeqNum,
|
||||
SendingTime = md.SendingTime,
|
||||
TransactTime = md.TransactTime,
|
||||
SecurityDesc = entry.security_desc(),
|
||||
Md = md,
|
||||
SecurityDesc = entry.security_desc(),
|
||||
Md = md,
|
||||
};
|
||||
if(entry.MdPriceLevel == 1)
|
||||
hist.TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] };
|
||||
@@ -235,25 +254,26 @@ namespace bts.history
|
||||
table.InsertOne( hist );
|
||||
}
|
||||
|
||||
// void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<MBOMessage> mbo)
|
||||
// {
|
||||
// string tname = table_name(secdesc);
|
||||
// var table = _db.GetCollection<HistoryRow>(tname);
|
||||
// var orders = mbo.Select(x =>
|
||||
// {
|
||||
// HistoryRow hist = new HistoryRow
|
||||
// {
|
||||
// TradeDate = md.TradeDate,
|
||||
// MsgSeqNum = md.MsgSeqNum,
|
||||
// SendingTime = md.SendingTime,
|
||||
// TransactTime = md.TransactTime,
|
||||
// SecurityDesc = x.SecurityDesc,
|
||||
// Order = x,
|
||||
// };
|
||||
// return hist;
|
||||
// });
|
||||
// table.InsertMany(orders);
|
||||
// }
|
||||
void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<MBOMessage> mbo)
|
||||
{
|
||||
string tname = table_name(secdesc);
|
||||
var table = _db.GetCollection<HistoryRow>(tname);
|
||||
var orders = mbo.Select(x =>
|
||||
{
|
||||
HistoryRow hist = new HistoryRow
|
||||
{
|
||||
TradeDate = md.TradeDate,
|
||||
MsgSeqNum = md.MsgSeqNum,
|
||||
SendingTime = md.SendingTime,
|
||||
TransactTime = md.TransactTime,
|
||||
SecurityDesc = x.SecurityDesc,
|
||||
Order = x,
|
||||
Md = md,
|
||||
};
|
||||
return hist;
|
||||
});
|
||||
table.InsertMany(orders);
|
||||
}
|
||||
|
||||
public async void Dispose()
|
||||
{
|
||||
|
||||
@@ -189,7 +189,7 @@ namespace bts.history
|
||||
return null;
|
||||
}
|
||||
|
||||
public IDictionary<string, List<MBOMessage>> extract_mbo(MarketDataIncrementalRefresh md)
|
||||
static public IDictionary<string, List<MBOMessage>> extract_mbo(MarketDataIncrementalRefresh md)
|
||||
{
|
||||
Dictionary<string, List<MBOMessage>> cache = new Dictionary<string, List<MBOMessage>>();
|
||||
foreach (var order in md.OrderIdEntries)
|
||||
|
||||
@@ -178,7 +178,7 @@ message HistoryRow
|
||||
co2.BookMessage book = 50;
|
||||
co2.TopBookMessage top_book = 55;
|
||||
co2.TradeMessage trade = 60;
|
||||
MBOMessage order = 70; //deprecated
|
||||
MBOMessage order = 70;
|
||||
|
||||
MarketDataIncrementalRefresh md = 80;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user