thought better of two tables, this makes simulation way easier
This commit is contained in:
1 parent
b7ad777de4
commit
69612eaee7
7 files changed
+77
-152
No files matched your search
Vendored
+1
-1
@@ -109,7 +109,7 @@
|
||||
"--rend", "11/26/2019",
|
||||
"--rsymbol", "ZN",
|
||||
"--rsecuritytype", "Synthetic",
|
||||
"--post_process_db", "roll",
|
||||
"--post_process_db", "roll2",
|
||||
"--post_process_db_host", "sparkvm",
|
||||
"--post_process_db_port", "27017",
|
||||
"--post_process_db_user", "mongo",
|
||||
|
||||
@@ -10,31 +10,9 @@ namespace bts.history
|
||||
DateTime transact_time { get ; }
|
||||
DateTime timestamp();
|
||||
}
|
||||
|
||||
public interface IMBP : ITimestamped
|
||||
{
|
||||
string SecurityDesc { get ; }
|
||||
co2.BookMessage Book { get ; }
|
||||
co2.TopBookMessage TopBook { get ; }
|
||||
co2.TradeMessage Trade { get ; }
|
||||
}
|
||||
|
||||
|
||||
public interface IMBO : ITimestamped
|
||||
{
|
||||
MDUpdateAction OrderUpdateAction { get ; }
|
||||
ulong OrderId { get; }
|
||||
uint SecurityId { get; }
|
||||
string SecurityDesc { get; }
|
||||
ulong MdOrderPriority { get; }
|
||||
|
||||
double MdEntryPx { get; }
|
||||
uint MdPriceLevel { get; }
|
||||
ulong MdDisplayQty { get; }
|
||||
}
|
||||
|
||||
|
||||
|
||||
public sealed partial class HistoryRow : IMBP
|
||||
public sealed partial class HistoryRow : ITimestamped
|
||||
{
|
||||
public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } }
|
||||
public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } }
|
||||
@@ -45,14 +23,4 @@ namespace bts.history
|
||||
}
|
||||
}
|
||||
|
||||
public sealed partial class HistoryOrderRow : IMBO
|
||||
{
|
||||
public DateTime trade_date { get { return this.TradeDate.ToDateTime(); } set { this.TradeDate = Timestamp.FromDateTime(value); } }
|
||||
public DateTime sending_time { get { return this.SendingTime.ToDateTime(); } set { this.SendingTime = Timestamp.FromDateTime(value); } }
|
||||
public DateTime transact_time { get { return this.TransactTime.ToDateTime(); } set { this.TransactTime = Timestamp.FromDateTime(value); } }
|
||||
public DateTime timestamp()
|
||||
{
|
||||
return this.sending_time;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -54,8 +54,9 @@ namespace bts.history
|
||||
}
|
||||
}
|
||||
|
||||
public interface IAlgo {}
|
||||
|
||||
abstract class AlgoBase
|
||||
abstract class AlgoBase : IAlgo
|
||||
{
|
||||
public readonly Guid guid = Guid.NewGuid();
|
||||
public ulong order_key { get; private set;}
|
||||
|
||||
@@ -16,10 +16,6 @@ namespace bts.history
|
||||
{
|
||||
return md.SecurityDesc;
|
||||
}
|
||||
public static string security_desc(this HistoryOrderRow md)
|
||||
{
|
||||
return md.SecurityDesc;
|
||||
}
|
||||
public static bool less(this Timestamp lhs, Timestamp rhs)
|
||||
{
|
||||
return lhs.Seconds * 1e9 + lhs.Nanos < rhs.Seconds * 1e9 + rhs.Nanos;
|
||||
|
||||
@@ -18,7 +18,7 @@ namespace bts.history
|
||||
{
|
||||
public MarketDataIncrementalRefresh md;
|
||||
public MDEntry book_or_trade_entry;
|
||||
public IEnumerable<HistoryOrderRow> mbo;
|
||||
public IEnumerable<MBOMessage> mbo;
|
||||
}
|
||||
|
||||
readonly DataStreamer _stream;
|
||||
@@ -28,8 +28,6 @@ namespace bts.history
|
||||
readonly CancellationTokenSource _cts = new CancellationTokenSource();
|
||||
bool _done_reading;
|
||||
|
||||
readonly ConcurrentDictionary<string, ConcurrentQueue<Operation>> _book_operations =
|
||||
new ConcurrentDictionary<string, ConcurrentQueue<Operation>>();
|
||||
readonly ConcurrentDictionary<string, ConcurrentQueue<Operation>> _mbo_operations =
|
||||
new ConcurrentDictionary<string, ConcurrentQueue<Operation>>();
|
||||
|
||||
@@ -37,7 +35,6 @@ namespace bts.history
|
||||
|
||||
readonly MongoClient _client;
|
||||
readonly IMongoDatabase _db;
|
||||
const bool SHOULD_PROCESS_ORDERS = true;
|
||||
public MongoGenerator(string post_process_db, string post_process_db_host, string post_process_db_port,
|
||||
string user = "mongo", string pass = "enter")
|
||||
{
|
||||
@@ -67,23 +64,18 @@ namespace bts.history
|
||||
int count = 0; Stopwatch watch = new Stopwatch();
|
||||
#pragma warning disable CS4014
|
||||
watch.Start();
|
||||
|
||||
//start_line_processor().ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted);
|
||||
|
||||
await _stream.process_md_stream_by_line(query, (sectype, _, line) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var md = ProtoFix.parse_fix<MarketDataIncrementalRefresh>(line);
|
||||
send_books_and_trades(md);
|
||||
if(SHOULD_PROCESS_ORDERS)
|
||||
send_orders(md);
|
||||
send(md);
|
||||
}
|
||||
catch (QuickFix.MessageParseError ex)
|
||||
{
|
||||
Log.FIX.warn("Message parse error: " + ex.ToString());
|
||||
}
|
||||
// _lines.Enqueue(line);
|
||||
if (++count % 10_000 == 1)
|
||||
{
|
||||
var qlen = get_qlen();
|
||||
@@ -111,57 +103,13 @@ namespace bts.history
|
||||
|
||||
int get_qlen()
|
||||
{
|
||||
var qlen = _book_operations.Values.Select(x => x.Count).Sum();
|
||||
var blen = _mbo_operations.Values.Select(x => x.Count).Sum();
|
||||
return qlen + blen;
|
||||
return blen;
|
||||
}
|
||||
|
||||
private void send_books_and_trades(MarketDataIncrementalRefresh md)
|
||||
private void send(MarketDataIncrementalRefresh md)
|
||||
{
|
||||
foreach (var entry in md.MdEntries)
|
||||
{
|
||||
if (!_book_operations.TryGetValue(entry.security_desc(), out ConcurrentQueue<Operation> q))
|
||||
{
|
||||
_book_operations[entry.security_desc()] = q = new ConcurrentQueue<Operation>();
|
||||
start_book_processor(entry.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
|
||||
}
|
||||
q.Enqueue(new Operation { md = md, book_or_trade_entry = entry });
|
||||
}
|
||||
}
|
||||
async Task start_book_processor(string secdesc)
|
||||
{
|
||||
var t = Task.Run( async () =>
|
||||
{
|
||||
using (var session = await _client.StartSessionAsync())
|
||||
{
|
||||
while(!_cts.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(1);
|
||||
if(!_book_operations.TryGetValue(secdesc, out ConcurrentQueue<Operation> operations))
|
||||
continue;
|
||||
int count = 0;
|
||||
session.StartTransaction();
|
||||
while(operations.TryDequeue(out Operation oper))
|
||||
{
|
||||
Log.assert(oper.book_or_trade_entry != null);
|
||||
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
|
||||
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
|
||||
if (count++ > 10_000)
|
||||
break;
|
||||
}
|
||||
session.CommitTransactionAsync();
|
||||
if(_done_reading && operations.Count == 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}, _cts.Token);
|
||||
_qtask.Add( t );
|
||||
await t;
|
||||
}
|
||||
|
||||
private void send_orders(MarketDataIncrementalRefresh md)
|
||||
{
|
||||
Dictionary<string, List<HistoryOrderRow>> cache = new Dictionary<string, List<HistoryOrderRow>>();
|
||||
Dictionary<string, List<MBOMessage>> cache = new Dictionary<string, List<MBOMessage>>();
|
||||
foreach (var order in md.OrderIdEntries)
|
||||
{
|
||||
if (order.ReferenceId == 0)
|
||||
@@ -172,7 +120,6 @@ namespace bts.history
|
||||
|
||||
MDEntry mdval = md.MdEntries[mdidx];
|
||||
|
||||
//now merge this order stuff into an MDEntry because we can't do templates....no I don't miss c++
|
||||
mdval = mdval.Clone();
|
||||
mdval.LastQty = order.LastQty;
|
||||
mdval.MdDisplayQty = order.MdDisplayQty;
|
||||
@@ -198,26 +145,32 @@ namespace bts.history
|
||||
_mbo_operations[pair.Key].Enqueue(new Operation { md = md, mbo = pair.Value });
|
||||
}
|
||||
|
||||
foreach (var mdval in md.MdEntries)
|
||||
{
|
||||
if (!_mbo_operations.TryGetValue(mdval.security_desc(), out ConcurrentQueue<Operation> q))
|
||||
{
|
||||
_mbo_operations[mdval.security_desc()] = q = new ConcurrentQueue<Operation>();
|
||||
start_mbo_processor(mdval.security_desc()).ContinueWith(t => { throw t.Exception; }, TaskContinuationOptions.OnlyOnFaulted); ;
|
||||
}
|
||||
q.Enqueue(new Operation { md = md, book_or_trade_entry = mdval });
|
||||
}
|
||||
|
||||
//each order in here can be for a different secdesc
|
||||
void cache_mbo(MDEntry order)
|
||||
{
|
||||
HistoryOrderRow mbo = new HistoryOrderRow();
|
||||
mbo.TradeDate = md.TradeDate;
|
||||
mbo.MsgSeqNum = md.MsgSeqNum;
|
||||
mbo.SendingTime = md.SendingTime;
|
||||
mbo.TransactTime = md.TransactTime;
|
||||
|
||||
MBOMessage mbo = new MBOMessage();
|
||||
mbo.OrderUpdateAction = order.MdUpdateAction;
|
||||
mbo.OrderId = order.OrderId;
|
||||
mbo.SecurityId = order.SecurityId;
|
||||
mbo.SecurityDesc = order.security_desc();
|
||||
mbo.MdOrderPriority = order.MdOrderPriority;
|
||||
|
||||
|
||||
mbo.MdEntryPx = order.MdEntryPx;
|
||||
mbo.MdPriceLevel = order.MdPriceLevel;
|
||||
mbo.MdDisplayQty = order.MdDisplayQty;
|
||||
if (!cache.TryGetValue(mbo.SecurityDesc, out List<HistoryOrderRow> list))
|
||||
cache[mbo.SecurityDesc] = list = new List<HistoryOrderRow>();
|
||||
|
||||
if (!cache.TryGetValue(order.security_desc(), out List<MBOMessage> list))
|
||||
cache[order.security_desc()] = list = new List<MBOMessage>();
|
||||
list.Add(mbo);
|
||||
}
|
||||
}
|
||||
@@ -236,10 +189,18 @@ namespace bts.history
|
||||
session.StartTransaction();
|
||||
while (operations.TryDequeue(out Operation oper))
|
||||
{
|
||||
Log.assert(oper.mbo != null);
|
||||
write_mbo(secdesc, oper.md, oper.mbo);
|
||||
if(oper.mbo != null)
|
||||
{
|
||||
write_mbo(secdesc, oper.md, oper.mbo);
|
||||
}
|
||||
else if(oper.book_or_trade_entry != null)
|
||||
{
|
||||
write_book_or_trade_entry(oper.md, oper.book_or_trade_entry);
|
||||
Log.assert(oper.book_or_trade_entry.security_desc() == secdesc);
|
||||
}
|
||||
Log.assert(oper.mbo == null || oper.book_or_trade_entry == null);
|
||||
if (count++ > 50_000)
|
||||
break;
|
||||
break;
|
||||
}
|
||||
session.CommitTransactionAsync();
|
||||
if(_done_reading && operations.Count == 0)
|
||||
@@ -307,9 +268,10 @@ namespace bts.history
|
||||
MsgSeqNum = md.MsgSeqNum,
|
||||
SendingTime = md.SendingTime,
|
||||
TransactTime = md.TransactTime,
|
||||
SecurityDesc = entry.security_desc(),
|
||||
TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] },
|
||||
};
|
||||
SecurityDesc = entry.security_desc(),
|
||||
};
|
||||
if(entry.MdPriceLevel == 1)
|
||||
hist.TopBook = new TopBookMessage { Bid = book.Bids[0], Ask = book.Asks[0] };
|
||||
if (OrderBook.quote_event_done(md))
|
||||
hist.Book = book;
|
||||
|
||||
@@ -317,11 +279,24 @@ namespace bts.history
|
||||
table.InsertOne( hist );
|
||||
}
|
||||
|
||||
void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<HistoryOrderRow> mbo)
|
||||
void write_mbo(string secdesc, MarketDataIncrementalRefresh md, IEnumerable<MBOMessage> mbo)
|
||||
{
|
||||
string tname = table_name(secdesc) + "_MBO";
|
||||
var table = _db.GetCollection<HistoryOrderRow>(tname);
|
||||
table.InsertMany(mbo);
|
||||
string tname = table_name(secdesc);
|
||||
var table = _db.GetCollection<HistoryRow>(tname);
|
||||
var orders = mbo.Select(x =>
|
||||
{
|
||||
HistoryRow hist = new HistoryRow
|
||||
{
|
||||
TradeDate = md.TradeDate,
|
||||
MsgSeqNum = md.MsgSeqNum,
|
||||
SendingTime = md.SendingTime,
|
||||
TransactTime = md.TransactTime,
|
||||
SecurityDesc = x.SecurityDesc,
|
||||
Order = x,
|
||||
};
|
||||
return hist;
|
||||
});
|
||||
table.InsertMany(orders);
|
||||
}
|
||||
|
||||
public async void Dispose()
|
||||
|
||||
@@ -50,18 +50,7 @@ namespace bts.history
|
||||
m.UnmapMember(x => x.SendingTime);
|
||||
m.UnmapMember(x => x.TransactTime);
|
||||
m.SetIgnoreExtraElements(true);
|
||||
});
|
||||
BsonClassMap.RegisterClassMap<HistoryOrderRow>(m =>
|
||||
{
|
||||
m.AutoMap();
|
||||
m.MapMember(x => x.trade_date).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
|
||||
m.MapMember(x => x.sending_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
|
||||
m.MapMember(x => x.transact_time).SetSerializer(new DateTimeSerializer(DateTimeKind.Utc, MongoDB.Bson.BsonType.Document));
|
||||
m.UnmapMember(x => x.TradeDate);
|
||||
m.UnmapMember(x => x.SendingTime);
|
||||
m.UnmapMember(x => x.TransactTime);
|
||||
m.SetIgnoreExtraElements(true);
|
||||
});
|
||||
});
|
||||
//BsonSerializer.RegisterSerializationProvider(new SerializationProvider());
|
||||
}
|
||||
|
||||
@@ -94,13 +83,13 @@ namespace bts.history
|
||||
return res;
|
||||
}
|
||||
|
||||
public IEnumerable<OrderOrFill> run_algo(AlgoBase algo, IFillModel fill_model, uint latency,
|
||||
IEnumerable<IMBP> bookstream, IEnumerable<IMBO> orderstream)
|
||||
public IEnumerable<OrderOrFill> run_algo(IAlgo algo, IFillModel fill_model, uint latency,
|
||||
IEnumerable<HistoryRow> datastream)
|
||||
{
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public interface IFillModel {}
|
||||
public static class QueryExtensions
|
||||
{
|
||||
static public IMongoQueryable<T> where<T>(this IMongoQueryable<T> q, string where)
|
||||
|
||||
@@ -154,26 +154,8 @@ message ProcessedBook
|
||||
co2.BookMessage book = 50;
|
||||
}
|
||||
|
||||
message HistoryRow
|
||||
message MBOMessage
|
||||
{
|
||||
google.protobuf.Timestamp trade_date = 5;
|
||||
uint32 msg_seq_num = 10;
|
||||
google.protobuf.Timestamp sending_time = 20;
|
||||
google.protobuf.Timestamp transact_time = 30;
|
||||
|
||||
string security_desc = 40;
|
||||
co2.BookMessage book = 50;
|
||||
co2.TopBookMessage top_book = 55;
|
||||
co2.TradeMessage trade = 60;
|
||||
}
|
||||
|
||||
message HistoryOrderRow
|
||||
{
|
||||
google.protobuf.Timestamp trade_date = 5;
|
||||
uint32 msg_seq_num = 10;
|
||||
google.protobuf.Timestamp sending_time = 20;
|
||||
google.protobuf.Timestamp transact_time = 30;
|
||||
|
||||
MDUpdateAction order_update_action = 279;
|
||||
uint64 order_id = 37;
|
||||
uint32 security_id = 48;
|
||||
@@ -185,6 +167,20 @@ message HistoryOrderRow
|
||||
uint64 md_display_qty = 37706;
|
||||
}
|
||||
|
||||
message HistoryRow
|
||||
{
|
||||
google.protobuf.Timestamp trade_date = 5;
|
||||
uint32 msg_seq_num = 10;
|
||||
google.protobuf.Timestamp sending_time = 20;
|
||||
google.protobuf.Timestamp transact_time = 30;
|
||||
|
||||
string security_desc = 40;
|
||||
co2.BookMessage book = 50;
|
||||
co2.TopBookMessage top_book = 55;
|
||||
co2.TradeMessage trade = 60;
|
||||
MBOMessage order = 70;
|
||||
}
|
||||
|
||||
service HistoricalDataService
|
||||
{
|
||||
rpc get_trading_slices(SlicesQuery) returns (Slices);
|
||||
|
||||
Reference in new issue
Block a user