diff --git a/src/client/spark_historical/FillModel/FIFOModel.cs b/src/client/spark_historical/FillModel/FIFOModel.cs index 5006e87ee..32de2b5d7 100644 --- a/src/client/spark_historical/FillModel/FIFOModel.cs +++ b/src/client/spark_historical/FillModel/FIFOModel.cs @@ -87,7 +87,15 @@ namespace bts.history public void process_external_book(Timestamp transact_time, string security_desc, co2.BookMessage b) { process_queue(transact_time); - var book = get_book(security_desc); + var book = get_book(security_desc); + + book.process_external_book(transact_time, b); + // double bb = mbts.co2.NativePriceHelpers.to_double (b.Bids[0].Mantissa, (sbyte)b.Bids[0].Exponent); + // double ba = mbts.co2.NativePriceHelpers.to_double(b.Asks[0].Mantissa, (sbyte)b.Asks[0].Exponent); + // double bbb = book.bid_at(0); + // double baa = book.ask_at(0); + // if(bb != bbb && !double.IsNaN(bbb) || ba != baa && !double.IsNaN(baa)) + // Log.Testing.error("order book and book book don't match " + _mismatch++); book.match_crosses(transact_time); } diff --git a/src/client/spark_historical/FillModel/FillEntry.cs b/src/client/spark_historical/FillModel/FillEntry.cs index c709db3fa..3e4100c6d 100644 --- a/src/client/spark_historical/FillModel/FillEntry.cs +++ b/src/client/spark_historical/FillModel/FillEntry.cs @@ -31,10 +31,11 @@ namespace bts.history public double bid { get; set; } public uint asksize { get; set; } public double ask { get; set; } + public string tag { get; set; } public override string ToString() { - return $"{timestamp.ToDateTime().ToLongDateString()} {timestamp.ToDateTime().ToString("H:mm:ss.ffffff")} {order.tag ?? "Our "} {order.security_desc} {side} {price} {size} -- {bidsize} {bid} at {ask} { asksize } queue: {piq} of {queue_len}"; + return $"{timestamp.ToDateTime().ToLongDateString()} {timestamp.ToDateTime().ToString("H:mm:ss.ffffff")} {order.tag ?? "Our "} {order.security_desc} {side} {price} {size} -- {bidsize} {bid} at {ask} { asksize } queue: {piq} of {queue_len} {tag}"; } } } \ No newline at end of file diff --git a/src/client/spark_historical/FillModel/IFillModel.cs b/src/client/spark_historical/FillModel/IFillModel.cs index cb844c3bd..6fde8dff4 100644 --- a/src/client/spark_historical/FillModel/IFillModel.cs +++ b/src/client/spark_historical/FillModel/IFillModel.cs @@ -4,23 +4,20 @@ using MathNet.Numerics.Statistics; namespace bts.history { + public interface IFillModel + { + void send(OrderEntry order); + void update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = uint.MinValue); + void cancel(string security_desc, Timestamp asof, ulong order_id); + (int, int) get_my_piq(string security_desc, ulong orderid); -public interface IFillModel -{ - void send(OrderEntry order); - void update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = uint.MinValue); - void cancel(string security_desc, Timestamp asof, ulong order_id); - (int, int) get_my_piq(string security_desc, ulong orderid); - - void process_external_order(Timestamp transact_time, MBOMessage mbo); - void process_external_trade(Timestamp transact_time, string security_desc, co2.TradeMessage trade); - void process_external_book(Timestamp transact_time, string security_desc, co2.BookMessage book); - - //TODO sucks - double tick_size(string security_desc); - bool operations_are_in_flight { get; } - Bucket bucket { get; } -} - + void process_external_order(Timestamp transact_time, MBOMessage mbo); + void process_external_trade(Timestamp transact_time, string security_desc, co2.TradeMessage trade); + void process_external_book(Timestamp transact_time, string security_desc, co2.BookMessage book); + //TODO sucks + double tick_size(string security_desc); + bool operations_are_in_flight { get; } + Bucket bucket { get; } + } } \ No newline at end of file diff --git a/src/client/spark_historical/FillModel/MBOBook.cs b/src/client/spark_historical/FillModel/MBOBook.cs index 8e795671a..05827cfe5 100644 --- a/src/client/spark_historical/FillModel/MBOBook.cs +++ b/src/client/spark_historical/FillModel/MBOBook.cs @@ -15,43 +15,45 @@ namespace bts.history { public string security_desc { get; private set; } - //price, priority + //price -> queue, queue above is priority -> order readonly Dictionary _mbobuy = new Dictionary(); readonly Dictionary _mbosell = new Dictionary(); - //orderid to { price, priority } + //orderid -> { side, price, priority } readonly Dictionary> _mbokey = new Dictionary>(); readonly SortedList _myorders = new SortedList(); readonly SortedList _mystatus = new SortedList(); + BookMessage _book; public double bid_at(int level) { - //return _book.bid_at(level); - return _mbobuy.Keys.OrderByDescending( x => x).Skip(level).First(); + return _book.bid_at(level); + //var b = _mbobuy.Keys.OrderByDescending(x => x).Skip(level).FirstOrDefault(x => true, double.NaN); + //return b; } public double ask_at(int level) { - //return _book.ask_at(level); - return _mbosell.Keys.OrderBy(x => x).Skip(level).First(); + return _book.ask_at(level); + //var a = _mbosell.Keys.OrderBy(x => x).Skip(level).FirstOrDefault(x => true, double.NaN); + //return a; } - uint bidsize_at(int level) + public uint bidsize_at(int level) { - //return _book.bidsize_at(level); - double bid = bid_at(level); - int sz = _mbobuy[bid].Values.Sum( x => (int)x.size ); - return (uint)sz; + return _book.bidsize_at(level); + //double bid = bid_at(level); + //int sz = _mbobuy[bid].Values.Sum( x => (int)x.size ); + //return (uint)sz; } - uint asksize_at(int level) + public uint asksize_at(int level) { - //return _book.asksize_at(level); - double ask = ask_at(level); - int sz = _mbosell[ask].Values.Sum(x => (int)x.size); - return (uint)sz; + return _book.asksize_at(level); + //double ask = ask_at(level); + //int sz = _mbosell[ask].Values.Sum(x => (int)x.size); + //return (uint)sz; } - public MBOBook(string security_desc) { this.security_desc = security_desc; @@ -181,6 +183,8 @@ namespace bts.history { if (!q.TryGetValue(priority_key, out OrderEntry order)) Log.assert(false); + if(order.deleted) + Log.Testing.warn("fill for deleted order, probably ok " + tradeentry.OrderId); //if the priority is greater than the priority of any of my orders, fill me first //the real order still gets decremented later by CME which isn't quite right TODO List tfills = new List(); @@ -211,7 +215,12 @@ namespace bts.history { process_mbo(ts, order); } - + + public void process_external_book(Timestamp transact_time, BookMessage b) + { + _book = b; + } + public void match_crosses(Timestamp ts) { List fills = new List(); @@ -220,7 +229,7 @@ namespace bts.history if (myorder.market_order) { double px = myorder.side == Side.Buy ? ask_at(0) : bid_at(0); - var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0); + var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0, "MARKET"); fills.Add(f); } else @@ -298,7 +307,7 @@ namespace bts.history return _mbosell; } - FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string dbg = null) + FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string tag = null) { var f = new FillEntry(ts, myorder, myorder.side, price, qty); f.piq = piq; @@ -307,11 +316,12 @@ namespace bts.history f.bid = bid_at(0); f.ask = ask_at(0); f.asksize = asksize_at(0); + f.tag = tag; Log.assert(qty <= myorder.size); myorder.size -= qty; if (myorder.size == 0) { - Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (dbg ?? "")); + Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? "")); f.fully_filled = true; } else @@ -442,20 +452,20 @@ namespace bts.history if (x.order_priority != order.MdOrderPriority) { Log.assert(false, "kd dec 12 2019, this is keyed on priority so this doens't make sense"); - var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority); - if (myorder != null) - { - //target order got better, go with - if (order.MdOrderPriority < myorder.order_priority) - { - _myorders.Remove(myorder.order_id); - myorder.order_priority = order.MdOrderPriority; - _myorders[myorder.order_id] = myorder; - } - else //target order got worse, attach to prior - slide_order_up(currq, order); - } - x.order_priority = myorder.order_priority; + // var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority); + // if (myorder != null) + // { + // //target order got better, go with + // if (order.MdOrderPriority < myorder.order_priority) + // { + // _myorders.Remove(myorder.order_id); + // myorder.order_priority = order.MdOrderPriority; + // _myorders[myorder.order_id] = myorder; + // } + // else //target order got worse, attach to prior + // slide_order_up(currq, order); + // } + // x.order_priority = myorder.order_priority; } x.price = order.MdEntryPx; x.size = order.MdDisplayQty; @@ -470,11 +480,11 @@ namespace bts.history if (currq == null) Log.assert(true); if (currq != null && currq.TryGetValue(order.MdOrderPriority, out OrderEntry removed_order)) - { + { removed_order.deleted = true; slide_order_up(currq, order); - //currq.Remove(order.MdOrderPriority); - //we can't delete the orders because we don't know if we are going to get the trade or the delete first + //can't take this out because this could actually be a trade + //currq.Remove(order.MdOrderPriority); } } } diff --git a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/roll-checkpoint.ipynb b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/roll-checkpoint.ipynb index b517dabb1..d3ae4deb4 100644 --- a/src/client/spark_historical/Notebooks/.ipynb_checkpoints/roll-checkpoint.ipynb +++ b/src/client/spark_historical/Notebooks/.ipynb_checkpoints/roll-checkpoint.ipynb @@ -112,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": { "scrolled": true }, @@ -131,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "metadata": { "scrolled": true }, @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -152,6 +152,7 @@ "{\n", " readonly AlgoResults _res;\n", " public string security_desc => _res.security_desc;\n", + " public string transact_time => _res.data?.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n", " public MarketBook market_book { get;}\n", " public MarketTrade market_trade { get;} \n", " public OurOrder our_order { get; }\n", @@ -214,13 +215,14 @@ " public double? our_fill_price => _res.fill?.price;\n", " public ulong? our_fill_size => _res.fill?.size;\n", " public int? our_fill_piq => _res.fill?.piq; \n", + " public string tag => _res.fill?.tag;\n", " }\n", "}" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -240,7 +242,7 @@ }, "outputs": [], "source": [ - "var fill_model = new FIFOModel(new Bucket(20_000, 20_001));\n", + "var fill_model = new FIFOModel(new Bucket(0, 1));\n", "var stream = run_algo(algo_type, fill_model, algo_params,\n", " bts.api.Side.Undefined, 0.0, 10, security_list, \n", " new[] {c2f, c3f, c1f}, x => x.Select(xx => new AlgoReport(xx)) );\n", diff --git a/src/client/spark_historical/Notebooks/roll.ipynb b/src/client/spark_historical/Notebooks/roll.ipynb index b73e337aa..d3ae4deb4 100644 --- a/src/client/spark_historical/Notebooks/roll.ipynb +++ b/src/client/spark_historical/Notebooks/roll.ipynb @@ -152,6 +152,7 @@ "{\n", " readonly AlgoResults _res;\n", " public string security_desc => _res.security_desc;\n", + " public string transact_time => _res.data?.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n", " public MarketBook market_book { get;}\n", " public MarketTrade market_trade { get;} \n", " public OurOrder our_order { get; }\n", @@ -214,6 +215,7 @@ " public double? our_fill_price => _res.fill?.price;\n", " public ulong? our_fill_size => _res.fill?.size;\n", " public int? our_fill_piq => _res.fill?.piq; \n", + " public string tag => _res.fill?.tag;\n", " }\n", "}" ] @@ -240,7 +242,7 @@ }, "outputs": [], "source": [ - "var fill_model = new FIFOModel(new Bucket(20_000, 20_001));\n", + "var fill_model = new FIFOModel(new Bucket(0, 1));\n", "var stream = run_algo(algo_type, fill_model, algo_params,\n", " bts.api.Side.Undefined, 0.0, 10, security_list, \n", " new[] {c2f, c3f, c1f}, x => x.Select(xx => new AlgoReport(xx)) );\n", diff --git a/src/client/spark_historical/Query/Query.cs b/src/client/spark_historical/Query/Query.cs index ba65eeda6..1ba121a97 100644 --- a/src/client/spark_historical/Query/Query.cs +++ b/src/client/spark_historical/Query/Query.cs @@ -94,7 +94,7 @@ namespace bts.history { val = function(valcache, min); } - catch(InvalidOperationException) + catch(System.IO.EndOfStreamException) { break; } @@ -165,30 +165,28 @@ namespace bts.history Dictionary last_trade = new Dictionary(); Dictionary last_book = new Dictionary(); - IAlgo algo = AlgoFactory.algo_factory(algo_type, fill_model, security_list, algo_params); - AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow), - (h, f) => q.Add(new AlgoResults(f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc) )), - (h, s) => q.Add(new AlgoResults(s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc)) )); + IAlgo algo = null; int count = 0; var ienum = stream_data(datastream, (T[] valcache, int min) => { q.Clear(); - if(count++ > framelimit) - throw new InvalidOperationException(); - if(algo.is_completed()) + var secdesc = security_list[min]; + var curr = valcache[min]; + + if(count == 0 || algo.is_completed()) { algo = AlgoFactory.algo_factory(algo_type, fill_model, security_list, algo_params); AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow), - (h, f) => q.Add(new AlgoResults(f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))), - (h, s) => q.Add(new AlgoResults(s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc)))); - } - var secdesc = security_list[min]; - var curr = valcache[min]; - + (h, f) => q.Add(new AlgoResults(curr, f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))), + (h, s) => q.Add(new AlgoResults(curr, s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc)))); + } + if (count++ > framelimit) + throw new System.IO.EndOfStreamException(); + + var orders = MBOBook.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders); if(curr.MdEntry.MdEntryType == MDEntryType.Bid || curr.MdEntry.MdEntryType == MDEntryType.Offer) - { - var orders = MBOBook.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders); + { foreach (var order in orders) algo.fill_model.process_external_order(curr.Md.TransactTime, order); } @@ -203,6 +201,21 @@ namespace bts.history if(curr.Trade != null) { algo.fill_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade); + foreach(var order in orders) + { + if(curr.Trade.OrderId != order.OrderId) + { + var clone = curr.Trade.Clone(); + clone.OrderId = order.OrderId; + clone.Key = order.SecurityId; + clone.AggressorSide = order.MdEntryType == + MDEntryType.Bid ? AggressorSide.Buy : (order.MdEntryType == MDEntryType.Offer ? AggressorSide.Sell : AggressorSide.None); + var ip = IntPrice.from_double(order.MdEntryPx); + clone.Mantissa = ip.mantissa; + clone.Exponent = ip.exponent; + algo.fill_model.process_external_trade(curr.Md.TransactTime, secdesc, clone); + } + } algo.process_trades(secdesc, curr.MdEntry.SecurityId, new [] {curr.Trade}, curr.Md); last_trade[secdesc] = curr.Trade; @@ -219,20 +232,23 @@ namespace bts.history public class AlgoResults { - public AlgoResults(OrderStatus ord, TradeMessage trd, BookMessage book) + public AlgoResults(IHasTimestamp h, OrderStatus ord, TradeMessage trd, BookMessage book) { + this.data = h; this.security_desc = ord.security_desc; this.order = ord; this.trade = trd; this.book = book; } - public AlgoResults(FillEntry fil, TradeMessage trd, BookMessage book) + public AlgoResults(IHasTimestamp h, FillEntry fil, TradeMessage trd, BookMessage book) { + this.data = h; this.security_desc = fil.order.security_desc; this.fill = fil; this.trade = trd; this.book = book; } + public IHasTimestamp data { get; } public string security_desc { get; } public OrderStatus order { get; } public FillEntry fill { get; }