diff --git a/src/client/spark_historical/FillModel/FIFOFillModel.cs b/src/client/spark_historical/FillModel/FIFOFillModel.cs index 301da64d3..ce43999d0 100644 --- a/src/client/spark_historical/FillModel/FIFOFillModel.cs +++ b/src/client/spark_historical/FillModel/FIFOFillModel.cs @@ -11,7 +11,7 @@ using bts.api; namespace bts.history { - public class FIFOFillModel : IFillModel + public class FIFOFillModel : FillModelBase, IFillModel { readonly SortedList _myorders = new SortedList(HistoricalExtensions.timestamp_comparer); @@ -19,28 +19,18 @@ namespace bts.history readonly Dictionary> _exchangeorders = new Dictionary>(); - readonly Dictionary _books = new Dictionary(); - readonly Dictionary _disqual = new Dictionary(); - public IEnumerable exchange_orders_raw => _exchangeorders.SelectValues(_exchangeorders.Keys).SelectMany(x => x); - public IEnumerable my_orders => _myorders.Values; - public event Action on_fill; - public event Action on_status_due_to_fill; - public Bucket outgoing_latency { get; } + public IEnumerable my_orders => _myorders.Values; - public FIFOFillModel(Bucket outgoing_latency) + public FIFOFillModel(Bucket outgoing_latency) : base(outgoing_latency) { - this.outgoing_latency = outgoing_latency; } - - public void reset() + + public override void reset() { + base.reset(); _myorders.Clear(); _exchangeorders.Clear(); - _books.Clear(); - _disqual.Clear(); - on_fill = null; - on_status_due_to_fill = null; } public void record_order_status(OrderStatus stat) @@ -180,24 +170,6 @@ namespace bts.history } } - List get_mbo(ulong id) - { - if (!_exchangeorders.TryGetValue(id, out List list)) - { - _exchangeorders[id] = list = new List(); - } - return list; - } - - co2.BookMessage get_book(string security_desc) - { - if (!_books.TryGetValue(security_desc, out co2.BookMessage book)) - { - _books[security_desc] = book = new co2.BookMessage(); - } - return book; - } - static int binary_search(IList list, T value, Func selector, Comparer comparer = null) { if (list == null) @@ -228,95 +200,14 @@ namespace bts.history return binary_search(sorted_list.Keys, key, x => x, comp); } - bool check_cross(OrderStatus status) + List get_mbo(ulong id) { - var fills = check_my_cross(status); - if (fills.Count > 0) - return fills.Any( x => x.fully_filled); - return false; - } - - List check_my_cross(OrderStatus status) - { - List fills = new List(); - var book = get_book(status.order.security_desc); - if (status.order.side == Side.Buy) + if (!_exchangeorders.TryGetValue(id, out List list)) { - int level = 0; int nasks = Math.Min(book.Asks.Count, 10); - while (status.state.Quantity > 0 && level < nasks) - { - var currask = book.ask_at(level); - if (status.state.Price >= currask || status.order.market_order) - { - var qty = Math.Min(status.state.Quantity, book.asksize_at(level)); - if (qty > 0) - fills.Add(generate_fill(status.TransactTime, status, currask, (uint)qty, book, "CROSS")); - } - else - break; - level++; - } + _exchangeorders[id] = list = new List(); } - else - { - int level = 0; int nbids = Math.Min(book.Bids.Count, 10); - while (status.state.Quantity > 0 && level < nbids) - { - var currbid = book.bid_at(level); - if (status.state.Price <= currbid || status.order.market_order) - { - var qty = Math.Min(status.state.Quantity, book.bidsize_at(level)); - if (qty > 0) - fills.Add(generate_fill(status.TransactTime, status, currbid, (uint)qty, book, "CROSS")); - } - else - break; - level++; - } - } - return fills; + return list; } - FillEntry generate_fill(Timestamp asof, OrderStatus status, double price, uint qty, co2.BookMessage book = null, string tag = null) - { - var f = new FillEntry(asof, status.order, status.order.side, price, qty); - f.SendingTime = f.TransactTime.add_nanos((int)outgoing_latency.LowerBound); - // f.piq = piq; - // f.queue_len = qlen; - f.bidsize = book?.bidsize_at(0) ?? 0; - f.bid = book?.bid_at(0) ?? 0; - f.ask = book?.ask_at(0) ?? 0; - f.asksize = book?.asksize_at(0) ?? 0; - f.tag = tag; - if(qty > status.state.Quantity) - throw new InvalidOperationException($"too much fill {qty} working quantity {status.state.Quantity}"); - status.state.Quantity -= qty; - if (status.state.Quantity == 0) - { - Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? "")); - f.fully_filled = true; - } - else - { - Log.Testing.info("Partial FILL " + Stringizer.to_string(f)); - } - var clone = status.clone(); - clone.TransactTime = asof; - clone.SendingTime = clone.TransactTime.add_nanos((int)outgoing_latency.LowerBound); - clone.state.LastPrice = f.price; - clone.state.AvgPrice = status.state.AvgPrice * status.state.FilledQuantity + f.price * f.size; - clone.state.FilledQuantity += (uint)f.size; - clone.state.AvgPrice /= status.state.FilledQuantity; - if (f.fully_filled) - { - clone.state.Status = api.orderman.OrderStatus.Completed; - _disqual[status.order.order_id] = int.MaxValue; - } - if (this.on_status_due_to_fill != null) - this.on_status_due_to_fill(clone); - if (this.on_fill != null) - this.on_fill(f); - return f; - } } } \ No newline at end of file diff --git a/src/client/spark_historical/FillModel/FillModelBase.cs b/src/client/spark_historical/FillModel/FillModelBase.cs new file mode 100644 index 000000000..e0b6a9d87 --- /dev/null +++ b/src/client/spark_historical/FillModel/FillModelBase.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using bts.api; +using Google.Protobuf.WellKnownTypes; +using MathNet.Numerics.Statistics; +using mbts.co2; + +namespace bts.history +{ + public class FillModelBase + { + readonly protected Dictionary _books = new Dictionary(); + readonly protected Dictionary _disqual = new Dictionary(); + + public event Action on_fill; + public event Action on_status_due_to_fill; + public Bucket outgoing_latency { get; } + + public FillModelBase(Bucket outgoing_latency) + { + this.outgoing_latency = outgoing_latency; + } + + public virtual void reset() + { + _books.Clear(); + _disqual.Clear(); + on_fill = null; + on_status_due_to_fill = null; + } + + co2.BookMessage get_book(string security_desc) + { + if (!_books.TryGetValue(security_desc, out co2.BookMessage book)) + { + _books[security_desc] = book = new co2.BookMessage(); + } + return book; + } + + protected bool check_cross(OrderStatus status) + { + var fills = check_my_cross(status); + if (fills.Count > 0) + return fills.Any(x => x.fully_filled); + return false; + } + + List check_my_cross(OrderStatus status) + { + List fills = new List(); + var book = get_book(status.order.security_desc); + if (status.order.side == Side.Buy) + { + int level = 0; int nasks = Math.Min(book.Asks.Count, 10); + while (status.state.Quantity > 0 && level < nasks) + { + var currask = book.ask_at(level); + if (status.state.Price >= currask || status.order.market_order) + { + var qty = Math.Min(status.state.Quantity, book.asksize_at(level)); + if (qty > 0) + fills.Add(generate_fill(status.TransactTime, status, currask, (uint)qty, book, "CROSS")); + } + else + break; + level++; + } + } + else + { + int level = 0; int nbids = Math.Min(book.Bids.Count, 10); + while (status.state.Quantity > 0 && level < nbids) + { + var currbid = book.bid_at(level); + if (status.state.Price <= currbid || status.order.market_order) + { + var qty = Math.Min(status.state.Quantity, book.bidsize_at(level)); + if (qty > 0) + fills.Add(generate_fill(status.TransactTime, status, currbid, (uint)qty, book, "CROSS")); + } + else + break; + level++; + } + } + return fills; + } + + protected FillEntry generate_fill(Timestamp asof, OrderStatus status, double price, uint qty, co2.BookMessage book = null, string tag = null) + { + var f = new FillEntry(asof, status.order, status.order.side, price, qty); + f.SendingTime = f.TransactTime.add_nanos((int)outgoing_latency.LowerBound); + // f.piq = piq; + // f.queue_len = qlen; + f.bidsize = book?.bidsize_at(0) ?? 0; + f.bid = book?.bid_at(0) ?? 0; + f.ask = book?.ask_at(0) ?? 0; + f.asksize = book?.asksize_at(0) ?? 0; + f.tag = tag; + if (qty > status.state.Quantity) + throw new InvalidOperationException($"too much fill {qty} working quantity {status.state.Quantity}"); + status.state.Quantity -= qty; + if (status.state.Quantity == 0) + { + Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? "")); + f.fully_filled = true; + } + else + { + Log.Testing.info("Partial FILL " + Stringizer.to_string(f)); + } + var clone = status.clone(); + clone.TransactTime = asof; + clone.SendingTime = clone.TransactTime.add_nanos((int)outgoing_latency.LowerBound); + clone.state.LastPrice = f.price; + clone.state.AvgPrice = status.state.AvgPrice * status.state.FilledQuantity + f.price * f.size; + clone.state.FilledQuantity += (uint)f.size; + clone.state.AvgPrice /= status.state.FilledQuantity; + if (f.fully_filled) + { + clone.state.Status = api.orderman.OrderStatus.Completed; + _disqual[status.order.order_id] = int.MaxValue; + } + if (this.on_status_due_to_fill != null) + this.on_status_due_to_fill(clone); + if (this.on_fill != null) + this.on_fill(f); + return f; + } + + } +} \ No newline at end of file diff --git a/src/client/spark_historical/FillModel/KFillModel.cs b/src/client/spark_historical/FillModel/KFillModel.cs new file mode 100644 index 000000000..b80faebfe --- /dev/null +++ b/src/client/spark_historical/FillModel/KFillModel.cs @@ -0,0 +1,388 @@ + +using System; +using System.Collections.Generic; +using System.Linq; +using bts.api; +using Google.Protobuf.WellKnownTypes; +using bts.co2; +using mbts.co2; + +using QueueType = System.Collections.Generic.SortedList; +using MathNet.Numerics.Statistics; + +namespace bts.history +{ + public class MBOBook + { + public string security_desc { get; private set; } + + //price -> queue, queue above is priority -> order + readonly Dictionary _mbobuy = new Dictionary(); + readonly Dictionary _mbosell = new Dictionary(); + //orderid -> { side, price, priority } + readonly Dictionary> _mbokey = new Dictionary>(); + readonly SortedList _myorders = new SortedList(); + readonly SortedList _mystatus = new SortedList(); + BookMessage _book; + + public double bid_at(int level) + { + return _book.bid_at(level); + //var b = _mbobuy.Keys.OrderByDescending(x => x).Skip(level).FirstOrDefault(x => true, double.NaN); + //return b; + } + + public double ask_at(int level) + { + return _book.ask_at(level); + //var a = _mbosell.Keys.OrderBy(x => x).Skip(level).FirstOrDefault(x => true, double.NaN); + //return a; + } + + public uint bidsize_at(int level) + { + return _book.bidsize_at(level); + //double bid = bid_at(level); + //int sz = _mbobuy[bid].Values.Sum( x => (int)x.size ); + //return (uint)sz; + } + + public uint asksize_at(int level) + { + return _book.asksize_at(level); + //double ask = ask_at(level); + //int sz = _mbosell[ask].Values.Sum(x => (int)x.size); + //return (uint)sz; + } + + public MBOBook(string security_desc) + { + this.security_desc = security_desc; + } + + public (int, int) get_my_piq(ulong orderid) + { + int piq = 0, count = 0; + if (_myorders.TryGetValue(orderid, out OrderEntry myorder)) + { + var mbo = get_mbo(myorder.side); + if (mbo.TryGetValue(myorder.price, out QueueType q)) + { + foreach (var ent in q.Values) + { + if (!ent.deleted) + { + if (ent.order_priority tfills = new List(); + foreach (var myorder in _myorders.Values) + { + if (order.price == myorder.price && side == myorder.side) + { + if (priority_key >= myorder.order_priority) + { + Log.assert(!order.deleted); + var(piq, count) = get_my_piq(myorder.order_id); + // int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count; + var f = generate_fill(ts, myorder, order.price, (uint)Math.Min(tradeentry.Size, myorder.size), + piq, count, $" {piq} of {count} "); + tfills.Add(f); + } + } + } + post_process_fill(tfills); + } + } + + public void process_one_mbo(Timestamp ts, IMBOMessage order) + { + process_mbo(ts, order); + } + + public void process_external_book(Timestamp transact_time, BookMessage b) + { + _book = b; + } + + public void match_crosses(Timestamp ts) + { + List fills = new List(); + foreach (var myorder in _myorders.Values) + { + if (myorder.market_order) + { + double px = myorder.side == Side.Buy ? ask_at(0) : bid_at(0); + var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0, "MARKET"); + fills.Add(f); + } + else + fills.AddRange(check_my_cross(ts, myorder)); + } + post_process_fill(fills); + } + + static public IDictionary> extract_mbo(MarketDataIncrementalRefresh md) + { + Dictionary> cache = new Dictionary>(); + foreach (var order in md.OrderIdEntries) + { + if (order.ReferenceId == 0) + continue; + // entries is indexed by 0 but ref is indexed by 1 :facepalm: + int mdidx = ((int)order.ReferenceId) - 1; + Log.assert((mdidx <= md.MdEntries.Count)); + + MDEntry mdval = md.MdEntries[mdidx]; + cache_mbo(MBOUtils.make_mbo(md, mdval, order) ); + } + foreach (var mdval in md.MdEntries) + { + if (mdval.OrderId > 0 && md.OrderIdEntries.Count == 0) + cache_mbo(MBOUtils.make_mbo(md, mdval, null) ); + } + + return cache; + + void cache_mbo(MBOMessage mbo) + { + if (!cache.TryGetValue(mbo.SecurityDesc, out List list)) + cache[mbo.SecurityDesc] = list = new List(); + list.Add(mbo); + } + } + + + Dictionary get_mbo(api.Side side) + { + if (side == Side.Buy) + return _mbobuy; + else + return _mbosell; + } + + FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string tag = null) + { + var f = new FillEntry(ts, myorder, myorder.side, price, qty); + f.bidsize = bidsize_at(0); + f.bid = bid_at(0); + f.ask = ask_at(0); + f.asksize = asksize_at(0); + f.tag = tag; + Log.assert(qty <= myorder.size); + myorder.size -= qty; + if (myorder.size == 0) + { + Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? "")); + f.fully_filled = true; + } + else + { + Log.Testing.info("Partial FILL " + Stringizer.to_string(f)); + } + return f; + } + + void post_process_fill(IEnumerable fills) + { + foreach (var f in fills) + { + if (f.fully_filled) + remove_my_order(f.order.order_id); + if (f.order.on_fill != null) + f.order.on_fill(f); + if(_mystatus.TryGetValue(f.order.order_id, out OrderStatus status)) + { + status.state.LastPrice = f.price; + status.state.AvgPrice = status.state.AvgPrice* status.state.FilledQuantity + f.price* f.size; + status.state.FilledQuantity += (uint) f.size; + status.state.AvgPrice /= status.state.FilledQuantity; + if (f.fully_filled) + status.state.Status = api.orderman.OrderStatus.Completed; + if(f.order.on_status != null) + f.order.on_status(status.clone()); + } + } + } + + OrderEntry find_order(ulong order_id) + { + OrderEntry order = null; + find_queue(order_id, out Side _, out double _, out ulong prior)?.TryGetValue(prior, out order); + return order; + } + + QueueType find_queue(ulong order_id, out api.Side side, out double price, out ulong priority_key) + { + QueueType q = null; side = Side.Undefined; price = double.NaN; priority_key = 0; + if (_mbokey.TryGetValue(order_id, out Tuple tup)) + { + var mbo = get_mbo(tup.Item1); + mbo.TryGetValue(tup.Item2, out q); + side = tup.Item1; + price = tup.Item2; + priority_key = tup.Item3; + } + return q; + } + + List check_my_cross(Timestamp ts, OrderEntry myorder) + { + List fills = new List(); + if (myorder.side == Side.Buy) + { + int level = 0; int nasks = Math.Min(_mbosell.Keys.Count, 10); + while (myorder.size > 0 && level= currask) + { + var qty = Math.Min(myorder.size, asksize_at(level)); + if (qty > 0) + fills.Add(generate_fill(ts, myorder, currask, (uint) qty, 0, 0, "CROSS")); + } + else + break; + level++; + } + } + else + { + int level = 0; int nbids = Math.Min(_mbobuy.Keys.Count, 10); + while (myorder.size > 0 && level 0) + fills.Add(generate_fill(ts, myorder, currbid, (uint) qty, 0, 0, "CROSS")); + } + else + break; + level++; + } + } + return fills; + } + + void process_mbo(Timestamp ts, IMBOMessage order) + { + if (order.MdEntryType != MDEntryType.Bid && order.MdEntryType != MDEntryType.Offer) + return; + var mbo = get_mbo(Utils.same_side(order.MdEntryType)); + if (order.OrderUpdateAction == MDUpdateAction.New) + { + var q = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType()); + var side = Utils.same_side(order.MdEntryType); + var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, order.ReferenceId, + this.security_desc, side, order.MdEntryPx, order.MdDisplayQty); + q.Add(order.MdOrderPriority, neworder); + _mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority); + } + else if (order.OrderUpdateAction == MDUpdateAction.Change) + { + var currq = find_queue(order.OrderId, out Side _, out double _, out ulong priority_key); + if (currq != null) + currq.Remove(priority_key); //this could have been previously deleted + var newq = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType()); + newq.AddOrUpdate(order.MdOrderPriority, + x => + { + var side = Utils.same_side(order.MdEntryType); var refid = 0u; + var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, refid, + this.security_desc, side, order.MdEntryPx, order.MdDisplayQty); + + Log.assert(neworder.order_priority == order.MdOrderPriority, "bad priority"); + _mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority); + return neworder; + }, + (id, x) => + { + //x.last_change = ts; + if (x.order_priority != order.MdOrderPriority) + { + Log.assert(false, "kd dec 12 2019, this is keyed on priority so this doens't make sense"); + // var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority); + // if (myorder != null) + // { + // //target order got better, go with + // if (order.MdOrderPriority < myorder.order_priority) + // { + // _myorders.Remove(myorder.order_id); + // myorder.order_priority = order.MdOrderPriority; + // _myorders[myorder.order_id] = myorder; + // } + // else //target order got worse, attach to prior + // slide_order_up(currq, order); + // } + // x.order_priority = myorder.order_priority; + } + x.price = order.MdEntryPx; + x.size = order.MdDisplayQty; + Log.assert(x.order_id == order.OrderId, "orderid invariant"); + _mbokey[x.order_id] = Tuple.Create(Utils.same_side(order.MdEntryType), x.price, x.order_priority); + return x; + }); + } + else if (order.OrderUpdateAction == MDUpdateAction.Delete) + { + var currq = find_queue(order.OrderId, out Side _, out double _, out ulong _); + if (currq == null) + Log.assert(true); + if (currq != null && currq.TryGetValue(order.MdOrderPriority, out OrderEntry removed_order)) + { + removed_order.deleted = true; + //removed_order.last_change = ts; + slide_order_up(currq, order); + //can't take this out because this could actually be a trade + //currq.Remove(order.MdOrderPriority); + } + } + } + + void slide_order_up(QueueType currq, IMBOMessage order) + { + var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == order.MdOrderPriority); + if (myorder != null) + { + _myorders.Remove(myorder.order_id); + var idx = currq.IndexOfKey(order.MdOrderPriority); + if (idx > 0) + { + var prev = currq.ElementAt(idx - 1); + myorder.order_priority = prev.Key; + } + else + myorder.order_priority = 999; + _myorders[myorder.order_id] = myorder; + } + } + } +} \ No newline at end of file