prorata start

This commit is contained in:
Kevin Darby committed 2019-12-19 23:47:41 -05:00
1 parent ddd8778bab
commit e6c6997e46
3 files changed
+532 -119

No files matched your search

@@ -11,7 +11,7 @@ using bts.api;
namespace bts.history
{
public class FIFOFillModel : IFillModel
public class FIFOFillModel : FillModelBase, IFillModel
{
readonly SortedList<Timestamp, OrderStatus> _myorders =
new SortedList<Timestamp, OrderStatus>(HistoricalExtensions.timestamp_comparer);
@@ -19,28 +19,18 @@ namespace bts.history
readonly Dictionary<ulong, List<MBOMessage>> _exchangeorders =
new Dictionary<ulong, List<MBOMessage>>();
readonly Dictionary<string, co2.BookMessage> _books = new Dictionary<string, co2.BookMessage>();
readonly Dictionary<ulong, uint> _disqual = new Dictionary<ulong, uint>();
public IEnumerable<MBOMessage> exchange_orders_raw => _exchangeorders.SelectValues(_exchangeorders.Keys).SelectMany(x => x);
public IEnumerable<OrderStatus> my_orders => _myorders.Values;
public event Action<FillEntry> on_fill;
public event Action<OrderStatus> on_status_due_to_fill;
public Bucket outgoing_latency { get; }
public IEnumerable<OrderStatus> my_orders => _myorders.Values;
public FIFOFillModel(Bucket outgoing_latency)
public FIFOFillModel(Bucket outgoing_latency) : base(outgoing_latency)
{
this.outgoing_latency = outgoing_latency;
}
public void reset()
public override void reset()
{
base.reset();
_myorders.Clear();
_exchangeorders.Clear();
_books.Clear();
_disqual.Clear();
on_fill = null;
on_status_due_to_fill = null;
}
public void record_order_status(OrderStatus stat)
@@ -180,24 +170,6 @@ namespace bts.history
}
}
List<MBOMessage> get_mbo(ulong id)
{
if (!_exchangeorders.TryGetValue(id, out List<MBOMessage> list))
{
_exchangeorders[id] = list = new List<MBOMessage>();
}
return list;
}
co2.BookMessage get_book(string security_desc)
{
if (!_books.TryGetValue(security_desc, out co2.BookMessage book))
{
_books[security_desc] = book = new co2.BookMessage();
}
return book;
}
static int binary_search<T,U>(IList<U> list, T value, Func<U, T> selector, Comparer<T> comparer = null)
{
if (list == null)
@@ -228,95 +200,14 @@ namespace bts.history
return binary_search(sorted_list.Keys, key, x => x, comp);
}
bool check_cross(OrderStatus status)
List<MBOMessage> get_mbo(ulong id)
{
var fills = check_my_cross(status);
if (fills.Count > 0)
return fills.Any( x => x.fully_filled);
return false;
}
List<FillEntry> check_my_cross(OrderStatus status)
{
List<FillEntry> fills = new List<FillEntry>();
var book = get_book(status.order.security_desc);
if (status.order.side == Side.Buy)
if (!_exchangeorders.TryGetValue(id, out List<MBOMessage> list))
{
int level = 0; int nasks = Math.Min(book.Asks.Count, 10);
while (status.state.Quantity > 0 && level < nasks)
{
var currask = book.ask_at(level);
if (status.state.Price >= currask || status.order.market_order)
{
var qty = Math.Min(status.state.Quantity, book.asksize_at(level));
if (qty > 0)
fills.Add(generate_fill(status.TransactTime, status, currask, (uint)qty, book, "CROSS"));
}
else
break;
level++;
}
_exchangeorders[id] = list = new List<MBOMessage>();
}
else
{
int level = 0; int nbids = Math.Min(book.Bids.Count, 10);
while (status.state.Quantity > 0 && level < nbids)
{
var currbid = book.bid_at(level);
if (status.state.Price <= currbid || status.order.market_order)
{
var qty = Math.Min(status.state.Quantity, book.bidsize_at(level));
if (qty > 0)
fills.Add(generate_fill(status.TransactTime, status, currbid, (uint)qty, book, "CROSS"));
}
else
break;
level++;
}
}
return fills;
return list;
}
FillEntry generate_fill(Timestamp asof, OrderStatus status, double price, uint qty, co2.BookMessage book = null, string tag = null)
{
var f = new FillEntry(asof, status.order, status.order.side, price, qty);
f.SendingTime = f.TransactTime.add_nanos((int)outgoing_latency.LowerBound);
// f.piq = piq;
// f.queue_len = qlen;
f.bidsize = book?.bidsize_at(0) ?? 0;
f.bid = book?.bid_at(0) ?? 0;
f.ask = book?.ask_at(0) ?? 0;
f.asksize = book?.asksize_at(0) ?? 0;
f.tag = tag;
if(qty > status.state.Quantity)
throw new InvalidOperationException($"too much fill {qty} working quantity {status.state.Quantity}");
status.state.Quantity -= qty;
if (status.state.Quantity == 0)
{
Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? ""));
f.fully_filled = true;
}
else
{
Log.Testing.info("Partial FILL " + Stringizer.to_string(f));
}
var clone = status.clone();
clone.TransactTime = asof;
clone.SendingTime = clone.TransactTime.add_nanos((int)outgoing_latency.LowerBound);
clone.state.LastPrice = f.price;
clone.state.AvgPrice = status.state.AvgPrice * status.state.FilledQuantity + f.price * f.size;
clone.state.FilledQuantity += (uint)f.size;
clone.state.AvgPrice /= status.state.FilledQuantity;
if (f.fully_filled)
{
clone.state.Status = api.orderman.OrderStatus.Completed;
_disqual[status.order.order_id] = int.MaxValue;
}
if (this.on_status_due_to_fill != null)
this.on_status_due_to_fill(clone);
if (this.on_fill != null)
this.on_fill(f);
return f;
}
}
}
@@ -0,0 +1,134 @@
using System;
using System.Collections.Generic;
using System.Linq;
using bts.api;
using Google.Protobuf.WellKnownTypes;
using MathNet.Numerics.Statistics;
using mbts.co2;
namespace bts.history
{
public class FillModelBase
{
readonly protected Dictionary<string, co2.BookMessage> _books = new Dictionary<string, co2.BookMessage>();
readonly protected Dictionary<ulong, uint> _disqual = new Dictionary<ulong, uint>();
public event Action<FillEntry> on_fill;
public event Action<OrderStatus> on_status_due_to_fill;
public Bucket outgoing_latency { get; }
public FillModelBase(Bucket outgoing_latency)
{
this.outgoing_latency = outgoing_latency;
}
public virtual void reset()
{
_books.Clear();
_disqual.Clear();
on_fill = null;
on_status_due_to_fill = null;
}
co2.BookMessage get_book(string security_desc)
{
if (!_books.TryGetValue(security_desc, out co2.BookMessage book))
{
_books[security_desc] = book = new co2.BookMessage();
}
return book;
}
protected bool check_cross(OrderStatus status)
{
var fills = check_my_cross(status);
if (fills.Count > 0)
return fills.Any(x => x.fully_filled);
return false;
}
List<FillEntry> check_my_cross(OrderStatus status)
{
List<FillEntry> fills = new List<FillEntry>();
var book = get_book(status.order.security_desc);
if (status.order.side == Side.Buy)
{
int level = 0; int nasks = Math.Min(book.Asks.Count, 10);
while (status.state.Quantity > 0 && level < nasks)
{
var currask = book.ask_at(level);
if (status.state.Price >= currask || status.order.market_order)
{
var qty = Math.Min(status.state.Quantity, book.asksize_at(level));
if (qty > 0)
fills.Add(generate_fill(status.TransactTime, status, currask, (uint)qty, book, "CROSS"));
}
else
break;
level++;
}
}
else
{
int level = 0; int nbids = Math.Min(book.Bids.Count, 10);
while (status.state.Quantity > 0 && level < nbids)
{
var currbid = book.bid_at(level);
if (status.state.Price <= currbid || status.order.market_order)
{
var qty = Math.Min(status.state.Quantity, book.bidsize_at(level));
if (qty > 0)
fills.Add(generate_fill(status.TransactTime, status, currbid, (uint)qty, book, "CROSS"));
}
else
break;
level++;
}
}
return fills;
}
protected FillEntry generate_fill(Timestamp asof, OrderStatus status, double price, uint qty, co2.BookMessage book = null, string tag = null)
{
var f = new FillEntry(asof, status.order, status.order.side, price, qty);
f.SendingTime = f.TransactTime.add_nanos((int)outgoing_latency.LowerBound);
// f.piq = piq;
// f.queue_len = qlen;
f.bidsize = book?.bidsize_at(0) ?? 0;
f.bid = book?.bid_at(0) ?? 0;
f.ask = book?.ask_at(0) ?? 0;
f.asksize = book?.asksize_at(0) ?? 0;
f.tag = tag;
if (qty > status.state.Quantity)
throw new InvalidOperationException($"too much fill {qty} working quantity {status.state.Quantity}");
status.state.Quantity -= qty;
if (status.state.Quantity == 0)
{
Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? ""));
f.fully_filled = true;
}
else
{
Log.Testing.info("Partial FILL " + Stringizer.to_string(f));
}
var clone = status.clone();
clone.TransactTime = asof;
clone.SendingTime = clone.TransactTime.add_nanos((int)outgoing_latency.LowerBound);
clone.state.LastPrice = f.price;
clone.state.AvgPrice = status.state.AvgPrice * status.state.FilledQuantity + f.price * f.size;
clone.state.FilledQuantity += (uint)f.size;
clone.state.AvgPrice /= status.state.FilledQuantity;
if (f.fully_filled)
{
clone.state.Status = api.orderman.OrderStatus.Completed;
_disqual[status.order.order_id] = int.MaxValue;
}
if (this.on_status_due_to_fill != null)
this.on_status_due_to_fill(clone);
if (this.on_fill != null)
this.on_fill(f);
return f;
}
}
}
@@ -0,0 +1,388 @@
using System;
using System.Collections.Generic;
using System.Linq;
using bts.api;
using Google.Protobuf.WellKnownTypes;
using bts.co2;
using mbts.co2;
using QueueType = System.Collections.Generic.SortedList<ulong, bts.history.OrderEntry>;
using MathNet.Numerics.Statistics;
namespace bts.history
{
public class MBOBook
{
public string security_desc { get; private set; }
//price -> queue, queue above is priority -> order
readonly Dictionary<double, QueueType> _mbobuy = new Dictionary<double, QueueType>();
readonly Dictionary<double, QueueType> _mbosell = new Dictionary<double, QueueType>();
//orderid -> { side, price, priority }
readonly Dictionary<ulong, Tuple<api.Side, double, ulong>> _mbokey = new Dictionary<ulong, Tuple<api.Side, double, ulong>>();
readonly SortedList<ulong, OrderEntry> _myorders = new SortedList<ulong, OrderEntry>();
readonly SortedList<ulong, OrderStatus> _mystatus = new SortedList<ulong, OrderStatus>();
BookMessage _book;
public double bid_at(int level)
{
return _book.bid_at(level);
//var b = _mbobuy.Keys.OrderByDescending(x => x).Skip(level).FirstOrDefault(x => true, double.NaN);
//return b;
}
public double ask_at(int level)
{
return _book.ask_at(level);
//var a = _mbosell.Keys.OrderBy(x => x).Skip(level).FirstOrDefault(x => true, double.NaN);
//return a;
}
public uint bidsize_at(int level)
{
return _book.bidsize_at(level);
//double bid = bid_at(level);
//int sz = _mbobuy[bid].Values.Sum( x => (int)x.size );
//return (uint)sz;
}
public uint asksize_at(int level)
{
return _book.asksize_at(level);
//double ask = ask_at(level);
//int sz = _mbosell[ask].Values.Sum(x => (int)x.size);
//return (uint)sz;
}
public MBOBook(string security_desc)
{
this.security_desc = security_desc;
}
public (int, int) get_my_piq(ulong orderid)
{
int piq = 0, count = 0;
if (_myorders.TryGetValue(orderid, out OrderEntry myorder))
{
var mbo = get_mbo(myorder.side);
if (mbo.TryGetValue(myorder.price, out QueueType q))
{
foreach (var ent in q.Values)
{
if (!ent.deleted)
{
if (ent.order_priority<myorder.order_priority)
piq++;
count++;
}
}
return (piq, count);
}
}
return (piq, count);
}
bool remove_my_order(ulong my_order_id)
{
return _myorders.Remove(my_order_id);
}
public void process_one_trade(Timestamp ts, co2.ITradeWithOrderIDAndSize tradeentry)
{
//look for the order that got filled, if we have it (could have been here since last Tuesday)
var q = find_queue(tradeentry.OrderId, out Side side, out double price, out ulong priority_key);
if (q != null)
{
if (!q.TryGetValue(priority_key, out OrderEntry order))
Log.assert(false);
if(order.deleted)
Log.Testing.warn("fill for deleted order, probably ok " + tradeentry.OrderId);
//if the priority is greater than the priority of any of my orders, fill me first
//the real order still gets decremented later by CME which isn't quite right TODO
List<FillEntry> tfills = new List<FillEntry>();
foreach (var myorder in _myorders.Values)
{
if (order.price == myorder.price && side == myorder.side)
{
if (priority_key >= myorder.order_priority)
{
Log.assert(!order.deleted);
var(piq, count) = get_my_piq(myorder.order_id);
// int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count;
var f = generate_fill(ts, myorder, order.price, (uint)Math.Min(tradeentry.Size, myorder.size),
piq, count, $" {piq} of {count} ");
tfills.Add(f);
}
}
}
post_process_fill(tfills);
}
}
public void process_one_mbo(Timestamp ts, IMBOMessage order)
{
process_mbo(ts, order);
}
public void process_external_book(Timestamp transact_time, BookMessage b)
{
_book = b;
}
public void match_crosses(Timestamp ts)
{
List<FillEntry> fills = new List<FillEntry>();
foreach (var myorder in _myorders.Values)
{
if (myorder.market_order)
{
double px = myorder.side == Side.Buy ? ask_at(0) : bid_at(0);
var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0, "MARKET");
fills.Add(f);
}
else
fills.AddRange(check_my_cross(ts, myorder));
}
post_process_fill(fills);
}
static public IDictionary<string, List<MBOMessage>> extract_mbo(MarketDataIncrementalRefresh md)
{
Dictionary<string, List<MBOMessage>> cache = new Dictionary<string, List<MBOMessage>>();
foreach (var order in md.OrderIdEntries)
{
if (order.ReferenceId == 0)
continue;
// entries is indexed by 0 but ref is indexed by 1 :facepalm:
int mdidx = ((int)order.ReferenceId) - 1;
Log.assert((mdidx <= md.MdEntries.Count));
MDEntry mdval = md.MdEntries[mdidx];
cache_mbo(MBOUtils.make_mbo(md, mdval, order) );
}
foreach (var mdval in md.MdEntries)
{
if (mdval.OrderId > 0 && md.OrderIdEntries.Count == 0)
cache_mbo(MBOUtils.make_mbo(md, mdval, null) );
}
return cache;
void cache_mbo(MBOMessage mbo)
{
if (!cache.TryGetValue(mbo.SecurityDesc, out List<MBOMessage> list))
cache[mbo.SecurityDesc] = list = new List<MBOMessage>();
list.Add(mbo);
}
}
Dictionary<double, QueueType> get_mbo(api.Side side)
{
if (side == Side.Buy)
return _mbobuy;
else
return _mbosell;
}
FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string tag = null)
{
var f = new FillEntry(ts, myorder, myorder.side, price, qty);
f.bidsize = bidsize_at(0);
f.bid = bid_at(0);
f.ask = ask_at(0);
f.asksize = asksize_at(0);
f.tag = tag;
Log.assert(qty <= myorder.size);
myorder.size -= qty;
if (myorder.size == 0)
{
Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? ""));
f.fully_filled = true;
}
else
{
Log.Testing.info("Partial FILL " + Stringizer.to_string(f));
}
return f;
}
void post_process_fill(IEnumerable<FillEntry> fills)
{
foreach (var f in fills)
{
if (f.fully_filled)
remove_my_order(f.order.order_id);
if (f.order.on_fill != null)
f.order.on_fill(f);
if(_mystatus.TryGetValue(f.order.order_id, out OrderStatus status))
{
status.state.LastPrice = f.price;
status.state.AvgPrice = status.state.AvgPrice* status.state.FilledQuantity + f.price* f.size;
status.state.FilledQuantity += (uint) f.size;
status.state.AvgPrice /= status.state.FilledQuantity;
if (f.fully_filled)
status.state.Status = api.orderman.OrderStatus.Completed;
if(f.order.on_status != null)
f.order.on_status(status.clone());
}
}
}
OrderEntry find_order(ulong order_id)
{
OrderEntry order = null;
find_queue(order_id, out Side _, out double _, out ulong prior)?.TryGetValue(prior, out order);
return order;
}
QueueType find_queue(ulong order_id, out api.Side side, out double price, out ulong priority_key)
{
QueueType q = null; side = Side.Undefined; price = double.NaN; priority_key = 0;
if (_mbokey.TryGetValue(order_id, out Tuple<api.Side, double, ulong> tup))
{
var mbo = get_mbo(tup.Item1);
mbo.TryGetValue(tup.Item2, out q);
side = tup.Item1;
price = tup.Item2;
priority_key = tup.Item3;
}
return q;
}
List<FillEntry> check_my_cross(Timestamp ts, OrderEntry myorder)
{
List<FillEntry> fills = new List<FillEntry>();
if (myorder.side == Side.Buy)
{
int level = 0; int nasks = Math.Min(_mbosell.Keys.Count, 10);
while (myorder.size > 0 && level<nasks)
{
var currask = ask_at(level);
if (myorder.price >= currask)
{
var qty = Math.Min(myorder.size, asksize_at(level));
if (qty > 0)
fills.Add(generate_fill(ts, myorder, currask, (uint) qty, 0, 0, "CROSS"));
}
else
break;
level++;
}
}
else
{
int level = 0; int nbids = Math.Min(_mbobuy.Keys.Count, 10);
while (myorder.size > 0 && level<nbids)
{
var currbid = bid_at(level);
if (myorder.price <= currbid)
{
var qty = Math.Min(myorder.size, bidsize_at(level));
if (qty > 0)
fills.Add(generate_fill(ts, myorder, currbid, (uint) qty, 0, 0, "CROSS"));
}
else
break;
level++;
}
}
return fills;
}
void process_mbo(Timestamp ts, IMBOMessage order)
{
if (order.MdEntryType != MDEntryType.Bid && order.MdEntryType != MDEntryType.Offer)
return;
var mbo = get_mbo(Utils.same_side(order.MdEntryType));
if (order.OrderUpdateAction == MDUpdateAction.New)
{
var q = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
var side = Utils.same_side(order.MdEntryType);
var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, order.ReferenceId,
this.security_desc, side, order.MdEntryPx, order.MdDisplayQty);
q.Add(order.MdOrderPriority, neworder);
_mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority);
}
else if (order.OrderUpdateAction == MDUpdateAction.Change)
{
var currq = find_queue(order.OrderId, out Side _, out double _, out ulong priority_key);
if (currq != null)
currq.Remove(priority_key); //this could have been previously deleted
var newq = mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
newq.AddOrUpdate(order.MdOrderPriority,
x =>
{
var side = Utils.same_side(order.MdEntryType); var refid = 0u;
var neworder = new OrderEntry(ts, order.OrderId, order.MdOrderPriority, refid,
this.security_desc, side, order.MdEntryPx, order.MdDisplayQty);
Log.assert(neworder.order_priority == order.MdOrderPriority, "bad priority");
_mbokey[order.OrderId] = Tuple.Create(Utils.same_side(order.MdEntryType), order.MdEntryPx, order.MdOrderPriority);
return neworder;
},
(id, x) =>
{
//x.last_change = ts;
if (x.order_priority != order.MdOrderPriority)
{
Log.assert(false, "kd dec 12 2019, this is keyed on priority so this doens't make sense");
// var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority);
// if (myorder != null)
// {
// //target order got better, go with
// if (order.MdOrderPriority < myorder.order_priority)
// {
// _myorders.Remove(myorder.order_id);
// myorder.order_priority = order.MdOrderPriority;
// _myorders[myorder.order_id] = myorder;
// }
// else //target order got worse, attach to prior
// slide_order_up(currq, order);
// }
// x.order_priority = myorder.order_priority;
}
x.price = order.MdEntryPx;
x.size = order.MdDisplayQty;
Log.assert(x.order_id == order.OrderId, "orderid invariant");
_mbokey[x.order_id] = Tuple.Create(Utils.same_side(order.MdEntryType), x.price, x.order_priority);
return x;
});
}
else if (order.OrderUpdateAction == MDUpdateAction.Delete)
{
var currq = find_queue(order.OrderId, out Side _, out double _, out ulong _);
if (currq == null)
Log.assert(true);
if (currq != null && currq.TryGetValue(order.MdOrderPriority, out OrderEntry removed_order))
{
removed_order.deleted = true;
//removed_order.last_change = ts;
slide_order_up(currq, order);
//can't take this out because this could actually be a trade
//currq.Remove(order.MdOrderPriority);
}
}
}
void slide_order_up(QueueType currq, IMBOMessage order)
{
var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == order.MdOrderPriority);
if (myorder != null)
{
_myorders.Remove(myorder.order_id);
var idx = currq.IndexOfKey(order.MdOrderPriority);
if (idx > 0)
{
var prev = currq.ElementAt(idx - 1);
myorder.order_priority = prev.Key;
}
else
myorder.order_priority = 999;
_myorders[myorder.order_id] = myorder;
}
}
}
}