seems like we really can't count on the order of trade vs delete, so we have to use the price book to approximate crosses

This commit is contained in:
Kevin Darby committed 2019-12-12 22:46:12 -05:00
1 parent 4f59c1bb47
commit 7fe20129d2
7 files changed
+117 -81

No files matched your search

@@ -87,7 +87,15 @@ namespace bts.history
public void process_external_book(Timestamp transact_time, string security_desc, co2.BookMessage b)
{
process_queue(transact_time);
var book = get_book(security_desc);
var book = get_book(security_desc);
book.process_external_book(transact_time, b);
// double bb = mbts.co2.NativePriceHelpers.to_double (b.Bids[0].Mantissa, (sbyte)b.Bids[0].Exponent);
// double ba = mbts.co2.NativePriceHelpers.to_double(b.Asks[0].Mantissa, (sbyte)b.Asks[0].Exponent);
// double bbb = book.bid_at(0);
// double baa = book.ask_at(0);
// if(bb != bbb && !double.IsNaN(bbb) || ba != baa && !double.IsNaN(baa))
// Log.Testing.error("order book and book book don't match " + _mismatch++);
book.match_crosses(transact_time);
}
@@ -31,10 +31,11 @@ namespace bts.history
public double bid { get; set; }
public uint asksize { get; set; }
public double ask { get; set; }
public string tag { get; set; }
public override string ToString()
{
return $"{timestamp.ToDateTime().ToLongDateString()} {timestamp.ToDateTime().ToString("H:mm:ss.ffffff")} {order.tag ?? "Our "} {order.security_desc} {side} {price} {size} -- {bidsize} {bid} at {ask} { asksize } queue: {piq} of {queue_len}";
return $"{timestamp.ToDateTime().ToLongDateString()} {timestamp.ToDateTime().ToString("H:mm:ss.ffffff")} {order.tag ?? "Our "} {order.security_desc} {side} {price} {size} -- {bidsize} {bid} at {ask} { asksize } queue: {piq} of {queue_len} {tag}";
}
}
}
@@ -4,23 +4,20 @@ using MathNet.Numerics.Statistics;
namespace bts.history
{
public interface IFillModel
{
void send(OrderEntry order);
void update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = uint.MinValue);
void cancel(string security_desc, Timestamp asof, ulong order_id);
(int, int) get_my_piq(string security_desc, ulong orderid);
public interface IFillModel
{
void send(OrderEntry order);
void update(string security_desc, Timestamp asof, ulong order_id, double price, uint size = uint.MinValue);
void cancel(string security_desc, Timestamp asof, ulong order_id);
(int, int) get_my_piq(string security_desc, ulong orderid);
void process_external_order(Timestamp transact_time, MBOMessage mbo);
void process_external_trade(Timestamp transact_time, string security_desc, co2.TradeMessage trade);
void process_external_book(Timestamp transact_time, string security_desc, co2.BookMessage book);
//TODO sucks
double tick_size(string security_desc);
bool operations_are_in_flight { get; }
Bucket bucket { get; }
}
void process_external_order(Timestamp transact_time, MBOMessage mbo);
void process_external_trade(Timestamp transact_time, string security_desc, co2.TradeMessage trade);
void process_external_book(Timestamp transact_time, string security_desc, co2.BookMessage book);
//TODO sucks
double tick_size(string security_desc);
bool operations_are_in_flight { get; }
Bucket bucket { get; }
}
}
@@ -15,43 +15,45 @@ namespace bts.history
{
public string security_desc { get; private set; }
//price, priority
//price -> queue, queue above is priority -> order
readonly Dictionary<double, QueueType> _mbobuy = new Dictionary<double, QueueType>();
readonly Dictionary<double, QueueType> _mbosell = new Dictionary<double, QueueType>();
//orderid to { price, priority }
//orderid -> { side, price, priority }
readonly Dictionary<ulong, Tuple<api.Side, double, ulong>> _mbokey = new Dictionary<ulong, Tuple<api.Side, double, ulong>>();
readonly SortedList<ulong, OrderEntry> _myorders = new SortedList<ulong, OrderEntry>();
readonly SortedList<ulong, OrderStatus> _mystatus = new SortedList<ulong, OrderStatus>();
BookMessage _book;
public double bid_at(int level)
{
//return _book.bid_at(level);
return _mbobuy.Keys.OrderByDescending( x => x).Skip(level).First();
return _book.bid_at(level);
//var b = _mbobuy.Keys.OrderByDescending(x => x).Skip(level).FirstOrDefault(x => true, double.NaN);
//return b;
}
public double ask_at(int level)
{
//return _book.ask_at(level);
return _mbosell.Keys.OrderBy(x => x).Skip(level).First();
return _book.ask_at(level);
//var a = _mbosell.Keys.OrderBy(x => x).Skip(level).FirstOrDefault(x => true, double.NaN);
//return a;
}
uint bidsize_at(int level)
public uint bidsize_at(int level)
{
//return _book.bidsize_at(level);
double bid = bid_at(level);
int sz = _mbobuy[bid].Values.Sum( x => (int)x.size );
return (uint)sz;
return _book.bidsize_at(level);
//double bid = bid_at(level);
//int sz = _mbobuy[bid].Values.Sum( x => (int)x.size );
//return (uint)sz;
}
uint asksize_at(int level)
public uint asksize_at(int level)
{
//return _book.asksize_at(level);
double ask = ask_at(level);
int sz = _mbosell[ask].Values.Sum(x => (int)x.size);
return (uint)sz;
return _book.asksize_at(level);
//double ask = ask_at(level);
//int sz = _mbosell[ask].Values.Sum(x => (int)x.size);
//return (uint)sz;
}
public MBOBook(string security_desc)
{
this.security_desc = security_desc;
@@ -181,6 +183,8 @@ namespace bts.history
{
if (!q.TryGetValue(priority_key, out OrderEntry order))
Log.assert(false);
if(order.deleted)
Log.Testing.warn("fill for deleted order, probably ok " + tradeentry.OrderId);
//if the priority is greater than the priority of any of my orders, fill me first
//the real order still gets decremented later by CME which isn't quite right TODO
List<FillEntry> tfills = new List<FillEntry>();
@@ -211,7 +215,12 @@ namespace bts.history
{
process_mbo(ts, order);
}
public void process_external_book(Timestamp transact_time, BookMessage b)
{
_book = b;
}
public void match_crosses(Timestamp ts)
{
List<FillEntry> fills = new List<FillEntry>();
@@ -220,7 +229,7 @@ namespace bts.history
if (myorder.market_order)
{
double px = myorder.side == Side.Buy ? ask_at(0) : bid_at(0);
var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0);
var f = generate_fill(ts, myorder, px, (uint)myorder.size, 0, 0, "MARKET");
fills.Add(f);
}
else
@@ -298,7 +307,7 @@ namespace bts.history
return _mbosell;
}
FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string dbg = null)
FillEntry generate_fill(Timestamp ts, OrderEntry myorder, double price, uint qty, int piq, int qlen, string tag = null)
{
var f = new FillEntry(ts, myorder, myorder.side, price, qty);
f.piq = piq;
@@ -307,11 +316,12 @@ namespace bts.history
f.bid = bid_at(0);
f.ask = ask_at(0);
f.asksize = asksize_at(0);
f.tag = tag;
Log.assert(qty <= myorder.size);
myorder.size -= qty;
if (myorder.size == 0)
{
Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (dbg ?? ""));
Log.Testing.info("Full FILL " + Stringizer.to_string(f) + (tag ?? ""));
f.fully_filled = true;
}
else
@@ -442,20 +452,20 @@ namespace bts.history
if (x.order_priority != order.MdOrderPriority)
{
Log.assert(false, "kd dec 12 2019, this is keyed on priority so this doens't make sense");
var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority);
if (myorder != null)
{
//target order got better, go with
if (order.MdOrderPriority < myorder.order_priority)
{
_myorders.Remove(myorder.order_id);
myorder.order_priority = order.MdOrderPriority;
_myorders[myorder.order_id] = myorder;
}
else //target order got worse, attach to prior
slide_order_up(currq, order);
}
x.order_priority = myorder.order_priority;
// var myorder = _myorders.Values.FirstOrDefault(o => o.order_priority == x.order_priority);
// if (myorder != null)
// {
// //target order got better, go with
// if (order.MdOrderPriority < myorder.order_priority)
// {
// _myorders.Remove(myorder.order_id);
// myorder.order_priority = order.MdOrderPriority;
// _myorders[myorder.order_id] = myorder;
// }
// else //target order got worse, attach to prior
// slide_order_up(currq, order);
// }
// x.order_priority = myorder.order_priority;
}
x.price = order.MdEntryPx;
x.size = order.MdDisplayQty;
@@ -470,11 +480,11 @@ namespace bts.history
if (currq == null)
Log.assert(true);
if (currq != null && currq.TryGetValue(order.MdOrderPriority, out OrderEntry removed_order))
{
{
removed_order.deleted = true;
slide_order_up(currq, order);
//currq.Remove(order.MdOrderPriority);
//we can't delete the orders because we don't know if we are going to get the trade or the delete first
//can't take this out because this could actually be a trade
//currq.Remove(order.MdOrderPriority);
}
}
}
@@ -112,7 +112,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 5,
"metadata": {
"scrolled": true
},
@@ -131,7 +131,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 6,
"metadata": {
"scrolled": true
},
@@ -144,7 +144,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -152,6 +152,7 @@
"{\n",
" readonly AlgoResults _res;\n",
" public string security_desc => _res.security_desc;\n",
" public string transact_time => _res.data?.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
" public MarketBook market_book { get;}\n",
" public MarketTrade market_trade { get;} \n",
" public OurOrder our_order { get; }\n",
@@ -214,13 +215,14 @@
" public double? our_fill_price => _res.fill?.price;\n",
" public ulong? our_fill_size => _res.fill?.size;\n",
" public int? our_fill_piq => _res.fill?.piq; \n",
" public string tag => _res.fill?.tag;\n",
" }\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -240,7 +242,7 @@
},
"outputs": [],
"source": [
"var fill_model = new FIFOModel(new Bucket(20_000, 20_001));\n",
"var fill_model = new FIFOModel(new Bucket(0, 1));\n",
"var stream = run_algo<HistoryRow,AlgoReport>(algo_type, fill_model, algo_params,\n",
" bts.api.Side.Undefined, 0.0, 10, security_list, \n",
" new[] {c2f, c3f, c1f}, x => x.Select(xx => new AlgoReport(xx)) );\n",
@@ -152,6 +152,7 @@
"{\n",
" readonly AlgoResults _res;\n",
" public string security_desc => _res.security_desc;\n",
" public string transact_time => _res.data?.transact_time.ToString(\"yyyy-MM-dd HH:mm:ss.ffffff\");\n",
" public MarketBook market_book { get;}\n",
" public MarketTrade market_trade { get;} \n",
" public OurOrder our_order { get; }\n",
@@ -214,6 +215,7 @@
" public double? our_fill_price => _res.fill?.price;\n",
" public ulong? our_fill_size => _res.fill?.size;\n",
" public int? our_fill_piq => _res.fill?.piq; \n",
" public string tag => _res.fill?.tag;\n",
" }\n",
"}"
]
@@ -240,7 +242,7 @@
},
"outputs": [],
"source": [
"var fill_model = new FIFOModel(new Bucket(20_000, 20_001));\n",
"var fill_model = new FIFOModel(new Bucket(0, 1));\n",
"var stream = run_algo<HistoryRow,AlgoReport>(algo_type, fill_model, algo_params,\n",
" bts.api.Side.Undefined, 0.0, 10, security_list, \n",
" new[] {c2f, c3f, c1f}, x => x.Select(xx => new AlgoReport(xx)) );\n",
+34 -18
View File
@@ -94,7 +94,7 @@ namespace bts.history
{
val = function(valcache, min);
}
catch(InvalidOperationException)
catch(System.IO.EndOfStreamException)
{
break;
}
@@ -165,30 +165,28 @@ namespace bts.history
Dictionary<string, TradeMessage> last_trade = new Dictionary<string, TradeMessage>();
Dictionary<string, BookMessage> last_book = new Dictionary<string, BookMessage>();
IAlgo algo = AlgoFactory.algo_factory(algo_type, fill_model, security_list, algo_params);
AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
(h, f) => q.Add(new AlgoResults(f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc) )),
(h, s) => q.Add(new AlgoResults(s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc)) ));
IAlgo algo = null;
int count = 0;
var ienum = stream_data(datastream, (T[] valcache, int min) =>
{
q.Clear();
if(count++ > framelimit)
throw new InvalidOperationException();
if(algo.is_completed())
var secdesc = security_list[min];
var curr = valcache[min];
if(count == 0 || algo.is_completed())
{
algo = AlgoFactory.algo_factory(algo_type, fill_model, security_list, algo_params);
AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
(h, f) => q.Add(new AlgoResults(f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))),
(h, s) => q.Add(new AlgoResults(s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc))));
}
var secdesc = security_list[min];
var curr = valcache[min];
(h, f) => q.Add(new AlgoResults(curr, f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))),
(h, s) => q.Add(new AlgoResults(curr, s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc))));
}
if (count++ > framelimit)
throw new System.IO.EndOfStreamException();
var orders = MBOBook.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders);
if(curr.MdEntry.MdEntryType == MDEntryType.Bid || curr.MdEntry.MdEntryType == MDEntryType.Offer)
{
var orders = MBOBook.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders);
{
foreach (var order in orders)
algo.fill_model.process_external_order(curr.Md.TransactTime, order);
}
@@ -203,6 +201,21 @@ namespace bts.history
if(curr.Trade != null)
{
algo.fill_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade);
foreach(var order in orders)
{
if(curr.Trade.OrderId != order.OrderId)
{
var clone = curr.Trade.Clone();
clone.OrderId = order.OrderId;
clone.Key = order.SecurityId;
clone.AggressorSide = order.MdEntryType ==
MDEntryType.Bid ? AggressorSide.Buy : (order.MdEntryType == MDEntryType.Offer ? AggressorSide.Sell : AggressorSide.None);
var ip = IntPrice.from_double(order.MdEntryPx);
clone.Mantissa = ip.mantissa;
clone.Exponent = ip.exponent;
algo.fill_model.process_external_trade(curr.Md.TransactTime, secdesc, clone);
}
}
algo.process_trades(secdesc, curr.MdEntry.SecurityId, new [] {curr.Trade}, curr.Md);
last_trade[secdesc] = curr.Trade;
@@ -219,20 +232,23 @@ namespace bts.history
public class AlgoResults
{
public AlgoResults(OrderStatus ord, TradeMessage trd, BookMessage book)
public AlgoResults(IHasTimestamp h, OrderStatus ord, TradeMessage trd, BookMessage book)
{
this.data = h;
this.security_desc = ord.security_desc;
this.order = ord;
this.trade = trd;
this.book = book;
}
public AlgoResults(FillEntry fil, TradeMessage trd, BookMessage book)
public AlgoResults(IHasTimestamp h, FillEntry fil, TradeMessage trd, BookMessage book)
{
this.data = h;
this.security_desc = fil.order.security_desc;
this.fill = fil;
this.trade = trd;
this.book = book;
}
public IHasTimestamp data { get; }
public string security_desc { get; }
public OrderStatus order { get; }
public FillEntry fill { get; }