using sendingTime instead of transactTime

This commit is contained in:
Kevin Darby committed 2019-10-16 14:45:13 -04:00
1 parent 90321045f6
commit a080a0ccaa
9 files changed
+146 -116

No files matched your search

@@ -17,10 +17,10 @@ namespace bts.history
{
class OrderEntry
{
public OrderEntry(Timestamp transact_time, ulong order_id, ulong order_priority, uint reference_id,
public OrderEntry(Timestamp timestamp, ulong order_id, ulong order_priority, uint reference_id,
api.Side side, double price, ulong size, Action<OrderEntry, FillEntry> on_fill = null)
{
this.transact_time = transact_time;
this.timestamp = timestamp;
this.order_id = order_id;
this.order_priority = order_priority;
this.reference_id = reference_id;
@@ -30,7 +30,7 @@ namespace bts.history
this.size = size;
this.on_fill = on_fill;
}
public Timestamp transact_time { get; set;}
public Timestamp timestamp { get; set;}
public ulong order_id { get; }
public ulong order_priority{ get; set;}
public uint reference_id { get; }
@@ -194,16 +194,17 @@ namespace bts.history
return _myorders.Remove(my_order_id);
}
public void process_md(Timestamp transact_time, string match_event,
IEnumerable<MDEntry> md, IEnumerable<OrderIDEntry> orders, string debug_fix = null)
public void process_md(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter, string debug_fix = null)
{
bool bidoffer = false, trade = false;
foreach (var entry in md)
foreach (var entry in md.MdEntries)
{
if(!filter(entry, null))
continue;
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
{
keep_book(_mbp, entry);
if (match_event == quote_event_done)
if (md.MatchEventIndicator == quote_event_done)
validate_book(_mbp, debug_fix);
bidoffer = true;
}
@@ -215,14 +216,15 @@ namespace bts.history
}
Log.assert( !(bidoffer && trade), "Saw trade and bids / offers in same md packet");
if(bidoffer)
process_orders(transact_time, orders, md);
process_orders(md, filter);
else if(trade)
process_trades(transact_time, orders);
process_trades(md);
}
void process_trades(Timestamp transact_time, IEnumerable<OrderIDEntry> trades)
public int better_order_filled, better_volume_fillled;
void process_trades(MarketDataIncrementalRefresh md)
{
foreach (var trade in trades)
foreach (var trade in md.OrderIdEntries)
{
//look for the order that got filled, if we have it (could have been here since last Tuesday)
var q = find_queue(trade.OrderId, out ulong priority_key);
@@ -235,21 +237,29 @@ namespace bts.history
List<OrderEntry> removes = new List<OrderEntry>();
foreach (var myorder in _myorders.Values)
{
if (order.price == myorder.price && priority_key >= myorder.order_priority)
if (order.price == myorder.price)
{
int piq = 0; int count = 0;
foreach(var entry in q)
if(priority_key >= myorder.order_priority)
{
if(entry.Key < myorder.order_priority && !entry.Value.deleted)
piq++;
if(!entry.Value.deleted)
count++;
int piq = 0; int count = 0;
foreach(var entry in q)
{
if(entry.Key < myorder.order_priority && !entry.Value.deleted)
piq++;
if(!entry.Value.deleted)
count++;
}
//int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count;
if (fill(myorder, trade, $" {piq} of {count}")) //full fill, remove our order from queue
removes.Add(myorder);
else
continue;
}
//int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count;
if (fill(myorder, trade, $" {piq} of {count}")) //full fill, remove our order from queue
removes.Add(myorder);
else
continue;
{
better_order_filled++;
better_volume_fillled += (int)trade.LastQty;
}
}
}
foreach (var r in removes)
@@ -263,32 +273,22 @@ namespace bts.history
}
}
static public api.Side same_side(MDEntryType mdval)
void process_orders(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter)
{
var side = (mdval == MDEntryType.Bid) ? api.Side.Buy : api.Side.Sell;
return side;
}
static public api.Side opposite_side(MDEntryType mdval)
{
var side = same_side(mdval);
return side == api.Side.Buy ? api.Side.Sell : api.Side.Buy;
}
void process_orders(Timestamp transact_time, IEnumerable<OrderIDEntry> orders, IEnumerable<MDEntry> reference)
{
foreach (var order in orders)
foreach (var order in md.OrderIdEntries)
{
// entries is indexed by 0 but ref is indexed by 1 :facepalm:
int mdidx = ((int)order.ReferenceId) - 1;
if(mdidx >= reference.Count()) //could be filtered
Log.assert((mdidx <= md.MdEntries.Count));
MDEntry mdval = md.MdEntries[mdidx];
if(!filter(mdval, order))
continue;
MDEntry mdval = reference.ElementAt( mdidx );
if (order.OrderUpdateAction == MDUpdateAction.New)
{
var q = _mbo.GetOrAdd(mdval.MdEntryPx, x => new QueueType());
var side = same_side(mdval.MdEntryType);
var neworder = new OrderEntry(transact_time, order.OrderId, order.MdOrderPriority, order.ReferenceId,
var neworder = new OrderEntry(md.timestamp(), order.OrderId, order.MdOrderPriority, order.ReferenceId,
side, mdval.MdEntryPx, order.MdDisplayQty);
q.Add(order.MdOrderPriority, neworder);
_mbokey[order.OrderId] = Tuple.Create(mdval.MdEntryPx, order.MdOrderPriority);
@@ -303,7 +303,7 @@ namespace bts.history
x =>
{
var side = same_side(mdval.MdEntryType);
var neworder = new OrderEntry(transact_time, order.OrderId, order.MdOrderPriority, order.ReferenceId,
var neworder = new OrderEntry(md.timestamp(), order.OrderId, order.MdOrderPriority, order.ReferenceId,
side, mdval.MdEntryPx, order.MdDisplayQty);
Log.assert(neworder.order_priority == order.MdOrderPriority, "bad priority");
@@ -312,7 +312,7 @@ namespace bts.history
(id, x) =>
{
Log.assert(x.order_id == order.OrderId, "orderid invariant");
x.transact_time = transact_time;
x.timestamp = md.timestamp();
x.order_priority = order.MdOrderPriority;
x.price = mdval.MdEntryPx;
x.size = order.MdDisplayQty;
@@ -346,7 +346,7 @@ namespace bts.history
return d;
}
public static void init_book(BookMessage book)
static internal void init_book(BookMessage book)
{
for (int i = 0; i < NativePriceHelpers.COCONUT_MAX_DEPTH; i++)
{
@@ -355,7 +355,19 @@ namespace bts.history
}
}
public static void keep_book(BookMessage book, MDEntry md)
static internal api.Side same_side(MDEntryType mdval)
{
var side = (mdval == MDEntryType.Bid) ? api.Side.Buy : api.Side.Sell;
return side;
}
static internal api.Side opposite_side(MDEntryType mdval)
{
var side = same_side(mdval);
return side == api.Side.Buy ? api.Side.Sell : api.Side.Buy;
}
static internal void keep_book(BookMessage book, MDEntry md)
{
if (md.MdPriceLevel <= 0)
return;
@@ -419,9 +431,9 @@ namespace bts.history
break;
}
}
}
}
public static bool validate_book(BookMessage book, string debug_fix = null)
static internal bool validate_book(BookMessage book, string debug_fix = null)
{
bool desc = true;
for (int i = 0; i < book.Bids.Count - 1; i++)
@@ -17,8 +17,8 @@ namespace bts.history
interface IAnalyzer
{
//this data comes in before analysis time, but is useful to create complete books etc
void pre_analyze(MarketDataIncrementalRefresh md);
void analyze(MarketDataIncrementalRefresh md);
void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null);
void analyze(MarketDataIncrementalRefresh md, string debug_fix = null);
}
class QueueAnalyzer : IAnalyzer
@@ -70,11 +70,11 @@ namespace bts.history
}
}
public void pre_analyze(MarketDataIncrementalRefresh md)
public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
{
}
public void analyze(MarketDataIncrementalRefresh md)
public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
{
var entries = md.MdEntries.Where(x => x.Symbol == _security_desc &&
(x.MdEntryType == _bid_or_ask) &&
@@ -86,7 +86,7 @@ namespace bts.history
if( (_bid_or_ask == MDEntryType.Bid && x.MdEntryPx > _lastentry.MdEntryPx) || //new top level bid price greater than old, market turn
(_bid_or_ask == MDEntryType.Offer && x.MdEntryPx < _lastentry.MdEntryPx) ) //new top level ask price less than old, market turn
{
record_data(md, x, ref _lastturntime, _lastpacket.TransactTime, ref _samples, _turndist, _turnhist, "TURN");
record_data(md, x, ref _lastturntime, _lastpacket.timestamp(), ref _samples, _turndist, _turnhist, "TURN");
_lastturn = x;
}
else if (x.MdEntryPx.AlmostEqual(_lastentry.MdEntryPx))
@@ -110,8 +110,8 @@ namespace bts.history
return;
const double sumthresh = 10_000;
const double sumthresh2 = 25_000;
last_time = md.TransactTime;
var delta = (md.TransactTime - ref_time);
last_time = md.timestamp();
var delta = (md.timestamp() - ref_time);
double nanos = delta.Seconds * 1e9 + delta.Nanos;
if (delta.Seconds == 0 && delta.Nanos > 0 && delta.Nanos <= _max_nanos)
{
@@ -15,7 +15,7 @@ namespace bts.history
{
class Reaction
{
public Timestamp transact_time;
public Timestamp transact_time, sending_time, timestamp;
public MDEntry catalyst;
public ReactionFunction function;
}
@@ -38,26 +38,26 @@ namespace bts.history
_factory = factory;
foreach (var hist in new[] { _hist } )
{
// hist.AddBucket(new Bucket(0, 1000.0.Decrement()));
// hist.AddBucket(new Bucket(1000, 2000.0.Decrement()));
// hist.AddBucket(new Bucket(2000, 3000.0.Decrement()));
// hist.AddBucket(new Bucket(3000, 4000.0.Decrement()));
// hist.AddBucket(new Bucket(4000, 5000.0.Decrement()));
// hist.AddBucket(new Bucket(5000, 6000.0.Decrement()));
// hist.AddBucket(new Bucket(6000, 7000.0.Decrement()));
// hist.AddBucket(new Bucket(7000, 8000.0.Decrement()));
// hist.AddBucket(new Bucket(8000, 9000.0.Decrement()));
// hist.AddBucket(new Bucket(9000, 10000.0.Decrement()));
// hist.AddBucket(new Bucket(10000, 20000.0.Decrement()));
// hist.AddBucket(new Bucket(20000, 30000.0.Decrement()));
// hist.AddBucket(new Bucket(30000, 40000.0.Decrement()));
// hist.AddBucket(new Bucket(40000, 50000.0.Decrement()));
// hist.AddBucket(new Bucket(50000, 60000.0.Decrement()));
// hist.AddBucket(new Bucket(60000, 70000.0.Decrement()));
// hist.AddBucket(new Bucket(70000, 80000.0.Decrement()));
// hist.AddBucket(new Bucket(80000, 90000.0.Decrement()));
// hist.AddBucket(new Bucket(90000, _max_nanos.Decrement()));
hist.AddBucket(new Bucket(0, 5000));
hist.AddBucket(new Bucket(0, 1000.0.Decrement()));
hist.AddBucket(new Bucket(1000, 2000.0.Decrement()));
hist.AddBucket(new Bucket(2000, 3000.0.Decrement()));
hist.AddBucket(new Bucket(3000, 4000.0.Decrement()));
hist.AddBucket(new Bucket(4000, 5000.0.Decrement()));
hist.AddBucket(new Bucket(5000, 6000.0.Decrement()));
hist.AddBucket(new Bucket(6000, 7000.0.Decrement()));
hist.AddBucket(new Bucket(7000, 8000.0.Decrement()));
hist.AddBucket(new Bucket(8000, 9000.0.Decrement()));
hist.AddBucket(new Bucket(9000, 10000.0.Decrement()));
hist.AddBucket(new Bucket(10000, 20000.0.Decrement()));
hist.AddBucket(new Bucket(20000, 30000.0.Decrement()));
hist.AddBucket(new Bucket(30000, 40000.0.Decrement()));
hist.AddBucket(new Bucket(40000, 50000.0.Decrement()));
hist.AddBucket(new Bucket(50000, 60000.0.Decrement()));
hist.AddBucket(new Bucket(60000, 70000.0.Decrement()));
hist.AddBucket(new Bucket(70000, 80000.0.Decrement()));
hist.AddBucket(new Bucket(80000, 90000.0.Decrement()));
hist.AddBucket(new Bucket(90000, _max_nanos.Decrement()));
// hist.AddBucket(new Bucket(0, 5000));
hist.LazySort();
for(int i = 0; i < hist.BucketCount; i++)
{
@@ -69,27 +69,26 @@ namespace bts.history
}
public void pre_analyze(MarketDataIncrementalRefresh md)
public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
{
var entries = _strategies.First().Value.filter_md(md);
//so we get complete books, we do this part before and during 'analysis time'
foreach (var x in _strategies.Values)
x.process_md(md.TransactTime, md.MatchEventIndicator, entries, md.OrderIdEntries);
x.process_md(md, x.filter_md, debug_fix);
}
int _samples;
public void analyze(MarketDataIncrementalRefresh md)
public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
{
var entries = _strategies.First().Value.filter_md(md);
//filter the entries for the security we're interested in
foreach (var x in entries)
foreach (var x in md.MdEntries)
{
foreach (var strat in _strategies)
{
var rx = strat.Value.check_for_catalyst(md.TransactTime, x, _lastentry, strat.Key); //all books have the same market values, our order positions differ
if(!strat.Value.filter_md(x))
continue;
var rx = strat.Value.check_for_catalyst(md.timestamp(), x, _lastentry, strat.Key); //all books have the same market values, our order positions differ
if (rx != null)
{
_reactions[strat.Key.LowerBound].Enqueue(new Reaction { transact_time = md.TransactTime, catalyst = x, function = rx } );
_reactions[strat.Key.LowerBound].Enqueue(
new Reaction { transact_time = md.TransactTime, sending_time = md.SendingTime, timestamp = md.timestamp(), catalyst = x, function = rx } );
}
}
@@ -97,29 +96,36 @@ namespace bts.history
//TODO put the reaction Q in each strategy
foreach(var ob in _strategies)
{
if (!ob.Value.filter_md(x))
continue;
//inside loop so it gets filtered
_lastentry = x;
if(_reactions.TryGetValue(ob.Key.LowerBound, out Queue<Reaction> rxq) && rxq.Count > 0)
{
Reaction rx = rxq.Peek();
var delta = (md.TransactTime - rx.transact_time);
var delta = (md.timestamp() - rx.timestamp);
double nanos = delta.Seconds * 1e9 + delta.Nanos;
if(nanos > 100 && nanos > ob.Key.LowerBound)
{
rx = _reactions[ob.Key.LowerBound].Dequeue();
rx.function(rx.transact_time, rx.catalyst, ob.Key);
rx.function(rx.timestamp, rx.catalyst, ob.Key);
}
}
//inside loop so this gets filtered
if (_samples % 1000 == 0)
{
var dt = md.TradeDate.ToDateTime();
Console.WriteLine($"{x.Symbol} { x.MdEntryType } {_samples } Trade date {dt.ToShortDateString()} {dt.ToShortTimeString()}");
Console.WriteLine($" Best Stats: {_strategies.First().Value.stats() } ");
Console.WriteLine($" Worst Stats: {_strategies.Last().Value.stats() } ");
Console.WriteLine($"{_security_desc} Histogram {Environment.NewLine + _hist.ToString() }");
Console.WriteLine($"----------------------------------------------------------------------------- {Environment.NewLine}");
}
}
_lastentry = x;
if (_samples++ % 1000 == 0)
{
Console.WriteLine($"{x.Symbol} { x.MdEntryType } {_samples } Trade date {md.TradeDate.ToDateTime().ToShortDateString()}");
Console.WriteLine($" Stats: {_strategies.First().Value.stats() } ");
Console.WriteLine($"{_security_desc} Histogram {Environment.NewLine + _hist.ToString() }");
Console.WriteLine($"----------------------------------------------------------------------------- {Environment.NewLine}");
}
_samples++;
}
foreach (var x in _strategies.Values)
x.process_md(md.TransactTime, md.MatchEventIndicator, entries, md.OrderIdEntries);
x.process_md(md, x.filter_md, debug_fix);
}
}
}
@@ -160,14 +160,14 @@ namespace bts.history
int count = 0;
await _server.process_md_stream<history.MarketDataIncrementalRefresh>(query, (md, line) =>
{
if (md.TransactTime.ToDateTime() < end)
if (md.timestamp()?.ToDateTime() < end)
{
if(should_send)
{
process(prod, md, line);
maybe_send(md, us);
}
do_anal(md, anal, (md.TransactTime.ToDateTime() < start));
do_anal(md, anal, (md.timestamp().ToDateTime() < start), line);
count++;
}
});
@@ -180,34 +180,34 @@ namespace bts.history
if (md.MatchEventIndicator == OrderBook.quote_event_done)
{
send(); //send the initial books
_time = md.TransactTime;
_time = md.timestamp();
}
else
return;
}
else
{
var ts = (md.TransactTime - _time).ToTimeSpan();
var ts = (md.timestamp() - _time).ToTimeSpan();
var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
if (diff > us && md.MatchEventIndicator == OrderBook.quote_event_done)
{
send(); //send the next book
_time = md.TransactTime;
_time = md.timestamp();
}
}
}
void do_anal(MarketDataIncrementalRefresh md, IEnumerable<IAnalyzer> anal, bool is_before_analysis_time)
void do_anal(MarketDataIncrementalRefresh md, IEnumerable<IAnalyzer> anal, bool is_before_analysis_time, string debug_fix = null)
{
if(is_before_analysis_time)
{
foreach(var d in anal)
d.pre_analyze(md);
d.pre_analyze(md, debug_fix);
}
else
{
foreach(var d in anal)
d.analyze(md);
d.analyze(md, debug_fix);
}
}
}
@@ -55,7 +55,7 @@ namespace bts.history
TradingDistribution dist = new TradingDistribution();
foreach(var md in str.Md)
{
if(md.TransactTime.Seconds >= request.Begin.Seconds && md.TransactTime.Seconds <= request.End.Seconds)
if(md.timestamp().Seconds >= request.Begin.Seconds && md.timestamp().Seconds <= request.End.Seconds)
{
//foreach(
}
@@ -0,0 +1,12 @@
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
static class HistoricalExtensions
{
public static Timestamp timestamp(this MarketDataIncrementalRefresh md)
{
return md.SendingTime;
}
}
}
@@ -54,7 +54,12 @@ namespace bts
static void set_field(IMessage msg, FieldDescriptor fd, QuickFix.FieldMap fix)
{
if (!fix.IsSetField(fd.FieldNumber))
return;
{
if(fix is QuickFix.Message fm && fm.Header.IsSetField(fd.FieldNumber))
fix = fm.Header;
else
return;
}
switch (fd.FieldType)
{
case FieldType.Bool:
@@ -33,7 +33,7 @@ namespace bts.history
MDEntry _joined_md;
co2.BookMessage _leanbook;
ulong _myjoinorderid, _myleanorderid;
int _tradecount, _tradesizecount, _turncount, _payupcount;
int _tradecount, _turncount, _payupcount;
readonly MDEntryType _bid_or_ask;
public JoinCoverLean(string security_desc) : this(security_desc, MDEntryType.Bid) { }
@@ -48,7 +48,7 @@ namespace bts.history
public override string stats()
{
return $"Trades: {_tradecount}, Turns { _turncount }, Payups { _payupcount }, Total Vol { _order_book.total_volume } ";
return $"Trades: {_tradecount}, Turns { _turncount }, Payups { _payupcount }, Total Vol { _order_book.total_volume } Better orders filled { _order_book.better_order_filled } Better volume { _order_book.better_volume_fillled} ";
}
public override ReactionFunction check_for_catalyst(Timestamp transact_time, MDEntry curr, MDEntry last, Bucket bucket)
@@ -44,23 +44,18 @@ namespace bts.history
return proceeds - cost;
}
public virtual void filter_md(MarketDataIncrementalRefresh md, out IEnumerable<MDEntry> entries, out IEnumerable<OrderIDEntry> orders )
public virtual bool filter_md(MDEntry x, OrderIDEntry associated_order = null)
{
entries = md.MdEntries.Where( x =>
return
(x.Symbol == security_desc || string.IsNullOrEmpty(security_desc)) &&
(x.MdEntryType == MDEntryType.Bid || x.MdEntryType == MDEntryType.Offer || x.MdEntryType == MDEntryType.Trade)
);
//need to filter the orders, and look at the orders along with the md
orders = md.OrderIdEntries.wh
return entries;
(x.MdEntryType == MDEntryType.Bid || x.MdEntryType == MDEntryType.Offer || x.MdEntryType == MDEntryType.Trade);
}
public abstract ReactionFunction check_for_catalyst(Timestamp transact_time, MDEntry curr, MDEntry last, Bucket bucket);
public void process_md(Timestamp transact_time, string match_event,
IEnumerable<MDEntry> md, IEnumerable<OrderIDEntry> orders, string debug_fix = null)
public void process_md(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter, string debug_fix = null)
{
_order_book.process_md(transact_time, match_event, md, orders, debug_fix);
_order_book.process_md(md, filter, debug_fix);
}
abstract public string stats();