From fdbbce9ff3893feecdeae7ff5d14cb9b20b95811 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 24 Oct 2019 18:25:06 -0400 Subject: [PATCH] remove concept of pre_book, instead separate the book arrangement and the trade / cross processing and layer in our orders by timestamp --- .vscode/launch.json | 2 +- .../sparkcmd/Cmd/Historical/Algos/AlgoBase.cs | 44 +++++++----- .../Cmd/Historical/Algos/CompositeAlgo.cs | 35 +++++----- .../Cmd/Historical/Algos/CoverAlgo.cs | 12 ++-- .../sparkcmd/Cmd/Historical/Algos/JoinAlgo.cs | 9 ++- .../sparkcmd/Cmd/Historical/Algos/LeanAlgo.cs | 9 +-- .../Cmd/Historical/Algos/LimitOrder.cs | 11 +-- .../sparkcmd/Cmd/Historical/Algos/NullAlgo.cs | 5 +- .../Cmd/Historical/Algos/RoundTripAlgo.cs | 16 ++--- .../sparkcmd/Cmd/Historical/Algos/TopAlgo.cs | 7 +- .../Cmd/Historical/Analyzers/AlgoAnalyzer.cs | 30 +++++--- .../Cmd/Historical/Analyzers/OrderBook.cs | 69 ++++++++++--------- .../Historical/HistoricalDataDownloader.cs | 43 ++++++++++++ .../Cmd/Historical/HistoricalDataStreamer.cs | 6 +- .../sparkcmd/Cmd/Historical/MBOParser.cs | 8 ++- 15 files changed, 192 insertions(+), 114 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 1abe2db9e..4d4c0e835 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -107,7 +107,7 @@ "historical", "--lskip", "18000000", "--rskip", "10/21/2019 02:30:00 PM", - "--rstart", "10/21/2019 06:30:00 PM", + "--rstart", "10/21/2019 03:30:00 PM", "--rend", "10/21/2019", "--rsymbol", "CL", "--rsecuritytype", "Spread", diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/AlgoBase.cs b/src/client/sparkcmd/Cmd/Historical/Algos/AlgoBase.cs index 5ca12789a..9b6ac1a47 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/AlgoBase.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/AlgoBase.cs @@ -3,10 +3,12 @@ using System.Collections.Generic; using System.Linq; using bts.api; using MathNet.Numerics.Statistics; +using Google.Protobuf.WellKnownTypes; using ReactionFunction = System.Action; namespace bts.history { @@ -15,6 +17,7 @@ namespace bts.history public co2.BookMessage pre_book; public co2.BookMessage curr_book; public MarketDataIncrementalRefresh pre_packet; + public Timestamp asof; } class Reaction @@ -24,6 +27,7 @@ namespace bts.history public MarketDataIncrementalRefresh packet; public string security_desc; public object param; + public Timestamp asof; } enum OrderState @@ -160,7 +164,7 @@ namespace bts.history double nanos = delta.Seconds * 1e9 + delta.Nanos; if(nanos >= this.bucket.LowerBound) { - rx.function(rx.packet, rx.security_desc, rx.param); + rx.function(rx.packet, rx.security_desc, rx.param, rx.asof); removes.Add(rx); } else if(nanos > 1) @@ -172,22 +176,28 @@ namespace bts.history protected void queue_catalyst(co2.BookMessage pre_book, co2.BookMessage curr_book, MarketDataIncrementalRefresh pre_packet) { - _catalysts.Add(new Catalyst { pre_book = pre_book, curr_book = curr_book, pre_packet = pre_packet }); + Timestamp asof = pre_packet.timestamp().Clone(); + asof.Nanos += (int)this.bucket.LowerBound; + _catalysts.Add(new Catalyst { pre_book = pre_book, curr_book = curr_book, pre_packet = pre_packet, asof = asof }); } protected virtual void queue_reaction(ReactionFunction reaction_function, MarketDataIncrementalRefresh packet, string security_desc, object param = null) { + Timestamp asof = packet.timestamp().Clone(); + asof.Nanos += (int)this.bucket.LowerBound; + _reactions.Add(new Reaction { function = reaction_function, packet = packet, - security_desc = security_desc, param = param }); + security_desc = security_desc, param = param, asof = asof }); } public abstract object report(); public abstract bool is_completed(); - public virtual Handle activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress = false, - Action more_on_fill = null, Action more_on_status = null) + public virtual Handle activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof, + Action more_on_fill = null, + Action more_on_status = null) where C : AlgoBase, new() { var child = new C(); @@ -203,25 +213,25 @@ namespace bts.history if(more_on_status != null) child.on_status += (s => more_on_status(h, s)); - child.on_activate(security_list, side, price, size, can_aggress); + child.on_activate(security_list, side, price, size, asof); return h; } - protected void call_on_activate(C child, string[] security_list, api.Side side, double price, uint size, bool can_aggress = false) + protected void call_on_activate(C child, string[] security_list, api.Side side, double price, uint size, Timestamp asof) where C : AlgoBase, new() { - child.on_activate(security_list, side, price, size, can_aggress); + child.on_activate(security_list, side, price, size, asof); } protected virtual void on_md(string security_desc, co2.BookMessage pre_book, co2.BookMessage curr_book, MarketDataIncrementalRefresh pre_packet) { } - protected abstract void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress); - protected abstract void on_update(double price, uint size = uint.MinValue, bool can_aggress = false); - protected virtual void on_update(Dictionary algo_params, double price, uint size = uint.MinValue, bool can_aggress = false) + protected abstract void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof); + protected abstract void on_update(double price, uint size = uint.MinValue, Timestamp asof = null); + protected virtual void on_update(Dictionary algo_params, double price, uint size = uint.MinValue, Timestamp asof = null) { foreach (var s in algo_params) this.set_param(s.Key, s.Value); - this.on_update(price, size, can_aggress); + this.on_update(price, size, asof); } protected abstract void on_cancel(); @@ -235,15 +245,15 @@ namespace bts.history public AlgoBase algo => _algo; public ulong order_id => _algo.order_id; - public void update(double price, uint size = uint.MinValue, bool can_aggress = false) + public void update(double price, uint size = uint.MinValue, Timestamp asof = null) { - _algo.on_update(price, size, can_aggress); + _algo.on_update(price, size, asof); } - public void update(Dictionary algo_params, double price, uint size = uint.MinValue, bool can_aggress = false) + public void update(Dictionary algo_params, double price, uint size = uint.MinValue, Timestamp asof = null) { foreach (var s in algo_params) _algo.set_param(s.Key, s.Value); - update(price, size, can_aggress); + update(price, size, asof); } public void cancel() { diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/CompositeAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/CompositeAlgo.cs index 0ccba7864..0d690eb37 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/CompositeAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/CompositeAlgo.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; using MathNet.Numerics.Statistics; - +using Google.Protobuf.WellKnownTypes; namespace bts.history { @@ -31,8 +31,9 @@ namespace bts.history { } - public override Handle activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress = false, - Action more_on_fill = null, Action more_on_status = null) + public override Handle activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof, + Action more_on_fill = null, + Action more_on_status = null) { if (typeof(Child) == typeof(C)) { @@ -48,33 +49,33 @@ namespace bts.history if(more_on_status != null) os += more_on_status; - var handle = base_activate(security_list, side, price, size, can_aggress, of, os); + var handle = base_activate(security_list, side, price, size, asof, of, os); _children.Add(handle); return handle; } else if(typeof(C) == this.GetType()) { //I am activating myself at the top - this.on_activate(security_list, side, price, size, can_aggress); + this.on_activate(security_list, side, price, size, asof); return new Handle(this); } else return null; } - protected Handle base_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress, + protected Handle base_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof, Action of, Action os) where C : AlgoBase, new() { - return base.activate(security_list, side, price, size, can_aggress, of, os); + return base.activate(security_list, side, price, size, asof, of, os); } - protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false) + protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null) { foreach(var child in _children) - child?.update(price, size, can_aggress); + child?.update(price, size, asof); } protected override void on_cancel() @@ -209,8 +210,9 @@ namespace bts.history sub.execute_with_latency(packet); } - public override Handle activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress = false, - Action more_on_fill = null, Action more_on_status = null) + public override Handle activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof, + Action more_on_fill = null, + Action more_on_status = null) { if (typeof(Hedge) == typeof(C)) { @@ -221,12 +223,12 @@ namespace bts.history if (more_on_status != null) os += more_on_status; - var handle = base_activate(security_list, side, price, size, can_aggress, of, os); + var handle = base_activate(security_list, side, price, size, asof, of, os); _hedges[handle] = (Hedge)handle.algo; return handle; } else - return base.activate(security_list, side, price, size, can_aggress, more_on_fill, more_on_status); + return base.activate(security_list, side, price, size, asof, more_on_fill, more_on_status); } @@ -236,7 +238,7 @@ namespace bts.history queue_reaction(send_hedge, packet, security_desc, param); } - private void send_hedge(MarketDataIncrementalRefresh packet, string security_desc, object param) + private void send_hedge(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof) { Action hedge_on_fill = (hand, f) => { @@ -256,8 +258,9 @@ namespace bts.history _hedges.Remove(hand); }; - ValueTuple tup = (ValueTuple)param; - var handle = this.activate(tup.Item1, tup.Item2, tup.Item3, tup.Item4, tup.Item5, hedge_on_fill, hedge_on_status); + ValueTuple tup = (ValueTuple)param; + var handle = this.activate(tup.Item1, tup.Item2, tup.Item3, tup.Item4, asof, + hedge_on_fill, hedge_on_status); var piq = this._order_book[security_desc].get_my_piq(handle.order_id); _hedges_in_transit--; diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/CoverAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/CoverAlgo.cs index b4969dcff..0e140db65 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/CoverAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/CoverAlgo.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using mbts.co2; + using bts.api; +using Google.Protobuf.WellKnownTypes; namespace bts.history { @@ -25,15 +27,15 @@ namespace bts.history return b.ask_at(1) - b.ask_at(0); } - protected override void on_activate(string[] security_list, Side side, double price, uint size, bool can_aggress) + protected override void on_activate(string[] security_list, Side side, double price, uint size, Timestamp asof) { _security_list = security_list; - _child = this.activate(security_list, side, price, size, can_aggress); + _child = this.activate(security_list, side, price, size, asof); } - protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false) + protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null) { - _child.update(price, size, can_aggress); + _child.update(price, size, asof); } protected override void on_cancel() @@ -58,7 +60,7 @@ namespace bts.history fill.price + this.tick_size(fill.order.security_desc) * this.hedge_ticks; queue_hedge(_packet, fill.order.security_desc, - (new[]{fill.order.security_desc}, side, price, size, false)); + (new[]{fill.order.security_desc}, side, price, size)); } } } \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/JoinAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/JoinAlgo.cs index e52bc1910..bbc7e1dfb 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/JoinAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/JoinAlgo.cs @@ -1,5 +1,6 @@ using System; +using Google.Protobuf.WellKnownTypes; using mbts.co2; namespace bts.history @@ -17,6 +18,7 @@ namespace bts.history OrderStatus _status; api.Side _myside; uint _mysize; + Timestamp _myasof; co2.BookMessage _snapshot, _currentbook; int _turncount, _pilecount; @@ -30,11 +32,12 @@ namespace bts.history public uint join_quantity{ get; set; } = 0; - protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress) + protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof) { _state = AlgoState.Begin; _myside = side; _mysize = size; + _myasof = asof; _child = null; } @@ -131,7 +134,7 @@ namespace bts.history void deploy_join_function(MarketDataIncrementalRefresh packet, - string security_desc, object param) + string security_desc, object param, Timestamp asof) { if(_state == AlgoState.Joined || _state == AlgoState.WaitingForJoin) { @@ -140,7 +143,7 @@ namespace bts.history double joinprice = (double) param; if(_child == null) { - _child = this.activate(new[]{security_desc}, _myside, joinprice, _mysize, false); + _child = this.activate(new[]{security_desc}, _myside, joinprice, _mysize, asof); _piq = this._order_book[security_desc].get_my_piq(_child.order_id); } else diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/LeanAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/LeanAlgo.cs index 0473f25c9..4a1e65761 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/LeanAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/LeanAlgo.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using mbts.co2; using System.Linq; +using Google.Protobuf.WellKnownTypes; namespace bts.history { @@ -29,13 +30,13 @@ namespace bts.history public double lean_factor { get; set; } = 3.0; public uint lean_quantity { get; set; } = 0; - protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress) + protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof) { _state = AlgoState.LeanWatch; _myside = side; _mysize = size; _myprice = price; - _child = this.activate(security_list, side, price, size, can_aggress); + _child = this.activate(security_list, side, price, size, asof); } public override bool is_completed() @@ -100,7 +101,7 @@ namespace bts.history } } - void payup_function(MarketDataIncrementalRefresh packet, string security_desc, object param) + void payup_function(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof) { if(_state == AlgoState.Payup) { @@ -111,7 +112,7 @@ namespace bts.history else newprice = _currentbook.all_asks().Max(); //this should precipitate a fill - _child.update(newprice, uint.MinValue, can_aggress: true); + _child.update(newprice, uint.MinValue, asof); Log.Testing.info($"payup_function {this.bucket.UpperBound} "); } diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/LimitOrder.cs b/src/client/sparkcmd/Cmd/Historical/Algos/LimitOrder.cs index f6e11558a..822c7e418 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/LimitOrder.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/LimitOrder.cs @@ -1,5 +1,6 @@ using System; using bts.api; +using Google.Protobuf.WellKnownTypes; namespace bts.history { @@ -21,13 +22,13 @@ namespace bts.history public override object report() { return string.Empty; } - protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress) + protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof) { _mysize = size; Log.assert(security_list.Length == 1); var security_desc = security_list[0]; _status = new OrderStatus { security_desc = security_desc, state = OrderState.Working, side = side, size = size, price = price }; - _myorder = this._order_book[security_desc].insert_my_order(side, price, size, f => + _myorder = this._order_book[security_desc].insert_my_order(asof, side, price, size, f => { if (f.fully_filled) { @@ -37,15 +38,15 @@ namespace bts.history this.on_fill(f); if (this.on_status != null) this.on_status(_status); - }, can_aggress); + }); if (this.on_status != null) this.on_status(_status); } - protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false) + protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null) { if(size == _mysize) size = uint.MinValue; - var ord = this._order_book[_myorder.security_desc].change_my_order(_myorder.order_id, price, size, can_aggress); + var ord = this._order_book[_myorder.security_desc].change_my_order(asof, _myorder.order_id, price, size); if (this.on_status != null && ord != null) { _status.size = (uint)ord.size; diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/NullAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/NullAlgo.cs index 0ecc3ab0c..517431f9f 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/NullAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/NullAlgo.cs @@ -1,6 +1,7 @@ using System; using bts.api; +using Google.Protobuf.WellKnownTypes; namespace bts.history { @@ -11,9 +12,9 @@ namespace bts.history public override object report() { return string.Empty; } - protected override void on_activate(string[] security_list, Side side, double price, uint size, bool can_aggress) { } + protected override void on_activate(string[] security_list, Side side, double price, uint size, Timestamp asof) { } - protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false) { } + protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null) { } protected override void on_cancel() { } } diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/RoundTripAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/RoundTripAlgo.cs index 6fc232fe9..c3cb297b0 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/RoundTripAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/RoundTripAlgo.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Google.Protobuf.WellKnownTypes; using mbts.co2; namespace bts.history @@ -41,7 +42,7 @@ namespace bts.history SortedList _hedge_factors = new SortedList(); - protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress) + protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof) { _security_list = security_list; for(int i = 0; i < _security_list.Length; i++) @@ -98,9 +99,8 @@ namespace bts.history var side = OrderBook.opposite_side(fill.order.side); uint size = (uint)(fill.size * this._hedge_factors[sec]); double price = (side == api.Side.Buy) ? _pre_book[sec].ask_at(0) : _pre_book[sec].bid_at(0); - bool can_aggress = true; queue_hedge(_packet, sec, - (new[] { sec }, side, price, size, can_aggress)); + (new[] { sec }, side, price, size)); } } @@ -155,23 +155,23 @@ namespace bts.history break; } } - void quote_function(MarketDataIncrementalRefresh packet, string qtleg, object param) + void quote_function(MarketDataIncrementalRefresh packet, string qtleg, object param, Timestamp asof) { (api.Side side, double qt, uint sz) tup = (ValueTuple) param; if (!double.IsNaN(tup.qt) && tup.sz > 0 && tup.side == api.Side.Buy) { if (_buychild == null) - _buychild = this.activate(new[] { qtleg }, tup.side, tup.qt, tup.sz, can_aggress: false); + _buychild = this.activate(new[] { qtleg }, tup.side, tup.qt, tup.sz, asof); else if (_buystatus.price != tup.qt && !_buystatus.is_terminal()) - _buychild.update(tup.qt, tup.sz, can_aggress: false); + _buychild.update(tup.qt, tup.sz, asof); _buypiq = _order_book[qtleg].get_my_piq(_buychild?.order_id ?? 0); } else if(!double.IsNaN(tup.qt) && tup.sz > 0 && tup.side == api.Side.Sell) { if (_sellchild == null) - _sellchild = this.activate(new[] { qtleg }, tup.side, tup.qt, tup.sz, can_aggress: false); + _sellchild = this.activate(new[] { qtleg }, tup.side, tup.qt, tup.sz, asof); else if (_sellstatus.price != tup.qt && !_sellstatus.is_terminal()) - _sellchild.update(tup.qt, tup.sz, can_aggress: false); + _sellchild.update(tup.qt, tup.sz, asof); _sellpiq = _order_book[qtleg].get_my_piq(_sellchild?.order_id ?? 0); } } diff --git a/src/client/sparkcmd/Cmd/Historical/Algos/TopAlgo.cs b/src/client/sparkcmd/Cmd/Historical/Algos/TopAlgo.cs index 4b01d9b4a..9beb52e73 100644 --- a/src/client/sparkcmd/Cmd/Historical/Algos/TopAlgo.cs +++ b/src/client/sparkcmd/Cmd/Historical/Algos/TopAlgo.cs @@ -1,5 +1,6 @@ using System; +using Google.Protobuf.WellKnownTypes; using mbts.co2; namespace bts.history @@ -26,12 +27,12 @@ namespace bts.history public double top_factor { get; set; } = 1.0; public uint top_quantity { get; set; } = 0; - protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress) + protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof) { _state = AlgoState.WaitingToCancel; _myside = side; _mysize = size; - _child = this.activate(security_list, side, price, size, can_aggress); + _child = this.activate(security_list, side, price, size, asof); } public override bool is_completed() @@ -81,7 +82,7 @@ namespace bts.history } } - void cancel_top_function(MarketDataIncrementalRefresh packet, string security_desc, object param) + void cancel_top_function(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof) { Log.assert(_state == AlgoState.CancelRequested || _state == AlgoState.Done); Log.Testing.info("cancel_top_function " + this.bucket.UpperBound); diff --git a/src/client/sparkcmd/Cmd/Historical/Analyzers/AlgoAnalyzer.cs b/src/client/sparkcmd/Cmd/Historical/Analyzers/AlgoAnalyzer.cs index 1e77dbb80..91e5f78c9 100644 --- a/src/client/sparkcmd/Cmd/Historical/Analyzers/AlgoAnalyzer.cs +++ b/src/client/sparkcmd/Cmd/Historical/Analyzers/AlgoAnalyzer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Google.Protobuf.WellKnownTypes; using MathNet.Numerics.Statistics; namespace bts.history @@ -15,7 +16,7 @@ namespace bts.history readonly SortedDictionary _shared_order_book = new SortedDictionary(); string[] _security_list; - public AlgoAnalyzer(string algo_security_list, Type algotype, string algo_params) + public AlgoAnalyzer(string algo_security_list, System.Type algotype, string algo_params) { _security_list = new string[] { algo_security_list }; if (algo_security_list.Contains(';')) @@ -49,7 +50,7 @@ namespace bts.history } } - AlgoBase algo_factory(Bucket bucket, string[] security_list, Type t, string algo_params) + AlgoBase algo_factory(Bucket bucket, string[] security_list, System.Type t, string algo_params) { var inst = (AlgoBase)Activator.CreateInstance(t); @@ -77,7 +78,8 @@ namespace bts.history var mi = algo.GetType().GetMethod(nameof(algo.activate)); var gmi = mi.MakeGenericMethod(algo.GetType()); - gmi.Invoke( algo, new object[]{ security_list, side, price, size, Type.Missing, Type.Missing, Type.Missing } ); + gmi.Invoke( algo, new object[]{ security_list, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow), + System.Type.Missing, System.Type.Missing } ); } public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null) @@ -89,15 +91,15 @@ namespace bts.history MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh(); public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null) { + //get the books right + process_md(md, debug_fix); + foreach(var ob in _shared_order_book) { //the last known good book from the last quote event indicator co2.BookMessage curr_book = ob.Value.current_book; - //the working book, with all msgs between curr_book and now - co2.BookMessage pre_book = ob.Value.working_book?.Clone(); - //add this final packet which the working book has not seen yet - OrderBook.keep_book(pre_book, md, (mde, oid) => mde.Symbol == ob.Key); - + //the working book, with all msgs between curr_book and now TODO duplciation + co2.BookMessage pre_book = ob.Value.current_book; foreach (var algo in _algos) { //check the market data for a 'catalyst' with (potential) latency @@ -110,9 +112,7 @@ namespace bts.history //maybe do the action if enough time has passed, update order book with my order algo.Value.algo.execute_with_latency(md); } - - //lay the actual orders over my orders if any - process_md(md, debug_fix); + process_trades(md, debug_fix); if (_samples++ % 1000 == 0) { @@ -148,5 +148,13 @@ namespace bts.history book.process_md(md, (mde, ord) => mde.Symbol == ob.Key, debug_fix); } } + void process_trades(MarketDataIncrementalRefresh md, string debug_fix = null) + { + foreach (var ob in _shared_order_book) + { + OrderBook book = (OrderBook)ob.Value; + book.process_trades(md, (mde, ord) => mde.Symbol == ob.Key); + } + } } } \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/Historical/Analyzers/OrderBook.cs b/src/client/sparkcmd/Cmd/Historical/Analyzers/OrderBook.cs index 986afee84..f8a084afd 100644 --- a/src/client/sparkcmd/Cmd/Historical/Analyzers/OrderBook.cs +++ b/src/client/sparkcmd/Cmd/Historical/Analyzers/OrderBook.cs @@ -7,6 +7,7 @@ using bts.co2; using mbts.co2; using QueueType = System.Collections.Generic.SortedList; +using MathNet.Numerics.Statistics; namespace bts.history { @@ -35,7 +36,6 @@ namespace bts.history public api.Side side { get; } public double price { get; set; } public ulong size { get; set; } - public bool can_aggress { get; set; } public Action on_fill { get; } public string tag { get; } public bool deleted { get; set; } @@ -75,8 +75,8 @@ namespace bts.history double last_trade_price { get; } (int, int) get_my_piq(ulong orderid); - OrderEntry insert_my_order(api.Side side, double price, uint size, Action on_fill, bool can_aggress); - OrderEntry change_my_order(ulong order_id, double price, uint size = uint.MinValue, bool can_aggress = false); + OrderEntry insert_my_order(Timestamp asof, api.Side side, double price, uint size, Action on_fill); + OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = uint.MinValue); bool cancel_my_order(ulong my_order_id); void validate_new_queue(AlgoBase algo, MarketDataIncrementalRefresh pre_book_packet, Func filter); } @@ -205,8 +205,6 @@ namespace bts.history List check_my_cross(OrderEntry myorder) { List fills = new List(); - if(!myorder.can_aggress) - return fills; if (myorder.side == Side.Buy) { int level = 0; int nasks = NativePriceHelpers.all_asks(this.current_book).Count; @@ -245,9 +243,8 @@ namespace bts.history //once you're in, they can't get in front of you const ulong _first_in_q = 999ul; - //TODO can_aggress is shitty; can't think of anything better right now - public OrderEntry insert_my_order(api.Side side, double price, uint size, - Action on_fill, bool can_aggress) + public OrderEntry insert_my_order(Timestamp asof, api.Side side, double price, uint size, + Action on_fill) { QueueType queue = get_mbo(side).GetOrAdd(price, x => { @@ -255,20 +252,22 @@ namespace bts.history return new QueueType(); }); - var ts = new Timestamp(); - var key = queue.Keys.LastOrDefault(_first_in_q); ++_myid; - OrderEntry myorder = new OrderEntry(ts, (ulong)_myid, key, 0, this.security_desc, side, price, size, on_fill); - myorder.can_aggress = can_aggress; + OrderEntry myorder = new OrderEntry(asof, (ulong)_myid, _first_in_q, 0, this.security_desc, side, price, size, on_fill); + + var key = queue.LastOrDefault(kvp => kvp.Value.timestamp.less_or_equal(asof), new KeyValuePair(0u, null)); + if (key.Value != null) + myorder.order_priority = key.Key; + else + myorder.order_priority = _first_in_q; + Log.Testing.info($"insert myorder {key} at price { price } piq { queue.Count } "); _myorders.Add((ulong)_myid, myorder); - // if(can_aggress) - // post_process_fill(check_my_cross(myorder)); //reentrant -- hedge can come in here return myorder; } - public OrderEntry change_my_order(ulong order_id, double price, uint size = uint.MinValue, bool can_aggress = false) + public OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = uint.MinValue) { if(!_myorders.TryGetValue(order_id, out OrderEntry myorder)) return null; @@ -276,16 +275,16 @@ namespace bts.history myorder.price = price; if(size > uint.MinValue) myorder.size = size; - myorder.can_aggress = can_aggress; var mbo = get_mbo(myorder.side); if(mbo.TryGetValue(price, out QueueType queue)) { - var key = queue.Keys.LastOrDefault(_first_in_q); - myorder.order_priority = key; + var key = queue.LastOrDefault( kvp => kvp.Value.timestamp.less_or_equal(asof), new KeyValuePair(0u, null)); + if(key.Value != null) + myorder.order_priority = key.Key; + else + myorder.order_priority = _first_in_q; Log.Testing.info($"update myorder {order_id} at price { price } piq { queue.Count } "); } - // if(can_aggress) - // post_process_fill( check_my_cross(myorder) ); return myorder; } @@ -338,19 +337,19 @@ namespace bts.history Log.assert( !(bidoffer && trade), "Saw trade and bids / offers in same md packet"); if(bidoffer) process_orders(md, filter); - else if(trade) - process_trades(md); - - //we can't do this, because if we are first, we don't know if we're joining or crossing - //TODO so we have can_aggress, which is shitty - List fills = new List(); - foreach(var myorder in _myorders.Values) - fills.AddRange(check_my_cross(myorder)); - post_process_fill(fills); } - void process_trades(MarketDataIncrementalRefresh md) + public void process_trades(MarketDataIncrementalRefresh md, Func filter) { + bool has_trade = false; + foreach (var entry in md.MdEntries) + { + if (!filter(entry, null)) + continue; + has_trade |= entry.MdEntryType == MDEntryType.Trade; + } + if(!has_trade) + return; foreach (var trade in md.OrderIdEntries) { //look for the order that got filled, if we have it (could have been here since last Tuesday) @@ -361,7 +360,7 @@ namespace bts.history continue; //if the priority is greater than the priority of any of my orders, fill me first //the real order still gets decremented later by CME which isn't quite right TODO - List fills = new List(); + List tfills = new List(); foreach (var myorder in _myorders.Values) { if (order.price >= myorder.price && myorder.side == Side.Sell || @@ -373,13 +372,17 @@ namespace bts.history // int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count; var f = generate_fill(myorder, order.price, (uint)Math.Min(trade.LastQty, myorder.size), piq, count, $" {piq} of {count} "); - fills.Add(f); + tfills.Add(f); } } } - post_process_fill(fills); + post_process_fill(tfills); } } + List fills = new List(); + foreach (var myorder in _myorders.Values) + fills.AddRange(check_my_cross(myorder)); + post_process_fill(fills); } void process_orders(MarketDataIncrementalRefresh md, Func filter) diff --git a/src/client/sparkcmd/Cmd/Historical/HistoricalDataDownloader.cs b/src/client/sparkcmd/Cmd/Historical/HistoricalDataDownloader.cs index dcea52e03..54f98cafd 100644 --- a/src/client/sparkcmd/Cmd/Historical/HistoricalDataDownloader.cs +++ b/src/client/sparkcmd/Cmd/Historical/HistoricalDataDownloader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Threading.Tasks; using Amazon; using Amazon.S3; @@ -128,12 +129,54 @@ namespace bts.history string key = make_key(dt, mic, symbol, st); return await download(key); } + + async Task download_s3_uncompress(string key) + { + var gz = await download(key); + var fullpath = _path + "/" + key.Replace(".gz", ""); + using (FileStream writer = new FileStream(fullpath, FileMode.CreateNew)) + { + using ( GZipStream ungz = new GZipStream(gz, CompressionMode.Decompress)) + { + ungz.CopyTo(writer); + writer.Seek(0, SeekOrigin.Begin); + return writer; + } + } + } + public async Task download(string key) { if (!is_in_cache(key)) return await download_s3(key); return cached(key); } + + public async Task download_uncompressed(string key) + { + var unkey = key.Replace(".gz", ""); + if (!is_in_cache(unkey)) + return await download_s3_uncompress(key); + return cached(unkey); + } + + public async Task download_uncompressed(DateTime dt, string symbol, api.SecurityType st) + { + var micconf = mbts.config.Configuration.get(); + var micmap = micconf.cme_mic; + if (micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic)) + { + return await download_uncompressed(dt, themic, symbol, st); + } + throw new NotSupportedException("Can't find mic for symbol " + symbol); + } + + public async Task download_uncompressed(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st) + { + string key = make_key(dt, mic, symbol, st); + return await download_uncompressed(key); + } + } } \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/Historical/HistoricalDataStreamer.cs b/src/client/sparkcmd/Cmd/Historical/HistoricalDataStreamer.cs index eb5455234..a0cd3cd44 100644 --- a/src/client/sparkcmd/Cmd/Historical/HistoricalDataStreamer.cs +++ b/src/client/sparkcmd/Cmd/Historical/HistoricalDataStreamer.cs @@ -22,8 +22,8 @@ namespace bts.history { if(request.SecurityType == SecurityType.Synthetic) { - var s3f = await _dl.download(d, request.Symbol, SecurityType.Future); - var s3s = await _dl.download(d, request.Symbol, SecurityType.Spread); + var s3f = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Future); + var s3s = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Spread); using (MBOParser fparser = new MBOParser(s3f)) { using(MBOParser sparser = new MBOParser(s3s)) @@ -44,7 +44,7 @@ namespace bts.history } else { - var s3 = await _dl.download(d, request.Symbol, request.SecurityType); + var s3 = await _dl.download_uncompressed(d, request.Symbol, request.SecurityType); using (MBOParser parser = new MBOParser(s3)) { foreach (var (mbo,line) in parser.next(null, null, request.Skip, request.LineSkip)) diff --git a/src/client/sparkcmd/Cmd/Historical/MBOParser.cs b/src/client/sparkcmd/Cmd/Historical/MBOParser.cs index 5ead56262..9af28a3b7 100644 --- a/src/client/sparkcmd/Cmd/Historical/MBOParser.cs +++ b/src/client/sparkcmd/Cmd/Historical/MBOParser.cs @@ -9,14 +9,14 @@ namespace bts.history { class MBOParser : IDisposable { - Stream _stream; + Stream _stream = null; StreamReader _reader; QuickFix.IMessageFactory _defaultMsgFactory = new QuickFix.DefaultMessageFactory(); public MBOParser(string path) : this(File.Open(path, FileMode.Open)) {} public MBOParser(Stream s3stream) { - _stream = new GZipStream( s3stream, CompressionMode.Decompress ); - _reader = new StreamReader(_stream, System.Text.Encoding.UTF8); + //_stream = new GZipStream( s3stream, CompressionMode.Decompress ); + _reader = new StreamReader(s3stream, System.Text.Encoding.UTF8); } int _line = 0; @@ -49,6 +49,8 @@ namespace bts.history Console.WriteLine("Skipping " + ts.ToDateTime()); continue; } + else + skip_before = null; } var md = ProtoFix.parse_fix(line); if(pred != null && !pred(md))