remove concept of pre_book, instead separate the book arrangement and the trade / cross processing and layer in our orders by timestamp

This commit is contained in:
Kevin Darby committed 2019-10-24 18:25:06 -04:00
1 parent d47ac768c4
commit fdbbce9ff3
15 files changed
+192 -114

No files matched your search

+1 -1
View File
@@ -107,7 +107,7 @@
"historical",
"--lskip", "18000000",
"--rskip", "10/21/2019 02:30:00 PM",
"--rstart", "10/21/2019 06:30:00 PM",
"--rstart", "10/21/2019 03:30:00 PM",
"--rend", "10/21/2019",
"--rsymbol", "CL",
"--rsecuritytype", "Spread",
@@ -3,10 +3,12 @@ using System.Collections.Generic;
using System.Linq;
using bts.api;
using MathNet.Numerics.Statistics;
using Google.Protobuf.WellKnownTypes;
using ReactionFunction = System.Action<bts.history.MarketDataIncrementalRefresh /*pre_book_packet*/,
string, /*security_desc*/
object /*param*/
object, /*param*/
Google.Protobuf.WellKnownTypes.Timestamp /* asof */
>;
namespace bts.history
{
@@ -15,6 +17,7 @@ namespace bts.history
public co2.BookMessage pre_book;
public co2.BookMessage curr_book;
public MarketDataIncrementalRefresh pre_packet;
public Timestamp asof;
}
class Reaction
@@ -24,6 +27,7 @@ namespace bts.history
public MarketDataIncrementalRefresh packet;
public string security_desc;
public object param;
public Timestamp asof;
}
enum OrderState
@@ -160,7 +164,7 @@ namespace bts.history
double nanos = delta.Seconds * 1e9 + delta.Nanos;
if(nanos >= this.bucket.LowerBound)
{
rx.function(rx.packet, rx.security_desc, rx.param);
rx.function(rx.packet, rx.security_desc, rx.param, rx.asof);
removes.Add(rx);
}
else if(nanos > 1)
@@ -172,22 +176,28 @@ namespace bts.history
protected void queue_catalyst(co2.BookMessage pre_book, co2.BookMessage curr_book, MarketDataIncrementalRefresh pre_packet)
{
_catalysts.Add(new Catalyst { pre_book = pre_book, curr_book = curr_book, pre_packet = pre_packet });
Timestamp asof = pre_packet.timestamp().Clone();
asof.Nanos += (int)this.bucket.LowerBound;
_catalysts.Add(new Catalyst { pre_book = pre_book, curr_book = curr_book, pre_packet = pre_packet, asof = asof });
}
protected virtual void queue_reaction(ReactionFunction reaction_function, MarketDataIncrementalRefresh packet,
string security_desc, object param = null)
{
Timestamp asof = packet.timestamp().Clone();
asof.Nanos += (int)this.bucket.LowerBound;
_reactions.Add(new Reaction { function = reaction_function, packet = packet,
security_desc = security_desc, param = param });
security_desc = security_desc, param = param, asof = asof });
}
public abstract object report();
public abstract bool is_completed();
public virtual Handle activate<C>(string[] security_list, api.Side side, double price, uint size, bool can_aggress = false,
Action<Handle, FillEntry> more_on_fill = null, Action<Handle, OrderStatus> more_on_status = null)
public virtual Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
Action<Handle, FillEntry> more_on_fill = null,
Action<Handle, OrderStatus> more_on_status = null)
where C : AlgoBase, new()
{
var child = new C();
@@ -203,25 +213,25 @@ namespace bts.history
if(more_on_status != null)
child.on_status += (s => more_on_status(h, s));
child.on_activate(security_list, side, price, size, can_aggress);
child.on_activate(security_list, side, price, size, asof);
return h;
}
protected void call_on_activate<C>(C child, string[] security_list, api.Side side, double price, uint size, bool can_aggress = false)
protected void call_on_activate<C>(C child, string[] security_list, api.Side side, double price, uint size, Timestamp asof)
where C : AlgoBase, new()
{
child.on_activate(security_list, side, price, size, can_aggress);
child.on_activate(security_list, side, price, size, asof);
}
protected virtual void on_md(string security_desc, co2.BookMessage pre_book, co2.BookMessage curr_book,
MarketDataIncrementalRefresh pre_packet) { }
protected abstract void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress);
protected abstract void on_update(double price, uint size = uint.MinValue, bool can_aggress = false);
protected virtual void on_update(Dictionary<string, string> algo_params, double price, uint size = uint.MinValue, bool can_aggress = false)
protected abstract void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof);
protected abstract void on_update(double price, uint size = uint.MinValue, Timestamp asof = null);
protected virtual void on_update(Dictionary<string, string> algo_params, double price, uint size = uint.MinValue, Timestamp asof = null)
{
foreach (var s in algo_params)
this.set_param(s.Key, s.Value);
this.on_update(price, size, can_aggress);
this.on_update(price, size, asof);
}
protected abstract void on_cancel();
@@ -235,15 +245,15 @@ namespace bts.history
public AlgoBase algo => _algo;
public ulong order_id => _algo.order_id;
public void update(double price, uint size = uint.MinValue, bool can_aggress = false)
public void update(double price, uint size = uint.MinValue, Timestamp asof = null)
{
_algo.on_update(price, size, can_aggress);
_algo.on_update(price, size, asof);
}
public void update(Dictionary<string, string> algo_params, double price, uint size = uint.MinValue, bool can_aggress = false)
public void update(Dictionary<string, string> algo_params, double price, uint size = uint.MinValue, Timestamp asof = null)
{
foreach (var s in algo_params)
_algo.set_param(s.Key, s.Value);
update(price, size, can_aggress);
update(price, size, asof);
}
public void cancel()
{
@@ -2,7 +2,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using MathNet.Numerics.Statistics;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
@@ -31,8 +31,9 @@ namespace bts.history
{
}
public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, bool can_aggress = false,
Action<Handle, FillEntry> more_on_fill = null, Action<Handle, OrderStatus> more_on_status = null)
public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
Action<Handle, FillEntry> more_on_fill = null,
Action<Handle, OrderStatus> more_on_status = null)
{
if (typeof(Child) == typeof(C))
{
@@ -48,33 +49,33 @@ namespace bts.history
if(more_on_status != null)
os += more_on_status;
var handle = base_activate<Child>(security_list, side, price, size, can_aggress, of, os);
var handle = base_activate<Child>(security_list, side, price, size, asof, of, os);
_children.Add(handle);
return handle;
}
else if(typeof(C) == this.GetType())
{
//I am activating myself at the top
this.on_activate(security_list, side, price, size, can_aggress);
this.on_activate(security_list, side, price, size, asof);
return new Handle(this);
}
else
return null;
}
protected Handle base_activate<C>(string[] security_list, api.Side side, double price, uint size, bool can_aggress,
protected Handle base_activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
Action<Handle, FillEntry> of, Action<Handle, OrderStatus> os)
where C : AlgoBase, new()
{
return base.activate<C>(security_list, side, price, size, can_aggress, of, os);
return base.activate<C>(security_list, side, price, size, asof, of, os);
}
protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false)
protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null)
{
foreach(var child in _children)
child?.update(price, size, can_aggress);
child?.update(price, size, asof);
}
protected override void on_cancel()
@@ -209,8 +210,9 @@ namespace bts.history
sub.execute_with_latency(packet);
}
public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, bool can_aggress = false,
Action<Handle, FillEntry> more_on_fill = null, Action<Handle, OrderStatus> more_on_status = null)
public override Handle activate<C>(string[] security_list, api.Side side, double price, uint size, Timestamp asof,
Action<Handle, FillEntry> more_on_fill = null,
Action<Handle, OrderStatus> more_on_status = null)
{
if (typeof(Hedge) == typeof(C))
{
@@ -221,12 +223,12 @@ namespace bts.history
if (more_on_status != null)
os += more_on_status;
var handle = base_activate<Hedge>(security_list, side, price, size, can_aggress, of, os);
var handle = base_activate<Hedge>(security_list, side, price, size, asof, of, os);
_hedges[handle] = (Hedge)handle.algo;
return handle;
}
else
return base.activate<C>(security_list, side, price, size, can_aggress, more_on_fill, more_on_status);
return base.activate<C>(security_list, side, price, size, asof, more_on_fill, more_on_status);
}
@@ -236,7 +238,7 @@ namespace bts.history
queue_reaction(send_hedge, packet, security_desc, param);
}
private void send_hedge(MarketDataIncrementalRefresh packet, string security_desc, object param)
private void send_hedge(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof)
{
Action<Handle, FillEntry> hedge_on_fill = (hand, f) =>
{
@@ -256,8 +258,9 @@ namespace bts.history
_hedges.Remove(hand);
};
ValueTuple<string[], api.Side, double, uint, bool> tup = (ValueTuple<string[], api.Side, double, uint, bool>)param;
var handle = this.activate<Hedge>(tup.Item1, tup.Item2, tup.Item3, tup.Item4, tup.Item5, hedge_on_fill, hedge_on_status);
ValueTuple<string[], api.Side, double, uint> tup = (ValueTuple<string[], api.Side, double, uint>)param;
var handle = this.activate<Hedge>(tup.Item1, tup.Item2, tup.Item3, tup.Item4, asof,
hedge_on_fill, hedge_on_status);
var piq = this._order_book[security_desc].get_my_piq(handle.order_id);
_hedges_in_transit--;
@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using mbts.co2;
using bts.api;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
@@ -25,15 +27,15 @@ namespace bts.history
return b.ask_at(1) - b.ask_at(0);
}
protected override void on_activate(string[] security_list, Side side, double price, uint size, bool can_aggress)
protected override void on_activate(string[] security_list, Side side, double price, uint size, Timestamp asof)
{
_security_list = security_list;
_child = this.activate<Child>(security_list, side, price, size, can_aggress);
_child = this.activate<Child>(security_list, side, price, size, asof);
}
protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false)
protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null)
{
_child.update(price, size, can_aggress);
_child.update(price, size, asof);
}
protected override void on_cancel()
@@ -58,7 +60,7 @@ namespace bts.history
fill.price + this.tick_size(fill.order.security_desc) * this.hedge_ticks;
queue_hedge(_packet, fill.order.security_desc,
(new[]{fill.order.security_desc}, side, price, size, false));
(new[]{fill.order.security_desc}, side, price, size));
}
}
}
@@ -1,5 +1,6 @@
using System;
using Google.Protobuf.WellKnownTypes;
using mbts.co2;
namespace bts.history
@@ -17,6 +18,7 @@ namespace bts.history
OrderStatus _status;
api.Side _myside;
uint _mysize;
Timestamp _myasof;
co2.BookMessage _snapshot, _currentbook;
int _turncount, _pilecount;
@@ -30,11 +32,12 @@ namespace bts.history
public uint join_quantity{ get; set; } = 0;
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress)
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof)
{
_state = AlgoState.Begin;
_myside = side;
_mysize = size;
_myasof = asof;
_child = null;
}
@@ -131,7 +134,7 @@ namespace bts.history
void deploy_join_function(MarketDataIncrementalRefresh packet,
string security_desc, object param)
string security_desc, object param, Timestamp asof)
{
if(_state == AlgoState.Joined || _state == AlgoState.WaitingForJoin)
{
@@ -140,7 +143,7 @@ namespace bts.history
double joinprice = (double) param;
if(_child == null)
{
_child = this.activate<Child>(new[]{security_desc}, _myside, joinprice, _mysize, false);
_child = this.activate<Child>(new[]{security_desc}, _myside, joinprice, _mysize, asof);
_piq = this._order_book[security_desc].get_my_piq(_child.order_id);
}
else
@@ -3,6 +3,7 @@ using System;
using System.Collections.Generic;
using mbts.co2;
using System.Linq;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
@@ -29,13 +30,13 @@ namespace bts.history
public double lean_factor { get; set; } = 3.0;
public uint lean_quantity { get; set; } = 0;
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress)
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof)
{
_state = AlgoState.LeanWatch;
_myside = side;
_mysize = size;
_myprice = price;
_child = this.activate<Child>(security_list, side, price, size, can_aggress);
_child = this.activate<Child>(security_list, side, price, size, asof);
}
public override bool is_completed()
@@ -100,7 +101,7 @@ namespace bts.history
}
}
void payup_function(MarketDataIncrementalRefresh packet, string security_desc, object param)
void payup_function(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof)
{
if(_state == AlgoState.Payup)
{
@@ -111,7 +112,7 @@ namespace bts.history
else
newprice = _currentbook.all_asks().Max();
//this should precipitate a fill
_child.update(newprice, uint.MinValue, can_aggress: true);
_child.update(newprice, uint.MinValue, asof);
Log.Testing.info($"payup_function {this.bucket.UpperBound} ");
}
@@ -1,5 +1,6 @@
using System;
using bts.api;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
@@ -21,13 +22,13 @@ namespace bts.history
public override object report() { return string.Empty; }
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress)
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof)
{
_mysize = size;
Log.assert(security_list.Length == 1);
var security_desc = security_list[0];
_status = new OrderStatus { security_desc = security_desc, state = OrderState.Working, side = side, size = size, price = price };
_myorder = this._order_book[security_desc].insert_my_order(side, price, size, f =>
_myorder = this._order_book[security_desc].insert_my_order(asof, side, price, size, f =>
{
if (f.fully_filled)
{
@@ -37,15 +38,15 @@ namespace bts.history
this.on_fill(f);
if (this.on_status != null)
this.on_status(_status);
}, can_aggress);
});
if (this.on_status != null)
this.on_status(_status);
}
protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false)
protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null)
{
if(size == _mysize)
size = uint.MinValue;
var ord = this._order_book[_myorder.security_desc].change_my_order(_myorder.order_id, price, size, can_aggress);
var ord = this._order_book[_myorder.security_desc].change_my_order(asof, _myorder.order_id, price, size);
if (this.on_status != null && ord != null)
{
_status.size = (uint)ord.size;
@@ -1,6 +1,7 @@
using System;
using bts.api;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
@@ -11,9 +12,9 @@ namespace bts.history
public override object report() { return string.Empty; }
protected override void on_activate(string[] security_list, Side side, double price, uint size, bool can_aggress) { }
protected override void on_activate(string[] security_list, Side side, double price, uint size, Timestamp asof) { }
protected override void on_update(double price, uint size = uint.MinValue, bool can_aggress = false) { }
protected override void on_update(double price, uint size = uint.MinValue, Timestamp asof = null) { }
protected override void on_cancel() { }
}
@@ -2,6 +2,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Google.Protobuf.WellKnownTypes;
using mbts.co2;
namespace bts.history
@@ -41,7 +42,7 @@ namespace bts.history
SortedList<string, double> _hedge_factors = new SortedList<string, double>();
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress)
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof)
{
_security_list = security_list;
for(int i = 0; i < _security_list.Length; i++)
@@ -98,9 +99,8 @@ namespace bts.history
var side = OrderBook.opposite_side(fill.order.side);
uint size = (uint)(fill.size * this._hedge_factors[sec]);
double price = (side == api.Side.Buy) ? _pre_book[sec].ask_at(0) : _pre_book[sec].bid_at(0);
bool can_aggress = true;
queue_hedge(_packet, sec,
(new[] { sec }, side, price, size, can_aggress));
(new[] { sec }, side, price, size));
}
}
@@ -155,23 +155,23 @@ namespace bts.history
break;
}
}
void quote_function(MarketDataIncrementalRefresh packet, string qtleg, object param)
void quote_function(MarketDataIncrementalRefresh packet, string qtleg, object param, Timestamp asof)
{
(api.Side side, double qt, uint sz) tup = (ValueTuple<api.Side, double, uint>) param;
if (!double.IsNaN(tup.qt) && tup.sz > 0 && tup.side == api.Side.Buy)
{
if (_buychild == null)
_buychild = this.activate<Child>(new[] { qtleg }, tup.side, tup.qt, tup.sz, can_aggress: false);
_buychild = this.activate<Child>(new[] { qtleg }, tup.side, tup.qt, tup.sz, asof);
else if (_buystatus.price != tup.qt && !_buystatus.is_terminal())
_buychild.update(tup.qt, tup.sz, can_aggress: false);
_buychild.update(tup.qt, tup.sz, asof);
_buypiq = _order_book[qtleg].get_my_piq(_buychild?.order_id ?? 0);
}
else if(!double.IsNaN(tup.qt) && tup.sz > 0 && tup.side == api.Side.Sell)
{
if (_sellchild == null)
_sellchild = this.activate<Child>(new[] { qtleg }, tup.side, tup.qt, tup.sz, can_aggress: false);
_sellchild = this.activate<Child>(new[] { qtleg }, tup.side, tup.qt, tup.sz, asof);
else if (_sellstatus.price != tup.qt && !_sellstatus.is_terminal())
_sellchild.update(tup.qt, tup.sz, can_aggress: false);
_sellchild.update(tup.qt, tup.sz, asof);
_sellpiq = _order_book[qtleg].get_my_piq(_sellchild?.order_id ?? 0);
}
}
@@ -1,5 +1,6 @@
using System;
using Google.Protobuf.WellKnownTypes;
using mbts.co2;
namespace bts.history
@@ -26,12 +27,12 @@ namespace bts.history
public double top_factor { get; set; } = 1.0;
public uint top_quantity { get; set; } = 0;
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, bool can_aggress)
protected override void on_activate(string[] security_list, api.Side side, double price, uint size, Timestamp asof)
{
_state = AlgoState.WaitingToCancel;
_myside = side;
_mysize = size;
_child = this.activate<Child>(security_list, side, price, size, can_aggress);
_child = this.activate<Child>(security_list, side, price, size, asof);
}
public override bool is_completed()
@@ -81,7 +82,7 @@ namespace bts.history
}
}
void cancel_top_function(MarketDataIncrementalRefresh packet, string security_desc, object param)
void cancel_top_function(MarketDataIncrementalRefresh packet, string security_desc, object param, Timestamp asof)
{
Log.assert(_state == AlgoState.CancelRequested || _state == AlgoState.Done);
Log.Testing.info("cancel_top_function " + this.bucket.UpperBound);
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Google.Protobuf.WellKnownTypes;
using MathNet.Numerics.Statistics;
namespace bts.history
@@ -15,7 +16,7 @@ namespace bts.history
readonly SortedDictionary<string, IOrderBook> _shared_order_book = new SortedDictionary<string, IOrderBook>();
string[] _security_list;
public AlgoAnalyzer(string algo_security_list, Type algotype, string algo_params)
public AlgoAnalyzer(string algo_security_list, System.Type algotype, string algo_params)
{
_security_list = new string[] { algo_security_list };
if (algo_security_list.Contains(';'))
@@ -49,7 +50,7 @@ namespace bts.history
}
}
AlgoBase algo_factory(Bucket bucket, string[] security_list, Type t, string algo_params)
AlgoBase algo_factory(Bucket bucket, string[] security_list, System.Type t, string algo_params)
{
var inst = (AlgoBase)Activator.CreateInstance(t);
@@ -77,7 +78,8 @@ namespace bts.history
var mi = algo.GetType().GetMethod(nameof(algo.activate));
var gmi = mi.MakeGenericMethod(algo.GetType());
gmi.Invoke( algo, new object[]{ security_list, side, price, size, Type.Missing, Type.Missing, Type.Missing } );
gmi.Invoke( algo, new object[]{ security_list, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
System.Type.Missing, System.Type.Missing } );
}
public void pre_analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
@@ -89,15 +91,15 @@ namespace bts.history
MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh();
public void analyze(MarketDataIncrementalRefresh md, string debug_fix = null)
{
//get the books right
process_md(md, debug_fix);
foreach(var ob in _shared_order_book)
{
//the last known good book from the last quote event indicator
co2.BookMessage curr_book = ob.Value.current_book;
//the working book, with all msgs between curr_book and now
co2.BookMessage pre_book = ob.Value.working_book?.Clone();
//add this final packet which the working book has not seen yet
OrderBook.keep_book(pre_book, md, (mde, oid) => mde.Symbol == ob.Key);
//the working book, with all msgs between curr_book and now TODO duplciation
co2.BookMessage pre_book = ob.Value.current_book;
foreach (var algo in _algos)
{
//check the market data for a 'catalyst' with (potential) latency
@@ -110,9 +112,7 @@ namespace bts.history
//maybe do the action if enough time has passed, update order book with my order
algo.Value.algo.execute_with_latency(md);
}
//lay the actual orders over my orders if any
process_md(md, debug_fix);
process_trades(md, debug_fix);
if (_samples++ % 1000 == 0)
{
@@ -148,5 +148,13 @@ namespace bts.history
book.process_md(md, (mde, ord) => mde.Symbol == ob.Key, debug_fix);
}
}
void process_trades(MarketDataIncrementalRefresh md, string debug_fix = null)
{
foreach (var ob in _shared_order_book)
{
OrderBook book = (OrderBook)ob.Value;
book.process_trades(md, (mde, ord) => mde.Symbol == ob.Key);
}
}
}
}
@@ -7,6 +7,7 @@ using bts.co2;
using mbts.co2;
using QueueType = System.Collections.Generic.SortedList<ulong, bts.history.OrderEntry>;
using MathNet.Numerics.Statistics;
namespace bts.history
{
@@ -35,7 +36,6 @@ namespace bts.history
public api.Side side { get; }
public double price { get; set; }
public ulong size { get; set; }
public bool can_aggress { get; set; }
public Action<FillEntry> on_fill { get; }
public string tag { get; }
public bool deleted { get; set; }
@@ -75,8 +75,8 @@ namespace bts.history
double last_trade_price { get; }
(int, int) get_my_piq(ulong orderid);
OrderEntry insert_my_order(api.Side side, double price, uint size, Action<FillEntry> on_fill, bool can_aggress);
OrderEntry change_my_order(ulong order_id, double price, uint size = uint.MinValue, bool can_aggress = false);
OrderEntry insert_my_order(Timestamp asof, api.Side side, double price, uint size, Action<FillEntry> on_fill);
OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = uint.MinValue);
bool cancel_my_order(ulong my_order_id);
void validate_new_queue(AlgoBase algo, MarketDataIncrementalRefresh pre_book_packet, Func<MDEntry, OrderIDEntry, bool> filter);
}
@@ -205,8 +205,6 @@ namespace bts.history
List<FillEntry> check_my_cross(OrderEntry myorder)
{
List<FillEntry> fills = new List<FillEntry>();
if(!myorder.can_aggress)
return fills;
if (myorder.side == Side.Buy)
{
int level = 0; int nasks = NativePriceHelpers.all_asks(this.current_book).Count;
@@ -245,9 +243,8 @@ namespace bts.history
//once you're in, they can't get in front of you
const ulong _first_in_q = 999ul;
//TODO can_aggress is shitty; can't think of anything better right now
public OrderEntry insert_my_order(api.Side side, double price, uint size,
Action<FillEntry> on_fill, bool can_aggress)
public OrderEntry insert_my_order(Timestamp asof, api.Side side, double price, uint size,
Action<FillEntry> on_fill)
{
QueueType queue = get_mbo(side).GetOrAdd(price, x =>
{
@@ -255,20 +252,22 @@ namespace bts.history
return new QueueType();
});
var ts = new Timestamp();
var key = queue.Keys.LastOrDefault(_first_in_q);
++_myid;
OrderEntry myorder = new OrderEntry(ts, (ulong)_myid, key, 0, this.security_desc, side, price, size, on_fill);
myorder.can_aggress = can_aggress;
OrderEntry myorder = new OrderEntry(asof, (ulong)_myid, _first_in_q, 0, this.security_desc, side, price, size, on_fill);
var key = queue.LastOrDefault(kvp => kvp.Value.timestamp.less_or_equal(asof), new KeyValuePair<ulong, OrderEntry>(0u, null));
if (key.Value != null)
myorder.order_priority = key.Key;
else
myorder.order_priority = _first_in_q;
Log.Testing.info($"insert myorder {key} at price { price } piq { queue.Count } ");
_myorders.Add((ulong)_myid, myorder);
// if(can_aggress)
// post_process_fill(check_my_cross(myorder)); //reentrant -- hedge can come in here
return myorder;
}
public OrderEntry change_my_order(ulong order_id, double price, uint size = uint.MinValue, bool can_aggress = false)
public OrderEntry change_my_order(Timestamp asof, ulong order_id, double price, uint size = uint.MinValue)
{
if(!_myorders.TryGetValue(order_id, out OrderEntry myorder))
return null;
@@ -276,16 +275,16 @@ namespace bts.history
myorder.price = price;
if(size > uint.MinValue)
myorder.size = size;
myorder.can_aggress = can_aggress;
var mbo = get_mbo(myorder.side);
if(mbo.TryGetValue(price, out QueueType queue))
{
var key = queue.Keys.LastOrDefault(_first_in_q);
myorder.order_priority = key;
var key = queue.LastOrDefault( kvp => kvp.Value.timestamp.less_or_equal(asof), new KeyValuePair<ulong, OrderEntry>(0u, null));
if(key.Value != null)
myorder.order_priority = key.Key;
else
myorder.order_priority = _first_in_q;
Log.Testing.info($"update myorder {order_id} at price { price } piq { queue.Count } ");
}
// if(can_aggress)
// post_process_fill( check_my_cross(myorder) );
return myorder;
}
@@ -338,19 +337,19 @@ namespace bts.history
Log.assert( !(bidoffer && trade), "Saw trade and bids / offers in same md packet");
if(bidoffer)
process_orders(md, filter);
else if(trade)
process_trades(md);
//we can't do this, because if we are first, we don't know if we're joining or crossing
//TODO so we have can_aggress, which is shitty
List<FillEntry> fills = new List<FillEntry>();
foreach(var myorder in _myorders.Values)
fills.AddRange(check_my_cross(myorder));
post_process_fill(fills);
}
void process_trades(MarketDataIncrementalRefresh md)
public void process_trades(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter)
{
bool has_trade = false;
foreach (var entry in md.MdEntries)
{
if (!filter(entry, null))
continue;
has_trade |= entry.MdEntryType == MDEntryType.Trade;
}
if(!has_trade)
return;
foreach (var trade in md.OrderIdEntries)
{
//look for the order that got filled, if we have it (could have been here since last Tuesday)
@@ -361,7 +360,7 @@ namespace bts.history
continue;
//if the priority is greater than the priority of any of my orders, fill me first
//the real order still gets decremented later by CME which isn't quite right TODO
List<FillEntry> fills = new List<FillEntry>();
List<FillEntry> tfills = new List<FillEntry>();
foreach (var myorder in _myorders.Values)
{
if (order.price >= myorder.price && myorder.side == Side.Sell ||
@@ -373,13 +372,17 @@ namespace bts.history
// int piq = q.IndexOfKey(myorder.order_priority); int count = q.Count;
var f = generate_fill(myorder, order.price, (uint)Math.Min(trade.LastQty, myorder.size),
piq, count, $" {piq} of {count} ");
fills.Add(f);
tfills.Add(f);
}
}
}
post_process_fill(fills);
post_process_fill(tfills);
}
}
List<FillEntry> fills = new List<FillEntry>();
foreach (var myorder in _myorders.Values)
fills.AddRange(check_my_cross(myorder));
post_process_fill(fills);
}
void process_orders(MarketDataIncrementalRefresh md, Func<MDEntry, OrderIDEntry, bool> filter)
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Amazon;
using Amazon.S3;
@@ -128,12 +129,54 @@ namespace bts.history
string key = make_key(dt, mic, symbol, st);
return await download(key);
}
async Task<Stream> download_s3_uncompress(string key)
{
var gz = await download(key);
var fullpath = _path + "/" + key.Replace(".gz", "");
using (FileStream writer = new FileStream(fullpath, FileMode.CreateNew))
{
using ( GZipStream ungz = new GZipStream(gz, CompressionMode.Decompress))
{
ungz.CopyTo(writer);
writer.Seek(0, SeekOrigin.Begin);
return writer;
}
}
}
public async Task<Stream> download(string key)
{
if (!is_in_cache(key))
return await download_s3(key);
return cached(key);
}
public async Task<Stream> download_uncompressed(string key)
{
var unkey = key.Replace(".gz", "");
if (!is_in_cache(unkey))
return await download_s3_uncompress(key);
return cached(unkey);
}
public async Task<Stream> download_uncompressed(DateTime dt, string symbol, api.SecurityType st)
{
var micconf = mbts.config.Configuration.get<bts.config.CmeSymbolMics>();
var micmap = micconf.cme_mic;
if (micmap.TryGetValue(symbol.ToUpper(), out api.CMEMics themic))
{
return await download_uncompressed(dt, themic, symbol, st);
}
throw new NotSupportedException("Can't find mic for symbol " + symbol);
}
public async Task<Stream> download_uncompressed(DateTime dt, api.CMEMics mic, string symbol, api.SecurityType st)
{
string key = make_key(dt, mic, symbol, st);
return await download_uncompressed(key);
}
}
}
@@ -22,8 +22,8 @@ namespace bts.history
{
if(request.SecurityType == SecurityType.Synthetic)
{
var s3f = await _dl.download(d, request.Symbol, SecurityType.Future);
var s3s = await _dl.download(d, request.Symbol, SecurityType.Spread);
var s3f = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Future);
var s3s = await _dl.download_uncompressed(d, request.Symbol, SecurityType.Spread);
using (MBOParser fparser = new MBOParser(s3f))
{
using(MBOParser sparser = new MBOParser(s3s))
@@ -44,7 +44,7 @@ namespace bts.history
}
else
{
var s3 = await _dl.download(d, request.Symbol, request.SecurityType);
var s3 = await _dl.download_uncompressed(d, request.Symbol, request.SecurityType);
using (MBOParser parser = new MBOParser(s3))
{
foreach (var (mbo,line) in parser.next<T>(null, null, request.Skip, request.LineSkip))
@@ -9,14 +9,14 @@ namespace bts.history
{
class MBOParser : IDisposable
{
Stream _stream;
Stream _stream = null;
StreamReader _reader;
QuickFix.IMessageFactory _defaultMsgFactory = new QuickFix.DefaultMessageFactory();
public MBOParser(string path) : this(File.Open(path, FileMode.Open)) {}
public MBOParser(Stream s3stream)
{
_stream = new GZipStream( s3stream, CompressionMode.Decompress );
_reader = new StreamReader(_stream, System.Text.Encoding.UTF8);
//_stream = new GZipStream( s3stream, CompressionMode.Decompress );
_reader = new StreamReader(s3stream, System.Text.Encoding.UTF8);
}
int _line = 0;
@@ -49,6 +49,8 @@ namespace bts.history
Console.WriteLine("Skipping " + ts.ToDateTime());
continue;
}
else
skip_before = null;
}
var md = ProtoFix.parse_fix<T>(line);
if(pred != null && !pred(md))