arbitraged spread and futures streams, separated replay from sending

This commit is contained in:
Kevin Darby committed 2019-10-18 22:13:30 -04:00
1 parent f3541b6c7f
commit ded1ad42e7
8 files changed
+331 -193

No files matched your search

+2 -2
View File
@@ -108,8 +108,8 @@
"--rstart", "09/30/2019",
"--rend", "09/30/2019",
"--rsymbol", "ZC",
"--rsecuritytype", "Future",
"--algo_security", "ZCZ9",
"--rsecuritytype", "Synthetic",
"--algo_security", "ZCZ9;ZCH0;ZCZ9-ZCH0",
"--algo_type", "JoinCoverLean",
"--algo_params", "size=1;lean_factor=3.5;join_factor=3.0",
],
@@ -30,15 +30,15 @@ namespace bts.history
abstract class AlgoBase
{
public Bucket bucket { get; private set;}
public string security_desc { get; private set; }
public uint security_id { get; private set; }
public Bucket bucket { get; private set;}
public Action<FillEntry> on_fill;
protected OrderBook _order_book;
List<Catalyst> _catalysts = new List<Catalyst>();
List<Reaction> _reactions = new List<Reaction>();
MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh();
Dictionary<string, int> _filter = new Dictionary<string, int>();
public AlgoBase()
{
@@ -49,10 +49,15 @@ namespace bts.history
public IReadOnlyDictionary<string, string> algo_params => _algo_params;
public virtual void init(OrderBook shared_book, Bucket bucket, string security_desc)
{
{
_order_book = shared_book;
this.bucket = bucket;
this.security_desc = security_desc;
if(security_desc.Contains(';'))
foreach(var sec in security_desc.Split(';'))
_filter[sec] = 0;
else
_filter[security_desc] = 0;
}
public virtual void init(Bucket bucket, string security_desc)
@@ -96,7 +101,7 @@ namespace bts.history
public virtual void analyze_with_latency(MarketDataIncrementalRefresh packet)
{
if(this.bucket.LowerBound == 0)
if(this.bucket.LowerBound == 0)
{
on_md(packet, _lastpacket, filter_md);
}
@@ -143,16 +148,13 @@ namespace bts.history
public virtual void post_process(MarketDataIncrementalRefresh packet, string debug_fix = null)
{
//but here we need the filter, after we lay our orders in queue
_order_book.process_md(packet, filter_md, debug_fix);
}
protected virtual bool filter_md(MDEntry x, OrderIDEntry associated_order = null)
{
bool yes =
(x.Symbol == security_desc || string.IsNullOrEmpty(security_desc)) &&
(x.MdEntryType == MDEntryType.Bid || x.MdEntryType == MDEntryType.Offer || x.MdEntryType == MDEntryType.Trade);
bool yes = _filter.ContainsKey(x.Symbol);
return yes;
}
@@ -390,6 +390,8 @@ namespace bts.history
{
foreach (var order in md.OrderIdEntries)
{
if(order.ReferenceId == 0)
continue;
// entries is indexed by 0 but ref is indexed by 1 :facepalm:
int mdidx = ((int)order.ReferenceId) - 1;
Log.assert((mdidx <= md.MdEntries.Count));
@@ -416,6 +418,8 @@ namespace bts.history
void process_one_mbo(MarketDataIncrementalRefresh md, MDEntry order)
{
if(!(order.MdEntryType == MDEntryType.Bid || order.MdEntryType == MDEntryType.Offer))
return;
if (order.MdUpdateAction == MDUpdateAction.New)
{
var q = _mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
@@ -16,120 +16,15 @@ namespace bts.history
{
class HistoricalDataReplayer
{
readonly HistoricalDataServiceServer _server;
readonly bts.co2.MarketDataReceiverService _mdservice;
Google.Protobuf.WellKnownTypes.Timestamp _time;
Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
public HistoricalDataReplayer(ServiceContext context)
readonly HistoricalDataStreamer _stream;
public HistoricalDataReplayer()
{
_server = new HistoricalDataServiceServer(context);
_mdservice = context.locate<bts.co2.MarketDataReceiverService>();
}
bts.api.ResultMessage send()
{
co2.MDReply md = new co2.MDReply();
md.Channels.AddRange(_channels.Values);
var x = _mdservice.push_md(md);
return x;
}
co2.ChannelUpdate create(Product prod, MDEntry md)
{
var chanup = new co2.ChannelUpdate()
{
Key = md.SecurityId, //TODO join with the secdefs that I don't have
Security = new Security()
{
Id = md.SecurityId,
Symbol = prod.Symbol,
Exchange = Venue.Glbx,
ContractSpec = new ContractSpecification()
{
MainFraction = 32,
SubFraction = 2,
PriceBreak = 0,
LowTick = 0.015625,
HighTick = 0.015625,
CashPerPoint = 1000.0,
PriceDisplayFormat = 3,
PriceMultiplier = 1.0,
},
Glbx = new GLBXExchangeData()
{
SecurityDesc = md.Symbol,
ExchangeId = md.SecurityId,
CmeSymbol = prod.Symbol,
}
}
};
if (prod.Type == SecurityType.Future)
{
chanup.Security.Future = new FutureContract()
{
ExpirationDay = 1,
ExpirationMonth = 9,
ExpirationYear = 2019
};
}
else if (prod.Type == SecurityType.Spread)
{
chanup.Security.Spread = new SpreadContract()
{
Normalized = true
};
}
else
System.Diagnostics.Trace.Assert(false, "unknown contract type");
return chanup;
}
void process(Product prod, MarketDataIncrementalRefresh mdr, string fix)
{
if (!_channels.TryGetValue(prod, out co2.ChannelReply chan))
{
chan = new co2.ChannelReply()
{
Platform = co2.Platform.Mdp3Onixs,
Channel = new co2.SecurityChannel()
{
Entitled = true,
Exchange = prod.Exchange,
Type = prod.Type,
Symbol = prod.Symbol,
Flags = (uint)co2.MDRequest.Types.Flags.Direct
},
};
_channels.Add(prod, chan);
}
int count = 0;
foreach (var md in mdr.MdEntries)
{
var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId);
if (upd == null)
{
upd = create(prod, md);
upd.Direct = new BookMessage();
OrderBook.init_book(upd.Direct);
chan.Updates.Add(upd);
}
OrderBook.keep_book(upd.Direct, md);
count++;
if (mdr.MatchEventIndicator == OrderBook.quote_event_done)
OrderBook.validate_book(upd.Direct, fix);
}
_stream = new HistoricalDataStreamer();
}
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
HistoricalDataOptions o)
{
bool should_send = o.should_send;
int us = o.dt_us;
if(_mdservice == null)
System.Diagnostics.Trace.Assert(!should_send);
HistoricalDataOptions o)
{
var query = new MDQuery
{
Exchange = Venue.Glbx,
@@ -140,19 +35,19 @@ namespace bts.history
};
List<IAnalyzer> anal = new List<IAnalyzer>();
string algo_security = o.algo_security;
if(!string.IsNullOrEmpty(algo_security))
if (!string.IsNullOrEmpty(algo_security))
{
var t = System.Type.GetType(o.algo_type);
if(t == null)
if (t == null)
t = System.Type.GetType("bts.history." + o.algo_type);
if(t != null)
if (t != null)
{
Func<Bucket, string, AlgoBase> func = (Bucket bucket, string security_desc) =>
Func<Bucket, string, AlgoBase> func = (Bucket bucket, string security_desc) =>
{
var inst = (AlgoBase)Activator.CreateInstance(t);
inst.init(bucket, algo_security);
string[] param = o.algo_params.Split(new[]{ ';' }, StringSplitOptions.RemoveEmptyEntries);
foreach(var par in param)
string[] param = o.algo_params.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var par in param)
{
string[] tup = par.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
Log.assert(tup.Length == 2);
@@ -163,59 +58,51 @@ namespace bts.history
anal.Add(new AlgoAnalyzer(algo_security, func));
}
else
anal.Add(new QueueAnalyzer(algo_security));
anal.Add(new QueueAnalyzer(algo_security));
}
api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol };
int count = 0;
await _server.process_md_stream<history.MarketDataIncrementalRefresh>(query, (md, line) =>
SortedList<decimal, Tuple<SecurityType, history.MarketDataIncrementalRefresh, string>> arb =
new SortedList<decimal, Tuple<SecurityType, MarketDataIncrementalRefresh, string>>();
await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
{
if (md.timestamp().ToDateTime() < end)
{
if(should_send)
bool preanal = (md.timestamp().ToDateTime() < start);
if (query.SecurityType == SecurityType.Synthetic)
{
process(prod, md, line);
maybe_send(md, us);
}
do_anal(md, anal, (md.timestamp().ToDateTime() < start), line);
count++;
decimal ts = (decimal)(md.timestamp().Seconds * 1e9) + md.timestamp().Nanos;
if(arb.ContainsKey(ts))
flush();
arb.Add(ts, Tuple.Create(sectype, md, line));
if (arb.Any(x => x.Value.Item1 == SecurityType.Future) &&
arb.Any(x => x.Value.Item1 == SecurityType.Spread) &&
arb.Last().Key != ts)
{
flush();
}
void flush()
{
foreach (var aa in arb)
do_anal(aa.Value.Item2, anal, preanal, aa.Value.Item3);
arb.Clear();
}
}
else
do_anal(md, anal, preanal, line);
}
});
}
void maybe_send(history.MarketDataIncrementalRefresh md, int us)
{
if (_time == null)
{
if (md.MatchEventIndicator == OrderBook.quote_event_done)
{
send(); //send the initial books
_time = md.timestamp();
}
else
return;
}
else
{
var ts = (md.timestamp() - _time).ToTimeSpan();
var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
if (diff > us && md.MatchEventIndicator == OrderBook.quote_event_done)
{
send(); //send the next book
_time = md.timestamp();
}
}
}
void do_anal(MarketDataIncrementalRefresh md, IEnumerable<IAnalyzer> anal, bool is_before_analysis_time, string debug_fix = null)
{
if(is_before_analysis_time)
if (is_before_analysis_time)
{
foreach(var d in anal)
foreach (var d in anal)
d.pre_analyze(md, debug_fix);
}
else
{
foreach(var d in anal)
foreach (var d in anal)
d.analyze(md, debug_fix);
}
}
@@ -0,0 +1,176 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using bts.api.core;
using bts.md;
using System.Linq;
using System.IO;
using System.Threading.Tasks;
using bts.api;
using Google.Protobuf.WellKnownTypes;
using bts.co2;
using mbts.co2;
using MathNet.Numerics.Statistics;
namespace bts.history
{
class HistoricalDataSender
{
readonly HistoricalDataStreamer _stream;
readonly bts.co2.MarketDataReceiverService _mdservice;
Google.Protobuf.WellKnownTypes.Timestamp _time;
Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
public HistoricalDataSender(ServiceContext context)
{
_stream = new HistoricalDataStreamer();
_mdservice = context.locate<bts.co2.MarketDataReceiverService>();
}
bts.api.ResultMessage send()
{
co2.MDReply md = new co2.MDReply();
md.Channels.AddRange(_channels.Values);
var x = _mdservice.push_md(md);
return x;
}
co2.ChannelUpdate create(Product prod, MDEntry md)
{
var chanup = new co2.ChannelUpdate()
{
Key = md.SecurityId, //TODO join with the secdefs that I don't have
Security = new Security()
{
Id = md.SecurityId,
Symbol = prod.Symbol,
Exchange = Venue.Glbx,
ContractSpec = new ContractSpecification()
{
MainFraction = 32,
SubFraction = 2,
PriceBreak = 0,
LowTick = 0.015625,
HighTick = 0.015625,
CashPerPoint = 1000.0,
PriceDisplayFormat = 3,
PriceMultiplier = 1.0,
},
Glbx = new GLBXExchangeData()
{
SecurityDesc = md.Symbol,
ExchangeId = md.SecurityId,
CmeSymbol = prod.Symbol,
}
}
};
if (prod.Type == SecurityType.Future)
{
chanup.Security.Future = new FutureContract()
{
ExpirationDay = 1,
ExpirationMonth = 9,
ExpirationYear = 2019
};
}
else if (prod.Type == SecurityType.Spread)
{
chanup.Security.Spread = new SpreadContract()
{
Normalized = true
};
}
else
System.Diagnostics.Trace.Assert(false, "unknown contract type");
return chanup;
}
void process(Product prod, MarketDataIncrementalRefresh mdr, string fix)
{
if (!_channels.TryGetValue(prod, out co2.ChannelReply chan))
{
chan = new co2.ChannelReply()
{
Platform = co2.Platform.Mdp3Onixs,
Channel = new co2.SecurityChannel()
{
Entitled = true,
Exchange = prod.Exchange,
Type = prod.Type,
Symbol = prod.Symbol,
Flags = (uint)co2.MDRequest.Types.Flags.Direct
},
};
_channels.Add(prod, chan);
}
int count = 0;
foreach (var md in mdr.MdEntries)
{
var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId);
if (upd == null)
{
upd = create(prod, md);
upd.Direct = new BookMessage();
OrderBook.init_book(upd.Direct);
chan.Updates.Add(upd);
}
OrderBook.keep_book(upd.Direct, md);
count++;
if (mdr.MatchEventIndicator == OrderBook.quote_event_done)
OrderBook.validate_book(upd.Direct, fix);
}
}
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
HistoricalDataOptions o)
{
int us = o.dt_us;
var query = new MDQuery
{
Exchange = Venue.Glbx,
Symbol = symbol,
SecurityType = st,
Begin = Timestamp.FromDateTime(DateTime.SpecifyKind(start, DateTimeKind.Utc)),
End = Timestamp.FromDateTime(DateTime.SpecifyKind(end, DateTimeKind.Utc))
};
api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol };
int count = 0;
await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
{
if (md.timestamp().ToDateTime() < end)
{
process(prod, md, line);
maybe_send(md, us);
count++;
}
});
}
void maybe_send(history.MarketDataIncrementalRefresh md, int us)
{
if (_time == null)
{
if (md.MatchEventIndicator == OrderBook.quote_event_done)
{
send(); //send the initial books
_time = md.timestamp();
}
else
return;
}
else
{
var ts = (md.timestamp() - _time).ToTimeSpan();
var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
if (diff > us && md.MatchEventIndicator == OrderBook.quote_event_done)
{
send(); //send the next book
_time = md.timestamp();
}
}
}
}
}
@@ -7,12 +7,13 @@ using System.Linq;
using System.IO;
using System.Threading.Tasks;
using bts.api;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
class HistoricalDataServiceServer : HistoricalDataServiceBase
{
readonly HistoricalDataDownloader _dl = new HistoricalDataDownloader();
readonly HistoricalDataStreamer _stream = new HistoricalDataStreamer();
public HistoricalDataServiceServer(ServiceContext context) : base(context, "")
{
}
@@ -22,32 +23,10 @@ namespace bts.history
if(request.Exchange != Venue.Glbx)
throw new NotSupportedException("HistoricalDataService only supports glbx right now");
var str = new MDStream();
await process_md_stream<history.MarketDataIncrementalRefresh>(request, (mbo, _) => str.Md.Add(mbo));
await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(request, (sectype, mbo, _) => str.Md.Add(mbo));
return str;
}
async public Task process_md_stream<T>(MDQuery request, Action<T, string> action) where T : Google.Protobuf.IMessage, new()
{
if (request.Exchange != Venue.Glbx)
throw new NotSupportedException("HistoricalDataService only supports glbx right now");
var str = new MDStream();
DateTime d = request.Begin.ToDateTime();
while (d <= request.End.ToDateTime())
{
try
{
var s3 = await _dl.download(d, request.Symbol, request.SecurityType);
using (MBOParser parser = new MBOParser(s3))
{
foreach (var (mbo,line) in parser.next<T>())
action(mbo, line.Replace('\u0001', '|'));
}
}
catch (Amazon.S3.AmazonS3Exception) { }
d = d.AddDays(1.0).Date;
}
}
async public override Task<TradingDistribution> get_trading_distribution(TradingDistributionQuery request)
{
var str = await get_md_stream(new MDQuery { Symbol = request.Symbol, Exchange = request.Exchange, SecurityType = request.SecurityType,
@@ -65,6 +44,15 @@ namespace bts.history
async public override Task<Slices> get_trading_slices(SlicesQuery request)
{
var str = await get_md_stream(new MDQuery
{
Symbol = request.Symbol,
Exchange = request.Exchange,
SecurityType = request.SecurityType,
Begin = request.Begin,
End = request.End
});
throw new NotImplementedException();
}
}
@@ -0,0 +1,69 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using bts.api.core;
using bts.md;
using System.Linq;
using System.IO;
using System.Threading.Tasks;
using bts.api;
using Google.Protobuf.WellKnownTypes;
namespace bts.history
{
class HistoricalDataStreamer
{
readonly HistoricalDataDownloader _dl = new HistoricalDataDownloader();
public HistoricalDataStreamer()
{
}
#pragma warning disable CS0183
async public Task process_md_stream<T>(MDQuery request, Action<SecurityType, T, string> action) where T : Google.Protobuf.IMessage, new()
{
if (request.Exchange != Venue.Glbx)
throw new NotSupportedException("HistoricalDataService only supports glbx right now");
var str = new MDStream();
DateTime d = request.Begin.ToDateTime();
while (d <= request.End.ToDateTime())
{
try
{
if(request.SecurityType == SecurityType.Synthetic)
{
var s3f = await _dl.download(d, request.Symbol, SecurityType.Future);
var s3s = await _dl.download(d, request.Symbol, SecurityType.Spread);
using (MBOParser fparser = new MBOParser(s3f))
{
using(MBOParser sparser = new MBOParser(s3s))
{
var fut = fparser.next<T>().GetEnumerator();
var spr = sparser.next<T>().GetEnumerator();
bool futgo , sprgo;
while( (futgo = fut.MoveNext()) is bool &&
(sprgo = spr.MoveNext()) is bool )
{
if(futgo)
action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2.Replace('\u0001', '|'));
if(sprgo)
action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2.Replace('\u0001', '|'));
}
}
}
}
else
{
var s3 = await _dl.download(d, request.Symbol, request.SecurityType);
using (MBOParser parser = new MBOParser(s3))
{
foreach (var (mbo,line) in parser.next<T>())
action(request.SecurityType, mbo, line.Replace('\u0001', '|'));
}
}
}
catch (Amazon.S3.AmazonS3Exception) { }
d = d.AddDays(1.0).Date;
}
}
}
}
@@ -63,7 +63,7 @@ namespace bts.history
[Option('1', "algo_security", Required = false, HelpText = "Security Desc i.e. ZCZ9 on which to run stat analysis")]
public string algo_security { get; set; }
[Option('2', "algo_type", Required = false, Default = "", HelpText = "which algo to run")]
[Option('2', "algo_type", Required = false, Default = "", HelpText = "which algo to run, leave blank for QueueAnalyzer")]
public string algo_type { get; set; }
[Option('3', "algo_params", Required = false, Default = "size=1;", HelpText = "semicolon dellimited list of key=value pairs to parameterize algo")]
@@ -82,6 +82,21 @@ namespace bts.history
{
Console.WriteLine(o.ToString());
if(o.should_send)
{
var start = DateTime.Parse(o.replaystart);
var end = DateTime.Parse(o.replayend).AddDays(1.0); //incl
SecurityType st = SecurityType.Future;
if (!string.IsNullOrEmpty(o.replaysecurity))
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecurity, true);
using (var ctx = new ServiceContext())
{
var rep = new HistoricalDataSender(ctx);
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
}
return 0;
}
if (!string.IsNullOrEmpty(o.replayend) && !string.IsNullOrEmpty(o.replaystart) &&
!string.IsNullOrEmpty(o.replaysymbol) )
{
@@ -90,13 +105,10 @@ namespace bts.history
SecurityType st = SecurityType.Future;
if(!string.IsNullOrEmpty(o.replaysecurity))
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecurity, true);
using (var ctx = new ServiceContext())
{
var rep = new HistoricalDataReplayer(ctx);
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
}
var rep = new HistoricalDataReplayer();
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
return 0;
}
}
if (!string.IsNullOrEmpty(o.s3key))
{