arbitraged spread and futures streams, separated replay from sending
This commit is contained in:
1 parent
f3541b6c7f
commit
ded1ad42e7
8 files changed
+331
-193
No files matched your search
Vendored
+2
-2
@@ -108,8 +108,8 @@
|
||||
"--rstart", "09/30/2019",
|
||||
"--rend", "09/30/2019",
|
||||
"--rsymbol", "ZC",
|
||||
"--rsecuritytype", "Future",
|
||||
"--algo_security", "ZCZ9",
|
||||
"--rsecuritytype", "Synthetic",
|
||||
"--algo_security", "ZCZ9;ZCH0;ZCZ9-ZCH0",
|
||||
"--algo_type", "JoinCoverLean",
|
||||
"--algo_params", "size=1;lean_factor=3.5;join_factor=3.0",
|
||||
],
|
||||
|
||||
@@ -30,15 +30,15 @@ namespace bts.history
|
||||
|
||||
abstract class AlgoBase
|
||||
{
|
||||
public Bucket bucket { get; private set;}
|
||||
public string security_desc { get; private set; }
|
||||
public uint security_id { get; private set; }
|
||||
public Bucket bucket { get; private set;}
|
||||
public Action<FillEntry> on_fill;
|
||||
|
||||
protected OrderBook _order_book;
|
||||
List<Catalyst> _catalysts = new List<Catalyst>();
|
||||
List<Reaction> _reactions = new List<Reaction>();
|
||||
MarketDataIncrementalRefresh _lastpacket = new MarketDataIncrementalRefresh();
|
||||
Dictionary<string, int> _filter = new Dictionary<string, int>();
|
||||
|
||||
public AlgoBase()
|
||||
{
|
||||
@@ -49,10 +49,15 @@ namespace bts.history
|
||||
public IReadOnlyDictionary<string, string> algo_params => _algo_params;
|
||||
|
||||
public virtual void init(OrderBook shared_book, Bucket bucket, string security_desc)
|
||||
{
|
||||
{
|
||||
_order_book = shared_book;
|
||||
this.bucket = bucket;
|
||||
this.security_desc = security_desc;
|
||||
if(security_desc.Contains(';'))
|
||||
foreach(var sec in security_desc.Split(';'))
|
||||
_filter[sec] = 0;
|
||||
else
|
||||
_filter[security_desc] = 0;
|
||||
}
|
||||
|
||||
public virtual void init(Bucket bucket, string security_desc)
|
||||
@@ -96,7 +101,7 @@ namespace bts.history
|
||||
|
||||
public virtual void analyze_with_latency(MarketDataIncrementalRefresh packet)
|
||||
{
|
||||
if(this.bucket.LowerBound == 0)
|
||||
if(this.bucket.LowerBound == 0)
|
||||
{
|
||||
on_md(packet, _lastpacket, filter_md);
|
||||
}
|
||||
@@ -143,16 +148,13 @@ namespace bts.history
|
||||
|
||||
public virtual void post_process(MarketDataIncrementalRefresh packet, string debug_fix = null)
|
||||
{
|
||||
//but here we need the filter, after we lay our orders in queue
|
||||
_order_book.process_md(packet, filter_md, debug_fix);
|
||||
}
|
||||
|
||||
|
||||
protected virtual bool filter_md(MDEntry x, OrderIDEntry associated_order = null)
|
||||
{
|
||||
bool yes =
|
||||
(x.Symbol == security_desc || string.IsNullOrEmpty(security_desc)) &&
|
||||
(x.MdEntryType == MDEntryType.Bid || x.MdEntryType == MDEntryType.Offer || x.MdEntryType == MDEntryType.Trade);
|
||||
bool yes = _filter.ContainsKey(x.Symbol);
|
||||
return yes;
|
||||
}
|
||||
|
||||
|
||||
@@ -390,6 +390,8 @@ namespace bts.history
|
||||
{
|
||||
foreach (var order in md.OrderIdEntries)
|
||||
{
|
||||
if(order.ReferenceId == 0)
|
||||
continue;
|
||||
// entries is indexed by 0 but ref is indexed by 1 :facepalm:
|
||||
int mdidx = ((int)order.ReferenceId) - 1;
|
||||
Log.assert((mdidx <= md.MdEntries.Count));
|
||||
@@ -416,6 +418,8 @@ namespace bts.history
|
||||
|
||||
void process_one_mbo(MarketDataIncrementalRefresh md, MDEntry order)
|
||||
{
|
||||
if(!(order.MdEntryType == MDEntryType.Bid || order.MdEntryType == MDEntryType.Offer))
|
||||
return;
|
||||
if (order.MdUpdateAction == MDUpdateAction.New)
|
||||
{
|
||||
var q = _mbo.GetOrAdd(order.MdEntryPx, x => new QueueType());
|
||||
|
||||
@@ -16,120 +16,15 @@ namespace bts.history
|
||||
{
|
||||
class HistoricalDataReplayer
|
||||
{
|
||||
readonly HistoricalDataServiceServer _server;
|
||||
readonly bts.co2.MarketDataReceiverService _mdservice;
|
||||
|
||||
Google.Protobuf.WellKnownTypes.Timestamp _time;
|
||||
Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
|
||||
|
||||
public HistoricalDataReplayer(ServiceContext context)
|
||||
readonly HistoricalDataStreamer _stream;
|
||||
public HistoricalDataReplayer()
|
||||
{
|
||||
_server = new HistoricalDataServiceServer(context);
|
||||
_mdservice = context.locate<bts.co2.MarketDataReceiverService>();
|
||||
}
|
||||
|
||||
bts.api.ResultMessage send()
|
||||
{
|
||||
co2.MDReply md = new co2.MDReply();
|
||||
md.Channels.AddRange(_channels.Values);
|
||||
var x = _mdservice.push_md(md);
|
||||
return x;
|
||||
}
|
||||
|
||||
co2.ChannelUpdate create(Product prod, MDEntry md)
|
||||
{
|
||||
var chanup = new co2.ChannelUpdate()
|
||||
{
|
||||
Key = md.SecurityId, //TODO join with the secdefs that I don't have
|
||||
Security = new Security()
|
||||
{
|
||||
Id = md.SecurityId,
|
||||
Symbol = prod.Symbol,
|
||||
Exchange = Venue.Glbx,
|
||||
ContractSpec = new ContractSpecification()
|
||||
{
|
||||
MainFraction = 32,
|
||||
SubFraction = 2,
|
||||
PriceBreak = 0,
|
||||
LowTick = 0.015625,
|
||||
HighTick = 0.015625,
|
||||
CashPerPoint = 1000.0,
|
||||
PriceDisplayFormat = 3,
|
||||
PriceMultiplier = 1.0,
|
||||
},
|
||||
Glbx = new GLBXExchangeData()
|
||||
{
|
||||
SecurityDesc = md.Symbol,
|
||||
ExchangeId = md.SecurityId,
|
||||
CmeSymbol = prod.Symbol,
|
||||
}
|
||||
}
|
||||
};
|
||||
if (prod.Type == SecurityType.Future)
|
||||
{
|
||||
chanup.Security.Future = new FutureContract()
|
||||
{
|
||||
ExpirationDay = 1,
|
||||
ExpirationMonth = 9,
|
||||
ExpirationYear = 2019
|
||||
};
|
||||
}
|
||||
else if (prod.Type == SecurityType.Spread)
|
||||
{
|
||||
chanup.Security.Spread = new SpreadContract()
|
||||
{
|
||||
Normalized = true
|
||||
};
|
||||
}
|
||||
else
|
||||
System.Diagnostics.Trace.Assert(false, "unknown contract type");
|
||||
return chanup;
|
||||
}
|
||||
|
||||
void process(Product prod, MarketDataIncrementalRefresh mdr, string fix)
|
||||
{
|
||||
if (!_channels.TryGetValue(prod, out co2.ChannelReply chan))
|
||||
{
|
||||
chan = new co2.ChannelReply()
|
||||
{
|
||||
Platform = co2.Platform.Mdp3Onixs,
|
||||
Channel = new co2.SecurityChannel()
|
||||
{
|
||||
Entitled = true,
|
||||
Exchange = prod.Exchange,
|
||||
Type = prod.Type,
|
||||
Symbol = prod.Symbol,
|
||||
Flags = (uint)co2.MDRequest.Types.Flags.Direct
|
||||
},
|
||||
};
|
||||
_channels.Add(prod, chan);
|
||||
}
|
||||
int count = 0;
|
||||
|
||||
foreach (var md in mdr.MdEntries)
|
||||
{
|
||||
var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId);
|
||||
if (upd == null)
|
||||
{
|
||||
upd = create(prod, md);
|
||||
upd.Direct = new BookMessage();
|
||||
OrderBook.init_book(upd.Direct);
|
||||
chan.Updates.Add(upd);
|
||||
}
|
||||
OrderBook.keep_book(upd.Direct, md);
|
||||
count++;
|
||||
if (mdr.MatchEventIndicator == OrderBook.quote_event_done)
|
||||
OrderBook.validate_book(upd.Direct, fix);
|
||||
}
|
||||
_stream = new HistoricalDataStreamer();
|
||||
}
|
||||
|
||||
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
|
||||
HistoricalDataOptions o)
|
||||
{
|
||||
bool should_send = o.should_send;
|
||||
int us = o.dt_us;
|
||||
if(_mdservice == null)
|
||||
System.Diagnostics.Trace.Assert(!should_send);
|
||||
HistoricalDataOptions o)
|
||||
{
|
||||
var query = new MDQuery
|
||||
{
|
||||
Exchange = Venue.Glbx,
|
||||
@@ -140,19 +35,19 @@ namespace bts.history
|
||||
};
|
||||
List<IAnalyzer> anal = new List<IAnalyzer>();
|
||||
string algo_security = o.algo_security;
|
||||
if(!string.IsNullOrEmpty(algo_security))
|
||||
if (!string.IsNullOrEmpty(algo_security))
|
||||
{
|
||||
var t = System.Type.GetType(o.algo_type);
|
||||
if(t == null)
|
||||
if (t == null)
|
||||
t = System.Type.GetType("bts.history." + o.algo_type);
|
||||
if(t != null)
|
||||
if (t != null)
|
||||
{
|
||||
Func<Bucket, string, AlgoBase> func = (Bucket bucket, string security_desc) =>
|
||||
Func<Bucket, string, AlgoBase> func = (Bucket bucket, string security_desc) =>
|
||||
{
|
||||
var inst = (AlgoBase)Activator.CreateInstance(t);
|
||||
inst.init(bucket, algo_security);
|
||||
string[] param = o.algo_params.Split(new[]{ ';' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
foreach(var par in param)
|
||||
string[] param = o.algo_params.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
foreach (var par in param)
|
||||
{
|
||||
string[] tup = par.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
Log.assert(tup.Length == 2);
|
||||
@@ -163,59 +58,51 @@ namespace bts.history
|
||||
anal.Add(new AlgoAnalyzer(algo_security, func));
|
||||
}
|
||||
else
|
||||
anal.Add(new QueueAnalyzer(algo_security));
|
||||
anal.Add(new QueueAnalyzer(algo_security));
|
||||
}
|
||||
api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol };
|
||||
int count = 0;
|
||||
await _server.process_md_stream<history.MarketDataIncrementalRefresh>(query, (md, line) =>
|
||||
SortedList<decimal, Tuple<SecurityType, history.MarketDataIncrementalRefresh, string>> arb =
|
||||
new SortedList<decimal, Tuple<SecurityType, MarketDataIncrementalRefresh, string>>();
|
||||
await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
|
||||
{
|
||||
if (md.timestamp().ToDateTime() < end)
|
||||
{
|
||||
if(should_send)
|
||||
bool preanal = (md.timestamp().ToDateTime() < start);
|
||||
if (query.SecurityType == SecurityType.Synthetic)
|
||||
{
|
||||
process(prod, md, line);
|
||||
maybe_send(md, us);
|
||||
}
|
||||
do_anal(md, anal, (md.timestamp().ToDateTime() < start), line);
|
||||
count++;
|
||||
decimal ts = (decimal)(md.timestamp().Seconds * 1e9) + md.timestamp().Nanos;
|
||||
if(arb.ContainsKey(ts))
|
||||
flush();
|
||||
arb.Add(ts, Tuple.Create(sectype, md, line));
|
||||
if (arb.Any(x => x.Value.Item1 == SecurityType.Future) &&
|
||||
arb.Any(x => x.Value.Item1 == SecurityType.Spread) &&
|
||||
arb.Last().Key != ts)
|
||||
{
|
||||
flush();
|
||||
}
|
||||
void flush()
|
||||
{
|
||||
foreach (var aa in arb)
|
||||
do_anal(aa.Value.Item2, anal, preanal, aa.Value.Item3);
|
||||
arb.Clear();
|
||||
}
|
||||
}
|
||||
else
|
||||
do_anal(md, anal, preanal, line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void maybe_send(history.MarketDataIncrementalRefresh md, int us)
|
||||
{
|
||||
if (_time == null)
|
||||
{
|
||||
if (md.MatchEventIndicator == OrderBook.quote_event_done)
|
||||
{
|
||||
send(); //send the initial books
|
||||
_time = md.timestamp();
|
||||
}
|
||||
else
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
var ts = (md.timestamp() - _time).ToTimeSpan();
|
||||
var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
|
||||
if (diff > us && md.MatchEventIndicator == OrderBook.quote_event_done)
|
||||
{
|
||||
send(); //send the next book
|
||||
_time = md.timestamp();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void do_anal(MarketDataIncrementalRefresh md, IEnumerable<IAnalyzer> anal, bool is_before_analysis_time, string debug_fix = null)
|
||||
{
|
||||
if(is_before_analysis_time)
|
||||
if (is_before_analysis_time)
|
||||
{
|
||||
foreach(var d in anal)
|
||||
foreach (var d in anal)
|
||||
d.pre_analyze(md, debug_fix);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach(var d in anal)
|
||||
foreach (var d in anal)
|
||||
d.analyze(md, debug_fix);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using bts.api.core;
|
||||
using bts.md;
|
||||
using System.Linq;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using bts.api;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using bts.co2;
|
||||
using mbts.co2;
|
||||
using MathNet.Numerics.Statistics;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataSender
|
||||
{
|
||||
readonly HistoricalDataStreamer _stream;
|
||||
readonly bts.co2.MarketDataReceiverService _mdservice;
|
||||
|
||||
Google.Protobuf.WellKnownTypes.Timestamp _time;
|
||||
Dictionary<Product, co2.ChannelReply> _channels = new Dictionary<Product, co2.ChannelReply>();
|
||||
|
||||
public HistoricalDataSender(ServiceContext context)
|
||||
{
|
||||
_stream = new HistoricalDataStreamer();
|
||||
_mdservice = context.locate<bts.co2.MarketDataReceiverService>();
|
||||
}
|
||||
|
||||
bts.api.ResultMessage send()
|
||||
{
|
||||
co2.MDReply md = new co2.MDReply();
|
||||
md.Channels.AddRange(_channels.Values);
|
||||
var x = _mdservice.push_md(md);
|
||||
return x;
|
||||
}
|
||||
|
||||
co2.ChannelUpdate create(Product prod, MDEntry md)
|
||||
{
|
||||
var chanup = new co2.ChannelUpdate()
|
||||
{
|
||||
Key = md.SecurityId, //TODO join with the secdefs that I don't have
|
||||
Security = new Security()
|
||||
{
|
||||
Id = md.SecurityId,
|
||||
Symbol = prod.Symbol,
|
||||
Exchange = Venue.Glbx,
|
||||
ContractSpec = new ContractSpecification()
|
||||
{
|
||||
MainFraction = 32,
|
||||
SubFraction = 2,
|
||||
PriceBreak = 0,
|
||||
LowTick = 0.015625,
|
||||
HighTick = 0.015625,
|
||||
CashPerPoint = 1000.0,
|
||||
PriceDisplayFormat = 3,
|
||||
PriceMultiplier = 1.0,
|
||||
},
|
||||
Glbx = new GLBXExchangeData()
|
||||
{
|
||||
SecurityDesc = md.Symbol,
|
||||
ExchangeId = md.SecurityId,
|
||||
CmeSymbol = prod.Symbol,
|
||||
}
|
||||
}
|
||||
};
|
||||
if (prod.Type == SecurityType.Future)
|
||||
{
|
||||
chanup.Security.Future = new FutureContract()
|
||||
{
|
||||
ExpirationDay = 1,
|
||||
ExpirationMonth = 9,
|
||||
ExpirationYear = 2019
|
||||
};
|
||||
}
|
||||
else if (prod.Type == SecurityType.Spread)
|
||||
{
|
||||
chanup.Security.Spread = new SpreadContract()
|
||||
{
|
||||
Normalized = true
|
||||
};
|
||||
}
|
||||
else
|
||||
System.Diagnostics.Trace.Assert(false, "unknown contract type");
|
||||
return chanup;
|
||||
}
|
||||
|
||||
void process(Product prod, MarketDataIncrementalRefresh mdr, string fix)
|
||||
{
|
||||
if (!_channels.TryGetValue(prod, out co2.ChannelReply chan))
|
||||
{
|
||||
chan = new co2.ChannelReply()
|
||||
{
|
||||
Platform = co2.Platform.Mdp3Onixs,
|
||||
Channel = new co2.SecurityChannel()
|
||||
{
|
||||
Entitled = true,
|
||||
Exchange = prod.Exchange,
|
||||
Type = prod.Type,
|
||||
Symbol = prod.Symbol,
|
||||
Flags = (uint)co2.MDRequest.Types.Flags.Direct
|
||||
},
|
||||
};
|
||||
_channels.Add(prod, chan);
|
||||
}
|
||||
int count = 0;
|
||||
|
||||
foreach (var md in mdr.MdEntries)
|
||||
{
|
||||
var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId);
|
||||
if (upd == null)
|
||||
{
|
||||
upd = create(prod, md);
|
||||
upd.Direct = new BookMessage();
|
||||
OrderBook.init_book(upd.Direct);
|
||||
chan.Updates.Add(upd);
|
||||
}
|
||||
OrderBook.keep_book(upd.Direct, md);
|
||||
count++;
|
||||
if (mdr.MatchEventIndicator == OrderBook.quote_event_done)
|
||||
OrderBook.validate_book(upd.Direct, fix);
|
||||
}
|
||||
}
|
||||
|
||||
async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st,
|
||||
HistoricalDataOptions o)
|
||||
{
|
||||
int us = o.dt_us;
|
||||
var query = new MDQuery
|
||||
{
|
||||
Exchange = Venue.Glbx,
|
||||
Symbol = symbol,
|
||||
SecurityType = st,
|
||||
Begin = Timestamp.FromDateTime(DateTime.SpecifyKind(start, DateTimeKind.Utc)),
|
||||
End = Timestamp.FromDateTime(DateTime.SpecifyKind(end, DateTimeKind.Utc))
|
||||
};
|
||||
api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol };
|
||||
int count = 0;
|
||||
await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(query, (sectype, md, line) =>
|
||||
{
|
||||
if (md.timestamp().ToDateTime() < end)
|
||||
{
|
||||
process(prod, md, line);
|
||||
maybe_send(md, us);
|
||||
count++;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void maybe_send(history.MarketDataIncrementalRefresh md, int us)
|
||||
{
|
||||
if (_time == null)
|
||||
{
|
||||
if (md.MatchEventIndicator == OrderBook.quote_event_done)
|
||||
{
|
||||
send(); //send the initial books
|
||||
_time = md.timestamp();
|
||||
}
|
||||
else
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
var ts = (md.timestamp() - _time).ToTimeSpan();
|
||||
var diff = ts.Ticks / (TimeSpan.TicksPerMillisecond / 1000);
|
||||
if (diff > us && md.MatchEventIndicator == OrderBook.quote_event_done)
|
||||
{
|
||||
send(); //send the next book
|
||||
_time = md.timestamp();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,13 @@ using System.Linq;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using bts.api;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataServiceServer : HistoricalDataServiceBase
|
||||
{
|
||||
readonly HistoricalDataDownloader _dl = new HistoricalDataDownloader();
|
||||
readonly HistoricalDataStreamer _stream = new HistoricalDataStreamer();
|
||||
public HistoricalDataServiceServer(ServiceContext context) : base(context, "")
|
||||
{
|
||||
}
|
||||
@@ -22,32 +23,10 @@ namespace bts.history
|
||||
if(request.Exchange != Venue.Glbx)
|
||||
throw new NotSupportedException("HistoricalDataService only supports glbx right now");
|
||||
var str = new MDStream();
|
||||
await process_md_stream<history.MarketDataIncrementalRefresh>(request, (mbo, _) => str.Md.Add(mbo));
|
||||
await _stream.process_md_stream<history.MarketDataIncrementalRefresh>(request, (sectype, mbo, _) => str.Md.Add(mbo));
|
||||
return str;
|
||||
}
|
||||
|
||||
async public Task process_md_stream<T>(MDQuery request, Action<T, string> action) where T : Google.Protobuf.IMessage, new()
|
||||
{
|
||||
if (request.Exchange != Venue.Glbx)
|
||||
throw new NotSupportedException("HistoricalDataService only supports glbx right now");
|
||||
var str = new MDStream();
|
||||
DateTime d = request.Begin.ToDateTime();
|
||||
while (d <= request.End.ToDateTime())
|
||||
{
|
||||
try
|
||||
{
|
||||
var s3 = await _dl.download(d, request.Symbol, request.SecurityType);
|
||||
using (MBOParser parser = new MBOParser(s3))
|
||||
{
|
||||
foreach (var (mbo,line) in parser.next<T>())
|
||||
action(mbo, line.Replace('\u0001', '|'));
|
||||
}
|
||||
}
|
||||
catch (Amazon.S3.AmazonS3Exception) { }
|
||||
d = d.AddDays(1.0).Date;
|
||||
}
|
||||
}
|
||||
|
||||
async public override Task<TradingDistribution> get_trading_distribution(TradingDistributionQuery request)
|
||||
{
|
||||
var str = await get_md_stream(new MDQuery { Symbol = request.Symbol, Exchange = request.Exchange, SecurityType = request.SecurityType,
|
||||
@@ -65,6 +44,15 @@ namespace bts.history
|
||||
|
||||
async public override Task<Slices> get_trading_slices(SlicesQuery request)
|
||||
{
|
||||
var str = await get_md_stream(new MDQuery
|
||||
{
|
||||
Symbol = request.Symbol,
|
||||
Exchange = request.Exchange,
|
||||
SecurityType = request.SecurityType,
|
||||
Begin = request.Begin,
|
||||
End = request.End
|
||||
});
|
||||
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
using bts.api.core;
|
||||
using bts.md;
|
||||
using System.Linq;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using bts.api;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
|
||||
namespace bts.history
|
||||
{
|
||||
class HistoricalDataStreamer
|
||||
{
|
||||
readonly HistoricalDataDownloader _dl = new HistoricalDataDownloader();
|
||||
public HistoricalDataStreamer()
|
||||
{
|
||||
}
|
||||
|
||||
#pragma warning disable CS0183
|
||||
async public Task process_md_stream<T>(MDQuery request, Action<SecurityType, T, string> action) where T : Google.Protobuf.IMessage, new()
|
||||
{
|
||||
if (request.Exchange != Venue.Glbx)
|
||||
throw new NotSupportedException("HistoricalDataService only supports glbx right now");
|
||||
var str = new MDStream();
|
||||
DateTime d = request.Begin.ToDateTime();
|
||||
while (d <= request.End.ToDateTime())
|
||||
{
|
||||
try
|
||||
{
|
||||
if(request.SecurityType == SecurityType.Synthetic)
|
||||
{
|
||||
var s3f = await _dl.download(d, request.Symbol, SecurityType.Future);
|
||||
var s3s = await _dl.download(d, request.Symbol, SecurityType.Spread);
|
||||
using (MBOParser fparser = new MBOParser(s3f))
|
||||
{
|
||||
using(MBOParser sparser = new MBOParser(s3s))
|
||||
{
|
||||
var fut = fparser.next<T>().GetEnumerator();
|
||||
var spr = sparser.next<T>().GetEnumerator();
|
||||
bool futgo , sprgo;
|
||||
while( (futgo = fut.MoveNext()) is bool &&
|
||||
(sprgo = spr.MoveNext()) is bool )
|
||||
{
|
||||
if(futgo)
|
||||
action(SecurityType.Future, fut.Current.Item1, fut.Current.Item2.Replace('\u0001', '|'));
|
||||
if(sprgo)
|
||||
action(SecurityType.Spread, spr.Current.Item1, spr.Current.Item2.Replace('\u0001', '|'));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var s3 = await _dl.download(d, request.Symbol, request.SecurityType);
|
||||
using (MBOParser parser = new MBOParser(s3))
|
||||
{
|
||||
foreach (var (mbo,line) in parser.next<T>())
|
||||
action(request.SecurityType, mbo, line.Replace('\u0001', '|'));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Amazon.S3.AmazonS3Exception) { }
|
||||
d = d.AddDays(1.0).Date;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,7 +63,7 @@ namespace bts.history
|
||||
[Option('1', "algo_security", Required = false, HelpText = "Security Desc i.e. ZCZ9 on which to run stat analysis")]
|
||||
public string algo_security { get; set; }
|
||||
|
||||
[Option('2', "algo_type", Required = false, Default = "", HelpText = "which algo to run")]
|
||||
[Option('2', "algo_type", Required = false, Default = "", HelpText = "which algo to run, leave blank for QueueAnalyzer")]
|
||||
public string algo_type { get; set; }
|
||||
|
||||
[Option('3', "algo_params", Required = false, Default = "size=1;", HelpText = "semicolon dellimited list of key=value pairs to parameterize algo")]
|
||||
@@ -82,6 +82,21 @@ namespace bts.history
|
||||
{
|
||||
Console.WriteLine(o.ToString());
|
||||
|
||||
if(o.should_send)
|
||||
{
|
||||
var start = DateTime.Parse(o.replaystart);
|
||||
var end = DateTime.Parse(o.replayend).AddDays(1.0); //incl
|
||||
SecurityType st = SecurityType.Future;
|
||||
if (!string.IsNullOrEmpty(o.replaysecurity))
|
||||
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecurity, true);
|
||||
using (var ctx = new ServiceContext())
|
||||
{
|
||||
var rep = new HistoricalDataSender(ctx);
|
||||
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(o.replayend) && !string.IsNullOrEmpty(o.replaystart) &&
|
||||
!string.IsNullOrEmpty(o.replaysymbol) )
|
||||
{
|
||||
@@ -90,13 +105,10 @@ namespace bts.history
|
||||
SecurityType st = SecurityType.Future;
|
||||
if(!string.IsNullOrEmpty(o.replaysecurity))
|
||||
st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecurity, true);
|
||||
using (var ctx = new ServiceContext())
|
||||
{
|
||||
var rep = new HistoricalDataReplayer(ctx);
|
||||
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
|
||||
}
|
||||
var rep = new HistoricalDataReplayer();
|
||||
rep.replay(start, end, o.replaysymbol.ToUpper(), st, o).GetAwaiter().GetResult();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(o.s3key))
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user