Files
spark/src/client/spark_historical/Query/Query.cs
T
2019-12-13 14:15:29 -05:00

369 lines
16 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Linq.Dynamic.Core;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Google.Protobuf.WellKnownTypes;
using bts.co2;
using mbts.co2;
using MathNet.Numerics.Statistics;
namespace bts.history
{
public class Query
{
readonly IMongoClient _client;
readonly IMongoDatabase _db;
public Query(string database, string host = "sparkvm", int port = 27017,
string user = "mongo", string pass = "enter")
{
MongoGenerator.setup_bson();
var conn = $@"mongodb://{user}:{pass}@{host}:{port}/admin";
_client = new MongoClient(conn);
_db = _client.GetDatabase(database);
}
public IMongoCollection<T> collection<T>(string collection)
{
var c = _db.GetCollection<T>(collection);
return c;
}
public IMongoQueryable<T> queryable<T>(string collection)
{
var coll = _db.GetCollection<T>(collection);
IMongoQueryable<T> res = queryable(coll);
return res;
}
public IMongoQueryable<T> queryable<T>(IMongoCollection<T> collection) =>
collection.AsQueryable(new AggregateOptions { AllowDiskUse = true } );
public IMongoQueryable<H> timeseries<H>(IMongoCollection<H> collection, DateTime begin, DateTime end, bool use_sending_time = true)
where H : IHasTimestamp
{
IMongoQueryable<H> res = queryable(collection);
Expression<Func<H, bool>> timeexpr = null;
begin = begin.ToUniversalTime();
end = end.ToUniversalTime();
if(use_sending_time)
timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end;
else
timeexpr = (H h) => h.transact_time >= begin && h.transact_time <= end;
res = res.Where( timeexpr );
return res;
}
//so I can get into a local linq through the jupyter nb
public static IEnumerable<T> local<T>(IMongoQueryable<T> q)
{
return q.AsEnumerable<T>();
}
public static IEnumerable<U> stream_data<T, U>(IEnumerable<T>[] datastream, Func<T[], int, IEnumerable<U>> function )
where T : class, IHasTimestamp
where U : class
{
T[] readcache = new T[datastream.Length];
IEnumerator<T>[] en = new IEnumerator<T>[datastream.Length];
for (int i = 0; i < datastream.Length; i++)
{
en[i] = datastream[i].GetEnumerator();
en[i].MoveNext();
readcache[i] = en[i].Current;
}
T[] valcache = new T[datastream.Length];
var comparer = Comparer<T>.Create((x, y) => x.sending_time.CompareTo(y.sending_time));
while (true)
{
int min = readcache.IndexOfMin(comparer);
valcache[min] = readcache[min];
if (en[min].MoveNext())
readcache[min] = en[min].Current;
else
{
//this read entry is completed
var max = DateTime.SpecifyKind(DateTime.MaxValue, DateTimeKind.Utc);
readcache[min].sending_time = max;
if (readcache.All(x => x.sending_time == max))
break;
continue;
}
IEnumerable<U> val = null;
try
{
val = function(valcache, min);
}
catch(System.IO.EndOfStreamException)
{
break;
}
foreach (var v in val)
yield return v;
}
}
public static IEnumerable<U> create_synthetic_top<T, U>(IEnumerable<T>[] datastream,
double[] ratio, double[] price_mult,
Func<T, bool> pred,
Func<T[], T, U> selector)
where T : class, IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>
where U : class
{
if(price_mult == null)
price_mult = ratio;
return stream_data(datastream, (T[] valcache, int min) =>
{
if(valcache.All( x => x != default(T) ))
{
double bid = 0.0, ask = 0.0; uint bsize = uint.MaxValue, asize = uint.MaxValue;
for(int z = 0; z < valcache.Length; z++)
{
double r = ratio[z];
double p = price_mult[z];
var b = valcache[z].TopBook.Bid;
var a = valcache[z].TopBook.Ask;
if(r > 0.0)
{
bid += b.price() * p;
bsize = Math.Min( bsize, (uint)(b.Size / Math.Abs(r)) );
ask += a.price() * p;
asize = Math.Min( asize, (uint)(a.Size / Math.Abs(r)) );
}
else //sell this leg, look at bid for bid
{
bid += a.price() * p;
bsize = Math.Min( bsize, (uint)(a.Size / Math.Abs(r)) );
ask += b.price() * p;
asize = Math.Min( asize, (uint)(b.Size / Math.Abs(r)) );
}
}
var bi = IntPrice.from_double(bid); var ai = IntPrice.from_double(ask);
var newthing = valcache[min].Clone();
newthing.TopBook = new TopBookMessage
{
Bid = new PriceSizeMessage { Size = bsize, Mantissa = bi.mantissa, Exponent = bi.exponent },
Ask = new PriceSizeMessage { Size = asize, Mantissa = ai.mantissa, Exponent = ai.exponent }
};
if(pred == null || pred(newthing))
return new[] {selector(valcache, newthing) };
}
return Enumerable.Empty<U>();
});
}
static public IEnumerable<MBOMessage> exchange_orders<T>(IEnumerable<T> datastream, int framelimit = int.MaxValue)
where T : class, IHasMdEntry, IHasMBO, IHasTimestamp
{
int count = 0;
return stream_data(new []{datastream}, (T[] valcache, int min) =>
{
var curr = valcache[min];
if (count++ > framelimit)
throw new System.IO.EndOfStreamException();
var entry = curr.MdEntry;
if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer)
{
var orders = MBOUtils.extract_mbo(entry, curr.EntryIndex, curr.Orders);
return orders;
}
else
return Enumerable.Empty<MBOMessage>();
});
}
static public IEnumerable<OrderStatus> algo_orders<T>(System.Type algo_type, uint latency, string algo_params,
api.Side side, double price, uint size,
string[] security_list, IEnumerable<T>[] datastream,
int framelimit = int.MaxValue)
where T : class, IHasBook, IHasTrade, IHasTimestamp
{
var q = new List<OrderStatus>();
IAlgo algo = null;
int count = 0; var matching_model = new NullMatchingModel(new Bucket(latency, latency +1));
var ienum = stream_data(datastream, (T[] valcache, int min) =>
{
q.Clear();
var secdesc = security_list[min];
var curr = valcache[min];
if (count == 0 || algo.is_completed()) //spreader case will never complete
{
algo = AlgoFactory.algo_factory(algo_type, matching_model, security_list, algo_params);
AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
(h, f) => {Log.assert(false, "null model should have no fills"); } ,
(h, s) => q.Add(s));
}
if (count++ > framelimit)
throw new System.IO.EndOfStreamException();
if (curr.Book != null)
{
algo.matching_model.process_external_book(curr.Md.TransactTime, secdesc, curr.Book);
algo.process_book(secdesc, curr.MdEntry.SecurityId, curr.Book, curr.Book, curr.Md);
}
if (curr.Trade != null)
{
algo.matching_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade);
algo.process_trades(secdesc, curr.MdEntry.SecurityId, new[] { curr.Trade }, curr.Md);
}
return q;
});
return ienum;
}
static public IEnumerable<FillEntry> model_passive_fills<T>(IMatchingModel matching_model,
IEnumerable<MBOMessage> exchange_orders,
IEnumerable<OrderStatus> algo_orders,
IEnumerable<T> datastream, int framelimit = int.MaxValue)
where T : class, IHasMBO, IHasTrade, IHasTimestamp
{
var q = new List<FillEntry>();
int count = 0;
var ienum = stream_data(new[]{datastream}, (T[] valcache, int min) =>
{
q.Clear();
var curr = valcache[min];
if (count++ > framelimit)
throw new System.IO.EndOfStreamException();
if (curr.Trade != null)
{
Log.assert(curr.MdEntry.MdEntryType == MDEntryType.Trade);
var trades = MBOUtils.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders);
}
return q;
});
return ienum;
}
static public IEnumerable<U> run_algo<T, U>(System.Type algo_type, IMatchingModel matching_model, string algo_params,
api.Side side, double price, uint size,
string[] security_list, IEnumerable<T>[] datastream,
Func<IEnumerable<AlgoResults>, IEnumerable<U>> selector = null,
int framelimit = int.MaxValue)
where T : class, IHasBook, IHasTrade, IHasMBO, IHasTimestamp
{
var q = new List<AlgoResults>();
Dictionary<string, TradeMessage> last_trade = new Dictionary<string, TradeMessage>();
Dictionary<string, BookMessage> last_book = new Dictionary<string, BookMessage>();
IAlgo algo = null;
int count = 0;
var ienum = stream_data(datastream, (T[] valcache, int min) =>
{
q.Clear();
var secdesc = security_list[min];
var curr = valcache[min];
if(count == 0 || algo.is_completed())
{
algo = AlgoFactory.algo_factory(algo_type, matching_model, security_list, algo_params);
AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
(h, f) => q.Add(new AlgoResults(curr, f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))),
(h, s) => q.Add(new AlgoResults(curr, s, last_trade.GetValueOrDefault(s.order.security_desc), last_book.GetValueOrDefault(s.order.security_desc))));
}
if (count++ > framelimit)
throw new System.IO.EndOfStreamException();
var orders = MBOUtils.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders);
if(curr.MdEntry.MdEntryType == MDEntryType.Bid || curr.MdEntry.MdEntryType == MDEntryType.Offer)
{
foreach (var order in orders)
algo.matching_model.process_external_order(curr.Md.TransactTime, order);
}
if(curr.Book != null)
{
algo.matching_model.process_external_book(curr.Md.TransactTime, secdesc, curr.Book);
algo.process_book(secdesc, curr.MdEntry.SecurityId, curr.Book, curr.Book, curr.Md);
Log.assert(curr.Trade == null, "new schema");
last_book[secdesc] = curr.Book;
}
if(curr.Trade != null)
{
algo.matching_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade);
foreach(var order in orders)
{
if(curr.Trade.OrderId != order.OrderId)
{
var clone = curr.Trade.Clone();
clone.OrderId = order.OrderId;
clone.Key = order.SecurityId;
clone.AggressorSide = order.MdEntryType ==
MDEntryType.Bid ? AggressorSide.Buy : (order.MdEntryType == MDEntryType.Offer ? AggressorSide.Sell : AggressorSide.None);
var ip = IntPrice.from_double(order.MdEntryPx);
clone.Mantissa = ip.mantissa;
clone.Exponent = ip.exponent;
algo.matching_model.process_external_trade(curr.Md.TransactTime, secdesc, clone);
}
}
algo.process_trades(secdesc, curr.MdEntry.SecurityId, new [] {curr.Trade}, curr.Md);
last_trade[secdesc] = curr.Trade;
}
return q;
});
if(selector != null)
return selector(ienum);
else
return ienum.Select( xx => (U)(object)xx);
}
}
public class AlgoResults
{
public AlgoResults(IHasTimestamp h, OrderStatus ord, TradeMessage trd, BookMessage book)
{
this.data = h;
this.order = ord;
this.trade = trd;
this.book = book;
}
public AlgoResults(IHasTimestamp h, FillEntry fil, TradeMessage trd, BookMessage book)
{
this.data = h;
this.fill = fil;
this.trade = trd;
this.book = book;
}
public IHasTimestamp data { get; }
public string security_desc { get; }
public OrderStatus order { get; }
public FillEntry fill { get; }
public co2.BookMessage book { get; }
public co2.TradeMessage trade { get; }
}
public static class QueryExtensions
{
static public IMongoQueryable<T> where<T>(this IMongoQueryable<T> q, string where)
{
var expr = DynamicExpressionParser.ParseLambda<T, bool>(ParsingConfig.Default, false, where);
var res = q.Where(expr);
return res;
}
static public IMongoQueryable<T> order_by<T>(this IMongoQueryable<T> q, string orderby)
{
var expr = DynamicExpressionParser.ParseLambda<T, bool>(ParsingConfig.Default, false, orderby);
var res = q.OrderBy(expr);
return res;
}
static public DateTime to_datetime(this Timestamp ts)
{
return DateTime.SpecifyKind(ts.ToDateTime(), DateTimeKind.Utc);
}
}
}