using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Linq.Dynamic.Core; using MongoDB.Driver; using MongoDB.Driver.Linq; using Google.Protobuf.WellKnownTypes; using bts.co2; using mbts.co2; using MathNet.Numerics.Statistics; namespace bts.history { public class Query { readonly IMongoClient _client; readonly IMongoDatabase _db; public Query(string database, string host = "sparkvm", int port = 27017, string user = "mongo", string pass = "enter") { MongoGenerator.setup_bson(); var conn = $@"mongodb://{user}:{pass}@{host}:{port}/admin"; _client = new MongoClient(conn); _db = _client.GetDatabase(database); } public IMongoCollection collection(string collection) { var c = _db.GetCollection(collection); return c; } public IMongoQueryable queryable(string collection) { var coll = _db.GetCollection(collection); IMongoQueryable res = queryable(coll); return res; } public IMongoQueryable queryable(IMongoCollection collection) => collection.AsQueryable(new AggregateOptions { AllowDiskUse = true } ); public IMongoQueryable timeseries(IMongoCollection collection, DateTime begin, DateTime end, bool use_sending_time = true) where H : IHasTimestamp { IMongoQueryable res = queryable(collection); Expression> timeexpr = null; begin = begin.ToUniversalTime(); end = end.ToUniversalTime(); if(use_sending_time) timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end; else timeexpr = (H h) => h.transact_time >= begin && h.transact_time <= end; res = res.Where( timeexpr ); return res; } //so I can get into a local linq through the jupyter nb public static IEnumerable local(IMongoQueryable q) { return q.AsEnumerable(); } public static IEnumerable stream_data(IEnumerable[] datastream, Func> function ) where T : class, IHasTimestamp where U : class { T[] readcache = new T[datastream.Length]; IEnumerator[] en = new IEnumerator[datastream.Length]; for (int i = 0; i < datastream.Length; i++) { en[i] = datastream[i].GetEnumerator(); en[i].MoveNext(); readcache[i] = en[i].Current; } T[] valcache = new T[datastream.Length]; var comparer = Comparer.Create((x, y) => x.sending_time.CompareTo(y.sending_time)); while (true) { int min = readcache.IndexOfMin(comparer); valcache[min] = readcache[min]; if (en[min].MoveNext()) readcache[min] = en[min].Current; else { //this read entry is completed var max = DateTime.SpecifyKind(DateTime.MaxValue, DateTimeKind.Utc); readcache[min].sending_time = max; if (readcache.All(x => x.sending_time == max)) break; continue; } IEnumerable val = null; try { val = function(valcache, min); } catch(System.IO.EndOfStreamException) { break; } foreach (var v in val) yield return v; } } public static IEnumerable create_synthetic_top(IEnumerable[] datastream, double[] ratio, double[] price_mult, Func pred, Func selector) where T : class, IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable where U : class { if(price_mult == null) price_mult = ratio; return stream_data(datastream, (T[] valcache, int min) => { if(valcache.All( x => x != default(T) )) { double bid = 0.0, ask = 0.0; uint bsize = uint.MaxValue, asize = uint.MaxValue; for(int z = 0; z < valcache.Length; z++) { double r = ratio[z]; double p = price_mult[z]; var b = valcache[z].TopBook.Bid; var a = valcache[z].TopBook.Ask; if(r > 0.0) { bid += b.price() * p; bsize = Math.Min( bsize, (uint)(b.Size / Math.Abs(r)) ); ask += a.price() * p; asize = Math.Min( asize, (uint)(a.Size / Math.Abs(r)) ); } else //sell this leg, look at bid for bid { bid += a.price() * p; bsize = Math.Min( bsize, (uint)(a.Size / Math.Abs(r)) ); ask += b.price() * p; asize = Math.Min( asize, (uint)(b.Size / Math.Abs(r)) ); } } var bi = IntPrice.from_double(bid); var ai = IntPrice.from_double(ask); var newthing = valcache[min].Clone(); newthing.TopBook = new TopBookMessage { Bid = new PriceSizeMessage { Size = bsize, Mantissa = bi.mantissa, Exponent = bi.exponent }, Ask = new PriceSizeMessage { Size = asize, Mantissa = ai.mantissa, Exponent = ai.exponent } }; if(pred == null || pred(newthing)) return new[] {selector(valcache, newthing) }; } return Enumerable.Empty(); }); } static public IEnumerable exchange_orders(IEnumerable datastream, int framelimit = int.MaxValue) where T : class, IHasMdEntry, IHasMBO, IHasTimestamp { int count = 0; return stream_data(new []{datastream}, (T[] valcache, int min) => { var curr = valcache[min]; if (count++ > framelimit) throw new System.IO.EndOfStreamException(); var entry = curr.MdEntry; if (entry.MdEntryType == MDEntryType.Bid || entry.MdEntryType == MDEntryType.Offer) { var orders = MBOUtils.extract_mbo(entry, curr.EntryIndex, curr.Orders); return orders; } else return Enumerable.Empty(); }); } static public IEnumerable algo_orders(System.Type algo_type, uint latency, string algo_params, api.Side side, double price, uint size, string[] security_list, IEnumerable[] datastream, int framelimit = int.MaxValue) where T : class, IHasBook, IHasTrade, IHasTimestamp { var q = new List(); IAlgo algo = null; int count = 0; var matching_model = new NullMatchingModel(new Bucket(latency, latency +1)); var ienum = stream_data(datastream, (T[] valcache, int min) => { q.Clear(); var secdesc = security_list[min]; var curr = valcache[min]; if (count == 0 || algo.is_completed()) //spreader case will never complete { algo = AlgoFactory.algo_factory(algo_type, matching_model, security_list, algo_params); AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow), (h, f) => {Log.assert(false, "null model should have no fills"); } , (h, s) => q.Add(s)); } if (count++ > framelimit) throw new System.IO.EndOfStreamException(); if (curr.Book != null) { algo.matching_model.process_external_book(curr.Md.TransactTime, secdesc, curr.Book); algo.process_book(secdesc, curr.MdEntry.SecurityId, curr.Book, curr.Book, curr.Md); } if (curr.Trade != null) { algo.matching_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade); algo.process_trades(secdesc, curr.MdEntry.SecurityId, new[] { curr.Trade }, curr.Md); } return q; }); return ienum; } static public IEnumerable model_passive_fills(IMatchingModel matching_model, IEnumerable exchange_orders, IEnumerable algo_orders, IEnumerable datastream, int framelimit = int.MaxValue) where T : class, IHasMBO, IHasTrade, IHasTimestamp { var q = new List(); int count = 0; var ienum = stream_data(new[]{datastream}, (T[] valcache, int min) => { q.Clear(); var curr = valcache[min]; if (count++ > framelimit) throw new System.IO.EndOfStreamException(); if (curr.Trade != null) { Log.assert(curr.MdEntry.MdEntryType == MDEntryType.Trade); var trades = MBOUtils.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders); } return q; }); return ienum; } static public IEnumerable run_algo(System.Type algo_type, IMatchingModel matching_model, string algo_params, api.Side side, double price, uint size, string[] security_list, IEnumerable[] datastream, Func, IEnumerable> selector = null, int framelimit = int.MaxValue) where T : class, IHasBook, IHasTrade, IHasMBO, IHasTimestamp { var q = new List(); Dictionary last_trade = new Dictionary(); Dictionary last_book = new Dictionary(); IAlgo algo = null; int count = 0; var ienum = stream_data(datastream, (T[] valcache, int min) => { q.Clear(); var secdesc = security_list[min]; var curr = valcache[min]; if(count == 0 || algo.is_completed()) { algo = AlgoFactory.algo_factory(algo_type, matching_model, security_list, algo_params); AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow), (h, f) => q.Add(new AlgoResults(curr, f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))), (h, s) => q.Add(new AlgoResults(curr, s, last_trade.GetValueOrDefault(s.order.security_desc), last_book.GetValueOrDefault(s.order.security_desc)))); } if (count++ > framelimit) throw new System.IO.EndOfStreamException(); var orders = MBOUtils.extract_mbo(curr.MdEntry, curr.EntryIndex, curr.Orders); if(curr.MdEntry.MdEntryType == MDEntryType.Bid || curr.MdEntry.MdEntryType == MDEntryType.Offer) { foreach (var order in orders) algo.matching_model.process_external_order(curr.Md.TransactTime, order); } if(curr.Book != null) { algo.matching_model.process_external_book(curr.Md.TransactTime, secdesc, curr.Book); algo.process_book(secdesc, curr.MdEntry.SecurityId, curr.Book, curr.Book, curr.Md); Log.assert(curr.Trade == null, "new schema"); last_book[secdesc] = curr.Book; } if(curr.Trade != null) { algo.matching_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade); foreach(var order in orders) { if(curr.Trade.OrderId != order.OrderId) { var clone = curr.Trade.Clone(); clone.OrderId = order.OrderId; clone.Key = order.SecurityId; clone.AggressorSide = order.MdEntryType == MDEntryType.Bid ? AggressorSide.Buy : (order.MdEntryType == MDEntryType.Offer ? AggressorSide.Sell : AggressorSide.None); var ip = IntPrice.from_double(order.MdEntryPx); clone.Mantissa = ip.mantissa; clone.Exponent = ip.exponent; algo.matching_model.process_external_trade(curr.Md.TransactTime, secdesc, clone); } } algo.process_trades(secdesc, curr.MdEntry.SecurityId, new [] {curr.Trade}, curr.Md); last_trade[secdesc] = curr.Trade; } return q; }); if(selector != null) return selector(ienum); else return ienum.Select( xx => (U)(object)xx); } } public class AlgoResults { public AlgoResults(IHasTimestamp h, OrderStatus ord, TradeMessage trd, BookMessage book) { this.data = h; this.order = ord; this.trade = trd; this.book = book; } public AlgoResults(IHasTimestamp h, FillEntry fil, TradeMessage trd, BookMessage book) { this.data = h; this.fill = fil; this.trade = trd; this.book = book; } public IHasTimestamp data { get; } public string security_desc { get; } public OrderStatus order { get; } public FillEntry fill { get; } public co2.BookMessage book { get; } public co2.TradeMessage trade { get; } } public static class QueryExtensions { static public IMongoQueryable where(this IMongoQueryable q, string where) { var expr = DynamicExpressionParser.ParseLambda(ParsingConfig.Default, false, where); var res = q.Where(expr); return res; } static public IMongoQueryable order_by(this IMongoQueryable q, string orderby) { var expr = DynamicExpressionParser.ParseLambda(ParsingConfig.Default, false, orderby); var res = q.OrderBy(expr); return res; } static public DateTime to_datetime(this Timestamp ts) { return DateTime.SpecifyKind(ts.ToDateTime(), DateTimeKind.Utc); } } }