Files
spark/src/client/spark_historical/Query/Query.cs
T
2019-12-12 14:27:28 -05:00

257 lines
11 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Linq.Dynamic.Core;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Google.Protobuf.WellKnownTypes;
using bts.co2;
using mbts.co2;
using MathNet.Numerics.Statistics;
namespace bts.history
{
public class Query
{
readonly IMongoClient _client;
readonly IMongoDatabase _db;
public Query(string database, string host = "sparkvm", int port = 27017,
string user = "mongo", string pass = "enter")
{
MongoGenerator.setup_bson();
var conn = $@"mongodb://{user}:{pass}@{host}:{port}/admin";
_client = new MongoClient(conn);
_db = _client.GetDatabase(database);
}
public IMongoCollection<T> collection<T>(string collection)
{
var c = _db.GetCollection<T>(collection);
return c;
}
public IMongoQueryable<T> queryable<T>(string collection)
{
var coll = _db.GetCollection<T>(collection);
IMongoQueryable<T> res = coll.AsQueryable();
return res;
}
public IMongoQueryable<T> queryable<T>(IMongoCollection<T> collection) => collection.AsQueryable();
public IMongoQueryable<H> timeseries<H>(IMongoCollection<H> collection, DateTime begin, DateTime end, bool use_sending_time = true)
where H : IHasTimestamp
{
IMongoQueryable<H> res = collection.AsQueryable();
Expression<Func<H, bool>> timeexpr = null;
begin = begin.ToUniversalTime();
end = end.ToUniversalTime();
if(use_sending_time)
timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end;
else
timeexpr = (H h) => h.transact_time >= begin && h.transact_time <= end;
res = res.Where( timeexpr );
return res;
}
//so I can get into a local linq through the jupyter nb
public static IEnumerable<T> local<T>(IMongoQueryable<T> q)
{
return q.AsEnumerable<T>();
}
public static IEnumerable<U> stream_data<T, U>(IEnumerable<T>[] datastream, Func<T[], int, IEnumerable<U>> function )
where T : class, IHasTimestamp
where U : class
{
T[] readcache = new T[datastream.Length];
IEnumerator<T>[] en = new IEnumerator<T>[datastream.Length];
for (int i = 0; i < datastream.Length; i++)
{
en[i] = datastream[i].GetEnumerator();
en[i].MoveNext();
readcache[i] = en[i].Current;
}
T[] valcache = new T[datastream.Length];
var comparer = Comparer<T>.Create((x, y) => x.sending_time.CompareTo(y.sending_time));
while (true)
{
int min = readcache.IndexOfMin(comparer);
valcache[min] = readcache[min];
if (en[min].MoveNext())
readcache[min] = en[min].Current;
else
{
//this read entry is completed
var max = DateTime.SpecifyKind(DateTime.MaxValue, DateTimeKind.Utc);
readcache[min].sending_time = max;
if (readcache.All(x => x.sending_time == max))
break;
continue;
}
IEnumerable<U> val = null;
try
{
val = function(valcache, min);
}
catch(InvalidOperationException)
{
break;
}
foreach (var v in val)
yield return v;
}
}
public static IEnumerable<U> create_synthetic_top<T, U>(IEnumerable<T>[] datastream,
double[] ratio, double[] price_mult,
Func<T, bool> pred,
Func<T[], T, U> selector)
where T : class, IHasTopBook, IHasTimestamp, Google.Protobuf.IDeepCloneable<T>
where U : class
{
if(price_mult == null)
price_mult = ratio;
return stream_data(datastream, (T[] valcache, int min) =>
{
if(valcache.All( x => x != default(T) ))
{
double bid = 0.0, ask = 0.0; uint bsize = uint.MaxValue, asize = uint.MaxValue;
for(int z = 0; z < valcache.Length; z++)
{
double r = ratio[z];
double p = price_mult[z];
var b = valcache[z].TopBook.Bid;
var a = valcache[z].TopBook.Ask;
if(r > 0.0)
{
bid += b.price() * p;
bsize = Math.Min( bsize, (uint)(b.Size / Math.Abs(r)) );
ask += a.price() * p;
asize = Math.Min( asize, (uint)(a.Size / Math.Abs(r)) );
}
else //sell this leg, look at bid for bid
{
bid += a.price() * p;
bsize = Math.Min( bsize, (uint)(a.Size / Math.Abs(r)) );
ask += b.price() * p;
asize = Math.Min( asize, (uint)(b.Size / Math.Abs(r)) );
}
}
var bi = IntPrice.from_double(bid); var ai = IntPrice.from_double(ask);
var newthing = valcache[min].Clone();
newthing.TopBook = new TopBookMessage
{
Bid = new PriceSizeMessage { Size = bsize, Mantissa = bi.mantissa, Exponent = bi.exponent },
Ask = new PriceSizeMessage { Size = asize, Mantissa = ai.mantissa, Exponent = ai.exponent }
};
if(pred == null || pred(newthing))
return new[] {selector(valcache, newthing) };
}
return Enumerable.Empty<U>();
});
}
static public IEnumerable<U> run_algo<T, U>(System.Type algo_type, IFillModel fill_model, string algo_params,
api.Side side, double price, uint size,
string[] security_list, IEnumerable<T>[] datastream,
Func<IEnumerable<AlgoResults>, IEnumerable<U>> selector = null,
int framelimit = int.MaxValue)
where T : class, IHasBook, IHasTrade, IHasMBO, IHasTimestamp
{
var q = new List<AlgoResults>();
Dictionary<string, TradeMessage> last_trade = new Dictionary<string, TradeMessage>();
Dictionary<string, BookMessage> last_book = new Dictionary<string, BookMessage>();
IAlgo algo = AlgoFactory.algo_factory(algo_type, fill_model, security_list, algo_params);
AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
(h, f) => q.Add(new AlgoResults(f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc) )),
(h, s) => q.Add(new AlgoResults(s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc)) ));
int count = 0;
var ienum = stream_data(datastream, (T[] valcache, int min) =>
{
q.Clear();
if(count++ > framelimit)
throw new InvalidOperationException();
if(algo.is_completed())
{
algo = AlgoFactory.algo_factory(algo_type, fill_model, security_list, algo_params);
AlgoBase.activate_self((AlgoBase)algo, side, price, size, Timestamp.FromDateTime(DateTime.UtcNow),
(h, f) => q.Add(new AlgoResults(f, last_trade.GetValueOrDefault(f.order.security_desc), last_book.GetValueOrDefault(f.order.security_desc))),
(h, s) => q.Add(new AlgoResults(s, last_trade.GetValueOrDefault(s.security_desc), last_book.GetValueOrDefault(s.security_desc))));
}
var secdesc = security_list[min];
var curr = valcache[min];
if(curr.Book != null && curr.Trade == null)
{
algo.process_book(secdesc, curr.MdEntry.SecurityId, curr.Book, curr.Book, curr.Md);
last_book[secdesc] = curr.Book;
}
else if(curr.Trade != null)
{
algo.process_trades(secdesc, curr.MdEntry.SecurityId, new [] {curr.Trade}, curr.Md);
algo.fill_model.process_external_trade(curr.Md.TransactTime, secdesc, curr.Trade);
last_trade[secdesc] = curr.Trade;
}
else if(curr.Order != null)
{
algo.fill_model.process_external_order(curr.Md.TransactTime, curr.Order);
}
return q;
});
if(selector != null)
return selector(ienum);
else
return (IEnumerable<U>)ienum;
}
}
public class AlgoResults
{
public AlgoResults(OrderStatus ord, TradeMessage trd, BookMessage book)
{
this.security_desc = ord.security_desc;
this.order = ord;
this.trade = trd;
this.book = book;
}
public AlgoResults(FillEntry fil, TradeMessage trd, BookMessage book)
{
this.security_desc = fil.order.security_desc;
this.fill = fil;
this.trade = trd;
this.book = book;
}
public string security_desc { get; }
public OrderStatus order { get; }
public FillEntry fill { get; }
public co2.BookMessage book { get; }
public co2.TradeMessage trade { get; }
}
public static class QueryExtensions
{
static public IMongoQueryable<T> where<T>(this IMongoQueryable<T> q, string where)
{
var expr = DynamicExpressionParser.ParseLambda<T, bool>(ParsingConfig.Default, false, where);
var res = q.Where(expr);
return res;
}
static public IMongoQueryable<T> order_by<T>(this IMongoQueryable<T> q, string orderby)
{
var expr = DynamicExpressionParser.ParseLambda<T, bool>(ParsingConfig.Default, false, orderby);
var res = q.OrderBy(expr);
return res;
}
static public DateTime to_datetime(this Timestamp ts)
{
return DateTime.SpecifyKind(ts.ToDateTime(), DateTimeKind.Utc);
}
}
}