diff --git a/src/client/spark_historical/StaticAnalysis/CircleAnalyzer.cs b/src/client/spark_historical/StaticAnalysis/CircleAnalyzer.cs index be69fa367..cefe13e83 100644 --- a/src/client/spark_historical/StaticAnalysis/CircleAnalyzer.cs +++ b/src/client/spark_historical/StaticAnalysis/CircleAnalyzer.cs @@ -16,7 +16,7 @@ using System.Collections.Concurrent; // date-range, product, path (or file), optional spread namespace bts.history{ - class CircleAnalyzer{ + class CircleAnalyzer{ public CircleAnalyzer(HistoricalDataOptions hd_options){ Console.WriteLine("Circle:Hit key to continue:"); Console.ReadKey(); // product, path, book, trades, orders @@ -27,8 +27,13 @@ namespace bts.history{ var book_file = path+args[2]; var trades_file = path+args[3]; var orders_file = path+args[4]; + // test + // for(int i=0;i<5;i++) { + // DataIOHelper.load_books_speed(book_file,symbol,true); + // //DataIOHelper.load_books(book_file,symbol,true); + // } var order_map = DataIOHelper.load_orders(orders_file); - var order_queue_times = order_map.ToDictionary( + var ord_times_map = order_map.ToDictionary( kv => kv.Key, kv => MarketQueries.get_order_queue_time(kv.Value)); var trade_map = DataIOHelper.load_trades(trades_file); @@ -38,7 +43,8 @@ namespace bts.history{ spr_book_map.Keys); List spr_trades = new List(); foreach(var s in circ_spreads) - _analyze_circle(s,spr_book_map,trade_map, ref spr_trades); + if(s.Contains("CL_G0M0N0")) + _analyze_circle(s,spr_book_map,trade_map,ord_times_map,ref spr_trades); } public void analyze(){ @@ -49,6 +55,7 @@ namespace bts.history{ string synth_txt, IDictionary md_map, IDictionary> trd_map, + IDictionary ord_times_map, ref List spr_trades){ var ord = MarketQueriesHelper.parse_synthetic(synth_txt); // alternates quoting per leg @@ -59,7 +66,7 @@ namespace bts.history{ else kv.Value.IsActiveQuoting = false; } - spr_trades.AddRange(_analyze_circle(synth_txt, ord, md_map, trd_map)); + spr_trades.AddRange(_analyze_circle(synth_txt,ord,md_map,trd_map,ord_times_map)); } } @@ -67,14 +74,19 @@ namespace bts.history{ string synth_txt, SpreaderOrderDefinition spr, IDictionary md_map, - IDictionary> trd_map){ + IDictionary> trd_map, + IDictionary ord_times_map){ var t1 = DateTime.Now; List rv = new List(); - rv.AddRange(MarketQueries.find_spreader_fills(spr,null,md_map,trd_map,api.Side.Buy,-0.001)); - rv.AddRange(MarketQueries.find_spreader_fills(spr,null,md_map,trd_map,api.Side.Sell,0.001)); + rv.AddRange(MarketQueries.find_spreader_fills(spr,null,md_map,trd_map,ord_times_map,api.Side.Buy,-0.001)); + rv.AddRange(MarketQueries.find_spreader_fills(spr,null,md_map,trd_map,ord_times_map,api.Side.Sell,0.001)); var t2 = DateTime.Now; var secs = (t2-t1).TotalSeconds; - Console.WriteLine("Circle: {0} {1} Spreads {2} Secs", synth_txt, rv.Count, secs); + foreach (var r in rv) + Console.WriteLine("Circle: {0} {1}/{2} Spreads", + synth_txt, + r.filtered_fills.Count(), + r.quote_fills.Count()); return rv; } } diff --git a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs index a795c2474..469965c51 100644 --- a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs +++ b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs @@ -63,6 +63,26 @@ namespace bts.history{ return ret; } + public static void load_books_speed(string file_name, + string symbol, + bool top_only){ + DateTime t1 = DateTime.Now; + var rv = new Dictionary>(); + int ln_no = 0; + var lines = new ConcurrentBag(); + var books = new ConcurrentBag(); + using (StreamReader sr = new StreamReader(file_name)) { + string line; + while ((line = sr.ReadLine()) != null) + if(ln_no++>0) + lines.Add(line); + } + Parallel.ForEach (lines, l => + books.Add (_csv_to_processedbook(l,true,top_only))); + Console.WriteLine ("{0} secs to load {1} lines", + (DateTime.Now - t1).TotalSeconds,ln_no); + } + public static IDictionary> load_trades( string file_name){ var rv = new Dictionary>(); @@ -132,7 +152,7 @@ namespace bts.history{ Log.assert(fractional_ns < Int32.MaxValue); Log.assert(quotient_secs < Int32.MaxValue); Int64 tot_seconds_epoch = Convert.ToInt64( - (new DateTime(yy,mm,dd,hh,mins,Convert.ToInt32(quotient_secs)) - + (new DateTime(yy,mm,dd,hh,mins,Convert.ToInt32(quotient_secs)) - new DateTime(1970,1,1)).TotalSeconds); return new Timestamp(){ Seconds = tot_seconds_epoch, diff --git a/src/client/spark_historical/StaticAnalysis/MarketQueries.cs b/src/client/spark_historical/StaticAnalysis/MarketQueries.cs index 52b46b789..839794d6f 100644 --- a/src/client/spark_historical/StaticAnalysis/MarketQueries.cs +++ b/src/client/spark_historical/StaticAnalysis/MarketQueries.cs @@ -24,14 +24,18 @@ namespace bts.history{ public class SpreaderTrade{ public readonly IEnumerable hedge_books; public readonly IEnumerable quote_fills; + public readonly IEnumerable filtered_fills; public readonly Side side; - public SpreaderTrade (IEnumerable hedge_books_a, + public SpreaderTrade (Timerange r, + IEnumerable hedge_books_a, IEnumerable quote_fills_a, - Side side_a){ + Side side_a, + IDictionary ord_times_map){ hedge_books = hedge_books_a; quote_fills = quote_fills_a; + filtered_fills = MarketQueries.restrict_to_range(r,quote_fills,ord_times_map); side = side_a; - } + } } class MarketQueries{ @@ -164,6 +168,7 @@ namespace bts.history{ Timerange r, IDictionary md_map, IDictionary> trd_map, + IDictionary ord_times_map, api.Side side, double price){ List rv = new List(); @@ -185,7 +190,7 @@ namespace bts.history{ trades_in_tr = trades_in_tr.Where(tr => conform_to_price(spr, side, price, s ,tr)); if(trades_in_tr.Count()>0) - rv.Add(new SpreaderTrade(s,trades_in_tr,side)); + rv.Add(new SpreaderTrade(int_r,s,trades_in_tr,side,ord_times_map)); } }); } @@ -247,6 +252,18 @@ namespace bts.history{ return MarketQueriesHelper.ref_min_dt; } + public static IEnumerable restrict_to_range( + Timerange r, + IEnumerable trades, + IDictionary ord_time_map){ + return trades.Where(t => { + Timestamp ord_time; + if(ord_time_map.TryGetValue(t.RestingOid, out ord_time)) + return MarketQueriesHelper.within(r,t.TransactTime); + return false; + }); + } + static IEnumerable> compute_fifo_fills ( Timerange time_range, double order_price, diff --git a/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs b/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs index 3269b4470..45fde749c 100644 --- a/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs +++ b/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs @@ -113,6 +113,10 @@ namespace bts.history{ Timestamp y){ return !_is_before(x,y); } + public static bool within (Timerange r, Timestamp t){ + return _is_before_or_same(r.T1,t) && + _is_after_or_same(r.T2,t); + } // x-y ns public static Int64 diff_ns (Timestamp x, Timestamp y){ @@ -529,6 +533,29 @@ namespace bts.history{ Log.assert(c == full.Count()); return rv; } + public static uint bin_search_nearest (IEnumerable sorted_items, + T to_find, + Func diff, + Tuple range = null){ + range = range == null ? + Tuple.Create((uint)0, (uint)sorted_items.Count()-1) : range; + + if (range.Item2 - range.Item1 <= 2){ + uint rv = 0; + for (var i=range.Item1; i<= range.Item2; i++){ + var curr_min = diff(to_find,sorted_items.ElementAt((int)rv)); + var diff_i = diff(to_find,sorted_items.ElementAt((int)i)); + if(diff_i <= curr_min) + rv = i; + } + return rv; + } + else{ + + } + + return 0; + } #endregion } } \ No newline at end of file