From 542b8e946d9fabccb9f8aef3c6392cbce34d03e4 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Mon, 13 Jan 2020 11:16:06 -0500 Subject: [PATCH] mass sniper analysis --- .../StaticAnalysis/DataIOHelper.cs | 23 +++++--- .../StaticAnalysis/MarketQueriesHelper.cs | 9 ++-- .../SpreaderTickSniperAnalyzer.cs | 53 ++++++++++--------- src/server/sparkd/cmds/ParseHistoricalFIX.cpp | 3 -- 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs index 9f82d8b88..04c119cad 100644 --- a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs +++ b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs @@ -53,6 +53,13 @@ namespace bts.history{ DateTime t1 = DateTime.Now; string file_name = String.Format("{0}/{1}", get_data_path(dt), String.Format("{0}_{1}", symbol,"books.csv")); + return load_books(file_name,symbol,top_only); + } + + public static IDictionary load_books(string file_name, + string symbol, + bool top_only){ + DateTime t1 = DateTime.Now; var rv = new Dictionary>(); int ln_no = 0; using (StreamReader sr = new StreamReader(file_name)) { @@ -73,6 +80,8 @@ namespace bts.history{ pbook.BookSeqNum)); else contract_books.Add(pbook); + if(ln_no % 100000 == 0) + Console.WriteLine("{0}: Read {1} lines", symbol, ln_no); } } } @@ -142,19 +151,21 @@ namespace bts.history{ uint iter_levels = top_only ? 1 : n_levels; for (uint n=0; n test_entry = (psm, is_bid) => { if(psm.Size>0){ - Log.assert(psm.Mantissa > last_price, - String.Format("{0}: {1} Order", pb.MsgSeqNum, - is_bid ? "Bid" : "Ask")); + // TODO (check for market close time) + // Log.assert(psm.Mantissa > last_price, + // String.Format("{0}: {1} {2}>{3}", + // pb.MsgSeqNum, + // (is_bid ? "Bid" : "Ask"), + // psm.Mantissa, last_price)); last_price = psm.Mantissa; } }; diff --git a/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs b/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs index 552e4560e..c82d08cab 100644 --- a/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs +++ b/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs @@ -13,40 +13,47 @@ using Google.Protobuf.WellKnownTypes; using System.Threading.Tasks; using System.Collections.Concurrent; -namespace bts.history{ + +// date-range, product, path (or file), optional spread +namespace bts.history{ class SpreaderTickSniperAnalyzer{ + public SpreaderTickSniperAnalyzer(HistoricalDataOptions hd_options){ Console.WriteLine("SpreaderTickSniperAnalyzer:Hit key to continue:"); Console.ReadKey(); - //SanityTest.run_tests(); - // 1. processing args + // product, input_path, output_path string[] args = hd_options.algo_params.Split(','); - Log.assert(args.Count()==2||args.Count()==3); - DateTime dt; - Log.assert(DateTime.TryParse(args[0],out dt)); - string symbol = args[1], sspr_arg = String.Empty; - if(args.Count()==3) - sspr_arg = args[2]; - // 2. downloading all market data for symbol, dt - Console.WriteLine("Downloading all spread market data"); - IDictionary spr_book_map = DataIOHelper.load_books(symbol,dt,true); + Log.assert(args.Count()==3); + var symbol = args[0]; + var path = DataIOHelper.get_data_path(args[1]); + var output_path = DataIOHelper.get_data_path(args[2]); + foreach (string file in Directory.GetFiles(path)){ + string output_file = String.Format("{0}/{1}_sniper.csv", + output_path, + Path.GetFileNameWithoutExtension(file)); + _execute_single(file,symbol,output_file); + } + } + + public void analyze(){ + } + + private void _execute_single (string file_path, + string symbol, + string outputfile_path){ + IDictionary spr_book_map = + DataIOHelper.load_books(file_path,symbol,true); Console.WriteLine("Downloaded {0} books",spr_book_map.Count()); // 3. generating circular spreads Console.WriteLine("Generating circular spreads"); var circ_spreads = MarketQueriesHelper.generate_3_legged_circulars( symbol, spr_book_map.Keys); - circ_spreads = String.IsNullOrEmpty(sspr_arg) ? - circ_spreads : - circ_spreads.Where(s => s.StartsWith(sspr_arg)); Console.WriteLine("Generated {0} circular spreads", circ_spreads.Count()); var spread_map = circ_spreads.ToDictionary( k=>k, - k=>MarketQueriesHelper.parse_synthetic(k)); - StreamWriter tw = new StreamWriter( - String.Format("{0}/{1}_{2}.csv", - DataIOHelper.get_data_path(dt), - symbol, - "sniper")); + k=>MarketQueriesHelper.parse_synthetic(k)); + (new FileInfo(outputfile_path)).Directory.Create(); + StreamWriter tw = new StreamWriter(outputfile_path); // 4. compute the synthetic prices now string header = String.Empty; for (int i=0; i < 3; i++) @@ -58,10 +65,6 @@ namespace bts.history{ tw.Close(); } - - public void analyze(){ - } - private static void _evaluate_synthetic( IDictionary leg_book_map, SpreaderOrderDefinition s, diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp index 714cbf817..7c6f87e1a 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp @@ -437,9 +437,6 @@ void ParseHistoricalFIX::_execute_single(const std::string &file, size_t c = 0; auto ok = 0; auto start = Timer::now(); - - print_book_header(*books_stream); - while (std::getline(input, line)) { c++; auto parsed_ptr = parser.parse(InMsgBuffer(