diff --git a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs index 0cddee8ea..9f1bb4cc3 100644 --- a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs +++ b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs @@ -21,7 +21,13 @@ namespace bts.history{ return String.Format(@"{0}/{1}", Configuration.get().data_path, relative_path); - } + } + public static IEnumerable get_files_recursively (string dir){ + List rv = new List(Directory.GetFiles(dir)); + foreach (string d in Directory.GetDirectories(dir)) + rv.AddRange(get_files_recursively(d)); + return rv; + } public static IDictionary load_books(string file_name, string symbol, bool top_only){ @@ -116,7 +122,8 @@ namespace bts.history{ return rv; } - public static IDictionary> load_orders(string file_name){ + public static IDictionary> load_orders( + string file_name){ var t1 = DateTime.Now; var rv = new Dictionary>(); int ln_no = 0; @@ -139,7 +146,8 @@ namespace bts.history{ } Console.WriteLine("Loaded {0} orders in {1} secs", ln_no, (DateTime.Now-t1).TotalSeconds); - return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable); + return rv.ToDictionary(kv => kv.Key, kv => + kv.Value as IEnumerable); } static Timestamp _parse_time_string(string ts){ @@ -184,7 +192,7 @@ namespace bts.history{ if(!MarketQueriesHelper.is_single_prod_cal_spr(toks[sec_idx])) return null; ProcessedBook rv = new ProcessedBook(){Book = new BookMessage()}; - Log.assert(toks.Length == 5+4*n_levels); + Log.assert(toks.Length == 5+4*n_levels || toks.Length == 7+4*n_levels); rv.MsgSeqNum = UInt32.Parse(toks[i++]); rv.SendingTime = _parse_time_string(toks[i++]); rv.TransactTime = _parse_time_string(toks[i++]); diff --git a/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs b/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs index 767121bc4..14ea883fc 100644 --- a/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs +++ b/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs @@ -24,8 +24,9 @@ namespace bts.history{ Log.assert(args.Count()==3); var symbol = args[0]; var path = DataIOHelper.get_data_path(args[1]); - var output_path = DataIOHelper.get_data_path(args[2]); - var files = Directory.GetFiles(path); + var output_path = DataIOHelper.get_data_path(args[2]); + var files = DataIOHelper.get_files_recursively(path); + files = files.Where(f => f.Contains("raw_books.csv")); var price_map = files.ToDictionary(f => f, f => new List()); ushort prl_files = 10;