Sniper reads from multiple files recursively

This commit is contained in:
Taha Afzal committed 2020-03-02 18:32:19 -05:00
1 parent dec178f133
commit 59e66cf1ef
2 files changed
+15 -6

No files matched your search

@@ -21,7 +21,13 @@ namespace bts.history{
return String.Format(@"{0}/{1}",
Configuration.get<bts.config.System>().data_path,
relative_path);
}
}
public static IEnumerable<string> get_files_recursively (string dir){
List<string> rv = new List<string>(Directory.GetFiles(dir));
foreach (string d in Directory.GetDirectories(dir))
rv.AddRange(get_files_recursively(d));
return rv;
}
public static IDictionary<string,BookSeries> load_books(string file_name,
string symbol,
bool top_only){
@@ -116,7 +122,8 @@ namespace bts.history{
return rv;
}
public static IDictionary<UInt64, IEnumerable<OrderRow>> load_orders(string file_name){
public static IDictionary<UInt64, IEnumerable<OrderRow>> load_orders(
string file_name){
var t1 = DateTime.Now;
var rv = new Dictionary<UInt64, List<OrderRow>>();
int ln_no = 0;
@@ -139,7 +146,8 @@ namespace bts.history{
}
Console.WriteLine("Loaded {0} orders in {1} secs",
ln_no, (DateTime.Now-t1).TotalSeconds);
return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<OrderRow>);
return rv.ToDictionary(kv => kv.Key, kv =>
kv.Value as IEnumerable<OrderRow>);
}
static Timestamp _parse_time_string(string ts){
@@ -184,7 +192,7 @@ namespace bts.history{
if(!MarketQueriesHelper.is_single_prod_cal_spr(toks[sec_idx]))
return null;
ProcessedBook rv = new ProcessedBook(){Book = new BookMessage()};
Log.assert(toks.Length == 5+4*n_levels);
Log.assert(toks.Length == 5+4*n_levels || toks.Length == 7+4*n_levels);
rv.MsgSeqNum = UInt32.Parse(toks[i++]);
rv.SendingTime = _parse_time_string(toks[i++]);
rv.TransactTime = _parse_time_string(toks[i++]);
@@ -24,8 +24,9 @@ namespace bts.history{
Log.assert(args.Count()==3);
var symbol = args[0];
var path = DataIOHelper.get_data_path(args[1]);
var output_path = DataIOHelper.get_data_path(args[2]);
var files = Directory.GetFiles(path);
var output_path = DataIOHelper.get_data_path(args[2]);
var files = DataIOHelper.get_files_recursively(path);
files = files.Where(f => f.Contains("raw_books.csv"));
var price_map = files.ToDictionary(f => f, f =>
new List<SyntheticPrice>());
ushort prl_files = 10;