From 3d22a58bac5dfd6f2644b31893d6592eeecb34e9 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Fri, 10 Jan 2020 19:19:52 -0500 Subject: [PATCH] ParseHistoricalFIX takes a path and processes all files with name _raw in it --- src/server/sparkd/cmds/ParseHistoricalFIX.cpp | 68 ++++++++++++++----- src/server/sparkd/cmds/ParseHistoricalFIX.h | 3 + 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp index 287bcb130..1fbfe1f7b 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp @@ -5,6 +5,8 @@ #include "lib/jetfix/codec/Group.h" #include "lib/jetfix/codec/MessageTypes.h" #include "lib/utils/bts_chrono.h" +#include +#include #include #include @@ -381,13 +383,49 @@ void print_orders(uint32_t msg_id, MDIncrementalUpdate const &update, } } -int ParseHistoricalFIX::execute(std::vector const &args) { - std::ifstream input(args[0]); - std::ofstream books_stream(args[0] + "_books.csv"); - std::ofstream msgs_stream(args[0] + "_msgs.csv"); - std::ofstream trades_stream(args[0] + "_trades.csv"); - std::ofstream orders_stream(args[0] + "_orders.csv"); +void get_files(const std::string& dir, + std::vector& out_files){ + if(boost::filesystem::is_directory(dir)){ + boost::filesystem::path p(dir); + for(auto& entry : boost::make_iterator_range( + boost::filesystem::directory_iterator(p), {})){ + if(boost::filesystem::is_regular_file(entry)){ + auto file_name = entry.path().string(); + if(file_name.find("_raw") != std::string::npos && + file_name.find(".csv") == std::string::npos) + out_files.push_back(file_name); + } + } + } +} +int ParseHistoricalFIX::execute(std::vector const &args) { + // first arg is the path to where the files are + // second arg is optional to parse orders/trades or not + auto dir = args[0]; + auto process_orders = args.size()==1 ? false : (args[1]=="orders"); + std::vector file_names; + get_files(dir, file_names); + for (const auto& f : file_names) + _execute_single(f, process_orders); + return 0; +} + +void ParseHistoricalFIX::_execute_single(const std::string& file, + bool process_orders){ + std::cout << "Processing file:" << file << std::endl; + std::ifstream input(file); + std::unique_ptr books_stream, msgs_stream, trades_stream, orders_stream; + books_stream = std::make_unique(file + "_books.csv"); + print_book_header(*books_stream); + if(process_orders){ + msgs_stream = std::make_unique(file + "_msgs.csv"); + trades_stream = std::make_unique(file + "_trades.csv"); + orders_stream = std::make_unique(file + "_orders.csv"); + print_msg_headers(*msgs_stream); + print_trades_header(*trades_stream); + print_orders_header(*orders_stream); + } std::unordered_map books; TradeSummaryEvent current_trade_summary; @@ -399,10 +437,7 @@ int ParseHistoricalFIX::execute(std::vector const &args) { auto ok = 0; auto start = Timer::now(); - print_book_header(books_stream); - print_msg_headers(msgs_stream); - print_trades_header(trades_stream); - print_orders_header(orders_stream); + print_book_header(*books_stream); while (std::getline(input, line)) { c++; @@ -411,18 +446,19 @@ int ParseHistoricalFIX::execute(std::vector const &args) { if (parsed_ptr) { auto msg = std::dynamic_pointer_cast(parsed_ptr); - update_and_maybe_print_book(c,*msg, books, books_stream); - print_msgs(c, *msg, msgs_stream); - update_and_maybe_print_trades(c,*msg, current_trade_summary, - trades_stream); - print_orders(c,*msg, orders_stream); + update_and_maybe_print_book(c,*msg, books, *books_stream); + if(process_orders){ + print_msgs(c, *msg, *msgs_stream); + update_and_maybe_print_trades(c,*msg, current_trade_summary, + *trades_stream); + print_orders(c,*msg, *orders_stream); + } } ok += (bool)(parsed_ptr); } auto sec_duration = sec_diff(Timer::now(), start); std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok << " out of " << c << " time: " << sec_duration << " secs\n"; - return 0; } std::string ParseHistoricalFIX::help() { diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.h b/src/server/sparkd/cmds/ParseHistoricalFIX.h index 6b95106a6..bcce3eea1 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.h +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.h @@ -7,6 +7,9 @@ namespace cmd { class ParseHistoricalFIX : public ICmd { +private: + void _execute_single(const std::string& full_file_name, + bool process_orders); public: int execute(std::vector const &args); std::string help ();