ParseHistoricalFIX takes a path and processes all files with name _raw in it

This commit is contained in:
Taha Afzal committed 2020-01-10 19:19:52 -05:00
1 parent 33e46b2fb1
commit 3d22a58bac
2 files changed
+55 -16

No files matched your search

+52 -16
View File
@@ -5,6 +5,8 @@
#include "lib/jetfix/codec/Group.h"
#include "lib/jetfix/codec/MessageTypes.h"
#include "lib/utils/bts_chrono.h"
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <fstream>
#include <unordered_map>
@@ -381,13 +383,49 @@ void print_orders(uint32_t msg_id, MDIncrementalUpdate const &update,
}
}
int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
std::ifstream input(args[0]);
std::ofstream books_stream(args[0] + "_books.csv");
std::ofstream msgs_stream(args[0] + "_msgs.csv");
std::ofstream trades_stream(args[0] + "_trades.csv");
std::ofstream orders_stream(args[0] + "_orders.csv");
void get_files(const std::string& dir,
std::vector<std::string>& out_files){
if(boost::filesystem::is_directory(dir)){
boost::filesystem::path p(dir);
for(auto& entry : boost::make_iterator_range(
boost::filesystem::directory_iterator(p), {})){
if(boost::filesystem::is_regular_file(entry)){
auto file_name = entry.path().string();
if(file_name.find("_raw") != std::string::npos &&
file_name.find(".csv") == std::string::npos)
out_files.push_back(file_name);
}
}
}
}
int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
// first arg is the path to where the files are
// second arg is optional to parse orders/trades or not
auto dir = args[0];
auto process_orders = args.size()==1 ? false : (args[1]=="orders");
std::vector<std::string> file_names;
get_files(dir, file_names);
for (const auto& f : file_names)
_execute_single(f, process_orders);
return 0;
}
void ParseHistoricalFIX::_execute_single(const std::string& file,
bool process_orders){
std::cout << "Processing file:" << file << std::endl;
std::ifstream input(file);
std::unique_ptr<std::ofstream> books_stream, msgs_stream, trades_stream, orders_stream;
books_stream = std::make_unique<std::ofstream>(file + "_books.csv");
print_book_header(*books_stream);
if(process_orders){
msgs_stream = std::make_unique<std::ofstream>(file + "_msgs.csv");
trades_stream = std::make_unique<std::ofstream>(file + "_trades.csv");
orders_stream = std::make_unique<std::ofstream>(file + "_orders.csv");
print_msg_headers(*msgs_stream);
print_trades_header(*trades_stream);
print_orders_header(*orders_stream);
}
std::unordered_map<std::string, Book> books;
TradeSummaryEvent current_trade_summary;
@@ -399,10 +437,7 @@ int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
auto ok = 0;
auto start = Timer::now();
print_book_header(books_stream);
print_msg_headers(msgs_stream);
print_trades_header(trades_stream);
print_orders_header(orders_stream);
print_book_header(*books_stream);
while (std::getline(input, line)) {
c++;
@@ -411,18 +446,19 @@ int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
if (parsed_ptr) {
auto msg =
std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
update_and_maybe_print_book(c,*msg, books, books_stream);
print_msgs(c, *msg, msgs_stream);
update_and_maybe_print_trades(c,*msg, current_trade_summary,
trades_stream);
print_orders(c,*msg, orders_stream);
update_and_maybe_print_book(c,*msg, books, *books_stream);
if(process_orders){
print_msgs(c, *msg, *msgs_stream);
update_and_maybe_print_trades(c,*msg, current_trade_summary,
*trades_stream);
print_orders(c,*msg, *orders_stream);
}
}
ok += (bool)(parsed_ptr);
}
auto sec_duration = sec_diff(Timer::now(), start);
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
<< " out of " << c << " time: " << sec_duration << " secs\n";
return 0;
}
std::string ParseHistoricalFIX::help() {
@@ -7,6 +7,9 @@ namespace cmd {
class ParseHistoricalFIX : public ICmd
{
private:
void _execute_single(const std::string& full_file_name,
bool process_orders);
public:
int execute(std::vector<std::string> const &args);
std::string help ();