From e9159458809fce616d531ba99df80bfe9a46bb46 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 9 Jul 2020 14:27:22 -0400 Subject: [PATCH] Trying to replay fix logs. --- .../actors/GLBXiLink3ExecutionGateway.cpp | 18 +++-- .../orderman/actors/ilink3/IL3Dispatcher.h | 13 ++-- src/lib/orderman/actors/ilink3/IL3GW.h | 8 ++- src/lib/orderman/actors/ilink3/IL3Parser.cpp | 72 +++++++++++++++++-- src/lib/orderman/actors/ilink3/IL3Parser.h | 6 +- .../orderman/actors/ilink3/SessionStore.cpp | 23 +++--- 6 files changed, 110 insertions(+), 30 deletions(-) diff --git a/src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp b/src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp index df433ecbb..eea21a4d7 100644 --- a/src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp +++ b/src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp @@ -5,13 +5,13 @@ #include "lib/jetfix/common/OutFIX.h" #include "lib/jetfix/store/session_name.h" #include "lib/orderman/actors/GLBXExecutionGateway.h" +#include "lib/orderman/actors/ilink3/IL3GW.h" #include "lib/orderman/actors/ilink3/IL3MessageFramer.h" #include "lib/orderman/actors/ilink3/IL3Reactor.h" -#include "lib/orderman/actors/ilink3/IL3GW.h" +#include "lib/orderman/actors/ilink3/IL3SecDB.h" #include "lib/orderman/actors/ilink3/SessionStore.h" #include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/orderman_utils/fix_order_codec.h" -#include "lib/orderman/actors/ilink3/IL3SecDB.h" namespace bts { namespace orderman { @@ -31,7 +31,7 @@ struct GLBXiLink3ExecutionGateway::PImpl { : state(), cfg(cfg), reactor(), start_of_pause(), on_exe_report(), on_gw_status_change(), on_cancel_reject(), perf_log(log), db(o), gw_name(cfg.name()), session_store(), gw(), connection(nullptr), - connection_failures(0), started(false) { + connection_failures(0), started(false), has_replayed(false) { TCPEndpoint endpoint{cfg.socket_connect_host_ip(), (ushort)cfg.socket_connect_port()}; connection = &reactor.register_connection({endpoint}); @@ -54,6 +54,7 @@ struct GLBXiLink3ExecutionGateway::PImpl { IL3Connection *connection; size_t connection_failures; bool started; + bool has_replayed; }; GLBXiLink3ExecutionGateway::GLBXiLink3ExecutionGateway( @@ -131,9 +132,16 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) { c.failover_on_disconnect = false; me->session_store.emplace(me->db, sn, me->cfg); + if (!me->has_replayed) { + me->has_replayed = true; + auto &replay = me->session_store.value().replay(); + for (auto const &er : replay) + me->on_exe_report(er, {}); + } + c.raw_message_logger = - [store = &me->session_store.value()](auto d, auto sn,auto const &msg) { - store->async_store_raw_msg(d, sn,msg); + [store = &me->session_store.value()](auto d, auto sn, auto const &msg) { + store->async_store_raw_msg(d, sn, msg); }; me->gw.emplace(*me->connection, me->framer, &me->db, diff --git a/src/lib/orderman/actors/ilink3/IL3Dispatcher.h b/src/lib/orderman/actors/ilink3/IL3Dispatcher.h index 167e3f38b..e7e7cff79 100644 --- a/src/lib/orderman/actors/ilink3/IL3Dispatcher.h +++ b/src/lib/orderman/actors/ilink3/IL3Dispatcher.h @@ -9,12 +9,17 @@ namespace ilink3 { template void handle(Handler& h, const char* data, uint16_t len) { + const FramingHeader* fheader = reinterpret_cast(data); + iLinkBinary::MessageHeader mheader { + (char*)data, sizeof(FramingHeader), fheader->message_size, + iLinkBinary::MessageHeader::sbeSchemaVersion() }; + T t { const_cast(data), sizeof(FramingHeader) + iLinkBinary::MessageHeader::encodedLength(), - len, - T::sbeBlockLength(), - T::sbeSchemaVersion() }; + len, + mheader.blockLength(), + mheader.version()}; h.handle(t); } @@ -132,4 +137,4 @@ size_t consume_messages( return read; } -}}}//namespace \ No newline at end of file +}}}//namespace diff --git a/src/lib/orderman/actors/ilink3/IL3GW.h b/src/lib/orderman/actors/ilink3/IL3GW.h index 184054005..22ceb1603 100644 --- a/src/lib/orderman/actors/ilink3/IL3GW.h +++ b/src/lib/orderman/actors/ilink3/IL3GW.h @@ -88,7 +88,7 @@ class IL3GW : IL3ConnectionHandler { public: IL3GW(IL3Connection &, IL3MessageFramer &, IL3SecDB *, const Session &, - const Config &, config::JetFIX::Connection const &); + const Config &, config::JetFIX::Connection const &); void connect(); void send_order(const CreateOrderMsg &); @@ -138,8 +138,10 @@ class IL3GW : IL3ConnectionHandler { std::stringstream buffer; buffer << msg; result.json = buffer.str(); - result.binary = - std::string(msg.buffer() + msg.offset(), msg.decodeLength()); + result.binary = std::string( + msg.buffer() + msg.offset() - + iLinkBinary::MessageHeader::encodedLength(), + msg.decodeLength() + iLinkBinary::MessageHeader::encodedLength()); return result; }; } diff --git a/src/lib/orderman/actors/ilink3/IL3Parser.cpp b/src/lib/orderman/actors/ilink3/IL3Parser.cpp index a8f0ba02a..422c236aa 100644 --- a/src/lib/orderman/actors/ilink3/IL3Parser.cpp +++ b/src/lib/orderman/actors/ilink3/IL3Parser.cpp @@ -97,6 +97,10 @@ void IL3Parser::parse_trade(const T &msg, ExecutionReport *data) { data->filled_quantity = Quantity{msg.cumQty()}; } +bool IL3Parser::has_account() const { + return strlen(most_recent_account.value().data()); +} + bool IL3Parser::parse(const iLinkBinary::ExecutionReportNew522 &msg, ExecutionReport *data) { parse_common(msg, data); @@ -114,12 +118,70 @@ bool IL3Parser::parse(const iLinkBinary::ExecutionReportNew522 &msg, return true; } -std::variant IL3Parser::parse(std::string_view const&) { - return {std::monostate ()}; +template +bool try_parse_exe_report_helper(IL3Parser &parser, std::string_view const &msg, + ExecutionReport *out) { + + iLinkBinary::MessageHeader mheader{ + (char *)msg.data(), 0, msg.size(), + iLinkBinary::MessageHeader::sbeSchemaVersion()}; + + if (mheader.templateId() == ER::sbeTemplateId()) { + return parser.parse( + ER{(char *)msg.data(), iLinkBinary::MessageHeader::encodedLength(), + msg.size(), mheader.blockLength(), mheader.version()}, + out); + } + return false; } - + +std::optional +IL3Parser::try_parse_exe_report(std::string_view const &msg) { + using namespace iLinkBinary; + + if (has_account()) { + ExecutionReport report; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + if (try_parse_exe_report_helper( + *this, msg, &report)) + return {report}; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + if (try_parse_exe_report_helper( + *this, msg, &report)) + return {report}; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + if (try_parse_exe_report_helper(*this, msg, + &report)) + return {report}; + } + + iLinkBinary::MessageHeader mheader{ + (char *)msg.data(), 0, msg.size(), + iLinkBinary::MessageHeader::sbeSchemaVersion()}; + + if (mheader.templateId() == + PartyDetailsDefinitionRequestAck519::sbeTemplateId()) + set_account(PartyDetailsDefinitionRequestAck519{ + (char *)msg.data(), iLinkBinary::MessageHeader::encodedLength(), + msg.size(), mheader.blockLength(), mheader.version()}); + return {}; +} + bool IL3Parser::parse(const iLinkBinary::ExecutionReportReject523 &msg, ExecutionReport *data) { parse_common(msg, data); diff --git a/src/lib/orderman/actors/ilink3/IL3Parser.h b/src/lib/orderman/actors/ilink3/IL3Parser.h index bb671447c..5ebec31a9 100644 --- a/src/lib/orderman/actors/ilink3/IL3Parser.h +++ b/src/lib/orderman/actors/ilink3/IL3Parser.h @@ -34,10 +34,8 @@ public: bool parse(const iLinkBinary::ExecutionReportCancel534&, ExecutionReport*); bool parse(const iLinkBinary::OrderCancelReject535&, CancelReject*); void set_account(const iLinkBinary::PartyDetailsDefinitionRequestAck519&); - - std::variant parse(std::string_view const&bytes); + bool has_account() const; + std::optional try_parse_exe_report(std::string_view const&bytes); private: template void parse_common(const T&, ExecutionReport*); template void parse_trade(const T&, ExecutionReport*); diff --git a/src/lib/orderman/actors/ilink3/SessionStore.cpp b/src/lib/orderman/actors/ilink3/SessionStore.cpp index 3031d57bf..73bda6a67 100644 --- a/src/lib/orderman/actors/ilink3/SessionStore.cpp +++ b/src/lib/orderman/actors/ilink3/SessionStore.cpp @@ -1,7 +1,7 @@ #include "SessionStore.h" #include "lib/jetfix/store/session_name.h" -#include "lib/orderman/actors/ilink3/IL3SecDB.h" #include "lib/orderman/actors/ilink3/IL3GW.h" +#include "lib/orderman/actors/ilink3/IL3SecDB.h" #include "lib/orderman/orderman_utils/ExecutionReport.h" #include "lib/utils/base64.hpp" #include "lib/utils/bts_chrono.h" @@ -49,7 +49,7 @@ static std::vector load_replay(IL3SecDB &db, config::JetFIX::Connection const &cfg, std::string const &path) { - IL3Parser parse(cfg,&db); + IL3Parser parser(cfg, &db); std::vector result; if (boost::filesystem::exists(path)) { @@ -66,14 +66,20 @@ load_replay(IL3SecDB &db, config::JetFIX::Connection const &cfg, f >> j; if (j["direction"] == "<") { auto ts = datetime::parse_FIX_timestamp(j["time"]); - if (!datetime::is_at_least_secs_old(ts, + if (true || !datetime::is_at_least_secs_old(ts, cfg.seconds_to_replay())) { + + auto msg_bytes = websocketpp::base64_decode(j["msg_sbe"]); + if (msg_bytes.size()) { + auto maybe_ex_rep = parser.try_parse_exe_report(msg_bytes); + if (maybe_ex_rep) + result.push_back(maybe_ex_rep.value()); + } } } } } } - return result; } @@ -81,7 +87,7 @@ static void save_raw_msg(std::ofstream &out, RawMessageDirection d, size_t sn, RawMessageData const &msg) { nlohmann::json j; j["direction"] = d == RawMessageDirection::INCOMING ? "<" : ">"; - j["sn"] = sn; + j["sn"] = sn; j["time"] = datetime::FIX_timestamp(datetime::now()); j["msg_sbe"] = websocketpp::base64_encode(msg.binary); try { @@ -156,11 +162,10 @@ void SessionStore::async_update_session(Session const &s) { Action{[this, s]() { save_session(session_path(me->store_path), s); }}); } -void SessionStore::async_store_raw_msg(RawMessageDirection d, - size_t sn, +void SessionStore::async_store_raw_msg(RawMessageDirection d, size_t sn, RawMsg const &ms) { - me->event_queue.push( - Action{[this, d, sn,ms]() { save_raw_msg(me->raw_msg_store, d, sn,ms()); }}); + me->event_queue.push(Action{ + [this, d, sn, ms]() { save_raw_msg(me->raw_msg_store, d, sn, ms()); }}); } void SessionStore::do_work() {