#include "CoreGateway.h" #include "lib/logging/logging.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/bts_chrono.h" #include "lib/utils/datetime.h" #include "lib/jetfix/common/OutFIX.h" #include "lib/jetfix/connection/AsyncReader.h" #include "lib/jetfix/connection/IConnection.h" #include "lib/jetfix/gateway/ICoreGatewayObserver.h" #include "lib/jetfix/gateway/Session.h" #include "lib/jetfix/store/IFIXStore.h" #include "lib/orderman/orderman_utils/orderman_common.h" #include "lib/tracing/bts_trace.h" #include #include #include #include #include #include #include namespace bts { namespace jetfix { enum class CoreGateway::State { OFF, PAUSED, CONNECTING, READY, WREADER_DISCONNECT, WSESSION_TERMINATE }; struct CoreGateway::SessionInfo { SessionInfo() : connection(), store(), reader(), session() {} std::unique_ptr connection; std::unique_ptr store; std::unique_ptr reader; std::unique_ptr session; }; struct CoreGateway::EventData { typedef void (CoreGateway::*Handler)(EventData const &ev); EventData(const EventData &src) : handler(src.handler), out_msgs(src.out_msgs), in_msg(src.in_msg), cid(src.cid), ts(src.ts) {} EventData() : handler(), out_msgs(), in_msg(), cid(orderman::NullTickToTradeCorID), ts() {} EventData(Handler handler) : handler(handler), out_msgs(), in_msg(), cid(), ts() {} EventData(Handler handler, orderman::TickToTradeCorID cid, OutFIXPtr const &out_msgs) : handler(handler), out_msgs(out_msgs), in_msg(), cid(cid), ts() {} EventData(Handler handler, InMsgBuffer const &in_msg) : handler(handler), out_msgs(), in_msg(in_msg), cid(orderman::NullTickToTradeCorID), ts() {} EventData(Handler handler, std::shared_ptr const &ts) : handler(handler), out_msgs(), in_msg(), cid(orderman::NullTickToTradeCorID), ts(ts) {} EventData &operator=(EventData &src) { handler = src.handler; out_msgs = src.out_msgs; in_msg = src.in_msg; cid = src.cid; ts = src.ts; return *this; } EventData &operator=(EventData &&src) { handler = std::move(src.handler); out_msgs = std::move(src.out_msgs); in_msg = std::move(src.in_msg); cid = src.cid; ts = std::move(src.ts); return *this; } Handler handler; OutFIXPtr out_msgs; InMsgBuffer in_msg; orderman::TickToTradeCorID cid; std::shared_ptr ts; }; struct CoreGateway::PImpl { PImpl(config::JetFIX::Connection const &cfg, ICoreGatewayObserver &observer, ConnectionFactory const &conn_factory, StoreFactory const &store_factory) : cfg(cfg), observer(observer), make_connection(conn_factory), make_store(store_factory), state(State::OFF), start_of_pause(), session_info(), event_queue(), timer_thread(), destroy(false), is_on(false), has_replayed(false), send_batch(), recv_batch(), connection_attempts(0) { send_batch.reserve(1024); recv_batch.reserve(1024); } const config::JetFIX::Connection cfg; ICoreGatewayObserver &observer; ConnectionFactory const make_connection; StoreFactory const make_store; State state; datetime::PTime start_of_pause; std::unique_ptr session_info; tbb::concurrent_queue event_queue; std::thread timer_thread; std::atomic destroy; bool is_on; bool has_replayed; std::vector send_batch; std::vector recv_batch; uint32_t connection_attempts; }; CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg, ICoreGatewayObserver &observer, ConnectionFactory const &conn_factory, StoreFactory const &store_factory) : me(new PImpl(cfg, observer, conn_factory, store_factory)) { me->timer_thread = std::thread(std::bind(&CoreGateway::call_time, this)); } CoreGateway::~CoreGateway() { stop(); while (me->state != State::OFF) { dispatch_events(); std::this_thread::sleep_for(100ms); } me->destroy = true; me->timer_thread.join(); } void CoreGateway::send(orderman::TickToTradeCorID cid, OutFIXPtr const &fix) { tracepoint(bts_trace, trace_order, (char *)"CoreGateway.send", 0, cid,0); me->event_queue.push(EventData(&CoreGateway::sync_send, cid, fix)); } void CoreGateway::send_hb() { me->event_queue.push(EventData(&CoreGateway::sync_send_hb)); } void CoreGateway::send_test_request() { me->event_queue.push(EventData(&CoreGateway::sync_send_test_request)); } void CoreGateway::start() { me->event_queue.push(EventData(&CoreGateway::sync_start)); } void CoreGateway::stop() { me->event_queue.push(EventData(&CoreGateway::sync_stop)); } void CoreGateway::session_ready() { me->event_queue.push(EventData(&CoreGateway::sync_session_ready)); } void CoreGateway::session_terminated() { me->event_queue.push(EventData(&CoreGateway::sync_session_terminated)); } void CoreGateway::reader_disconnected() { me->event_queue.push(EventData(&CoreGateway::sync_reader_disconnected)); } void CoreGateway::in_msg_raw(InMsgBuffer const &msg) { me->event_queue.push(EventData(&CoreGateway::sync_in_msg_raw, msg)); } void CoreGateway::in_msg_sequenced(InMsgBuffer const &msg) { me->event_queue.push(EventData(&CoreGateway::sync_in_msg_sequenced, msg)); } void CoreGateway::dispatch_events() { EventData ev; utils::TimeStamp cid; // we prioratize sends over receives while (me->event_queue.try_pop(ev)) { if (ev.out_msgs) { cid = ev.out_msgs->get_ts(jetfix::OutFIX::MD_WRITE_TS); me->send_batch.push_back(ev); } else me->recv_batch.push_back(ev); } if (me->send_batch.size() || me->recv_batch.size()) { for (auto &ev : me->send_batch) (this->*ev.handler)(ev); for (auto &ev : me->recv_batch) (this->*ev.handler)(ev); me->send_batch.clear(); me->recv_batch.clear(); } } void CoreGateway::call_time() { os::set_thread_name(fmt::format("{0}_timer", me->cfg.name()).c_str()); while (!me->destroy) { std::this_thread::sleep_for(10ms); auto ts = std::make_shared( datetime::FIX_timestamp(datetime::now())); me->event_queue.push(EventData(&CoreGateway::sync_time, ts)); // NOLINT } } void CoreGateway::sync_start(EventData const &) { switch (me->state) { case State::OFF: { me->is_on = true; pause_or_shutdown(); break; } case State::PAUSED: case State::CONNECTING: case State::READY: bts_assert(false); break; case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: bts_assert(!me->is_on); me->is_on = true; break; default: bts_assert(false); } } void CoreGateway::sync_stop(EventData const &) { switch (me->state) { case State::OFF: break; case State::PAUSED: me->is_on = false; transition(State::OFF); break; case State::CONNECTING: case State::READY: me->is_on = false; me->session_info->session->shutdown(); break; case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: me->is_on = false; break; default: bts_assert(false); } } void CoreGateway::sync_send(EventData const &ev) { tracepoint(bts_trace, trace_order, (char *)"CoreGateway.sync_send", 0, ev.cid,0); switch (me->state) { case State::OFF: bts_assert(false); break; case State::PAUSED: case State::CONNECTING: case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: break; case State::READY: me->session_info->session->send(ev.cid, ev.out_msgs); break; default: bts_assert(false); } } void CoreGateway::sync_send_hb(EventData const &) { switch (me->state) { case State::OFF: bts_assert(false); break; case State::PAUSED: case State::CONNECTING: case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: break; case State::READY: me->session_info->session->send_hb(); break; default: bts_assert(false); } } void CoreGateway::sync_send_test_request(EventData const &) { switch (me->state) { case State::OFF: bts_assert(false); break; case State::PAUSED: case State::CONNECTING: case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: break; case State::READY: me->session_info->session->send_test_request(); break; default: bts_assert(false); } } void CoreGateway::sync_session_ready(EventData const &) { switch (me->state) { case State::OFF: case State::PAUSED: case State::READY: case State::WREADER_DISCONNECT: bts_assert(false); break; case State::WSESSION_TERMINATE: break; case State::CONNECTING: transition(State::READY); me->observer.ready(); break; default: bts_assert(false); } } void CoreGateway::sync_session_terminated(EventData const &) { switch (me->state) { case State::OFF: case State::PAUSED: case State::WREADER_DISCONNECT: bts_assert(false); break; case State::WSESSION_TERMINATE: pause_or_shutdown(); break; case State::READY: me->session_info->session.reset(); me->session_info->connection->shutdown(); transition(State::WREADER_DISCONNECT); me->observer.disconnect(); break; case State::CONNECTING: me->session_info->session.reset(); me->session_info->connection->shutdown(); transition(State::WREADER_DISCONNECT); break; default: bts_assert(false); } } void CoreGateway::sync_reader_disconnected(EventData const &) { switch (me->state) { case State::OFF: case State::PAUSED: case State::WSESSION_TERMINATE: bts_assert(false); break; case State::READY: me->session_info->session.reset(); me->observer.disconnect(); transition(State::WSESSION_TERMINATE); break; case State::CONNECTING: me->session_info->session.reset(); transition(State::WSESSION_TERMINATE); break; case State::WREADER_DISCONNECT: pause_or_shutdown(); break; default: bts_assert(false); } } void CoreGateway::sync_in_msg_raw(EventData const &ev) { switch (me->state) { case State::OFF: case State::PAUSED: bts_assert(false); break; case State::CONNECTING: case State::READY: me->session_info->session->receive(ev.in_msg); break; case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: break; default: bts_assert(false); } } void CoreGateway::sync_in_msg_sequenced(EventData const &ev) { switch (me->state) { case State::OFF: case State::PAUSED: case State::WREADER_DISCONNECT: bts_assert(false); break; case State::CONNECTING: case State::READY: me->observer.msg(ev.in_msg); break; case State::WSESSION_TERMINATE: break; default: bts_assert(false); } } void CoreGateway::sync_time(EventData const &ev) { switch (me->state) { case State::OFF: break; case State::PAUSED: if (datetime::is_at_least_secs_old(me->start_of_pause, me->cfg.seconds_to_pause())) { LOG(Jetfix, Info, "Pause period ended for jetfix connection {}", me->cfg.name()); me->session_info = std::make_unique(); me->session_info->store.reset(me->make_store(me->cfg)); if (me->session_info->store->session_active()) { me->connection_attempts++; if (!me->has_replayed) { me->observer.replay(me->session_info->store->replay()); me->has_replayed = true; } if (me->connection_attempts > me->cfg.max_connection_attempts()) { me->session_info.reset(); me->start_of_pause = datetime::now(); LOG(Jetfix, Error, "Failed to connect to exchange gateway {} too many " "times. Giving up permanently. You must bounce " "the server to recover.", me->cfg.name()); break; } me->session_info->connection.reset(me->make_connection(me->cfg)); LOG(Jetfix, Warn, "Trying to connect to exchange for {} ({}/{})", me->cfg.name(), me->connection_attempts, me->cfg.max_connection_attempts()); if (me->session_info->connection->connect()) { LOG(Jetfix, Warn, "Connected to exchange gateway for {}", me->cfg.name()); me->session_info->session.reset( new Session(me->cfg, *me->session_info->connection, *me->session_info->store, *this, []() { return utils::TimeStamp::now(); })); me->session_info->reader.reset(new AsyncReader( *me->session_info->connection, me->cfg.name(), *this, me->cfg.msg_buffer_sz(), me->cfg.msg_high_water_mark())); me->session_info->reader->start(); me->session_info->session->start(); transition(State::CONNECTING); } else { me->session_info.reset(); me->start_of_pause = datetime::now(); LOG(Jetfix, Warn, "Failed to connect to exchange for {}. ({}/{}) ", me->cfg.name(), me->connection_attempts, me->cfg.max_connection_attempts()); } } else { me->session_info.reset(); me->start_of_pause = datetime::now(); } } break; case State::CONNECTING: case State::READY: me->session_info->session->time(ev.ts); break; case State::WREADER_DISCONNECT: case State::WSESSION_TERMINATE: break; default: bts_assert(false); } } void CoreGateway::pause_or_shutdown() { me->session_info.reset(); if (me->is_on) { me->start_of_pause = datetime::now(); transition(State::PAUSED); } else transition(State::OFF); } void CoreGateway::transition(State new_state) { LOG(Jetfix, Warn, "Transitioning Jetfix gateway {} from {} to {}", me->cfg.name(), std::string(state_name(me->state)), std::string(state_name(new_state))); me->state = new_state; check_state_invariants(); } void CoreGateway::check_state_invariants() { switch (me->state) { case State::OFF: bts_assert(!me->session_info); bts_assert(!me->is_on); return; case State::PAUSED: bts_assert(!me->session_info); bts_assert(me->is_on); return; case State::CONNECTING: case State::READY: bts_assert(me->session_info); bts_assert(me->session_info->session); bts_assert(me->is_on); return; case State::WREADER_DISCONNECT: bts_assert(me->session_info); bts_assert(!me->session_info->session); return; case State::WSESSION_TERMINATE: bts_assert(me->session_info); bts_assert(!me->session_info->session); break; default: bts_assert(false); return; } } const char *CoreGateway::state_name(State st) { switch (st) { case State::OFF: return "OFF"; case State::PAUSED: return "PAUSED"; case State::CONNECTING: return "CONNECTING"; case State::READY: return "READY"; case State::WREADER_DISCONNECT: return "WREADER_DISCONNECT"; case State::WSESSION_TERMINATE: return "WSESSION_TERMINATE"; default: bts_assert(false); } } } }