532 lines
15 KiB
C++
532 lines
15 KiB
C++
#include "CoreGateway.h"
|
|
|
|
#include "lib/jetfix/common/OutFIX.h"
|
|
#include "lib/jetfix/connection/AsyncReader.h"
|
|
#include "lib/jetfix/connection/IConnection.h"
|
|
#include "lib/jetfix/gateway/ICoreGatewayObserver.h"
|
|
#include "lib/jetfix/gateway/Session.h"
|
|
#include "lib/jetfix/store/IFIXStore.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/orderman/orderman_utils/orderman_common.h"
|
|
#include "lib/tracing/bts_trace.h"
|
|
#include "lib/utils/TimeStamp.h"
|
|
#include "lib/utils/bts_chrono.h"
|
|
#include "lib/utils/datetime.h"
|
|
#include "lib/utils/thread_manager.h"
|
|
#include <boost/lockfree/queue.hpp>
|
|
|
|
#include <atomic>
|
|
#include <boost/date_time/posix_time/posix_time.hpp>
|
|
#include <fmt/format.h>
|
|
#include <fstream>
|
|
#include <tbb/concurrent_queue.h>
|
|
#include <thread>
|
|
|
|
namespace bts {
|
|
namespace jetfix {
|
|
|
|
enum class CoreGateway::State {
|
|
OFF,
|
|
PAUSED,
|
|
CONNECTING,
|
|
READY,
|
|
WREADER_DISCONNECT,
|
|
WSESSION_TERMINATE
|
|
};
|
|
|
|
struct CoreGateway::SessionInfo {
|
|
SessionInfo() : connection(), store(), reader(), session() {}
|
|
|
|
std::unique_ptr<IConnection> connection;
|
|
std::unique_ptr<IFIXStore> store;
|
|
std::unique_ptr<AsyncReader> reader;
|
|
std::unique_ptr<Session> session;
|
|
};
|
|
|
|
struct CoreGateway::EventData {
|
|
typedef void (CoreGateway::*Handler)(EventData const &ev);
|
|
|
|
EventData(const EventData &src)
|
|
: handler(src.handler), out_msgs(src.out_msgs), in_msg(src.in_msg),
|
|
trace(src.trace), ts(src.ts) {}
|
|
|
|
EventData() : handler(), out_msgs(), in_msg(), trace(), ts() {}
|
|
EventData(Handler handler)
|
|
: handler(handler), out_msgs(), in_msg(), trace(), ts() {}
|
|
EventData(Handler handler, trace::CorID const &trace,
|
|
OutFIXPtr const &out_msgs)
|
|
: handler(handler), out_msgs(out_msgs), in_msg(), trace(trace), ts() {}
|
|
|
|
EventData(Handler handler, InMsgBuffer const &in_msg)
|
|
: handler(handler), out_msgs(), in_msg(in_msg), trace(), ts() {}
|
|
|
|
EventData(Handler handler, std::shared_ptr<std::string> const &ts)
|
|
: handler(handler), out_msgs(), in_msg(), trace(), ts(ts) {}
|
|
|
|
EventData &operator=(EventData &src) {
|
|
handler = src.handler;
|
|
out_msgs = src.out_msgs;
|
|
in_msg = src.in_msg;
|
|
trace = src.trace;
|
|
ts = src.ts;
|
|
return *this;
|
|
}
|
|
|
|
EventData &operator=(EventData &&src) {
|
|
handler = std::move(src.handler);
|
|
out_msgs = std::move(src.out_msgs);
|
|
in_msg = std::move(src.in_msg);
|
|
trace = src.trace;
|
|
ts = std::move(src.ts);
|
|
return *this;
|
|
}
|
|
Handler handler;
|
|
OutFIXPtr out_msgs;
|
|
InMsgBuffer in_msg;
|
|
trace::CorID trace;
|
|
std::shared_ptr<std::string> ts;
|
|
};
|
|
|
|
struct CoreGateway::PImpl {
|
|
PImpl(config::JetFIX::Connection const &cfg, ICoreGatewayObserver &observer,
|
|
ConnectionFactory const &conn_factory,
|
|
StoreFactory const &store_factory, trace::PerfLog &l)
|
|
: cfg(cfg), observer(observer), make_connection(conn_factory),
|
|
make_store(store_factory), state(State::OFF), start_of_pause(),
|
|
session_info(), event_queue(), timer_thread(), destroy(false),
|
|
is_on(false), has_replayed(false), send_batch(), recv_batch(),
|
|
connection_attempts(0), perf_log(l) {
|
|
send_batch.reserve(1024);
|
|
recv_batch.reserve(1024);
|
|
}
|
|
|
|
const config::JetFIX::Connection cfg;
|
|
ICoreGatewayObserver &observer;
|
|
ConnectionFactory const make_connection;
|
|
StoreFactory const make_store;
|
|
|
|
State state;
|
|
datetime::PTime start_of_pause;
|
|
std::unique_ptr<SessionInfo> session_info;
|
|
tbb::concurrent_queue<EventData> event_queue;
|
|
std::thread timer_thread;
|
|
std::atomic<bool> destroy;
|
|
bool is_on;
|
|
bool has_replayed;
|
|
std::vector<EventData> send_batch;
|
|
std::vector<EventData> recv_batch;
|
|
uint32_t connection_attempts;
|
|
trace::PerfLog &perf_log;
|
|
};
|
|
|
|
std::thread CoreGateway::make_timer_thread() {
|
|
auto thread_name = fmt::format("jf_tmr_{}", me->cfg.name());
|
|
return threading::make_thread(thread_name,
|
|
std::bind(&CoreGateway::call_time, this));
|
|
}
|
|
|
|
CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg,
|
|
ICoreGatewayObserver &observer,
|
|
ConnectionFactory const &conn_factory,
|
|
StoreFactory const &store_factory,
|
|
trace::PerfLog &perf_trace)
|
|
: me(new PImpl(cfg, observer, conn_factory, store_factory, perf_trace)) {
|
|
me->timer_thread = make_timer_thread();
|
|
}
|
|
|
|
CoreGateway::~CoreGateway() {
|
|
stop();
|
|
while (me->state != State::OFF) {
|
|
dispatch_events();
|
|
std::this_thread::sleep_for(100ms);
|
|
}
|
|
me->destroy = true;
|
|
me->timer_thread.join();
|
|
}
|
|
|
|
void CoreGateway::send(OutFIXPtr const &fix, trace::CorID trace) {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_send, trace, fix));
|
|
}
|
|
|
|
void CoreGateway::start() {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_start));
|
|
}
|
|
|
|
void CoreGateway::stop() {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_stop));
|
|
}
|
|
|
|
void CoreGateway::session_ready() {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_session_ready));
|
|
}
|
|
|
|
void CoreGateway::session_terminated() {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_session_terminated));
|
|
}
|
|
|
|
void CoreGateway::reader_disconnected() {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_reader_disconnected));
|
|
}
|
|
|
|
void CoreGateway::in_msg_raw(InMsgBuffer const &msg) {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_in_msg_raw, msg));
|
|
}
|
|
|
|
void CoreGateway::in_msg_sequenced(InMsgBuffer const &msg) {
|
|
me->event_queue.push(EventData(&CoreGateway::sync_in_msg_sequenced, msg));
|
|
}
|
|
|
|
void CoreGateway::dispatch_events() {
|
|
EventData ev;
|
|
// we prioratize sends over receives
|
|
while (me->event_queue.try_pop(ev)) {
|
|
if (ev.out_msgs) {
|
|
ev.out_msgs->get_ts(jetfix::OutFIX::MD_WRITE_TS);
|
|
me->send_batch.push_back(ev);
|
|
} else
|
|
me->recv_batch.push_back(ev);
|
|
}
|
|
|
|
if (me->send_batch.size() || me->recv_batch.size()) {
|
|
for (auto &ev : me->send_batch)
|
|
(this->*ev.handler)(ev);
|
|
for (auto &ev : me->recv_batch)
|
|
(this->*ev.handler)(ev);
|
|
me->send_batch.clear();
|
|
me->recv_batch.clear();
|
|
}
|
|
}
|
|
|
|
void CoreGateway::call_time() {
|
|
while (!me->destroy) {
|
|
std::this_thread::sleep_for(10ms);
|
|
auto ts = std::make_shared<std::string>(
|
|
datetime::FIX_timestamp(datetime::now()));
|
|
me->event_queue.push(EventData(&CoreGateway::sync_time, ts)); // NOLINT
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_start(EventData const &) {
|
|
switch (me->state) {
|
|
case State::OFF: {
|
|
me->is_on = true;
|
|
pause_or_shutdown();
|
|
break;
|
|
}
|
|
case State::PAUSED:
|
|
case State::CONNECTING:
|
|
case State::READY:
|
|
bts_assert(false);
|
|
break;
|
|
case State::WREADER_DISCONNECT:
|
|
case State::WSESSION_TERMINATE:
|
|
bts_assert(!me->is_on);
|
|
me->is_on = true;
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_stop(EventData const &) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
break;
|
|
case State::PAUSED:
|
|
me->is_on = false;
|
|
transition(State::OFF);
|
|
break;
|
|
case State::CONNECTING:
|
|
case State::READY:
|
|
me->is_on = false;
|
|
me->session_info->session->shutdown();
|
|
break;
|
|
case State::WREADER_DISCONNECT:
|
|
case State::WSESSION_TERMINATE:
|
|
me->is_on = false;
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_send(EventData const &ev) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
bts_assert(false);
|
|
break;
|
|
case State::PAUSED:
|
|
case State::CONNECTING:
|
|
case State::WREADER_DISCONNECT:
|
|
case State::WSESSION_TERMINATE:
|
|
break;
|
|
case State::READY:
|
|
me->session_info->session->send(ev.out_msgs, ev.trace);
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_session_ready(EventData const &) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
case State::PAUSED:
|
|
case State::READY:
|
|
case State::WREADER_DISCONNECT:
|
|
bts_assert(false);
|
|
break;
|
|
case State::WSESSION_TERMINATE:
|
|
break;
|
|
case State::CONNECTING:
|
|
transition(State::READY);
|
|
me->observer.ready();
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_session_terminated(EventData const &) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
case State::PAUSED:
|
|
case State::WREADER_DISCONNECT:
|
|
bts_assert(false);
|
|
break;
|
|
|
|
case State::WSESSION_TERMINATE:
|
|
pause_or_shutdown();
|
|
break;
|
|
|
|
case State::READY:
|
|
me->session_info->session.reset();
|
|
me->session_info->connection->shutdown();
|
|
transition(State::WREADER_DISCONNECT);
|
|
me->observer.disconnect();
|
|
break;
|
|
|
|
case State::CONNECTING:
|
|
me->session_info->session.reset();
|
|
me->session_info->connection->shutdown();
|
|
transition(State::WREADER_DISCONNECT);
|
|
break;
|
|
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_reader_disconnected(EventData const &) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
case State::PAUSED:
|
|
case State::WSESSION_TERMINATE:
|
|
bts_assert(false);
|
|
break;
|
|
|
|
case State::READY:
|
|
me->session_info->session.reset();
|
|
me->observer.disconnect();
|
|
transition(State::WSESSION_TERMINATE);
|
|
break;
|
|
|
|
case State::CONNECTING:
|
|
me->session_info->session.reset();
|
|
transition(State::WSESSION_TERMINATE);
|
|
break;
|
|
|
|
case State::WREADER_DISCONNECT:
|
|
pause_or_shutdown();
|
|
break;
|
|
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_in_msg_raw(EventData const &ev) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
case State::PAUSED:
|
|
bts_assert(false);
|
|
break;
|
|
case State::CONNECTING:
|
|
case State::READY: {
|
|
me->session_info->session->receive(ev.in_msg);
|
|
} break;
|
|
case State::WREADER_DISCONNECT:
|
|
case State::WSESSION_TERMINATE:
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_in_msg_sequenced(EventData const &ev) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
case State::PAUSED:
|
|
case State::WREADER_DISCONNECT:
|
|
bts_assert(false);
|
|
break;
|
|
case State::CONNECTING:
|
|
case State::READY:
|
|
me->observer.msg(ev.in_msg);
|
|
break;
|
|
case State::WSESSION_TERMINATE:
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::sync_time(EventData const &ev) {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
break;
|
|
case State::PAUSED:
|
|
if (datetime::is_at_least_secs_old(me->start_of_pause,
|
|
me->cfg.seconds_to_pause())) {
|
|
LOG(Jetfix, Info, "Pause period ended for jetfix connection {}",
|
|
me->cfg.name());
|
|
me->session_info = std::make_unique<SessionInfo>();
|
|
|
|
me->session_info->store.reset(me->make_store(me->cfg));
|
|
|
|
if (me->session_info->store->session_active()) {
|
|
me->connection_attempts++;
|
|
|
|
if (!me->has_replayed) {
|
|
auto msgs_to_replay = me->session_info->store->replay();
|
|
LOG(Jetfix, Warn, "Replaying {} FIX msgs",
|
|
msgs_to_replay.size());
|
|
me->observer.replay(msgs_to_replay);
|
|
me->has_replayed = true;
|
|
}
|
|
|
|
if (me->connection_attempts > me->cfg.max_connection_attempts()) {
|
|
me->session_info.reset();
|
|
me->start_of_pause = datetime::now();
|
|
LOG(Jetfix, Error,
|
|
"Failed to connect to exchange gateway {} too many "
|
|
"times. Giving up permanently. You must bounce "
|
|
"the server to recover.",
|
|
me->cfg.name());
|
|
break;
|
|
}
|
|
|
|
me->session_info->connection.reset(me->make_connection(me->cfg));
|
|
LOG(Jetfix, Warn, "Trying to connect to exchange for {} ({}/{})",
|
|
me->cfg.name(), me->connection_attempts,
|
|
me->cfg.max_connection_attempts());
|
|
if (me->session_info->connection->connect()) {
|
|
LOG(Jetfix, Warn, "Connected to exchange gateway for {}",
|
|
me->cfg.name());
|
|
|
|
me->session_info->session.reset(new Session(
|
|
me->cfg, *me->session_info->connection,
|
|
*me->session_info->store, *this,
|
|
[]() { return utils::TimeStamp::now(); }, me->perf_log));
|
|
me->session_info->reader.reset(new AsyncReader(
|
|
*me->session_info->connection, me->cfg.name(), *this,
|
|
me->perf_log, me->cfg.msg_buffer_sz(),
|
|
me->cfg.msg_high_water_mark()));
|
|
me->session_info->reader->start();
|
|
me->session_info->session->start();
|
|
transition(State::CONNECTING);
|
|
} else {
|
|
me->session_info.reset();
|
|
me->start_of_pause = datetime::now();
|
|
LOG(Jetfix, Warn,
|
|
"Failed to connect to exchange for {}. ({}/{}) ",
|
|
me->cfg.name(), me->connection_attempts,
|
|
me->cfg.max_connection_attempts());
|
|
}
|
|
} else {
|
|
me->session_info.reset();
|
|
me->start_of_pause = datetime::now();
|
|
}
|
|
}
|
|
break;
|
|
case State::CONNECTING:
|
|
case State::READY:
|
|
me->session_info->session->time(ev.ts);
|
|
break;
|
|
case State::WREADER_DISCONNECT:
|
|
case State::WSESSION_TERMINATE:
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
|
|
void CoreGateway::pause_or_shutdown() {
|
|
me->session_info.reset();
|
|
|
|
if (me->is_on) {
|
|
me->start_of_pause = datetime::now();
|
|
transition(State::PAUSED);
|
|
} else
|
|
transition(State::OFF);
|
|
}
|
|
|
|
void CoreGateway::transition(State new_state) {
|
|
LOG(Jetfix, Warn, "Transitioning Jetfix gateway {} from {} to {}",
|
|
me->cfg.name(), std::string(state_name(me->state)),
|
|
std::string(state_name(new_state)));
|
|
me->state = new_state;
|
|
check_state_invariants();
|
|
}
|
|
|
|
void CoreGateway::check_state_invariants() {
|
|
switch (me->state) {
|
|
case State::OFF:
|
|
bts_assert(!me->session_info);
|
|
bts_assert(!me->is_on);
|
|
return;
|
|
case State::PAUSED:
|
|
bts_assert(!me->session_info);
|
|
bts_assert(me->is_on);
|
|
return;
|
|
case State::CONNECTING:
|
|
case State::READY:
|
|
bts_assert(me->session_info);
|
|
bts_assert(me->session_info->session);
|
|
bts_assert(me->is_on);
|
|
return;
|
|
case State::WREADER_DISCONNECT:
|
|
bts_assert(me->session_info);
|
|
bts_assert(!me->session_info->session);
|
|
return;
|
|
case State::WSESSION_TERMINATE:
|
|
bts_assert(me->session_info);
|
|
bts_assert(!me->session_info->session);
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
return;
|
|
}
|
|
}
|
|
|
|
const char *CoreGateway::state_name(State st) {
|
|
switch (st) {
|
|
case State::OFF:
|
|
return "OFF";
|
|
case State::PAUSED:
|
|
return "PAUSED";
|
|
case State::CONNECTING:
|
|
return "CONNECTING";
|
|
case State::READY:
|
|
return "READY";
|
|
case State::WREADER_DISCONNECT:
|
|
return "WREADER_DISCONNECT";
|
|
case State::WSESSION_TERMINATE:
|
|
return "WSESSION_TERMINATE";
|
|
default:
|
|
bts_assert(false);
|
|
}
|
|
}
|
|
} // namespace jetfix
|
|
} // namespace bts
|