Files
spark/src/lib/jetfix/gateway/CoreGateway.cpp
T
2019-09-27 17:11:49 -04:00

532 lines
15 KiB
C++

#include "CoreGateway.h"
#include "lib/jetfix/common/OutFIX.h"
#include "lib/jetfix/connection/AsyncReader.h"
#include "lib/jetfix/connection/IConnection.h"
#include "lib/jetfix/gateway/ICoreGatewayObserver.h"
#include "lib/jetfix/gateway/Session.h"
#include "lib/jetfix/store/IFIXStore.h"
#include "lib/logging/logging.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/tracing/bts_trace.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/datetime.h"
#include "lib/utils/thread_manager.h"
#include <boost/lockfree/queue.hpp>
#include <atomic>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <fmt/format.h>
#include <fstream>
#include <tbb/concurrent_queue.h>
#include <thread>
namespace bts {
namespace jetfix {
enum class CoreGateway::State {
OFF,
PAUSED,
CONNECTING,
READY,
WREADER_DISCONNECT,
WSESSION_TERMINATE
};
struct CoreGateway::SessionInfo {
SessionInfo() : connection(), store(), reader(), session() {}
std::unique_ptr<IConnection> connection;
std::unique_ptr<IFIXStore> store;
std::unique_ptr<AsyncReader> reader;
std::unique_ptr<Session> session;
};
struct CoreGateway::EventData {
typedef void (CoreGateway::*Handler)(EventData const &ev);
EventData(const EventData &src)
: handler(src.handler), out_msgs(src.out_msgs), in_msg(src.in_msg),
trace(src.trace), ts(src.ts) {}
EventData() : handler(), out_msgs(), in_msg(), trace(), ts() {}
EventData(Handler handler)
: handler(handler), out_msgs(), in_msg(), trace(), ts() {}
EventData(Handler handler, trace::CorID const &trace,
OutFIXPtr const &out_msgs)
: handler(handler), out_msgs(out_msgs), in_msg(), trace(trace), ts() {}
EventData(Handler handler, InMsgBuffer const &in_msg)
: handler(handler), out_msgs(), in_msg(in_msg), trace(), ts() {}
EventData(Handler handler, std::shared_ptr<std::string> const &ts)
: handler(handler), out_msgs(), in_msg(), trace(), ts(ts) {}
EventData &operator=(EventData &src) {
handler = src.handler;
out_msgs = src.out_msgs;
in_msg = src.in_msg;
trace = src.trace;
ts = src.ts;
return *this;
}
EventData &operator=(EventData &&src) {
handler = std::move(src.handler);
out_msgs = std::move(src.out_msgs);
in_msg = std::move(src.in_msg);
trace = src.trace;
ts = std::move(src.ts);
return *this;
}
Handler handler;
OutFIXPtr out_msgs;
InMsgBuffer in_msg;
trace::CorID trace;
std::shared_ptr<std::string> ts;
};
struct CoreGateway::PImpl {
PImpl(config::JetFIX::Connection const &cfg, ICoreGatewayObserver &observer,
ConnectionFactory const &conn_factory,
StoreFactory const &store_factory, trace::PerfLog &l)
: cfg(cfg), observer(observer), make_connection(conn_factory),
make_store(store_factory), state(State::OFF), start_of_pause(),
session_info(), event_queue(), timer_thread(), destroy(false),
is_on(false), has_replayed(false), send_batch(), recv_batch(),
connection_attempts(0), perf_log(l) {
send_batch.reserve(1024);
recv_batch.reserve(1024);
}
const config::JetFIX::Connection cfg;
ICoreGatewayObserver &observer;
ConnectionFactory const make_connection;
StoreFactory const make_store;
State state;
datetime::PTime start_of_pause;
std::unique_ptr<SessionInfo> session_info;
tbb::concurrent_queue<EventData> event_queue;
std::thread timer_thread;
std::atomic<bool> destroy;
bool is_on;
bool has_replayed;
std::vector<EventData> send_batch;
std::vector<EventData> recv_batch;
uint32_t connection_attempts;
trace::PerfLog &perf_log;
};
std::thread CoreGateway::make_timer_thread() {
auto thread_name = fmt::format("jf_tmr_{}", me->cfg.name());
return threading::make_thread(thread_name,
std::bind(&CoreGateway::call_time, this));
}
CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg,
ICoreGatewayObserver &observer,
ConnectionFactory const &conn_factory,
StoreFactory const &store_factory,
trace::PerfLog &perf_trace)
: me(new PImpl(cfg, observer, conn_factory, store_factory, perf_trace)) {
me->timer_thread = make_timer_thread();
}
CoreGateway::~CoreGateway() {
stop();
while (me->state != State::OFF) {
dispatch_events();
std::this_thread::sleep_for(100ms);
}
me->destroy = true;
me->timer_thread.join();
}
void CoreGateway::send(OutFIXPtr const &fix, trace::CorID trace) {
me->event_queue.push(EventData(&CoreGateway::sync_send, trace, fix));
}
void CoreGateway::start() {
me->event_queue.push(EventData(&CoreGateway::sync_start));
}
void CoreGateway::stop() {
me->event_queue.push(EventData(&CoreGateway::sync_stop));
}
void CoreGateway::session_ready() {
me->event_queue.push(EventData(&CoreGateway::sync_session_ready));
}
void CoreGateway::session_terminated() {
me->event_queue.push(EventData(&CoreGateway::sync_session_terminated));
}
void CoreGateway::reader_disconnected() {
me->event_queue.push(EventData(&CoreGateway::sync_reader_disconnected));
}
void CoreGateway::in_msg_raw(InMsgBuffer const &msg) {
me->event_queue.push(EventData(&CoreGateway::sync_in_msg_raw, msg));
}
void CoreGateway::in_msg_sequenced(InMsgBuffer const &msg) {
me->event_queue.push(EventData(&CoreGateway::sync_in_msg_sequenced, msg));
}
void CoreGateway::dispatch_events() {
EventData ev;
// we prioratize sends over receives
while (me->event_queue.try_pop(ev)) {
if (ev.out_msgs) {
ev.out_msgs->get_ts(jetfix::OutFIX::MD_WRITE_TS);
me->send_batch.push_back(ev);
} else
me->recv_batch.push_back(ev);
}
if (me->send_batch.size() || me->recv_batch.size()) {
for (auto &ev : me->send_batch)
(this->*ev.handler)(ev);
for (auto &ev : me->recv_batch)
(this->*ev.handler)(ev);
me->send_batch.clear();
me->recv_batch.clear();
}
}
void CoreGateway::call_time() {
while (!me->destroy) {
std::this_thread::sleep_for(10ms);
auto ts = std::make_shared<std::string>(
datetime::FIX_timestamp(datetime::now()));
me->event_queue.push(EventData(&CoreGateway::sync_time, ts)); // NOLINT
}
}
void CoreGateway::sync_start(EventData const &) {
switch (me->state) {
case State::OFF: {
me->is_on = true;
pause_or_shutdown();
break;
}
case State::PAUSED:
case State::CONNECTING:
case State::READY:
bts_assert(false);
break;
case State::WREADER_DISCONNECT:
case State::WSESSION_TERMINATE:
bts_assert(!me->is_on);
me->is_on = true;
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_stop(EventData const &) {
switch (me->state) {
case State::OFF:
break;
case State::PAUSED:
me->is_on = false;
transition(State::OFF);
break;
case State::CONNECTING:
case State::READY:
me->is_on = false;
me->session_info->session->shutdown();
break;
case State::WREADER_DISCONNECT:
case State::WSESSION_TERMINATE:
me->is_on = false;
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_send(EventData const &ev) {
switch (me->state) {
case State::OFF:
bts_assert(false);
break;
case State::PAUSED:
case State::CONNECTING:
case State::WREADER_DISCONNECT:
case State::WSESSION_TERMINATE:
break;
case State::READY:
me->session_info->session->send(ev.out_msgs, ev.trace);
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_session_ready(EventData const &) {
switch (me->state) {
case State::OFF:
case State::PAUSED:
case State::READY:
case State::WREADER_DISCONNECT:
bts_assert(false);
break;
case State::WSESSION_TERMINATE:
break;
case State::CONNECTING:
transition(State::READY);
me->observer.ready();
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_session_terminated(EventData const &) {
switch (me->state) {
case State::OFF:
case State::PAUSED:
case State::WREADER_DISCONNECT:
bts_assert(false);
break;
case State::WSESSION_TERMINATE:
pause_or_shutdown();
break;
case State::READY:
me->session_info->session.reset();
me->session_info->connection->shutdown();
transition(State::WREADER_DISCONNECT);
me->observer.disconnect();
break;
case State::CONNECTING:
me->session_info->session.reset();
me->session_info->connection->shutdown();
transition(State::WREADER_DISCONNECT);
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_reader_disconnected(EventData const &) {
switch (me->state) {
case State::OFF:
case State::PAUSED:
case State::WSESSION_TERMINATE:
bts_assert(false);
break;
case State::READY:
me->session_info->session.reset();
me->observer.disconnect();
transition(State::WSESSION_TERMINATE);
break;
case State::CONNECTING:
me->session_info->session.reset();
transition(State::WSESSION_TERMINATE);
break;
case State::WREADER_DISCONNECT:
pause_or_shutdown();
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_in_msg_raw(EventData const &ev) {
switch (me->state) {
case State::OFF:
case State::PAUSED:
bts_assert(false);
break;
case State::CONNECTING:
case State::READY: {
me->session_info->session->receive(ev.in_msg);
} break;
case State::WREADER_DISCONNECT:
case State::WSESSION_TERMINATE:
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_in_msg_sequenced(EventData const &ev) {
switch (me->state) {
case State::OFF:
case State::PAUSED:
case State::WREADER_DISCONNECT:
bts_assert(false);
break;
case State::CONNECTING:
case State::READY:
me->observer.msg(ev.in_msg);
break;
case State::WSESSION_TERMINATE:
break;
default:
bts_assert(false);
}
}
void CoreGateway::sync_time(EventData const &ev) {
switch (me->state) {
case State::OFF:
break;
case State::PAUSED:
if (datetime::is_at_least_secs_old(me->start_of_pause,
me->cfg.seconds_to_pause())) {
LOG(Jetfix, Info, "Pause period ended for jetfix connection {}",
me->cfg.name());
me->session_info = std::make_unique<SessionInfo>();
me->session_info->store.reset(me->make_store(me->cfg));
if (me->session_info->store->session_active()) {
me->connection_attempts++;
if (!me->has_replayed) {
auto msgs_to_replay = me->session_info->store->replay();
LOG(Jetfix, Warn, "Replaying {} FIX msgs",
msgs_to_replay.size());
me->observer.replay(msgs_to_replay);
me->has_replayed = true;
}
if (me->connection_attempts > me->cfg.max_connection_attempts()) {
me->session_info.reset();
me->start_of_pause = datetime::now();
LOG(Jetfix, Error,
"Failed to connect to exchange gateway {} too many "
"times. Giving up permanently. You must bounce "
"the server to recover.",
me->cfg.name());
break;
}
me->session_info->connection.reset(me->make_connection(me->cfg));
LOG(Jetfix, Warn, "Trying to connect to exchange for {} ({}/{})",
me->cfg.name(), me->connection_attempts,
me->cfg.max_connection_attempts());
if (me->session_info->connection->connect()) {
LOG(Jetfix, Warn, "Connected to exchange gateway for {}",
me->cfg.name());
me->session_info->session.reset(new Session(
me->cfg, *me->session_info->connection,
*me->session_info->store, *this,
[]() { return utils::TimeStamp::now(); }, me->perf_log));
me->session_info->reader.reset(new AsyncReader(
*me->session_info->connection, me->cfg.name(), *this,
me->perf_log, me->cfg.msg_buffer_sz(),
me->cfg.msg_high_water_mark()));
me->session_info->reader->start();
me->session_info->session->start();
transition(State::CONNECTING);
} else {
me->session_info.reset();
me->start_of_pause = datetime::now();
LOG(Jetfix, Warn,
"Failed to connect to exchange for {}. ({}/{}) ",
me->cfg.name(), me->connection_attempts,
me->cfg.max_connection_attempts());
}
} else {
me->session_info.reset();
me->start_of_pause = datetime::now();
}
}
break;
case State::CONNECTING:
case State::READY:
me->session_info->session->time(ev.ts);
break;
case State::WREADER_DISCONNECT:
case State::WSESSION_TERMINATE:
break;
default:
bts_assert(false);
}
}
void CoreGateway::pause_or_shutdown() {
me->session_info.reset();
if (me->is_on) {
me->start_of_pause = datetime::now();
transition(State::PAUSED);
} else
transition(State::OFF);
}
void CoreGateway::transition(State new_state) {
LOG(Jetfix, Warn, "Transitioning Jetfix gateway {} from {} to {}",
me->cfg.name(), std::string(state_name(me->state)),
std::string(state_name(new_state)));
me->state = new_state;
check_state_invariants();
}
void CoreGateway::check_state_invariants() {
switch (me->state) {
case State::OFF:
bts_assert(!me->session_info);
bts_assert(!me->is_on);
return;
case State::PAUSED:
bts_assert(!me->session_info);
bts_assert(me->is_on);
return;
case State::CONNECTING:
case State::READY:
bts_assert(me->session_info);
bts_assert(me->session_info->session);
bts_assert(me->is_on);
return;
case State::WREADER_DISCONNECT:
bts_assert(me->session_info);
bts_assert(!me->session_info->session);
return;
case State::WSESSION_TERMINATE:
bts_assert(me->session_info);
bts_assert(!me->session_info->session);
break;
default:
bts_assert(false);
return;
}
}
const char *CoreGateway::state_name(State st) {
switch (st) {
case State::OFF:
return "OFF";
case State::PAUSED:
return "PAUSED";
case State::CONNECTING:
return "CONNECTING";
case State::READY:
return "READY";
case State::WREADER_DISCONNECT:
return "WREADER_DISCONNECT";
case State::WSESSION_TERMINATE:
return "WSESSION_TERMINATE";
default:
bts_assert(false);
}
}
} // namespace jetfix
} // namespace bts