Internalized ExecutionGateway inside OrderManager.

This commit is contained in:
Pedro Pinto committed 2018-11-30 01:55:27 +00:00
1 parent e8f797739c
commit d3043e9a19
10 files changed
+143 -160

No files matched your search

+30 -45
View File
@@ -26,8 +26,7 @@ using JFGw =
struct ExecutionGateway::PImpl {
PImpl(config::JetFIX::Connection const &cfg, PerfTraceLog &log)
: gw(cfg, log), exe_report_pool(), state(State::WAITING_FOR_SECURITIES),
securities(), securities_by_descriptor(), exe_report_stream(),
gw_status_change_stream(), cancel_reject_stream(), gw_name(cfg.name()),
securities(), securities_by_descriptor(), gw_name(cfg.name()),
cfg(cfg) {}
JFGw gw;
@@ -36,25 +35,19 @@ struct ExecutionGateway::PImpl {
std::unordered_map<SecurityKey::Value, api::Security const *> securities;
std::unordered_map<std::string, api::Security const *>
securities_by_descriptor;
mercury::Stream<NewExecutionReportEvent> exe_report_stream;
mercury::Stream<GatewayStatusChangeEvent> gw_status_change_stream;
mercury::Stream<CancelRejectEvent> cancel_reject_stream;
GatewayName gw_name;
config::JetFIX::Connection cfg;
OnExeReport on_exe_report;
OnGwStatusChange on_gw_status_change;
OnCancelReject on_cancel_reject;
};
ExecutionGateway::ExecutionGateway(config::JetFIX::Connection const &cfg,
PerfTraceLog &log)
: me(new PImpl(cfg, log)) {}
ExecutionGateway::~ExecutionGateway() {}
void ExecutionGateway::init() {
: me(new PImpl(cfg, log)) {
me->state = State::WAITING_FOR_SECURITIES;
subscribe<NewSecuritiesEvent>(
[this](auto const &sec_update) { this->on_securities(sec_update); });
subscribe<OutFIXMsgEvent>(
me->gw_name, [this](auto const &fix) { this->on_outgoing_fix(fix); });
}
ExecutionGateway::~ExecutionGateway() {}
void ExecutionGateway::on_securities(
const mercury::Update<NewSecuritiesEvent> &update) {
@@ -96,24 +89,22 @@ void ExecutionGateway::on_securities(
me->gw.on<fix::RejectMsg>() = [this](auto const &m) {
this->on_session_reject(m);
};
// stream of execution reports
me->exe_report_stream =
make_stream<NewExecutionReportEvent>(me->gw_name);
// stream for gateway status
me->gw_status_change_stream =
make_stream<GatewayStatusChangeEvent>(me->gw_name);
// stream for cancel rejects
me->cancel_reject_stream = make_stream<CancelRejectEvent>(me->gw_name);
// start the gateway
me->gw.start();
}
}
}
void ExecutionGateway::start() { me->gw.start(); }
OnExeReport &ExecutionGateway::on_exe_report() { return me->on_exe_report; }
OnGwStatusChange &ExecutionGateway::on_gw_status_change() {
return me->on_gw_status_change;
}
OnCancelReject &ExecutionGateway::on_cancel_reject() {
return me->on_cancel_reject;
}
void ExecutionGateway::on_exe_report(
const jetfix::fix::ExecutionReportMsg &msg) {
auto start = Timer::now();
@@ -260,7 +251,7 @@ void ExecutionGateway::on_exe_report(
data->msg = text_msg;
auto trace = msg.trace();
trace.exe_gw = utils::TimeStamp::now();
publish(me->exe_report_stream, data, trace);
me->on_exe_report(*data, trace);
LOG(Latency, Info,
"Finished handling of execution report for order {}. {}ns",
ok.value(), ns_diff(Timer::now(), start));
@@ -279,7 +270,7 @@ void ExecutionGateway::on_cancel_reject(
else if (msg.has<fix::CXlRejReason>())
reject_msg = fmt::format("Order cancel was rejected with error code {}",
msg.get<fix::CXlRejReason>());
publish(me->cancel_reject_stream, ok, reject_msg);
me->on_cancel_reject(ok, reject_msg);
}
void ExecutionGateway::on_connection_status_change(bool is_connected) {
@@ -287,8 +278,7 @@ void ExecutionGateway::on_connection_status_change(bool is_connected) {
me->gw_name.value(),
is_connected ? std::string("connected") : std::string("disconnected"));
publish(me->gw_status_change_stream,
GatewayStatusChangeEvent(me->gw_name, is_connected));
me->on_gw_status_change(is_connected);
}
void ExecutionGateway::on_business_reject(
@@ -308,21 +298,16 @@ void ExecutionGateway::on_session_reject(jetfix::fix::RejectMsg const &m) {
m.has<fix::Text>() ? m.get<fix::Text>() : std::string("Unknown"));
}
void ExecutionGateway::on_poll() { me->gw.dispatch_events(); }
void ExecutionGateway::dispatch_incoming_fix_messages() {
me->gw.dispatch_events();
}
void ExecutionGateway::on_outgoing_fix(
mercury::Update<OutFIXMsgEvent> const &msg) {
if (msg.events)
for (auto const &ev : *(msg.events)) {
LOG(Latency, Info, "on_outgoing_fix sn: {} ok: {}", ev.trace.md_sn,
ev.trace.order_key);
tracepoint(bts_trace, trace_order, (char *)"on_outgoing_fix", 0,
ev.trace.md_sn, 0);
auto trace = ev.trace;
trace.exe_gw = utils::TimeStamp::now();
me->gw.send(trace, ev.msg);
me->gw.dispatch_events();
}
void ExecutionGateway::send_fix(jetfix::OutFIXPtr const &fix,
PerfTrace &trace) {
LOG(Latency, Info, "on_outgoing_fix sn: {} ok: {}", trace.md_sn, trace.order_key);
trace.exe_gw = utils::TimeStamp::now();
me->gw.send(trace,fix);
me->gw.dispatch_events();
}
} // namespace orderman
+31 -24
View File
@@ -1,44 +1,51 @@
#pragma once
#include <memory>
#include "lib/config/protos/jetfix.pb.h"
#include "lib/mercury/actors/Actor.h"
#include "lib/jetfix/codec/MessageDefinitions.h"
#include "lib/jetfix/common/OutFIXPtr.h"
#include "lib/mercury/events/Update.h"
#include "lib/orderman/events/OutFIXMsgEvent.h"
#include "lib/mercury/actors/IPollableActor.h"
#include "lib/mercury/actors/PingPong.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include <memory>
namespace bts {
class PerfTraceLog;
namespace mercury { class PollEvent;}
namespace orderman {
class NewSecuritiesEvent;
class ExecutionReport;
using OnExeReport =
std::function<void(ExecutionReport const &, PerfTrace const &)>;
using OnGwStatusChange = std::function<void(bool)>;
using OnCancelReject = std::function<void(OrderKey, std::string const &)>;
// fix execution gateway for a single FIX session
class ExecutionGateway : public mercury::Actor,
public mercury::IPollableActor
{
public:
ExecutionGateway(config::JetFIX::Connection const&, PerfTraceLog& );
class ExecutionGateway {
public:
ExecutionGateway(config::JetFIX::Connection const &, PerfTraceLog &);
~ExecutionGateway();
void init() override;
private:
enum class State {WAITING_FOR_SECURITIES,READY};
void start();
void dispatch_incoming_fix_messages();
void on_securities(const mercury::Update<NewSecuritiesEvent> &);
void send_fix(jetfix::OutFIXPtr const &fix, PerfTrace &trace);
OnExeReport& on_exe_report();
OnGwStatusChange& on_gw_status_change();
OnCancelReject &on_cancel_reject();
private:
enum class State { WAITING_FOR_SECURITIES, READY };
struct PImpl;
std::unique_ptr<PImpl> me;
void on_poll() override;
void on_securities(const mercury::Update<NewSecuritiesEvent> &);
void on_exe_report(jetfix::fix::ExecutionReportMsg const&);
void on_cancel_reject(jetfix::fix::CancelRejectMsg const&);
// fix gateway events
void on_exe_report(jetfix::fix::ExecutionReportMsg const &);
void on_cancel_reject(jetfix::fix::CancelRejectMsg const &);
void on_connection_status_change(bool connected);
void on_business_reject(jetfix::fix::BusinessMessageRejectMsg const&);
void on_session_reject(jetfix::fix::RejectMsg const&);
void on_outgoing_fix(const mercury::Update<OutFIXMsgEvent> &);
void on_business_reject(jetfix::fix::BusinessMessageRejectMsg const &);
void on_session_reject(jetfix::fix::RejectMsg const &);
};
}}
} // namespace orderman
} // namespace bts
+3 -1
View File
@@ -49,7 +49,9 @@ class ServiceBridge : public mercury::Actor {
std::function<void(std::vector<api::orderman::Fill> const &)>;
ServiceBridge(OnReady const &, OnOrders const &, OnFills const &,
DynamicOrderRegistry const &, mercury::BrokerProxy &service_call_marshaller);
DynamicOrderRegistry const &,
mercury::BrokerProxy &service_call_marshaller);
void init() override;
~ServiceBridge();
@@ -31,8 +31,8 @@ enum class OrderManager::State {
};
struct OrderManager::PImpl {
PImpl(OrderManager &owner, DynamicOrderRegistry const &b)
: state(State::WAIT_ADMIN), services_data(owner, b) {}
PImpl(OrderManager &owner, DynamicOrderRegistry const &b, PerfTraceLog &l)
: state(State::WAIT_ADMIN), services_data(owner, b, l) {}
State state;
std::unordered_map<OrderKey, Order *> orders;
@@ -42,8 +42,8 @@ struct OrderManager::PImpl {
mercury::Stream<OrderManagerReadyEvent> pub_ready;
};
OrderManager::OrderManager(DynamicOrderRegistry const &b)
: me(new PImpl(*this, b)) {}
OrderManager::OrderManager(DynamicOrderRegistry const &b, PerfTraceLog &l)
: me(new PImpl(*this, b, l)) {}
OrderManager::~OrderManager() {}
void OrderManager::init() {
@@ -73,6 +73,11 @@ void OrderManager::on_admin_data(const mercury::Update<NewAdminDataEvent> &m) {
void OrderManager::on_securities(
const mercury::Update<NewSecuritiesEvent> &update) {
// forward to execution gateways
for (auto &osd : me->services_data.sessions)
osd->gw.on_securities(update);
if (update.events) {
// update our security db and accumulate for the sake of the limiter
@@ -348,8 +353,24 @@ void OrderManager::init_order_services() {
s->gateway_status_subs = std::make_unique<
ContentSubscription<GatewayStatusChangeEvent, NoKey>>(*this, gw_name);
// FIX output
s->out_fix_pub = make_stream<OutFIXMsgEvent>(gw_name);
// publishers for execution reports, gateway status and cancel
// reject evets
auto s_ptr = s.get();
s->exe_report_pub = make_stream<NewExecutionReportEvent>(gw_name);
s->gw.on_exe_report() = [s_ptr,this] (auto const &er, auto const &t) {
publish(s_ptr->exe_report_pub, &er,t);
};
s->gw_status_change_pub = make_stream<GatewayStatusChangeEvent>(gw_name);
s->gw.on_gw_status_change() = [s_ptr,this,gw_name] (bool b) {
publish(s_ptr->gw_status_change_pub, gw_name,b);
};
s->cancel_reject_pub = make_stream<CancelRejectEvent>(gw_name);
s->gw.on_cancel_reject() = [s_ptr,this] (auto const &k,auto const&msg) {
publish(s_ptr->cancel_reject_pub, k,msg);
};
// handlers are ready so we can start processing fix events
s->gw.start();
reload_fix_buffer(*s);
}
@@ -380,7 +401,7 @@ void OrderManager::init_order_services() {
me->services_data.settlement_request_pub =
make_stream<SettlementRequestEvent>();
me->services_data.report_order_state_pub =
make_stream<OrderStateUpdateEvent>();
make_stream<OrderStateUpdateEvent>();
me->pub_ready = make_stream<OrderManagerReadyEvent>();
// md subs are peculiar in that we try to make them on demand, so
@@ -408,26 +429,30 @@ void OrderManager::after_event() {
auto &fix_stream = osd->current_buffer->first_stream();
if (fix_stream.n_msgs()) {
fix_stream.close();
publish(osd->out_fix_pub, osd->current_buffer, osd->current_traces);
osd->current_traces.clear();
osd->gw.send_fix(osd->current_buffer, osd->current_trace);
reload_fix_buffer(*osd);
}
}
}
}
void OrderManager::reload_fix_buffer(SessionServicesData &dt) {
if (dt.free_buffers.size()) {
dt.current_buffer = dt.free_buffers.back();
dt.free_buffers.pop_back();
void OrderManager::reload_fix_buffer(SessionServicesData &osd) {
if (osd.free_buffers.size()) {
osd.current_buffer = osd.free_buffers.back();
osd.free_buffers.pop_back();
} else {
LOG(OrderMan, Warn,
"Ran out of fix buffers in {}. Will allocate directly. "
"This will have an impact on performance",
dt.cfg.name());
dt.current_buffer = std::make_shared<jetfix::OutFIX>(dt.cfg);
osd.cfg.name());
osd.current_buffer = std::make_shared<jetfix::OutFIX>(osd.cfg);
}
}
void OrderManager::on_poll() {
for (auto &osd : me->services_data.sessions)
osd->gw.dispatch_incoming_fix_messages();
}
} // namespace orderman
} // namespace bts
@@ -3,6 +3,7 @@
#include "lib/config/protos/jetfix.pb.h"
#include "lib/jetfix/common/OutFIXPtr.h"
#include "lib/mercury/actors/Actor.h"
#include "lib/mercury/actors/IPollableActor.h"
#include "lib/orderman/events/ActivateOrderEvent.h"
#include "lib/orderman/events/CancelOrderEvent.h"
#include "lib/orderman/events/CrackOrderEvent.h"
@@ -13,6 +14,7 @@
// fwd
namespace bts {
class PerfTraceLog;
namespace mercury {
class TimerEvent;
}
@@ -49,8 +51,6 @@ namespace orderman {
// - security creation events
// - timer updates
// - order state changes
// - order update/cancel/crack events
//
//
// Additionally handles order maintenance events:
// - create
@@ -58,9 +58,9 @@ namespace orderman {
// - crack
// - update
class OrderManager : public mercury::Actor {
class OrderManager : public mercury::Actor, public mercury::IPollableActor {
public:
OrderManager(DynamicOrderRegistry const &dyn_order_builder);
OrderManager(DynamicOrderRegistry const &dyn_order_builder, PerfTraceLog &);
~OrderManager();
void init() override;
@@ -80,6 +80,7 @@ class OrderManager : public mercury::Actor {
void on_gateway_state(const mercury::Update<GatewayStatusChangeEvent> &);
void on_timer(const mercury::Update<mercury::TimerEvent> &);
void on_admin_data(const mercury::Update<NewAdminDataEvent> &);
void on_poll() override;
// end external events
void init_order_services();
@@ -57,7 +57,9 @@ class SessionServices {
PerfTrace const &trace) {
auto &stream = _session_data->current_buffer->first_stream();
stream.new_msg(mt).append(f);
_session_data->current_traces.push_back(trace);
if (stream.n_msgs() == 1)
_session_data->current_trace = trace;
}
bool is_gateway_connected() const { return _session_data->gw_connected; }
@@ -6,7 +6,8 @@ namespace bts {
namespace orderman {
OrderServicesData::OrderServicesData(OrderManager &owner,
DynamicOrderRegistry const &b)
DynamicOrderRegistry const &b,
PerfTraceLog &l)
: owner(owner), order_router(), order_factory(sec_db, order_router, b),
last_timer_ev(
Timer::now(),
@@ -14,7 +15,8 @@ OrderServicesData::OrderServicesData(OrderManager &owner,
auto &all_connections = config::get<config::JetFIX>().connection();
for (auto const &c : all_connections)
sessions.push_back(std::make_unique<SessionServicesData>(c));
sessions.push_back(std::make_unique<SessionServicesData>(
c, l));
}
} // namespace orderman
} // namespace bts
@@ -7,11 +7,10 @@
#include "lib/orderman/order_factory/OrderFactory.h"
#include "lib/orderman/securities/MapSecurityDB.h"
#include "lib/utils/datetime.h"
#include "lib/mercury/events/TimerEvent.h"
#include "lib/orderman/actors/ExecutionGateway.h"
#include "lib/orderman/events/ActivateOrderEvent.h"
#include "lib/orderman/events/BookUpdateEvent.h"
#include "lib/orderman/events/SettlementUpdateEvent.h"
#include "lib/orderman/events/CancelOrderEvent.h"
#include "lib/orderman/events/CancelRejectEvent.h"
#include "lib/orderman/events/CrackOrderEvent.h"
@@ -21,15 +20,16 @@
#include "lib/orderman/events/CreateOrderRequestEvent.h"
#include "lib/orderman/events/GatewayStatusChangeEvent.h"
#include "lib/orderman/events/MDRequestEvent.h"
#include "lib/orderman/events/SettlementRequestEvent.h"
#include "lib/orderman/events/NewExecutionReportEvent.h"
#include "lib/orderman/events/NewSecuritiesEvent.h"
#include "lib/orderman/events/OrderStateUpdateEvent.h"
#include "lib/orderman/events/OutFIXMsgEvent.h"
#include "lib/orderman/events/SettlementRequestEvent.h"
#include "lib/orderman/events/SettlementUpdateEvent.h"
#include "lib/orderman/events/UpdateOrderEvent.h"
#include "lib/orderman/governor/Governor.h"
#include "lib/orderman/limiter/Limiter.h"
#include "lib/orderman/order_router/OrderRouter.h"
#include "lib/orderman/governor/Governor.h"
#include "lib/utils/datetime.h"
#include <boost/date_time/posix_time/posix_time.hpp>
@@ -44,19 +44,13 @@ struct ParentKey {
};
struct SessionServicesData {
SessionServicesData(config::JetFIX::Connection const&cfg)
: session(cfg.name()),
gw_connected(false),
cfg(cfg),
governor(cfg.max_order_msgs_per_window(),
cfg.window_duration_msecs())
{
current_traces.reserve(1000);
}
SessionServicesData(config::JetFIX::Connection const &cfg, PerfTraceLog &log)
: session(cfg.name()), gw_connected(false), cfg(cfg),
governor(cfg.max_order_msgs_per_window(), cfg.window_duration_msecs()),
gw(cfg, log) {}
std::string session;
std::vector<jetfix::OutFIXPtr> free_buffers;
mercury::Stream<OutFIXMsgEvent> out_fix_pub;
bool gw_connected;
config::JetFIX::Connection cfg;
std::unique_ptr<mercury::ContentSubscription<NewExecutionReportEvent>>
@@ -69,20 +63,26 @@ struct SessionServicesData {
Governor governor;
jetfix::OutFIXPtr current_buffer;
std::vector<PerfTrace> current_traces;
PerfTrace current_trace;
ExecutionGateway gw;
mercury::Stream<NewExecutionReportEvent> exe_report_pub;
mercury::Stream<GatewayStatusChangeEvent> gw_status_change_pub;
mercury::Stream<CancelRejectEvent> cancel_reject_pub;
};
class OrderServicesData {
public:
OrderServicesData(OrderManager &owner,DynamicOrderRegistry const&b);
public:
OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b,
PerfTraceLog &l);
OrderManager &owner;
securities::MapSecurityDB sec_db;
OrderRouter order_router;
Limiter limiter;
OrderFactory order_factory;
std::unique_ptr<mercury::ContentSubscription<BookUpdateEvent>> md_subs;
std::unique_ptr<mercury::ContentSubscription<SettlementUpdateEvent>> settlement_subs;
std::unique_ptr<mercury::ContentSubscription<SettlementUpdateEvent>>
settlement_subs;
std::unique_ptr<
mercury::ContentSubscription<mercury::TimerEvent, mercury::NoKey>>
time_subs;
@@ -105,5 +105,5 @@ public:
bts_assert(false);
}
};
}
}
} // namespace orderman
} // namespace bts
-32
View File
@@ -1,32 +0,0 @@
#pragma once
#include "lib/jetfix/common/OutFIXPtr.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/mercury/events/Event.h"
#include "lib/mercury/synopsis/CacheAll.h"
namespace bts {
namespace orderman {
class OutFIXMsgEvent : public mercury::Event<OutFIXMsgEvent> {
public:
using Tag = GatewayName;
// there is no need to store synopsis for this event because no one
// should be sending fix events (OutFIXMsgEvent) until they receive
// an event from the FIX gateway regarding the status of the FIX
// connection. Receipt of this connection event guarantees to
// potential publishers of OutFIXMsgEvent that the gateway is
// already subscribed to OutFIXMsgEvent and so no events will be
// missed.
using SynopsisPolicy = mercury::NoSynopsis;
OutFIXMsgEvent (jetfix::OutFIXPtr const&fix,std::vector<PerfTrace> const&traces) :
msg(fix),trace(traces[0])
{}
jetfix::OutFIXPtr msg;
PerfTrace trace;
};
}}
+5 -14
View File
@@ -9,7 +9,6 @@
#include "lib/mercury/executor/Executor.h"
#include "lib/metrics/metrics.h"
#include "lib/orderman/actors/AsyncFactory.h"
#include "lib/orderman/actors/ExecutionGateway.h"
#include "lib/orderman/actors/FillStore.h"
#include "lib/orderman/actors/MDSource.h"
#include "lib/orderman/actors/MDSupplementarySource.h"
@@ -90,11 +89,11 @@ ActorThread *find_thread_for_actor(
struct OrderServiceServer::PImpl {
PImpl(bts::api::ServiceContext &ctx,
std::shared_ptr<api::ISyntheticSecurityService> &ss,
DynamicOrderRegistry const &d, metrics::MetricsContainer &stats)
DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, PerfTraceLog &log)
: service_bridge(), service_ready(false), known_orders(), known_fills(),
publish_orders(), publish_fills(),
admin_service(ctx.locate<api::admin::AdminService>()), md_consumer(ss),
stats(stats), dyn_order_registry(d) {}
stats(stats), dyn_order_registry(d),log(log) {}
ServiceBridge *service_bridge;
@@ -110,6 +109,7 @@ struct OrderServiceServer::PImpl {
bts::md::MDConsumer md_consumer;
metrics::MetricsContainer &stats;
DynamicOrderRegistry const &dyn_order_registry;
PerfTraceLog &log;
};
OrderServiceServer::OrderServiceServer(
@@ -117,7 +117,7 @@ OrderServiceServer::OrderServiceServer(
std::shared_ptr<api::ISyntheticSecurityService> &ss,
DynamicOrderRegistry const &d, metrics::MetricsContainer &stats,
PerfTraceLog &log)
: api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, ss, d, stats)) {
: api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, ss, d, stats,log)) {
me->publish_orders = make_orders_publisher();
me->publish_fills = make_fills_publisher();
@@ -144,16 +144,7 @@ OrderServiceServer::OrderServiceServer(
bts_assert_msg(thread,
"Could not find a configured thread for OrderManager. See "
"the OrdermanCfg key");
thread->executor.spawn<OrderManager>(d);
// ExecutionGateway's
thread = find_thread_for_actor("ExecutionGateway", me->actor_threads);
bts_assert_msg(thread,
"Could not find a configured thread for ExecutionGateway "
"instances. See the OrdermanCfg key");
auto &all_connections = config::get<config::JetFIX>().connection();
for (auto const &c : all_connections)
thread->executor.spawn<ExecutionGateway>(c, log);
thread->executor.spawn<OrderManager>(d,me->log);
// Timer
thread = find_thread_for_actor("Timer", me->actor_threads);