From 7bf24f01c4851ff2b333e7273d4ffc19fa1bd33e Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Fri, 29 Dec 2017 18:11:04 -0500 Subject: [PATCH] Fixed a few bugs in the order manager. Almost there. --- src/server/sparkd/mercury/actors/Actor.h | 10 +- .../mercury/actors/ContentSubscription.h | 3 +- .../sparkd/mercury/synopsis/MergeByKey.h | 46 ++-- .../orderman2/actors/ExecutionGateway.cpp | 225 ++++++++++-------- .../sparkd/orderman2/actors/OrderStore.cpp | 4 +- .../actors/order_manager/OrderManager.cpp | 13 +- .../actors/order_manager/OrderManager.h | 32 +-- .../orderman2/server/OrderServiceServer2.cpp | 2 + 8 files changed, 184 insertions(+), 151 deletions(-) diff --git a/src/server/sparkd/mercury/actors/Actor.h b/src/server/sparkd/mercury/actors/Actor.h index 629179e9f..b50b0125f 100644 --- a/src/server/sparkd/mercury/actors/Actor.h +++ b/src/server/sparkd/mercury/actors/Actor.h @@ -53,7 +53,7 @@ class Actor : public std::enable_shared_from_this { return; LOG(Mercury, Diag, - "{} on event broker {} received update to {}. Payload: {}", + "{} on event broker {} received update to {}. Payload: {}", name(), broker_id().value(), TopicType(tag).to_str(), (size_t)payload); if (payload) @@ -196,10 +196,10 @@ class Actor : public std::enable_shared_from_this { template void publish_batch(Stream const &stream, C const &c) { - LOG(Mercury, Info, - "{} on event broker {} publishing batch {} on stream {}", name(), - broker_id().value(), std::string(utils::type_name()), - stream.sid.value()); + LOG(Mercury, Info, "{} on event broker {} publishing a batch of {} {} " + " on stream {}", + name(), broker_id().value(), c.size(), + std::string(utils::type_name()), stream.sid.value()); bts_assert(_broker); auto msg = _broker->alloc_internal_notification_msg(stream.sid); diff --git a/src/server/sparkd/mercury/actors/ContentSubscription.h b/src/server/sparkd/mercury/actors/ContentSubscription.h index 9df83adbb..c61fe135d 100644 --- a/src/server/sparkd/mercury/actors/ContentSubscription.h +++ b/src/server/sparkd/mercury/actors/ContentSubscription.h @@ -102,7 +102,8 @@ public: ~ContentSubscription() { bts_assert(!_dispatching); - _actor->unsubscribe(_sub); + if (_actor) + _actor->unsubscribe(_sub); } private: diff --git a/src/server/sparkd/mercury/synopsis/MergeByKey.h b/src/server/sparkd/mercury/synopsis/MergeByKey.h index bc7700f5d..66ca750bf 100644 --- a/src/server/sparkd/mercury/synopsis/MergeByKey.h +++ b/src/server/sparkd/mercury/synopsis/MergeByKey.h @@ -6,42 +6,34 @@ namespace bts { namespace mercury { -template -class MergeByKey : public ISynopsisBuilder -{ -public: - using Key = decltype( ((EV*) nullptr)->key()); - using TMap = std::unordered_map; - - MergeByKey() : - _synopsis(), - _empty(), - _dirty() - {clear_synopsis();} +template class MergeByKey : public ISynopsisBuilder { + public: + using Key = decltype(((EV *)nullptr)->key()); + using TMap = std::unordered_map; - EventPayload* get_synopsis () override - { + MergeByKey() : _synopsis(), _empty(), _dirty() { clear_synopsis(); } + + EventPayload *get_synopsis() override { if (!_empty) { if (_dirty) { - // regen synopsis from merged map + // regen synopsis from merged map _dirty = false; _synopsis.clear(); - auto &syn = _synopsis.unwrap(); + auto &syn = _synopsis.wrap_in_place(); for (auto const &kvp : _merged) syn.emplace_back(kvp.second); } return &_synopsis; - } - else + } else return nullptr; } - - void update(EventPayload const&pl) override - { - auto const &update = pl.unwrap (); + + void update(EventPayload const &pl) override { _dirty = true; - for (auto const &ev : update) - _merged.emplace(ev.key(),ev); + _empty = false; + auto const &update = pl.unwrap(); + for (auto const &ev : update) + _merged.emplace(ev.key(), ev); } void clear_synopsis() override { @@ -51,11 +43,11 @@ public: _dirty = true; }; -private: + private: EventPayload _synopsis; TMap _merged; bool _empty; bool _dirty; }; - -}} +} +} diff --git a/src/server/sparkd/orderman2/actors/ExecutionGateway.cpp b/src/server/sparkd/orderman2/actors/ExecutionGateway.cpp index f4b28fcc4..f734b8574 100644 --- a/src/server/sparkd/orderman2/actors/ExecutionGateway.cpp +++ b/src/server/sparkd/orderman2/actors/ExecutionGateway.cpp @@ -1,12 +1,12 @@ #include "ExecutionGateway.h" +#include "lib/utils/TimeStamp.h" #include "server/sparkd/jetfix/codec/MessageDefinitions.h" #include "server/sparkd/jetfix/gateway/JetFIXGateway.h" -#include "server/sparkd/orderman2/events/NewExecutionReportEvent.h" -#include "server/sparkd/orderman2/orderman2_utils/fix_order_codec.h" -#include "server/sparkd/orderman2/events/NewSecuritiesEvent.h" -#include "server/sparkd/orderman2/events/GatewayStatusChangeEvent.h" #include "server/sparkd/orderman2/events/CancelRejectEvent.h" -#include "lib/utils/TimeStamp.h" +#include "server/sparkd/orderman2/events/GatewayStatusChangeEvent.h" +#include "server/sparkd/orderman2/events/NewExecutionReportEvent.h" +#include "server/sparkd/orderman2/events/NewSecuritiesEvent.h" +#include "server/sparkd/orderman2/orderman2_utils/fix_order_codec.h" #include "lib/logging/logging.h" @@ -17,29 +17,20 @@ namespace orderman2 { using namespace jetfix; -using JFGw = JetFIXGateway; +using JFGw = + JetFIXGateway; -struct ExecutionGateway::PImpl -{ - PImpl(config::JetFIX::Connection const&cfg) : - gw(cfg), - exe_report_pool (), - state(State::WAITING_FOR_SECURITIES), - securities(), - exe_report_stream(), - gw_status_change_stream(), - cancel_reject_stream(), - gw_name(cfg.name()), - cfg (cfg) - {} +struct ExecutionGateway::PImpl { + PImpl(config::JetFIX::Connection const &cfg) + : gw(cfg), exe_report_pool(), state(State::WAITING_FOR_SECURITIES), + securities(), exe_report_stream(), gw_status_change_stream(), + cancel_reject_stream(), gw_name(cfg.name()), cfg(cfg) {} JFGw gw; boost::object_pool exe_report_pool; State state; - std::unordered_map securities; + std::unordered_map securities; mercury::Stream exe_report_stream; mercury::Stream gw_status_change_stream; mercury::Stream cancel_reject_stream; @@ -47,18 +38,20 @@ struct ExecutionGateway::PImpl config::JetFIX::Connection cfg; }; -ExecutionGateway::ExecutionGateway (config::JetFIX::Connection const&cfg) : me (new PImpl(cfg)) {} -ExecutionGateway::~ExecutionGateway () {} +ExecutionGateway::ExecutionGateway(config::JetFIX::Connection const &cfg) + : me(new PImpl(cfg)) {} +ExecutionGateway::~ExecutionGateway() {} -void ExecutionGateway::init() -{ +void ExecutionGateway::init() { me->state = State::WAITING_FOR_SECURITIES; - subscribe([this] (auto const&sec_update) {this->on_securities(sec_update);}); - subscribe(me->gw_name,[this] (auto const&fix) {this->on_outgoing_fix(fix);}); + subscribe( + [this](auto const &sec_update) { this->on_securities(sec_update); }); + subscribe( + me->gw_name, [this](auto const &fix) { this->on_outgoing_fix(fix); }); } -void ExecutionGateway::on_securities(const mercury::Update &update) -{ +void ExecutionGateway::on_securities( + const mercury::Update &update) { if (update.events) { for (auto const &ev : *(update.events)) { @@ -67,30 +60,43 @@ void ExecutionGateway::on_securities(const mercury::Update & } if (me->state == State::WAITING_FOR_SECURITIES) { - LOG(OrderMan,Warn, - "Received first set of {} securities. Proceeding with initialization of {}", - update.events->size(), - me->gw_name.value()); + LOG(OrderMan, Warn, "Received first set of {} securities. Proceeding " + "with initialization of execution gateway {}", + update.events->size(), me->gw_name.value()); // got our security synopsis, ready to start the GW me->state = State::READY; // subscribe to gateway events - me->gw.on() = [this] (auto const&msg) {this->on_exe_report(msg);}; - me->gw.on() = [this] (auto const&msg) {this->on_cancel_reject(msg);}; - me->gw.on_ready() = [this] () {this->on_connection_status_change(true);}; - me->gw.on_disconnect() = [this] () {this->on_connection_status_change(false);}; - me->gw.on() = [this] (auto const&m) {this->on_business_reject(m);}; - me->gw.on() = [this] (auto const&m) {this->on_session_reject(m);}; + me->gw.on() = [this](auto const &msg) { + this->on_exe_report(msg); + }; + me->gw.on() = [this](auto const &msg) { + this->on_cancel_reject(msg); + }; + me->gw.on_ready() = [this]() { + this->on_connection_status_change(true); + }; + me->gw.on_disconnect() = [this]() { + this->on_connection_status_change(false); + }; + me->gw.on() = [this](auto const &m) { + this->on_business_reject(m); + }; + me->gw.on() = [this](auto const &m) { + this->on_session_reject(m); + }; // stream of execution reports - me->exe_report_stream = make_stream(me->gw_name); + me->exe_report_stream = + make_stream(me->gw_name); // stream for gateway status - me->gw_status_change_stream = make_stream(me->gw_name); + me->gw_status_change_stream = + make_stream(me->gw_name); // stream for cancel rejects - me->cancel_reject_stream = make_stream (me->gw_name); + me->cancel_reject_stream = make_stream(me->gw_name); // start the gateway me->gw.start(); @@ -98,24 +104,27 @@ void ExecutionGateway::on_securities(const mercury::Update & } } -void ExecutionGateway::on_exe_report(const jetfix::fix::ExecutionReportMsg &msg) -{ +void ExecutionGateway::on_exe_report( + const jetfix::fix::ExecutionReportMsg &msg) { OrderKey ok; SecurityKey sk; - decode_clorid(msg.get(),&ok, &sk); + decode_clorid(msg.get(), &ok, &sk); auto sec = me->securities[sk.value()]; - if (!sec) - LOG(OrderMan,Error, + if (!sec) + LOG(OrderMan, Error, "Received execution report for an unknown security {}. Will not be " - " able to process this report. This is likely a bug that could lead to" - " lost fills. Please investigate.",sk.value()); - else { + " able to process this report. This is likely a bug that could lead " + "to" + " lost fills. Please investigate.", + sk.value()); + else { // ignore leg fills, we care only about the actual security fill - if (msg.has() && msg.get() == 2) + if (msg.has() && + msg.get() == 2) return; - + std::string text_msg; if (msg.has()) text_msg = msg.get(); @@ -124,80 +133,94 @@ void ExecutionGateway::on_exe_report(const jetfix::fix::ExecutionReportMsg &msg) auto data = me->exe_report_pool.construct(); // copy data from fix msg to our structure - str_to(msg.get(),&data->account_name); - str_to(msg.get(),&data->operator_id); + str_to(msg.get(), &data->account_name); + str_to(msg.get(), &data->operator_id); data->order_key = ok; data->security = sec; data->side = api::Side(msg.get()); - data->time_in_force = api::orderman2::TimeInForce(msg.get()); - data->order_type = api::orderman2::NativeOrderType(msg.get()); + data->time_in_force = + api::orderman2::TimeInForce(msg.get()); + data->order_type = + api::orderman2::NativeOrderType(msg.get()); data->exec_type = api::orderman2::ExecType(msg.get()); - data->order_status = api::orderman2::NativeOrderStatus(msg.get()); - data->manual_order_indicator = msg.get() ? - api::orderman2::MANUAL_ORDER_INDICATOR_YES : - api::orderman2::MANUAL_ORDER_INDICATOR_YES ; - str_to(msg.get(),&data->exchange_order_id); - data->price = securities::decode_exchange_price(*sec,Price (msg.get())); + data->order_status = + api::orderman2::NativeOrderStatus(msg.get()); + data->manual_order_indicator = + msg.get() + ? api::orderman2::MANUAL_ORDER_INDICATOR_YES + : api::orderman2::MANUAL_ORDER_INDICATOR_YES; + str_to(msg.get(), &data->exchange_order_id); + data->price = + securities::decode_exchange_price(*sec, Price(msg.get())); data->quantity = Quantity(msg.get()); data->filled_quantity = Quantity(msg.get()); - data->last_shares = msg.has()? Quantity(msg.get()):Quantity(0); - data->last_price = msg.has()? - securities::decode_exchange_price(*sec,Price(msg.get())) : Price(0); - data->avg_price = msg.has()? - securities::decode_exchange_price(*sec,Price(msg.get())) : Price(0); - str_to(msg.get(),&data->exchange_fill_id); - str_to(msg.get(),&data->transact_time); + data->last_shares = msg.has() + ? Quantity(msg.get()) + : Quantity(0); + data->last_price = msg.has() + ? securities::decode_exchange_price( + *sec, Price(msg.get())) + : Price(0); + data->avg_price = msg.has() + ? securities::decode_exchange_price( + *sec, Price(msg.get())) + : Price(0); + str_to(msg.get(), &data->exchange_fill_id); + str_to(msg.get(), &data->transact_time); data->cfg = &me->cfg; data->msg = text_msg; - publish(me->exe_report_stream,data); + publish(me->exe_report_stream, data); } } -void ExecutionGateway::on_cancel_reject(const jetfix::fix::CancelRejectMsg &msg) -{ +void ExecutionGateway::on_cancel_reject( + const jetfix::fix::CancelRejectMsg &msg) { OrderKey ok; SecurityKey sk; - decode_clorid(msg.get(),&ok, &sk); + decode_clorid(msg.get(), &ok, &sk); std::string reject_msg; if (msg.has()) reject_msg = msg.get(); else if (msg.has()) - reject_msg = fmt::format ("Order cancel was rejected with error code {}", msg.get()); - publish(me->cancel_reject_stream,ok,reject_msg); + reject_msg = fmt::format("Order cancel was rejected with error code {}", + msg.get()); + publish(me->cancel_reject_stream, ok, reject_msg); } -void ExecutionGateway::on_connection_status_change(bool is_connected) -{ - LOG(OrderMan,Info,"Connection state of {} changed to {}", - me->gw_name.value(), is_connected? std::string("connected") : std::string("disconnected")); - - publish(me->gw_status_change_stream,is_connected); +void ExecutionGateway::on_connection_status_change(bool is_connected) { + LOG(OrderMan, Info, "Connection state of {} changed to {}", + me->gw_name.value(), + is_connected ? std::string("connected") : std::string("disconnected")); + + publish(me->gw_status_change_stream, is_connected); } -void ExecutionGateway::on_business_reject(jetfix::fix::BusinessMessageRejectMsg const&m) -{ - LOG(OrderMan,Error,"Unexpected business reject in {} gateway . Reason: {}. This should be investigated", - me->gw_name.value(), m.has() ? m.get() : std::string ("Unknown")); +void ExecutionGateway::on_business_reject( + jetfix::fix::BusinessMessageRejectMsg const &m) { + LOG(OrderMan, Error, "Unexpected business reject in {} gateway . Reason: " + "{}. This should be investigated", + me->gw_name.value(), + m.has() ? m.get() : std::string("Unknown")); } -void ExecutionGateway::on_session_reject(jetfix::fix::RejectMsg const&m) -{ - LOG(OrderMan,Error,"Unexpected reject message in {} gateway . Reason: {}. This should be investigated", - me->gw_name.value(), m.has() ? m.get() : std::string ("Unknown")); +void ExecutionGateway::on_session_reject(jetfix::fix::RejectMsg const &m) { + LOG(OrderMan, Error, "Unexpected reject message in {} gateway . Reason: {}. " + "This should be investigated", + me->gw_name.value(), + m.has() ? m.get() : std::string("Unknown")); } -void ExecutionGateway::on_poll() -{me->gw.dispatch_events();} +void ExecutionGateway::on_poll() { me->gw.dispatch_events(); } -void ExecutionGateway::on_outgoing_fix( mercury::Update const&msg) -{ - if (msg.events) - for (auto const&ev : *(msg.events)) { - LOG(Latency,Info, "Gateway {} {}", ev.cid,utils::TimeStamp::now().to_uint64()); +void ExecutionGateway::on_outgoing_fix( + mercury::Update const &msg) { + if (msg.events) + for (auto const &ev : *(msg.events)) { + LOG(Latency, Info, "Gateway {} {}", ev.cid, + utils::TimeStamp::now().to_uint64()); me->gw.send(ev.cid, ev.msg); } } - -}} - +} +} diff --git a/src/server/sparkd/orderman2/actors/OrderStore.cpp b/src/server/sparkd/orderman2/actors/OrderStore.cpp index bd869d206..41b5c8416 100644 --- a/src/server/sparkd/orderman2/actors/OrderStore.cpp +++ b/src/server/sparkd/orderman2/actors/OrderStore.cpp @@ -45,11 +45,13 @@ void OrderStore::init() me->known_orders[ok] = std::move(order_record); } else - LOG(persist,Error, + LOG(OrderMan,Error, "Could not recover order {} from storage. " "This should be investigated", kvp.first); me->order_stream = make_stream (me->gw_name); + LOG(OrderMan,Info, + "Order Store published {} orders recoved from disk", all.size()); publish_batch(me->order_stream,stored_batch); subscribe ([this] (auto const&ev) {this->on_order_state_update(ev);}); } diff --git a/src/server/sparkd/orderman2/actors/order_manager/OrderManager.cpp b/src/server/sparkd/orderman2/actors/order_manager/OrderManager.cpp index dcab17478..82dd45d52 100644 --- a/src/server/sparkd/orderman2/actors/order_manager/OrderManager.cpp +++ b/src/server/sparkd/orderman2/actors/order_manager/OrderManager.cpp @@ -1,13 +1,13 @@ #include "OrderManager.h" #include "server/sparkd/jetfix/common/OutFIX.h" +#include "server/sparkd/mercury/events/TimerEvent.h" #include "server/sparkd/orderman2/actors/order_manager/OrderServices.h" #include "server/sparkd/orderman2/actors/order_manager/OrderServicesData.h" #include "server/sparkd/orderman2/events/OrderManagerReadyEvent.h" #include "server/sparkd/orderman2/order_factory/OrderFactory.h" #include "server/sparkd/orderman2/orders/Order.h" #include "server/sparkd/securities/ISecurityDB.h" -#include "server/sparkd/mercury/events/TimerEvent.h" #include @@ -66,6 +66,9 @@ void OrderManager::on_securities( } if (me->state == State::WAIT_SEC) { + LOG(OrderMan, Info, "Order manager received initial set of {} " + "securities. Waiting for orders now", + update.events->size()); // obtain recovered orders me->state = State::WAIT_ORDERS; me->stored_order_sub = subscribe( @@ -82,6 +85,10 @@ void OrderManager::on_order_recovery( // Got the first batch of persisted orders which has to match // the orders stored on disk (since we are the only ones // producing new orders) + LOG(OrderMan, Info, + "Order manager received initial set of {} persisted orders.", + update.events->size()); + for (auto const &ev : *(update.events)) { auto &order = me->services_data.order_factory.recover(*ev.order_record); @@ -196,6 +203,7 @@ void OrderManager::on_order_recovery( me->pub_ready = make_stream(me->gw_name); // signal that we are ready + LOG(OrderMan, Info, "Order manager is ready"); publish(me->pub_ready); } } @@ -240,6 +248,7 @@ void OrderManager::on_activate_order( order->on_activate(ev.price, ev.quantity, *serv_ptr); } } + void OrderManager::on_update_order(const mercury::Update &m) { if (m.events) for (auto const &ev : *m.events) { @@ -314,3 +323,5 @@ std::vector &OrderManager::free_buffers() { } } } + + diff --git a/src/server/sparkd/orderman2/actors/order_manager/OrderManager.h b/src/server/sparkd/orderman2/actors/order_manager/OrderManager.h index 2bed4c4c1..ff21de705 100644 --- a/src/server/sparkd/orderman2/actors/order_manager/OrderManager.h +++ b/src/server/sparkd/orderman2/actors/order_manager/OrderManager.h @@ -1,13 +1,15 @@ #pragma once -#include -#include "server/sparkd/mercury/actors/Actor.h" -#include "server/sparkd/jetfix/common/OutFIXPtr.h" #include "lib/config/protos/jetfix.pb.h" +#include "server/sparkd/jetfix/common/OutFIXPtr.h" +#include "server/sparkd/mercury/actors/Actor.h" +#include // fwd namespace bts { -namespace mercury {class TimerEvent;} +namespace mercury { +class TimerEvent; +} namespace orderman2 { class NewSecuritiesEvent; class NewExecutionReportEvent; @@ -18,7 +20,8 @@ class CrackOrderEvent; class StoredOrderEvent; class CreateFIXBufferReplyEvent; class GatewayStatusChangeEvent; -}} +} +} namespace bts { namespace orderman2 { @@ -30,26 +33,25 @@ namespace orderman2 { // - a security database (SecurityDB) // - subscriptions to interesting events such as: // - execution gateway messages (execution reports, rejects, etc) -// - market data book updates +// - market data book updates // - security creation events // - timer updates // - order state changes // - order update/cancel/crack events // -// Additionally handle or routes order maintenance events: +// Additionally handles order maintenance events: // - create // - cancel // - crack // - update -class OrderManager : public mercury::Actor -{ -public: - OrderManager(config::JetFIX::Connection const&); +class OrderManager : public mercury::Actor { + public: + OrderManager(config::JetFIX::Connection const &); ~OrderManager(); void init() override; -private: + private: void on_securities(const mercury::Update &); void on_order_recovery(const mercury::Update &); void on_execution_report(const mercury::Update &); @@ -60,12 +62,12 @@ private: void on_new_fix_buffer(const mercury::Update &); void on_gateway_state(const mercury::Update &); void on_timer(const mercury::Update &); - + std::vector &free_buffers(); enum class State; struct OrderInfo; struct PImpl; std::unique_ptr me; }; - -}} +} +} diff --git a/src/server/sparkd/orderman2/server/OrderServiceServer2.cpp b/src/server/sparkd/orderman2/server/OrderServiceServer2.cpp index 3a95419bb..d6101f103 100644 --- a/src/server/sparkd/orderman2/server/OrderServiceServer2.cpp +++ b/src/server/sparkd/orderman2/server/OrderServiceServer2.cpp @@ -13,6 +13,7 @@ #include "server/sparkd/mercury/actors/Timer.h" #include "server/sparkd/orderman2/actors/ServiceBridge.h" #include "lib/api/public/ServiceContext.h" +#include "server/sparkd/orderman2/actors/order_manager/OrderManager.h" namespace bts { namespace orderman2 { @@ -81,6 +82,7 @@ OrderServiceServer2::OrderServiceServer2 (bts::api::ServiceContext& ctx) : // fast actors me->fast_exe.spawn (); + me->fast_exe.spawn (conn); me->fast_exe.spawn (conn); me->fast_thread = std::thread([this] () { os::set_thread_name("fast_exe");