From 830b590e5b75bb80bbf007485260b6862443d2f8 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 15 Jul 2019 10:59:13 -0400 Subject: [PATCH] Algos now go through an orderly shutdown phase. During which they can still process child updates. --- .../orderman/orders/algo/AlgoWrapperOrder.cpp | 111 ++++++++++++++++-- .../orderman/orders/algo/AlgoWrapperOrder.h | 4 + src/lib/orderman/orders/algo/IAlgoServices.h | 7 ++ src/server/sparkd/cmds/QuickTest.cpp | 1 + 4 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp index c3a483969..81142a5ce 100644 --- a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp +++ b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp @@ -16,7 +16,13 @@ namespace orderman { using namespace api::orderman; template using Map = boost::container::flat_map; -enum class AlgoWrapperOrder::Status { INACTIVE, W_ATTACH, WORKING, COMPLETED }; +enum class AlgoWrapperOrder::Status { + INACTIVE, + W_ATTACH, + WORKING, + SHUTTING_DOWN, + COMPLETED +}; class ChildTemplate { public: @@ -62,6 +68,7 @@ struct AlgoWrapperOrder::PImpl { OrderKey attach_target = null_order_key(); std::optional pending_update; + std::optional> pending_terminate; OrderStatus external_status() const { return _state.status(); } void set_external_status(OrderStatus s) { @@ -222,6 +229,7 @@ OrderOpResult AlgoWrapperOrder::on_activate(Price p, Quantity q, return {}; case Status::W_ATTACH: + case Status::SHUTTING_DOWN: case Status::WORKING: bts_assert(false); } @@ -238,6 +246,7 @@ OrderOpResult AlgoWrapperOrder::on_update(Update const &update) { me->pending_update = update; return {}; case Status::COMPLETED: + case Status::SHUTTING_DOWN: return {}; case Status::WORKING: { me->set_price_quantity(Price(update.price()), @@ -275,6 +284,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_detach() { switch (me->status) { case Status::W_ATTACH: case Status::WORKING: + case Status::SHUTTING_DOWN: for (auto &[_, c] : me->children) { (void)_; c.order.mark_detached_from_parent(); @@ -309,6 +319,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_cancel() { break; } break; + case Status::SHUTTING_DOWN: case Status::COMPLETED: break; case Status::INACTIVE: @@ -347,6 +358,10 @@ Quantity AlgoWrapperOrder::total_fills() const { return Quantity(me->state().filled_quantity()); } +bool AlgoWrapperOrder::is_shutting_down() const { + return me->status == Status::SHUTTING_DOWN; +} + void AlgoWrapperOrder::subscribe_book(SecurityKey k) { if (!is_working()) return; @@ -485,7 +500,7 @@ void AlgoWrapperOrder::detach_all_children() { } void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) { - if (!is_working()) + if (!is_working() && !is_shutting_down()) return; ReportStateGuard _(*this); @@ -519,11 +534,14 @@ void AlgoWrapperOrder::cancel(std::string const &msg) { if (!is_working()) return; ReportStateGuard _(*this); - terminate(ORDER_STATUS_CANCELED, msg); + if (working_children_count() == 0) + terminate(ORDER_STATUS_CANCELED, msg); + else + shutdown(ORDER_STATUS_CANCELED, msg); } void AlgoWrapperOrder::terminate_error(std::string const &msg) { - if (!is_working() && me->status != Status::W_ATTACH) + if (!is_working() && !is_shutting_down() && me->status != Status::W_ATTACH) return; ReportStateGuard _(*this); // If we attached and an error was signaled we do not want to @@ -538,7 +556,11 @@ void AlgoWrapperOrder::complete() { if (!is_working()) return; ReportStateGuard _(*this); - terminate(ORDER_STATUS_COMPLETED); + + if (working_children_count() == 0) + terminate(ORDER_STATUS_COMPLETED); + else + shutdown(ORDER_STATUS_COMPLETED); } // -------------------------------------------------------------------------------- @@ -573,6 +595,7 @@ void AlgoWrapperOrder::on_attach_complete( break; case Status::COMPLETED: break; + case Status::SHUTTING_DOWN: case Status::INACTIVE: case Status::WORKING: bts_assert(false); @@ -587,6 +610,7 @@ void AlgoWrapperOrder::on_child_order_update( switch (me->status) { case Status::WORKING: case Status::W_ATTACH: + case Status::SHUTTING_DOWN: if (auto it = me->children.find(child_key); it != me->children.end()) { auto &child_record = it->second; switch (new_child_state.status()) { @@ -634,7 +658,29 @@ void AlgoWrapperOrder::on_child_order_update( bts_assert(false); return; } - bts_assert(false); + + if (me->status == Status::SHUTTING_DOWN) + try_complete_shutdown(); +} + +void AlgoWrapperOrder::try_complete_shutdown() { + bts_assert(me->status == Status::SHUTTING_DOWN); + bts_assert(me->children.size()); + bts_assert(me->pending_terminate.has_value()); + + if (working_children_count() == 0) { + LOG(OrderMan, Info, "AlgoOrder {} completed shutdown. Terminated", + key().value()); + auto st = me->pending_terminate.value().first; + auto msg = me->pending_terminate.value().second; + me->pending_terminate.reset(); + terminate(st, msg); + } else { + LOG(OrderMan, Info, + "AlgoOrder {} still has {} orders working. Shutdown still in " + "progress", + key().value(), working_children_count()); + } } void AlgoWrapperOrder::setup_transitions() { @@ -676,8 +722,20 @@ void AlgoWrapperOrder::setup_transitions() { // From WORKING me->transition_actions[{Status::WORKING, Status::COMPLETED}] = [this]() { cleanup(); + me->children.clear(); _transition(Status::COMPLETED); }; + me->transition_actions[{Status::WORKING, Status::SHUTTING_DOWN}] = [this]() { + cleanup(); + _transition(Status::SHUTTING_DOWN); + }; + + // From SHUTTING_DOWN + me->transition_actions[{Status::SHUTTING_DOWN, Status::COMPLETED}] = + [this]() { + me->children.clear(); + _transition(Status::COMPLETED); + }; } void AlgoWrapperOrder::check_invariants() { @@ -697,6 +755,7 @@ void AlgoWrapperOrder::check_invariants() { bts_assert(me->book_subs.empty()); bts_assert(me->children.empty()); bts_assert(!me->has_timer_sub); + bts_assert(!me->pending_terminate.has_value()); break; case Status::WORKING: @@ -704,10 +763,21 @@ void AlgoWrapperOrder::check_invariants() { bts_assert(me->security); bts_assert(me->external_status() == ORDER_STATUS_WORKING); bts_assert(!me->pending_update.has_value()); + bts_assert(!me->pending_terminate.has_value()); + break; + + case Status::SHUTTING_DOWN: + bts_assert(working_children_count()); + bts_assert(me->services); + bts_assert(me->security); + bts_assert(me->external_status() == ORDER_STATUS_WORKING); + bts_assert(!me->pending_update.has_value()); + bts_assert(me->pending_terminate.has_value()); break; case Status::COMPLETED: bts_assert(me->children.empty()); + bts_assert(!me->pending_update.has_value()); bts_assert(me->external_status() == ORDER_STATUS_CANCELED || me->external_status() == ORDER_STATUS_COMPLETED || me->external_status() == ORDER_STATUS_ERROR || @@ -715,6 +785,7 @@ void AlgoWrapperOrder::check_invariants() { bts_assert(me->book_subs.empty()); bts_assert(!me->has_timer_sub); bts_assert(!me->pending_update.has_value()); + bts_assert(!me->pending_terminate.has_value()); break; } } @@ -745,15 +816,25 @@ void AlgoWrapperOrder::terminate(api::orderman::OrderStatus st, transition(Status::COMPLETED); } +void AlgoWrapperOrder::shutdown(api::orderman::OrderStatus st, + std::string const &error) { + bts_assert(!me->pending_terminate.has_value()); + bts_assert(working_children_count()); + me->pending_terminate = std::make_pair(st, error); + LOG(OrderMan, Info, + "AlgoOrder {} is shutting down. Waiting for {} child orders to complete " + "before terminating", + key().value(), working_children_count()); + transition(Status::SHUTTING_DOWN); +} + void AlgoWrapperOrder::cleanup() { for (auto const &s : me->book_subs) { me->services->unsubscribe_from_book(s.second); me->book_subs.clear(); } - for (auto &c : me->children) { + for (auto &c : me->children) c.second.order.cancel(); - me->children.clear(); - } if (me->has_timer_sub) unsubscribe_timer(); me->pending_update.reset(); @@ -785,6 +866,16 @@ OrderKey AlgoWrapperOrder::index_and_launch_child(ChildOrder &&child, Price p, return k; } +size_t AlgoWrapperOrder::working_children_count() const { + auto working_children_count = 0; + for (auto &c : me->children) + if (!c.second.state.has_value() || + c.second.state.value().status == ChildStatus::WORKING) { + working_children_count++; + } + return working_children_count; +} + bool AlgoWrapperOrder::is_working() const { return me->status == Status::WORKING; } @@ -799,6 +890,8 @@ log::CharPtrWrapper AlgoWrapperOrder::state_name(Status st) { return {"INACTIVE"}; case Status::WORKING: return {"WORKING"}; + case Status::SHUTTING_DOWN: + return {"SHUTTING_DOWN"}; } bts_assert(false); return {""}; diff --git a/src/lib/orderman/orders/algo/AlgoWrapperOrder.h b/src/lib/orderman/orders/algo/AlgoWrapperOrder.h index f11a84a23..9cf34d7fe 100644 --- a/src/lib/orderman/orders/algo/AlgoWrapperOrder.h +++ b/src/lib/orderman/orders/algo/AlgoWrapperOrder.h @@ -40,6 +40,7 @@ class AlgoWrapperOrder : public Order, Price price() const override; Quantity quantity() const override; Quantity total_fills() const override; + bool is_shutting_down() const override; void subscribe_book(SecurityKey) override; void unsubscribe_book(SecurityKey) override; void subscribe_child_change(OrderKey) override; @@ -75,10 +76,13 @@ class AlgoWrapperOrder : public Order, void transition(Status); void _transition(Status); void terminate(api::orderman::OrderStatus, std::string const &error = ""); + void shutdown(api::orderman::OrderStatus, std::string const &error = ""); + void try_complete_shutdown(); void cleanup(); void update_child_record(api::orderman::OrderState const &, ChildRecord *); OrderKey index_and_launch_child(ChildOrder &&, Price, Quantity, bool); bool is_working() const ; + size_t working_children_count() const; log::CharPtrWrapper state_name(Status) ; ChildStatus map_child_status(api::orderman::OrderStatus) ; bool _is_attaching(api::orderman::OrderDefinition const &) const ; diff --git a/src/lib/orderman/orders/algo/IAlgoServices.h b/src/lib/orderman/orders/algo/IAlgoServices.h index 4d9cd2f2f..5e779730f 100644 --- a/src/lib/orderman/orders/algo/IAlgoServices.h +++ b/src/lib/orderman/orders/algo/IAlgoServices.h @@ -20,6 +20,13 @@ class IAlgoServices : public IBeforeLaunchAlgoServices { // Returns the filled quantity for this algo virtual Quantity total_fills() const = 0; + // Returns wheter the algo is in the process of shutting down (due + // to a cancel, an error, or just having completed its work). When + // algo is shutting down any requests to create, update or cancel + // child orders will be ignored. + virtual bool is_shutting_down() const = 0; + + // -------------------------------------------------------------- // Event Subscriptions // -------------------------------------------------------------- diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index f6f86be5a..6dd5d649f 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -27,6 +27,7 @@ struct Proxy : public IAlgoServices { Price price() const { return Price(0); } Quantity quantity() const { return Quantity(0); } Quantity total_fills() const { return Quantity(0); } + bool is_shutting_down() const {return false;} void subscribe_book(SecurityKey){}; void unsubscribe_book(SecurityKey) {} void subscribe_child_change(OrderKey) {}