diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index c0f100e53..ebcb98854 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,6 +4,19 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ +## 2.7.0 +### User Visible Changes + - Allowed algos to process "in-flight" fills while shutting down + - Changed the cover algo to continue to hedge while shutting down +### API Changes + - Changed the API so that algos do not fully terminate until all + their children are terminated. Algos that are in the process of + shutting down in response to a complete or cancel call will still + receive child updates, which they can use to calculate and report + an accurate fill count (during this shutdown stage algos cannot + create, cancel or update children). A new method IAlgoServices + method, is_shutting_down(), can be used to determine if an algo is + shutting down. ## 2.6.2 ### User Visible Changes diff --git a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp index c3a483969..368a25435 100644 --- a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp +++ b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp @@ -16,7 +16,13 @@ namespace orderman { using namespace api::orderman; template using Map = boost::container::flat_map; -enum class AlgoWrapperOrder::Status { INACTIVE, W_ATTACH, WORKING, COMPLETED }; +enum class AlgoWrapperOrder::Status { + INACTIVE, + W_ATTACH, + WORKING, + SHUTTING_DOWN, + COMPLETED +}; class ChildTemplate { public: @@ -62,6 +68,7 @@ struct AlgoWrapperOrder::PImpl { OrderKey attach_target = null_order_key(); std::optional pending_update; + std::optional> pending_terminate; OrderStatus external_status() const { return _state.status(); } void set_external_status(OrderStatus s) { @@ -222,6 +229,7 @@ OrderOpResult AlgoWrapperOrder::on_activate(Price p, Quantity q, return {}; case Status::W_ATTACH: + case Status::SHUTTING_DOWN: case Status::WORKING: bts_assert(false); } @@ -238,6 +246,7 @@ OrderOpResult AlgoWrapperOrder::on_update(Update const &update) { me->pending_update = update; return {}; case Status::COMPLETED: + case Status::SHUTTING_DOWN: return {}; case Status::WORKING: { me->set_price_quantity(Price(update.price()), @@ -275,6 +284,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_detach() { switch (me->status) { case Status::W_ATTACH: case Status::WORKING: + case Status::SHUTTING_DOWN: for (auto &[_, c] : me->children) { (void)_; c.order.mark_detached_from_parent(); @@ -309,6 +319,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_cancel() { break; } break; + case Status::SHUTTING_DOWN: case Status::COMPLETED: break; case Status::INACTIVE: @@ -347,13 +358,20 @@ Quantity AlgoWrapperOrder::total_fills() const { return Quantity(me->state().filled_quantity()); } +bool AlgoWrapperOrder::is_shutting_down() const { + return me->status == Status::SHUTTING_DOWN; +} + void AlgoWrapperOrder::subscribe_book(SecurityKey k) { if (!is_working()) return; bts_assert(me->book_subs.find(k) == me->book_subs.end()); me->book_subs[k] = me->services->subscribe_to_book( - k, [this, k](auto const &ev) { me->algo->on_book(k, ev.book); }); + k, [this, k](auto const &ev) { + ReportStateGuard _(*this); + me->algo->on_book(k, ev.book); + }); } void AlgoWrapperOrder::unsubscribe_book(SecurityKey k) { @@ -392,7 +410,10 @@ void AlgoWrapperOrder::subscribe_timer() { bts_assert(!me->has_timer_sub); me->has_timer_sub = true; me->services->subscribe_to_time( - [this](auto const &) { me->algo->on_timer(); }); + [this](auto const &) { + ReportStateGuard _(*this); + me->algo->on_timer(); + }); } void AlgoWrapperOrder::unsubscribe_timer() { @@ -485,10 +506,9 @@ void AlgoWrapperOrder::detach_all_children() { } void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) { - if (!is_working()) + if (!is_working() && !is_shutting_down()) return; - ReportStateGuard _(*this); auto previous_qty = me->state().filled_quantity(); if (previous_qty > q.value()) { LOG(OrderMan, Error, @@ -510,35 +530,37 @@ void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) { void AlgoWrapperOrder::set_ladder_visibility(bool is_visible) { if (!is_working()) return; - - ReportStateGuard _(*this); me->set_ladder_visibility(is_visible); } void AlgoWrapperOrder::cancel(std::string const &msg) { if (!is_working()) return; - ReportStateGuard _(*this); - terminate(ORDER_STATUS_CANCELED, msg); + if (working_children_count() == 0) + terminate(ORDER_STATUS_CANCELED, msg); + else + shutdown(ORDER_STATUS_CANCELED, msg); } void AlgoWrapperOrder::terminate_error(std::string const &msg) { - if (!is_working() && me->status != Status::W_ATTACH) + if (!is_working() && !is_shutting_down() && me->status != Status::W_ATTACH) return; - ReportStateGuard _(*this); // If we attached and an error was signaled we do not want to // cancel our child because that child might be a long running // order with a good queue position that we want to preserve if (is_attaching()) me->children.erase(me->attach_target); + terminate(ORDER_STATUS_ERROR, msg); } void AlgoWrapperOrder::complete() { if (!is_working()) return; - ReportStateGuard _(*this); - terminate(ORDER_STATUS_COMPLETED); + if (working_children_count() == 0) + terminate(ORDER_STATUS_COMPLETED); + else + shutdown(ORDER_STATUS_COMPLETED); } // -------------------------------------------------------------------------------- @@ -573,6 +595,7 @@ void AlgoWrapperOrder::on_attach_complete( break; case Status::COMPLETED: break; + case Status::SHUTTING_DOWN: case Status::INACTIVE: case Status::WORKING: bts_assert(false); @@ -587,6 +610,7 @@ void AlgoWrapperOrder::on_child_order_update( switch (me->status) { case Status::WORKING: case Status::W_ATTACH: + case Status::SHUTTING_DOWN: if (auto it = me->children.find(child_key); it != me->children.end()) { auto &child_record = it->second; switch (new_child_state.status()) { @@ -625,16 +649,38 @@ void AlgoWrapperOrder::on_child_order_update( break; } } else - terminate_error( - fmt::format("Unknown child order update: {}", child_key.value())); - return; + LOG(OrderMan, Info, + "AlgoOrder {} received update for presumably detached order {}. " + "Ignoring it", + key().value(), child_key.value()); + break; case Status::COMPLETED: return; case Status::INACTIVE: bts_assert(false); return; } - bts_assert(false); + + if (me->status == Status::SHUTTING_DOWN) + try_complete_shutdown(); +} + +void AlgoWrapperOrder::try_complete_shutdown() { + bts_assert(me->status == Status::SHUTTING_DOWN); + bts_assert(me->children.size()); + bts_assert(me->pending_terminate.has_value()); + + if (working_children_count() == 0) { + LOG(OrderMan, Info, "AlgoOrder {} completed shutdown. Terminated", + key().value()); + terminate(me->pending_terminate.value().first, + me->pending_terminate.value().second); + } else { + LOG(OrderMan, Info, + "AlgoOrder {} still has {} orders working. Shutdown still in " + "progress", + key().value(), working_children_count()); + } } void AlgoWrapperOrder::setup_transitions() { @@ -676,8 +722,21 @@ void AlgoWrapperOrder::setup_transitions() { // From WORKING me->transition_actions[{Status::WORKING, Status::COMPLETED}] = [this]() { cleanup(); + me->children.clear(); _transition(Status::COMPLETED); }; + me->transition_actions[{Status::WORKING, Status::SHUTTING_DOWN}] = [this]() { + cleanup(); + _transition(Status::SHUTTING_DOWN); + }; + + // From SHUTTING_DOWN + me->transition_actions[{Status::SHUTTING_DOWN, Status::COMPLETED}] = + [this]() { + me->children.clear(); + me->pending_terminate.reset(); + _transition(Status::COMPLETED); + }; } void AlgoWrapperOrder::check_invariants() { @@ -697,6 +756,7 @@ void AlgoWrapperOrder::check_invariants() { bts_assert(me->book_subs.empty()); bts_assert(me->children.empty()); bts_assert(!me->has_timer_sub); + bts_assert(!me->pending_terminate.has_value()); break; case Status::WORKING: @@ -704,10 +764,23 @@ void AlgoWrapperOrder::check_invariants() { bts_assert(me->security); bts_assert(me->external_status() == ORDER_STATUS_WORKING); bts_assert(!me->pending_update.has_value()); + bts_assert(!me->pending_terminate.has_value()); + break; + + case Status::SHUTTING_DOWN: + bts_assert(working_children_count()); + bts_assert(me->services); + bts_assert(me->security); + bts_assert(me->external_status() == ORDER_STATUS_WORKING); + bts_assert(me->book_subs.empty()); + bts_assert(!me->has_timer_sub); + bts_assert(!me->pending_update.has_value()); + bts_assert(me->pending_terminate.has_value()); break; case Status::COMPLETED: bts_assert(me->children.empty()); + bts_assert(!me->pending_update.has_value()); bts_assert(me->external_status() == ORDER_STATUS_CANCELED || me->external_status() == ORDER_STATUS_COMPLETED || me->external_status() == ORDER_STATUS_ERROR || @@ -715,6 +788,7 @@ void AlgoWrapperOrder::check_invariants() { bts_assert(me->book_subs.empty()); bts_assert(!me->has_timer_sub); bts_assert(!me->pending_update.has_value()); + bts_assert(!me->pending_terminate.has_value()); break; } } @@ -745,15 +819,25 @@ void AlgoWrapperOrder::terminate(api::orderman::OrderStatus st, transition(Status::COMPLETED); } +void AlgoWrapperOrder::shutdown(api::orderman::OrderStatus st, + std::string const &error) { + bts_assert(!me->pending_terminate.has_value()); + bts_assert(working_children_count()); + me->pending_terminate = std::make_pair(st, error); + LOG(OrderMan, Info, + "AlgoOrder {} is shutting down. Waiting for {} child orders to complete " + "before terminating", + key().value(), working_children_count()); + transition(Status::SHUTTING_DOWN); +} + void AlgoWrapperOrder::cleanup() { for (auto const &s : me->book_subs) { me->services->unsubscribe_from_book(s.second); me->book_subs.clear(); } - for (auto &c : me->children) { + for (auto &c : me->children) c.second.order.cancel(); - me->children.clear(); - } if (me->has_timer_sub) unsubscribe_timer(); me->pending_update.reset(); @@ -785,6 +869,16 @@ OrderKey AlgoWrapperOrder::index_and_launch_child(ChildOrder &&child, Price p, return k; } +size_t AlgoWrapperOrder::working_children_count() const { + auto working_children_count = 0; + for (auto &c : me->children) + if (!c.second.state.has_value() || + c.second.state.value().status == ChildStatus::WORKING) { + working_children_count++; + } + return working_children_count; +} + bool AlgoWrapperOrder::is_working() const { return me->status == Status::WORKING; } @@ -799,6 +893,8 @@ log::CharPtrWrapper AlgoWrapperOrder::state_name(Status st) { return {"INACTIVE"}; case Status::WORKING: return {"WORKING"}; + case Status::SHUTTING_DOWN: + return {"SHUTTING_DOWN"}; } bts_assert(false); return {""}; diff --git a/src/lib/orderman/orders/algo/AlgoWrapperOrder.h b/src/lib/orderman/orders/algo/AlgoWrapperOrder.h index f11a84a23..9cf34d7fe 100644 --- a/src/lib/orderman/orders/algo/AlgoWrapperOrder.h +++ b/src/lib/orderman/orders/algo/AlgoWrapperOrder.h @@ -40,6 +40,7 @@ class AlgoWrapperOrder : public Order, Price price() const override; Quantity quantity() const override; Quantity total_fills() const override; + bool is_shutting_down() const override; void subscribe_book(SecurityKey) override; void unsubscribe_book(SecurityKey) override; void subscribe_child_change(OrderKey) override; @@ -75,10 +76,13 @@ class AlgoWrapperOrder : public Order, void transition(Status); void _transition(Status); void terminate(api::orderman::OrderStatus, std::string const &error = ""); + void shutdown(api::orderman::OrderStatus, std::string const &error = ""); + void try_complete_shutdown(); void cleanup(); void update_child_record(api::orderman::OrderState const &, ChildRecord *); OrderKey index_and_launch_child(ChildOrder &&, Price, Quantity, bool); bool is_working() const ; + size_t working_children_count() const; log::CharPtrWrapper state_name(Status) ; ChildStatus map_child_status(api::orderman::OrderStatus) ; bool _is_attaching(api::orderman::OrderDefinition const &) const ; diff --git a/src/lib/orderman/orders/algo/IAlgoServices.h b/src/lib/orderman/orders/algo/IAlgoServices.h index 4d9cd2f2f..5e779730f 100644 --- a/src/lib/orderman/orders/algo/IAlgoServices.h +++ b/src/lib/orderman/orders/algo/IAlgoServices.h @@ -20,6 +20,13 @@ class IAlgoServices : public IBeforeLaunchAlgoServices { // Returns the filled quantity for this algo virtual Quantity total_fills() const = 0; + // Returns wheter the algo is in the process of shutting down (due + // to a cancel, an error, or just having completed its work). When + // algo is shutting down any requests to create, update or cancel + // child orders will be ignored. + virtual bool is_shutting_down() const = 0; + + // -------------------------------------------------------------- // Event Subscriptions // -------------------------------------------------------------- diff --git a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp index 783bcd960..6b112a9cb 100644 --- a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp @@ -10,18 +10,8 @@ namespace orderman { using namespace api::orderman; -enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE }; -const char *CoverAlgo::status_name(Status st) { - switch (st) { - case Status::WORKING: - return "WORKING"; - case Status::WAITING_FOR_HEDGES: - return "WAITING_FOR_HEDGES"; - case Status::DONE: - return "Done"; - } - return nullptr; -} +enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE, CANCELING }; + struct CoverAlgo::PImpl { IAlgoServices *algo; OrderKey child_key = null_order_key(); @@ -92,8 +82,11 @@ void CoverAlgo::do_launch_attached(IAlgoServices &algo, OrderKey k, on_child_change(k, st); } -void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) { - if (k == me->child_key) { +void CoverAlgo::on_covered_change(ChildState const &st) { + // First launch hedge if appropriate + switch (me->status) { + case Status::WORKING: + case Status::CANCELING: if (st.total_fills.value() > me->hedged_fills.value()) { auto hedge_qty = Quantity(st.total_fills.value() - me->hedged_fills.value()); @@ -116,6 +109,15 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) { hedge_qty.value(), hedge_price.value()); } algo().update_fill_state(st.total_fills, st.last_price); + break; + case Status::WAITING_FOR_HEDGES: + case Status::DONE: + break; + } + + // Now analyze child to figure out wheter we should terminate + switch (me->status) { + case Status::WORKING: switch (st.status) { case ChildStatus::WORKING: break; @@ -125,6 +127,7 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) { algo().order_key().value()); algo().set_ladder_visibility(false); me->status = Status::WAITING_FOR_HEDGES; + check_terminate(); break; case ChildStatus::CANCELED: LOG(OrderMan, Info, "CoverAlgo {}. Child was canceled, terminating", @@ -133,30 +136,60 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) { algo().terminate_error("Unexpected child cancel"); break; } - } else { + break; + case Status::CANCELING: switch (st.status) { case ChildStatus::WORKING: break; case ChildStatus::CANCELED: - if (me->cancel_on_external_hedge_cancel) { - LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating", - algo().order_key().value()); - me->status = Status::DONE; - algo().terminate_error("Unexpected child cancel"); - } else - me->hedges.erase( - std::remove(me->hedges.begin(), me->hedges.end(), k), - me->hedges.end()); - break; case ChildStatus::COMPLETED: - me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k), - me->hedges.end()); + LOG(OrderMan, Info, "CoverAlgo {}. Shutdown complete", + algo().order_key().value()); + me->status = Status::DONE; + algo().cancel(); break; } + break; + case Status::WAITING_FOR_HEDGES: + case Status::DONE: + break; } +} + +void CoverAlgo::on_hedge_change(OrderKey k, ChildState const &st) { + + // First adjust our hedge count or cancel if something went wrong + switch (st.status) { + case ChildStatus::WORKING: + break; + case ChildStatus::COMPLETED: + me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k), + me->hedges.end()); + break; + case ChildStatus::CANCELED: + if (me->cancel_on_external_hedge_cancel) { + LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating", + algo().order_key().value()); + me->status = Status::DONE; + algo().terminate_error("Unexpected child cancel"); + return; + } else + me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k), + me->hedges.end()); + break; + } + + // Now if we are waiting for hedges to complete, bail check_terminate(); } +void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) { + if (k == me->child_key) + on_covered_change(st); + else + on_hedge_change(k,st); +} + void CoverAlgo::do_update(api::orderman::Update const &update) { if (me->status == Status::WORKING) { LOG(OrderMan, Diag, "CoverAlgo order {} receiving update: {}", @@ -225,11 +258,45 @@ void CoverAlgo::check_terminate() { } IAlgoOrder::OnCancelResponse CoverAlgo::on_cancel() { - if (!me->cancel_hedges_on_cancel) - for (auto &k : me->hedges) - algo().detach_child(k); + switch (me->status) { + case Status::WORKING: { + if (!me->cancel_hedges_on_cancel) + for (auto &k : me->hedges) + algo().detach_child(k); + algo().cancel_child(me->child_key); + + LOG(OrderMan, Info, + "CoverAlgo {} received cancel. Canceling child and waiting for an ack", + algo().order_key()); + me->status = Status::CANCELING; + return IAlgoOrder::OnCancelResponse::IGNORE; + } + case Status::CANCELING: + return IAlgoOrder::OnCancelResponse::IGNORE; + case Status::WAITING_FOR_HEDGES: + return IAlgoOrder::OnCancelResponse::HONOR; + case Status::DONE: + bts_assert(false); + return IAlgoOrder::OnCancelResponse::HONOR; + } + + bts_assert(false); return IAlgoOrder::OnCancelResponse::HONOR; } +const char *CoverAlgo::status_name(Status st) { + switch (st) { + case Status::WORKING: + return "WORKING"; + case Status::WAITING_FOR_HEDGES: + return "WAITING_FOR_HEDGES"; + case Status::DONE: + return "DONE"; + case Status::CANCELING: + return "CANCELING"; + } + return nullptr; +} + } // namespace orderman } // namespace bts diff --git a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h index 24fb06cf5..86904a2fc 100644 --- a/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h +++ b/src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h @@ -31,6 +31,9 @@ private: void init_hedge_template(IBeforeLaunchAlgoServices &, api::orderman::CustomOrderDefinition const &); void check_terminate(); + + void on_hedge_change(OrderKey hedge, ChildState const&) ; + void on_covered_change(ChildState const&); const char *status_name(Status st); }; diff --git a/src/lib/orderman/orders/algo/algo_orders/IcebergAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/IcebergAlgo.cpp index f93231c57..897a366ff 100644 --- a/src/lib/orderman/orders/algo/algo_orders/IcebergAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/IcebergAlgo.cpp @@ -145,7 +145,7 @@ void IcebergAlgo::on_child_change(OrderKey k, ChildState const &st) { on_timer(); return; case ChildStatus::CANCELED: - algo().terminate_error("Unexpected child cancel"); + algo().cancel("Unexpected child cancel"); return; } } diff --git a/src/lib/orderman/orders/algo/doc/algo_wrapper_sm.png b/src/lib/orderman/orders/algo/doc/algo_wrapper_sm.png new file mode 100644 index 000000000..4e49734d6 Binary files /dev/null and b/src/lib/orderman/orders/algo/doc/algo_wrapper_sm.png differ diff --git a/src/lib/orderman/orders/algo/doc/state_machine.org b/src/lib/orderman/orders/algo/doc/state_machine.org new file mode 100644 index 000000000..5eaf1b7bf --- /dev/null +++ b/src/lib/orderman/orders/algo/doc/state_machine.org @@ -0,0 +1,39 @@ +#+TITLE: Algo Wrapper State Machien +#+OPTIONS: ^:nil +#+OPTIONS: H:2 +#+OPTIONS: ^:nil H:2 author:nil creator:nil timestamp:nil toc:nil +#+OPTIONS: html-postamble:nil + + +* State Machine +#+begin_src dot :file algo_wrapper_sm.png :cmdline -Kdot -Tpng +digraph G { + bgcolor="transparent"; + edge [ fontname=Helvetica, fontsize=12 ]; + + start [style=invisible]; + COMPLETED [style=filled] ; + start -> INACTIVE; + INACTIVE -> COMPLETED [label="qty=0|error"]; + INACTIVE -> W_ATTACH [label="ok & attach"]; + INACTIVE -> WORKING [label="ok & !attach"]; + W_ATTACH -> COMPLETED [label=attach_fail]; + W_ATTACH -> WORKING [label=attach_ok]; + WORKING -> COMPLETED [label="[wchild=0]cancel|complete"]; + WORKING -> COMPLETED [label=error]; + WORKING -> SHUTTING_DOWN [label="[wchild>0]cancel|complete"]; + SHUTTING_DOWN -> COMPLETED [label="wchild==0|error|on_detach"]; +} +#+END_SRC + +#+RESULTS: +[[file:algo_wrapper_sm.png]] + +* Invariants +| | attach_target | exernal_status | pending_update | pending_terminate | working_children | subs | +|---------------+---------------+--------------------------------------+----------------+-------------------+------------------+------| +| INACTIVE | | | | | | 0 | +| W_ATTACH | 1 | WORKING | | 0 | 0 | 0 | +| WORKING | | WORKING | 0 | 0 | | | +| SHUTTING_DOWN | | WORKING | 0 | 1 | >0 | 0 | +| COMPLETED | | CANCELED, COMPLETED, ERROR, REJECTED | 0 | 0 | 0 | 0 | diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index f6f86be5a..6dd5d649f 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -27,6 +27,7 @@ struct Proxy : public IAlgoServices { Price price() const { return Price(0); } Quantity quantity() const { return Quantity(0); } Quantity total_fills() const { return Quantity(0); } + bool is_shutting_down() const {return false;} void subscribe_book(SecurityKey){}; void unsubscribe_book(SecurityKey) {} void subscribe_child_change(OrderKey) {}