Merged in SP-132-make-algo-shutdown-orderly (pull request #305)

RTM. SP-132 make algo shutdown orderly
This commit is contained in:
Pedro Pinto committed 2019-07-17 14:23:09 +00:00
commit d65e00fade
10 files changed
+281 -51

No files matched your search

+13
View File
@@ -4,6 +4,19 @@ weight = 3
sort_by = "weight"
insert_anchor_links = "right"
+++
## 2.7.0
### User Visible Changes
- Allowed algos to process "in-flight" fills while shutting down
- Changed the cover algo to continue to hedge while shutting down
### API Changes
- Changed the API so that algos do not fully terminate until all
their children are terminated. Algos that are in the process of
shutting down in response to a complete or cancel call will still
receive child updates, which they can use to calculate and report
an accurate fill count (during this shutdown stage algos cannot
create, cancel or update children). A new method IAlgoServices
method, is_shutting_down(), can be used to determine if an algo is
shutting down.
## 2.6.2
### User Visible Changes
+116 -20
View File
@@ -16,7 +16,13 @@ namespace orderman {
using namespace api::orderman;
template <class K, class V> using Map = boost::container::flat_map<K, V>;
enum class AlgoWrapperOrder::Status { INACTIVE, W_ATTACH, WORKING, COMPLETED };
enum class AlgoWrapperOrder::Status {
INACTIVE,
W_ATTACH,
WORKING,
SHUTTING_DOWN,
COMPLETED
};
class ChildTemplate {
public:
@@ -62,6 +68,7 @@ struct AlgoWrapperOrder::PImpl {
OrderKey attach_target = null_order_key();
std::optional<api::orderman::Update> pending_update;
std::optional<std::pair<OrderStatus, std::string>> pending_terminate;
OrderStatus external_status() const { return _state.status(); }
void set_external_status(OrderStatus s) {
@@ -222,6 +229,7 @@ OrderOpResult AlgoWrapperOrder::on_activate(Price p, Quantity q,
return {};
case Status::W_ATTACH:
case Status::SHUTTING_DOWN:
case Status::WORKING:
bts_assert(false);
}
@@ -238,6 +246,7 @@ OrderOpResult AlgoWrapperOrder::on_update(Update const &update) {
me->pending_update = update;
return {};
case Status::COMPLETED:
case Status::SHUTTING_DOWN:
return {};
case Status::WORKING: {
me->set_price_quantity(Price(update.price()),
@@ -275,6 +284,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_detach() {
switch (me->status) {
case Status::W_ATTACH:
case Status::WORKING:
case Status::SHUTTING_DOWN:
for (auto &[_, c] : me->children) {
(void)_;
c.order.mark_detached_from_parent();
@@ -309,6 +319,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_cancel() {
break;
}
break;
case Status::SHUTTING_DOWN:
case Status::COMPLETED:
break;
case Status::INACTIVE:
@@ -347,13 +358,20 @@ Quantity AlgoWrapperOrder::total_fills() const {
return Quantity(me->state().filled_quantity());
}
bool AlgoWrapperOrder::is_shutting_down() const {
return me->status == Status::SHUTTING_DOWN;
}
void AlgoWrapperOrder::subscribe_book(SecurityKey k) {
if (!is_working())
return;
bts_assert(me->book_subs.find(k) == me->book_subs.end());
me->book_subs[k] = me->services->subscribe_to_book(
k, [this, k](auto const &ev) { me->algo->on_book(k, ev.book); });
k, [this, k](auto const &ev) {
ReportStateGuard _(*this);
me->algo->on_book(k, ev.book);
});
}
void AlgoWrapperOrder::unsubscribe_book(SecurityKey k) {
@@ -392,7 +410,10 @@ void AlgoWrapperOrder::subscribe_timer() {
bts_assert(!me->has_timer_sub);
me->has_timer_sub = true;
me->services->subscribe_to_time(
[this](auto const &) { me->algo->on_timer(); });
[this](auto const &) {
ReportStateGuard _(*this);
me->algo->on_timer();
});
}
void AlgoWrapperOrder::unsubscribe_timer() {
@@ -485,10 +506,9 @@ void AlgoWrapperOrder::detach_all_children() {
}
void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) {
if (!is_working())
if (!is_working() && !is_shutting_down())
return;
ReportStateGuard _(*this);
auto previous_qty = me->state().filled_quantity();
if (previous_qty > q.value()) {
LOG(OrderMan, Error,
@@ -510,35 +530,37 @@ void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) {
void AlgoWrapperOrder::set_ladder_visibility(bool is_visible) {
if (!is_working())
return;
ReportStateGuard _(*this);
me->set_ladder_visibility(is_visible);
}
void AlgoWrapperOrder::cancel(std::string const &msg) {
if (!is_working())
return;
ReportStateGuard _(*this);
terminate(ORDER_STATUS_CANCELED, msg);
if (working_children_count() == 0)
terminate(ORDER_STATUS_CANCELED, msg);
else
shutdown(ORDER_STATUS_CANCELED, msg);
}
void AlgoWrapperOrder::terminate_error(std::string const &msg) {
if (!is_working() && me->status != Status::W_ATTACH)
if (!is_working() && !is_shutting_down() && me->status != Status::W_ATTACH)
return;
ReportStateGuard _(*this);
// If we attached and an error was signaled we do not want to
// cancel our child because that child might be a long running
// order with a good queue position that we want to preserve
if (is_attaching())
me->children.erase(me->attach_target);
terminate(ORDER_STATUS_ERROR, msg);
}
void AlgoWrapperOrder::complete() {
if (!is_working())
return;
ReportStateGuard _(*this);
terminate(ORDER_STATUS_COMPLETED);
if (working_children_count() == 0)
terminate(ORDER_STATUS_COMPLETED);
else
shutdown(ORDER_STATUS_COMPLETED);
}
// --------------------------------------------------------------------------------
@@ -573,6 +595,7 @@ void AlgoWrapperOrder::on_attach_complete(
break;
case Status::COMPLETED:
break;
case Status::SHUTTING_DOWN:
case Status::INACTIVE:
case Status::WORKING:
bts_assert(false);
@@ -587,6 +610,7 @@ void AlgoWrapperOrder::on_child_order_update(
switch (me->status) {
case Status::WORKING:
case Status::W_ATTACH:
case Status::SHUTTING_DOWN:
if (auto it = me->children.find(child_key); it != me->children.end()) {
auto &child_record = it->second;
switch (new_child_state.status()) {
@@ -625,16 +649,38 @@ void AlgoWrapperOrder::on_child_order_update(
break;
}
} else
terminate_error(
fmt::format("Unknown child order update: {}", child_key.value()));
return;
LOG(OrderMan, Info,
"AlgoOrder {} received update for presumably detached order {}. "
"Ignoring it",
key().value(), child_key.value());
break;
case Status::COMPLETED:
return;
case Status::INACTIVE:
bts_assert(false);
return;
}
bts_assert(false);
if (me->status == Status::SHUTTING_DOWN)
try_complete_shutdown();
}
void AlgoWrapperOrder::try_complete_shutdown() {
bts_assert(me->status == Status::SHUTTING_DOWN);
bts_assert(me->children.size());
bts_assert(me->pending_terminate.has_value());
if (working_children_count() == 0) {
LOG(OrderMan, Info, "AlgoOrder {} completed shutdown. Terminated",
key().value());
terminate(me->pending_terminate.value().first,
me->pending_terminate.value().second);
} else {
LOG(OrderMan, Info,
"AlgoOrder {} still has {} orders working. Shutdown still in "
"progress",
key().value(), working_children_count());
}
}
void AlgoWrapperOrder::setup_transitions() {
@@ -676,8 +722,21 @@ void AlgoWrapperOrder::setup_transitions() {
// From WORKING
me->transition_actions[{Status::WORKING, Status::COMPLETED}] = [this]() {
cleanup();
me->children.clear();
_transition(Status::COMPLETED);
};
me->transition_actions[{Status::WORKING, Status::SHUTTING_DOWN}] = [this]() {
cleanup();
_transition(Status::SHUTTING_DOWN);
};
// From SHUTTING_DOWN
me->transition_actions[{Status::SHUTTING_DOWN, Status::COMPLETED}] =
[this]() {
me->children.clear();
me->pending_terminate.reset();
_transition(Status::COMPLETED);
};
}
void AlgoWrapperOrder::check_invariants() {
@@ -697,6 +756,7 @@ void AlgoWrapperOrder::check_invariants() {
bts_assert(me->book_subs.empty());
bts_assert(me->children.empty());
bts_assert(!me->has_timer_sub);
bts_assert(!me->pending_terminate.has_value());
break;
case Status::WORKING:
@@ -704,10 +764,23 @@ void AlgoWrapperOrder::check_invariants() {
bts_assert(me->security);
bts_assert(me->external_status() == ORDER_STATUS_WORKING);
bts_assert(!me->pending_update.has_value());
bts_assert(!me->pending_terminate.has_value());
break;
case Status::SHUTTING_DOWN:
bts_assert(working_children_count());
bts_assert(me->services);
bts_assert(me->security);
bts_assert(me->external_status() == ORDER_STATUS_WORKING);
bts_assert(me->book_subs.empty());
bts_assert(!me->has_timer_sub);
bts_assert(!me->pending_update.has_value());
bts_assert(me->pending_terminate.has_value());
break;
case Status::COMPLETED:
bts_assert(me->children.empty());
bts_assert(!me->pending_update.has_value());
bts_assert(me->external_status() == ORDER_STATUS_CANCELED ||
me->external_status() == ORDER_STATUS_COMPLETED ||
me->external_status() == ORDER_STATUS_ERROR ||
@@ -715,6 +788,7 @@ void AlgoWrapperOrder::check_invariants() {
bts_assert(me->book_subs.empty());
bts_assert(!me->has_timer_sub);
bts_assert(!me->pending_update.has_value());
bts_assert(!me->pending_terminate.has_value());
break;
}
}
@@ -745,15 +819,25 @@ void AlgoWrapperOrder::terminate(api::orderman::OrderStatus st,
transition(Status::COMPLETED);
}
void AlgoWrapperOrder::shutdown(api::orderman::OrderStatus st,
std::string const &error) {
bts_assert(!me->pending_terminate.has_value());
bts_assert(working_children_count());
me->pending_terminate = std::make_pair(st, error);
LOG(OrderMan, Info,
"AlgoOrder {} is shutting down. Waiting for {} child orders to complete "
"before terminating",
key().value(), working_children_count());
transition(Status::SHUTTING_DOWN);
}
void AlgoWrapperOrder::cleanup() {
for (auto const &s : me->book_subs) {
me->services->unsubscribe_from_book(s.second);
me->book_subs.clear();
}
for (auto &c : me->children) {
for (auto &c : me->children)
c.second.order.cancel();
me->children.clear();
}
if (me->has_timer_sub)
unsubscribe_timer();
me->pending_update.reset();
@@ -785,6 +869,16 @@ OrderKey AlgoWrapperOrder::index_and_launch_child(ChildOrder &&child, Price p,
return k;
}
size_t AlgoWrapperOrder::working_children_count() const {
auto working_children_count = 0;
for (auto &c : me->children)
if (!c.second.state.has_value() ||
c.second.state.value().status == ChildStatus::WORKING) {
working_children_count++;
}
return working_children_count;
}
bool AlgoWrapperOrder::is_working() const {
return me->status == Status::WORKING;
}
@@ -799,6 +893,8 @@ log::CharPtrWrapper AlgoWrapperOrder::state_name(Status st) {
return {"INACTIVE"};
case Status::WORKING:
return {"WORKING"};
case Status::SHUTTING_DOWN:
return {"SHUTTING_DOWN"};
}
bts_assert(false);
return {""};
@@ -40,6 +40,7 @@ class AlgoWrapperOrder : public Order,
Price price() const override;
Quantity quantity() const override;
Quantity total_fills() const override;
bool is_shutting_down() const override;
void subscribe_book(SecurityKey) override;
void unsubscribe_book(SecurityKey) override;
void subscribe_child_change(OrderKey) override;
@@ -75,10 +76,13 @@ class AlgoWrapperOrder : public Order,
void transition(Status);
void _transition(Status);
void terminate(api::orderman::OrderStatus, std::string const &error = "");
void shutdown(api::orderman::OrderStatus, std::string const &error = "");
void try_complete_shutdown();
void cleanup();
void update_child_record(api::orderman::OrderState const &, ChildRecord *);
OrderKey index_and_launch_child(ChildOrder &&, Price, Quantity, bool);
bool is_working() const ;
size_t working_children_count() const;
log::CharPtrWrapper state_name(Status) ;
ChildStatus map_child_status(api::orderman::OrderStatus) ;
bool _is_attaching(api::orderman::OrderDefinition const &) const ;
@@ -20,6 +20,13 @@ class IAlgoServices : public IBeforeLaunchAlgoServices {
// Returns the filled quantity for this algo
virtual Quantity total_fills() const = 0;
// Returns wheter the algo is in the process of shutting down (due
// to a cancel, an error, or just having completed its work). When
// algo is shutting down any requests to create, update or cancel
// child orders will be ignored.
virtual bool is_shutting_down() const = 0;
// --------------------------------------------------------------
// Event Subscriptions
// --------------------------------------------------------------
@@ -10,18 +10,8 @@ namespace orderman {
using namespace api::orderman;
enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE };
const char *CoverAlgo::status_name(Status st) {
switch (st) {
case Status::WORKING:
return "WORKING";
case Status::WAITING_FOR_HEDGES:
return "WAITING_FOR_HEDGES";
case Status::DONE:
return "Done";
}
return nullptr;
}
enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE, CANCELING };
struct CoverAlgo::PImpl {
IAlgoServices *algo;
OrderKey child_key = null_order_key();
@@ -92,8 +82,11 @@ void CoverAlgo::do_launch_attached(IAlgoServices &algo, OrderKey k,
on_child_change(k, st);
}
void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
if (k == me->child_key) {
void CoverAlgo::on_covered_change(ChildState const &st) {
// First launch hedge if appropriate
switch (me->status) {
case Status::WORKING:
case Status::CANCELING:
if (st.total_fills.value() > me->hedged_fills.value()) {
auto hedge_qty =
Quantity(st.total_fills.value() - me->hedged_fills.value());
@@ -116,6 +109,15 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
hedge_qty.value(), hedge_price.value());
}
algo().update_fill_state(st.total_fills, st.last_price);
break;
case Status::WAITING_FOR_HEDGES:
case Status::DONE:
break;
}
// Now analyze child to figure out wheter we should terminate
switch (me->status) {
case Status::WORKING:
switch (st.status) {
case ChildStatus::WORKING:
break;
@@ -125,6 +127,7 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
algo().order_key().value());
algo().set_ladder_visibility(false);
me->status = Status::WAITING_FOR_HEDGES;
check_terminate();
break;
case ChildStatus::CANCELED:
LOG(OrderMan, Info, "CoverAlgo {}. Child was canceled, terminating",
@@ -133,30 +136,60 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
algo().terminate_error("Unexpected child cancel");
break;
}
} else {
break;
case Status::CANCELING:
switch (st.status) {
case ChildStatus::WORKING:
break;
case ChildStatus::CANCELED:
if (me->cancel_on_external_hedge_cancel) {
LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating",
algo().order_key().value());
me->status = Status::DONE;
algo().terminate_error("Unexpected child cancel");
} else
me->hedges.erase(
std::remove(me->hedges.begin(), me->hedges.end(), k),
me->hedges.end());
break;
case ChildStatus::COMPLETED:
me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k),
me->hedges.end());
LOG(OrderMan, Info, "CoverAlgo {}. Shutdown complete",
algo().order_key().value());
me->status = Status::DONE;
algo().cancel();
break;
}
break;
case Status::WAITING_FOR_HEDGES:
case Status::DONE:
break;
}
}
void CoverAlgo::on_hedge_change(OrderKey k, ChildState const &st) {
// First adjust our hedge count or cancel if something went wrong
switch (st.status) {
case ChildStatus::WORKING:
break;
case ChildStatus::COMPLETED:
me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k),
me->hedges.end());
break;
case ChildStatus::CANCELED:
if (me->cancel_on_external_hedge_cancel) {
LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating",
algo().order_key().value());
me->status = Status::DONE;
algo().terminate_error("Unexpected child cancel");
return;
} else
me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k),
me->hedges.end());
break;
}
// Now if we are waiting for hedges to complete, bail
check_terminate();
}
void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
if (k == me->child_key)
on_covered_change(st);
else
on_hedge_change(k,st);
}
void CoverAlgo::do_update(api::orderman::Update const &update) {
if (me->status == Status::WORKING) {
LOG(OrderMan, Diag, "CoverAlgo order {} receiving update: {}",
@@ -225,11 +258,45 @@ void CoverAlgo::check_terminate() {
}
IAlgoOrder::OnCancelResponse CoverAlgo::on_cancel() {
if (!me->cancel_hedges_on_cancel)
for (auto &k : me->hedges)
algo().detach_child(k);
switch (me->status) {
case Status::WORKING: {
if (!me->cancel_hedges_on_cancel)
for (auto &k : me->hedges)
algo().detach_child(k);
algo().cancel_child(me->child_key);
LOG(OrderMan, Info,
"CoverAlgo {} received cancel. Canceling child and waiting for an ack",
algo().order_key());
me->status = Status::CANCELING;
return IAlgoOrder::OnCancelResponse::IGNORE;
}
case Status::CANCELING:
return IAlgoOrder::OnCancelResponse::IGNORE;
case Status::WAITING_FOR_HEDGES:
return IAlgoOrder::OnCancelResponse::HONOR;
case Status::DONE:
bts_assert(false);
return IAlgoOrder::OnCancelResponse::HONOR;
}
bts_assert(false);
return IAlgoOrder::OnCancelResponse::HONOR;
}
const char *CoverAlgo::status_name(Status st) {
switch (st) {
case Status::WORKING:
return "WORKING";
case Status::WAITING_FOR_HEDGES:
return "WAITING_FOR_HEDGES";
case Status::DONE:
return "DONE";
case Status::CANCELING:
return "CANCELING";
}
return nullptr;
}
} // namespace orderman
} // namespace bts
@@ -31,6 +31,9 @@ private:
void init_hedge_template(IBeforeLaunchAlgoServices &,
api::orderman::CustomOrderDefinition const &);
void check_terminate();
void on_hedge_change(OrderKey hedge, ChildState const&) ;
void on_covered_change(ChildState const&);
const char *status_name(Status st);
};
@@ -145,7 +145,7 @@ void IcebergAlgo::on_child_change(OrderKey k, ChildState const &st) {
on_timer();
return;
case ChildStatus::CANCELED:
algo().terminate_error("Unexpected child cancel");
algo().cancel("Unexpected child cancel");
return;
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

@@ -0,0 +1,39 @@
#+TITLE: Algo Wrapper State Machien
#+OPTIONS: ^:nil
#+OPTIONS: H:2
#+OPTIONS: ^:nil H:2 author:nil creator:nil timestamp:nil toc:nil
#+OPTIONS: html-postamble:nil
* State Machine
#+begin_src dot :file algo_wrapper_sm.png :cmdline -Kdot -Tpng
digraph G {
bgcolor="transparent";
edge [ fontname=Helvetica, fontsize=12 ];
start [style=invisible];
COMPLETED [style=filled] ;
start -> INACTIVE;
INACTIVE -> COMPLETED [label="qty=0|error"];
INACTIVE -> W_ATTACH [label="ok & attach"];
INACTIVE -> WORKING [label="ok & !attach"];
W_ATTACH -> COMPLETED [label=attach_fail];
W_ATTACH -> WORKING [label=attach_ok];
WORKING -> COMPLETED [label="[wchild=0]cancel|complete"];
WORKING -> COMPLETED [label=error];
WORKING -> SHUTTING_DOWN [label="[wchild>0]cancel|complete"];
SHUTTING_DOWN -> COMPLETED [label="wchild==0|error|on_detach"];
}
#+END_SRC
#+RESULTS:
[[file:algo_wrapper_sm.png]]
* Invariants
| | attach_target | exernal_status | pending_update | pending_terminate | working_children | subs |
|---------------+---------------+--------------------------------------+----------------+-------------------+------------------+------|
| INACTIVE | | | | | | 0 |
| W_ATTACH | 1 | WORKING | | 0 | 0 | 0 |
| WORKING | | WORKING | 0 | 0 | | |
| SHUTTING_DOWN | | WORKING | 0 | 1 | >0 | 0 |
| COMPLETED | | CANCELED, COMPLETED, ERROR, REJECTED | 0 | 0 | 0 | 0 |
+1
View File
@@ -27,6 +27,7 @@ struct Proxy : public IAlgoServices {
Price price() const { return Price(0); }
Quantity quantity() const { return Quantity(0); }
Quantity total_fills() const { return Quantity(0); }
bool is_shutting_down() const {return false;}
void subscribe_book(SecurityKey){};
void unsubscribe_book(SecurityKey) {}
void subscribe_child_change(OrderKey) {}