diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp index d75c2cf42..89457105a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp @@ -88,7 +88,7 @@ void SimOrderGateway:: place_order(SecurityKey skey, MDP3OrderPriority{}, md::IntPrice_from_double<7>(p.value()), MDP3Quantity{static_cast(q.value())}, - MDP3Quantity(0) + {} }, OrderChange::added }); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.cpp index 176b7ad96..d9174fb13 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.cpp @@ -153,6 +153,14 @@ FifoMergedSecurityMatcher* FifoMatcherContext::_get_or_add_matcher(MDP3SecurityI return m->second; } +std::tuple +FifoMatcherContext::find_order(MDP3SecurityID sec_id, MDP3OrderKey ok) { + auto* m = _get_matcher(sec_id); + if (m != nullptr) + return m->matcher()->find_order(ok); + return { nullptr, nullptr }; +} + /////////////////////// // order callbacks /////////////////////// diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.h index b30a3a3c0..99ed9fc8c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.h @@ -38,6 +38,7 @@ struct FifoMatcherContext { void on_order_change(const SmallOrderChange& change, std::vector* effects, FillDepot& fills); void print_all_bbos(std::ostream& out); std::optional has_same_orders(MDP3SecurityID, const Orders&); + std::tuple find_order(MDP3SecurityID, MDP3OrderKey); private: struct PImpl; @@ -82,6 +83,7 @@ struct FifoMatcherContext { OrderChange ch; ch.change = OrderChange::ChangeType::added; ch.order = kv.second; + bts_assert(ch.order.quantity.value() > 0); matcher->matcher()->on_order_change(ch); } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.cpp index 26475dc8d..4bf97b101 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.cpp @@ -131,16 +131,15 @@ void FifoMergedSecurityMatcher::on_order_change(const SmallOrderChange& ch, std: auto prev_ord_tup = matcher()->find_order(ch.order.key); auto prev_ord = std::get<1>(prev_ord_tup); if (prev_ord == nullptr) { - bts_assert_msg(_me->ignore_ops_on_deleted_orders, "unable to process order-remove event: failed to find order."); + bts_assert_msg(_me->ignore_ops_on_deleted_orders, "unable to process order-update event: failed to find order."); break; } if (orderman::price_comp::is_better(ch.order.side, ch.order.price, prev_ord->price)) { // price improved, try to fill the order auto new_ch = ch; - _fill(new_ch.order, effects, fills, prev_ord->ib.is_ib ? std::max(prev_ord->ib.show_qty, new_ch.order.quantity) : new_ch.order.quantity); - auto rem = new_ch.order.quantity; - PRINT_LINE(_log_header(), "updated order rem qty: ", rem, ", orig state: ", ch.order); - if (rem.value() == 0) { + _fill(new_ch.order, effects, fills, false); + // PRINT_LINE(_log_header(), " updated order rem qty: ", new_ch.order.quantity, ", orig state: ", ch.order); + if (new_ch.order.quantity.value() == 0) { matcher()->on_order_completed(ch.order.key); } else { matcher()->on_order_change(new_ch, effects); @@ -157,8 +156,8 @@ void FifoMergedSecurityMatcher::on_order_change(const SmallOrderChange& ch, std: } case OrderChange::ChangeType::added: { auto new_ch = ch; - _fill(new_ch.order, effects, fills, ch.order.quantity); - if (new_ch.order.quantity.value() > 0) + _fill(new_ch.order, effects, fills, false); + if (get_total_qty(new_ch.order).value() > 0) matcher()->on_order_change(new_ch); break; } @@ -284,21 +283,14 @@ void FifoMergedSecurityMatcher::merged_book( } } -void FifoMergedSecurityMatcher::_fill(MDP3Order& order, std::vector* effects, FillDepot& fills, MDP3Quantity show_qty) { - auto total_qty = order.iceberg_quantity.has_value() ? order.iceberg_quantity.value() - : order.quantity; +void FifoMergedSecurityMatcher::_fill(MDP3Order& order, std::vector* effects, FillDepot& fills, bool reset_show_qty) { if (order.iceberg_quantity.has_value()) { // the order's quantity cannot exceed the show qty (but it can be smaller). - if (show_qty < order.quantity) { - //bts_assert(show_qty >= order.quantity); - bts_assert_msg(false, FMT("ord: ", order, " ***SHOW QTY: ", show_qty)); - } + bts_assert(order.quantity <= order.iceberg_quantity.value().show); } - auto rem = _fill(order.key, order.side, order.price, total_qty, effects, fills); - if (order.iceberg_quantity.has_value()) { - order.iceberg_quantity = rem; - order.quantity = std::min(show_qty, rem); - } else order.quantity = rem; + auto rem = _fill(order.key, order.side, order.price, get_total_qty(order), effects, fills); + reset_show_qty |= rem == get_total_qty(order); + set_total_qty(order, rem, reset_show_qty); } MDP3Quantity FifoMergedSecurityMatcher::_fill( @@ -311,7 +303,7 @@ MDP3Quantity FifoMergedSecurityMatcher::_fill( // auto oside = bts::securities::other_side(side); bts::orderman::SidePriceComp oside_comp(oside); - // SUMMARIZE_TRANSACTION_NAMED(_order_fill_log, _log_header(), FMT("filling ", fmt_order(key, side, quantity, price))); + SUMMARIZE_TRANSACTION_NAMED(_order_fill_log, _log_header(), FMT("filling ", fmt_order(key, side, quantity, price))); while (quantity.value() > 0) { // Fill one price level at a time. // While one market may have the best 1st price level, diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.h index 5157d4489..bd5254d0e 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMergedSecurityMatcher.h @@ -39,7 +39,9 @@ struct FifoMergedSecurityMatcher { FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher&); FifoImpliedSecurityMatcher* _best_implied_matcher(api::Side); - void _fill(MDP3Order&, std::vector*, FillDepot&, MDP3Quantity ib_show_qty); + // the order's params (quantity, and iceberg_quantity if applicable) + // will be appropriately adjusted according to the number of fills. + void _fill(MDP3Order&, std::vector*, FillDepot&, bool reset_show_qty); MDP3Quantity _fill( MDP3OrderKey, api::Side, diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp index fbedf5e85..efaa217be 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp @@ -1,5 +1,7 @@ #include "FifoPriceLevelMatcher.h" #include "../../common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" namespace bts::spartan::mdp3 { @@ -31,13 +33,8 @@ void FifoPriceLevelMatcher::_logically_delete(MatcherOrder &o) { o.is_active = false; } void FifoPriceLevelMatcher::_add_order(const MDP3Order &ord) { - auto ib = - MatcherIB{ord.iceberg_quantity.has_value(), ord.quantity, - ord.iceberg_quantity.has_value() - ? ord.iceberg_quantity.value() - : ord.quantity}; MatcherOrder new_ord (ord.key,ord.side,ord.price,ord.quantity, - _priority++,ib); + _priority++,ord.iceberg_quantity); auto[p,res] = _master_order_map->try_emplace(new_ord.key,new_ord); MatcherOrder* new_ord_ptr = &p->second; _orders.push_back(new_ord_ptr); @@ -129,7 +126,7 @@ void FifoPriceLevelMatcher::_log_order(const MatcherOrder& ord, LOG(OrderMan, Diag, "{} key:{} side:{} qty:{} ib_qty:{} price:{}", msg, ord.key, static_cast(side()), ord.quantity, - ord.ib.is_ib? ord.ib.total_qty : MDP3Quantity(0), + ord.ib.has_value() ? ord.ib.value().total : MDP3Quantity(0), price()); } void FifoPriceLevelMatcher::_log_order(const MDP3Order& ord, @@ -138,8 +135,8 @@ void FifoPriceLevelMatcher::_log_order(const MDP3Order& ord, LOG(OrderMan, Diag, "{} key:{} side:{} qty:{} ib_qty:{} price:{}", msg, ord.key, static_cast(ord.side), ord.quantity, - ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value() - : MDP3Quantity(0), + ord.iceberg_quantity.has_value() ? FMT(ord.iceberg_quantity.value()) + : FMT(MDP3Quantity(0)), ord.price.price()); } // constructors /////////////////////////////////////////////////////////////// @@ -248,8 +245,11 @@ void FifoPriceLevelMatcher::on_order_update( _orders.push_back(&ord); // udpate value ord.quantity = upd_ord.quantity; - } else + } else { ord.quantity = upd_ord.quantity; + if (ord.ib.has_value() && upd_ord.iceberg_quantity.has_value() && ord.ib.value() != upd_ord.iceberg_quantity.value()) + ord.ib = upd_ord.iceberg_quantity; + } } void FifoPriceLevelMatcher::on_order_cancel(MatcherOrder& order, std::vector* res){ diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp index 6a90876e9..8bf9c9def 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp @@ -1,5 +1,7 @@ #include "FifoSecurityMatcher.h" #include "../../common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" #include namespace bts::spartan::mdp3 { @@ -16,7 +18,7 @@ inline bool _is_cross(const MDP3Order &ord, } inline MDP3Quantity total_quantity(const MDP3Order &ord) { - return ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value() + return ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value().total : ord.quantity; } @@ -107,21 +109,6 @@ void FifoSecurityMatcher::_on_new_order(const MDP3Order &ord, price_level = &*std::get<0>(p); } MDP3Order ord_corrected = ord; - if (ord.iceberg_quantity.has_value()) { - uint32_t tot = - static_cast(ord.iceberg_quantity.value().value()); - // this is not exactly correct - uint32_t show = static_cast(ord.quantity.value()); - uint32_t filled = tot-new_ord_rem; - ord_corrected.iceberg_quantity = MDP3Quantity(new_ord_rem); - uint32_t working; - if(filled < show) - working = show - filled; - else - working = std::min(new_ord_rem, show); - ord_corrected.quantity = MDP3Quantity(working); - } else - ord_corrected.quantity = MDP3Quantity(new_ord_rem); price_level->on_new_same_side_order(ord_corrected); } else{ @@ -331,7 +318,8 @@ FifoSecurityMatcher::has_same_orders(const Orders &ord_map) { auto match = plevel != nullptr && ord != nullptr && plevel->price() == exp_ord.price && plevel->side() == exp_ord.side && - ord->quantity == exp_ord.quantity; + ord->quantity == exp_ord.quantity && + ord->ib == exp_ord.iceberg_quantity; if (!match) { std::ostringstream buff; buff << "eid:" << _event_id << " n_ords:" << ord_map.size() @@ -343,10 +331,7 @@ FifoSecurityMatcher::has_same_orders(const Orders &ord_map) { buff << "Expected price:" << exp_ord.price << std::endl; buff << "Expected qty:" << exp_ord.quantity << std::endl; buff << "Expected side:" << exp_ord.side << std::endl; - uint32_t exp_ib_val = exp_ord.iceberg_quantity.has_value() - ? exp_ord.iceberg_quantity.value().value() - : 0; - buff << "Expected IB state:" << exp_ib_val << std::endl; + buff << "Expected IB state:" << exp_ord.iceberg_quantity << std::endl; } else { buff << "Order with Key:" << exp_ord.key.value() @@ -357,13 +342,8 @@ FifoSecurityMatcher::has_same_orders(const Orders &ord_map) { << " matcher:" << ord->quantity << std::endl; buff << "Expected side:" << exp_ord.side << " matcher:" << plevel->side() << std::endl; - uint32_t exp_ib_val = exp_ord.iceberg_quantity.has_value() - ? exp_ord.iceberg_quantity.value().value() - : 0, - matcher_ib_val = - ord->ib.is_ib ? ord->ib.total_qty.value() : 0; - buff << "Expected IB state:" << exp_ib_val - << " matcher:" << matcher_ib_val << std::endl; + buff << "Expected IB state:" << exp_ord.iceberg_quantity + << " matcher:" << ord->ib << std::endl; } LOG(OrderMan, Error, "FifoSecurityMatcher::has_same_orders failed:{}", buff.str()); @@ -418,10 +398,8 @@ void FifoSecurityMatcher::_log_trace_order_state(const std::string& msg){ auto[pl,o] = _find_order(k); if(pl != nullptr && o != nullptr) LOG(OrderMan, Diag, - "Order state msg:{} eid:{} key:{} qty:{} ib_show_qty:{} ib_total_qty:{} " - "price:{}", - msg, _event_id, o->key, o->quantity, o->ib.show_qty, o->ib.total_qty, - pl->price()); + "Order state msg:{} eid:{} key:{} qty:{} ib:{} price:{}", + msg, _event_id, o->key, o->quantity, o->ib, pl->price()); } } void FifoSecurityMatcher::_log_trace_order(const MDP3Order& ord, @@ -430,9 +408,7 @@ void FifoSecurityMatcher::_log_trace_order(const MDP3Order& ord, LOG(OrderMan, Diag, "{} eid:{} key:{} side:{} qty:{} ib_qty:{} price:{} book:{}", msg, _event_id, ord.key, static_cast(ord.side), ord.quantity, - ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value() - : MDP3Quantity(0), - ord.price.price(), print_book()); + ord.iceberg_quantity, ord.price.price(), print_book()); _log_trace_order_state(msg); } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h index e6662afbf..2de6711dd 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h @@ -62,6 +62,7 @@ public: void _on_order_update(const MDP3Order&, std::vector* = nullptr); void _on_order_remove(MDP3OrderKey, std::vector* = nullptr); + std::string _log_header() const; public: // cons/dest /////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////// diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp index 6cc310c1a..9d9728468 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp @@ -1,7 +1,10 @@ #include "Matcher.h" #include "MatchingCommon.h" #include +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" +#include #include #include @@ -10,20 +13,22 @@ using namespace google::protobuf::util; namespace bts::spartan::mdp3{ MergedMatcher::MergedMatcher() -: -_matcher_map(), -_trace(), -_match_count(0), -_match_count_this_per(0), -_per_start_time(TimeUtil::GetCurrentTime()){ +: MergedMatcher(std::unordered_map{}) {} + +MergedMatcher::MergedMatcher( + const std::unordered_map& icebergs) + : _matcher_map(), icebergs(), + _trace(), _match_count(0), _match_count_this_per(0), + _per_start_time(TimeUtil::GetCurrentTime()) { + for (const auto& ib : icebergs) + this->icebergs.emplace(ib); } MergedMatcher::MergedMatcher(MergedMatcher&& src) { _matcher_map = std::move(src._matcher_map); + icebergs = std::move(src.icebergs); } -MergedMatcher::MergedMatcher (std::vector const&) { } - void MergedMatcher::add_def(const InstrumentDefinition& def) { std::string ass(def.asset()); _asset_map.emplace(def.security_id(), ass); @@ -129,7 +134,38 @@ void MergedMatcher::match(ChangeSrc, // todo: make the fills param optional, as // it will often be more info than we care abt. FillDepot fills; - ass_matcher.on_order_change(c, res, fills); + auto new_ch = c; + switch (c.change) { + case OrderChange::ChangeType::added: + iceberg_helper::add_iceberg_annotation(new_ch.order, icebergs); + break; + case OrderChange::ChangeType::updated: { + auto* old_ord = std::get<1>(ass_matcher.find_order(c.order.security, c.order.key)); + if (old_ord == nullptr) { + // the matcher may not have this order anymore; + // it may have been filled already due to an algo shot. + continue; + } + OrderChange big_ch; + big_ch.change = c.change; + big_ch.previous_order = to_mdp3_order(*old_ord); + big_ch.order = c.order; + iceberg_helper::detect_show_qty_changes(big_ch, icebergs); + new_ch.order = big_ch.order; + break; + } + default: break; + } + ass_matcher.on_order_change(new_ch, res, fills); + // the DeducedIceberg's estimated quantity is essentially a fill count + // (obtained by sweeping the entire pcap). Since the order's iceberg + // props (total/show qty) are assigned in this method (anytime the order changes), + // the fills must be incrementally subtracted from the remaining quantity + // in our icebergs cache. That way, the iceberg total qty it will be correct + // the next time the order changes. + std::vector fills_vect; + fills.copy_to(fills_vect); + iceberg_helper::subtract_iceberg_fills(fills_vect.begin(), fills_vect.end(), icebergs); } // log/validate order updates for(const auto& u : *res) @@ -171,7 +207,6 @@ Book MergedMatcher::book(MDP3SecurityID id, api::Side s) { return it->second.matcher.book(id, s); } - ///////////////// // DirectMatcher //////////////// @@ -244,6 +279,5 @@ Book DirectMatcher::book(MDP3SecurityID id, api::Side s) { bts_assert(it != _matcher_map.end()); return it->second.matcher.book(s); } - } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h index 0733545c9..e5ab2da08 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h @@ -6,6 +6,7 @@ #include "FifoSecurityMatcher.h" #include "FifoMatcherContext.h" #include "MatchingCommon.h" +#include #include #include #include @@ -13,6 +14,7 @@ namespace bts::spartan::mdp3 { class InstrumentDefinition; +class DeducedIceberg; class MergedMatcher { private: @@ -23,6 +25,7 @@ private: std::map _asset_map; std::map _matcher_map; + boost::container::flat_map icebergs; Trace _trace; size_t _match_count; size_t _match_count_this_per; @@ -31,7 +34,7 @@ private: public: MergedMatcher (); - MergedMatcher (std::vector const&); + MergedMatcher(const std::unordered_map&); MergedMatcher (MergedMatcher&&); MergedMatcher &operator=(MergedMatcher&&) {return *this;} @@ -48,6 +51,7 @@ public: std::vector *); Book book(MDP3SecurityID, api::Side); + template void init(TIter first, TIter last) { while (first != last) { @@ -89,6 +93,7 @@ public: std::vector *); Book book(MDP3SecurityID, api::Side); + ImpliedBook implied_book(MDP3SecurityID, api::Side); }; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.cpp new file mode 100644 index 000000000..5cfa5c671 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.cpp @@ -0,0 +1,26 @@ +#include "MatchingCommon.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" + +namespace bts::spartan::mdp3 { +std::tuple MatcherOrder::cross(uint32_t cross_qty) { + auto [r, c] = compute_cross(quantity.value(), cross_qty); + bts_assert(quantity.value() - r == cross_qty - c); + auto filled = cross_qty - c; + quantity = MDP3Quantity(r); + if (ib.has_value()) { + ib.value().total -= MDP3Quantity(filled); + is_active = ib.value().total.value() > 0; + } else + is_active = quantity.value() > 0; + return std::make_tuple(r, c, filled); +} + +void MatcherOrder::reload_ib() { + if (ready_to_load_ib()) { + is_active = ib.value().total.value() > 0; + if (quantity.value() == 0) + quantity = std::min(ib.value().show, ib.value().total); + is_active = quantity.value() > 0; + } +} +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.h index 8eb48a72d..1971429cf 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.h @@ -4,6 +4,7 @@ // the templated classes (will fix later) #include "../../common/InstrumentSnapshot.h" #include "../simulator/AlgoEvent.h" +#include namespace bts::spartan::mdp3 { @@ -34,8 +35,7 @@ public: "Matcher incoming change:{} ok:{} side:{} price:{} qty:{} ib:{}", c.change, c.order.key, _side(c.order.side), c.order.price, c.order.quantity, - c.order.iceberg_quantity.has_value() ? - c.order.iceberg_quantity.value().value() : 0); + c.order.iceberg_quantity); } void log_outgoing_update(const OrderUpdate& u){ if(traced_orders.find(u.key) != traced_orders.end()) @@ -61,12 +61,6 @@ public: } }; -struct MatcherIB { - bool is_ib = false; - MDP3Quantity show_qty = MDP3Quantity(0); - MDP3Quantity total_qty = MDP3Quantity(0); -}; - struct MatcherOrder { static std::tuple compute_cross(uint32_t resting_qty, uint32_t cross_qty) { @@ -85,14 +79,14 @@ struct MatcherOrder { md::IntPrice price; MDP3Quantity quantity; uint32_t priority = 0; - MatcherIB ib; + std::optional ib; bool is_active = true; MatcherOrder(MDP3OrderKey key, api::Side side, md::IntPrice price, MDP3Quantity qty, // working qty out there uint32_t priority, - MatcherIB ib) + std::optional ib = {}) : key(key), side(side), @@ -100,58 +94,23 @@ struct MatcherOrder { quantity(qty), priority(priority), ib(ib) {} - MatcherOrder(MDP3OrderKey key, - api::Side side, - md::IntPrice price, - MDP3Quantity qty, // working qty out there - uint32_t priority) - : - MatcherOrder(key, - side, - price, - qty, - priority, - MatcherIB()) { - } // returns the remainder of this order, remainder of opp order // (ie the new qty NOT the filled qty, for that subtract from // original) - std::tuple cross(uint32_t cross_qty) { - auto [r, c] = - compute_cross(quantity.value(), cross_qty); - bts_assert(quantity.value() - r == cross_qty - c); - auto filled = cross_qty - c; - quantity = MDP3Quantity(r); - if (ib.is_ib){ - ib.total_qty.value() = ib.total_qty.value() - filled; - is_active = ib.total_qty.value() > 0; - } - else - is_active = quantity.value() > 0; - return std::make_tuple(r, c, filled); - } + std::tuple cross(uint32_t cross_qty); MDP3Quantity total_qty() const { - return ib.is_ib ? ib.total_qty : quantity; + return ib.has_value() ? ib.value().total : quantity; } MDP3Quantity working_qty() const { return quantity; } bool ready_to_load_ib() const { - return ib.is_ib && quantity.value() == 0 && ib.total_qty.value() > 0; - } - void reload_ib() { - if (ready_to_load_ib()) { - auto load_qty = std::min(ib.show_qty.value(), ib.total_qty.value()); - quantity = MDP3Quantity(load_qty); - is_active = true; - } + return ib.has_value() && quantity.value() == 0 && ib.value().total.value() > 0; } + void reload_ib(); }; inline std::ostream& operator <<(std::ostream& out, const MatcherOrder& ord) { - out << "[" << ord.key << "] " << ord.side << " " << ord.quantity << " @ " << ord.price << " (ice "; - if (ord.ib.is_ib) - out << "qty " << ord.ib.total_qty; - else out << "n/a"; - out << " active " << (ord.is_active ? "Y" : "N"); + out << "[" << ord.key << "] " << ord.side << " " << ord.quantity << " @ " << ord.price + << " (ice " << ord.ib << " active " << (ord.is_active ? "Y" : "N") << ")"; return out; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp new file mode 100644 index 000000000..9c456bc4f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp @@ -0,0 +1,137 @@ +#include "iceberg_helper.h" + +namespace bts::spartan::mdp3::iceberg_helper { +template +using flat_map = boost::container::flat_map; + +IcebergQuantity to_iceberg_qty(const DeducedIceberg& dib) { + // initializer lists with diff fields of the same type seem dangerous + // ...what if the declaration order changes? accidents happen. + IcebergQuantity ib_qty; + ib_qty.show = dib.show_quantity; + ib_qty.total = dib.estimated_quantity; + return ib_qty; +} + +void add_iceberg_annotation( + MDP3Order& order, + const flat_map& icebergs, + const flat_map& overrides) { + auto it = icebergs.find(order.key); + if (it != icebergs.end()) { + auto over_it = overrides.find(order.key); + auto ibq = to_iceberg_qty(it->second); + if (over_it != overrides.end()) + ibq = to_iceberg_qty(over_it->second); + if (order.iceberg_quantity != ibq) + order.iceberg_quantity = ibq; + // adjust order's actual quantity in accordance + // wih the new iceberg quantity + order.quantity = order.quantity.value() == 0 + ? std::min(ibq.show, ibq.total) + : std::min(order.quantity, std::min(ibq.show, ibq.total)); + } +} + +void add_iceberg_annotation( + MDP3Order& order, const flat_map& icebergs) { + flat_map overrides; + add_iceberg_annotation(order, icebergs, overrides); +} + +void detect_show_qty_changes( + const MDP3Event& ev, + flat_map& icebergs, + flat_map& show_qty_changes) { + std::set ibs_with_fills; + for (const auto* s : ev.affected_securities) { + for (auto f : s->fills()) { + auto ib_it = icebergs.find(f.key); + if (ib_it != icebergs.end()) + ibs_with_fills.insert(f.key); + } + } + // the show qty can change, most of these changes + // can be detected simply by looking for order updates + // that change the qty, when there are no fills + for (auto s : ev.affected_securities) { + for (auto ch : s->order_changes()) { + switch (ch.change) { + case OrderChange::ChangeType::updated: { + // if the qty changed without any fills, + // then the show qty must have changed. + if (ch.order.quantity == ch.previous_order.quantity) + continue; + auto ib_it = icebergs.find(ch.order.key); + if (ib_it == icebergs.end()) + continue; + if (ibs_with_fills.find(ch.order.key) != ibs_with_fills.end()) + continue; + auto new_dib = ib_it->second; + new_dib.estimated_quantity -= (new_dib.show_quantity - ch.order.quantity); + new_dib.show_quantity = ch.order.quantity; + show_qty_changes.insert_or_assign(new_dib.key, new_dib); + auto tmp = ib_it->second; + ib_it->second.estimated_quantity += + (ch.order.quantity - ch.previous_order.quantity); + PRINT_LINE("adjusting est qty: ", tmp, " --> ", ib_it->second); + break; + } + default: break; + } + } + } +} + +void detect_show_qty_changes( + OrderChange& ch, + flat_map& icebergs) { + // the show qty can change, most of these changes + // can be detected simply by looking for order updates + // that change the qty, when there are no fills + switch (ch.change) { + case OrderChange::ChangeType::updated: { + // if the qty changed without any fills, + // then the show qty must have changed. + if (ch.order.quantity == ch.previous_order.quantity) + break; + auto ib_it = icebergs.find(ch.order.key); + if (ib_it == icebergs.end()) + break; + auto qty_diff = ch.order.quantity - ch.previous_order.quantity; + ib_it->second.estimated_quantity += qty_diff; + ib_it->second.show_quantity = ch.order.quantity; + break; + } + default: break; + } +} + +void subtract_iceberg_fills( + const MDP3Event& ev, flat_map& icebergs) { + for (const auto* s : ev.affected_securities) { + for (auto f : s->fills()) { + auto ib_it = icebergs.find(f.key); + if (ib_it != icebergs.end()) + ib_it->second.estimated_quantity -= f.quantity; + } + } +} + +void add_iceberg_annotation( + const InstrumentSnapshot& s, + Orders& orders, + const flat_map& icebergs) { + // + for (auto ch : s.order_changes()) { + switch (ch.change) { + case OrderChange::added: + case OrderChange::updated: + add_iceberg_annotation(ch.order, icebergs); + orders[ch.order.key] = ch.order; + break; + case OrderChange::removed: orders.erase(ch.order.key); break; + } + } +} +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h new file mode 100644 index 000000000..2d235702b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h @@ -0,0 +1,61 @@ +#pragma once +#include "lib/orderman/orderman_utils/Formatter.h" +#include "lib/orderman/orderman_utils/PriceComp.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/fill_depot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/matcher_diagnostics.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/security_comp.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" +#include +#include +#include + +namespace bts::spartan::mdp3::iceberg_helper { +IcebergQuantity to_iceberg_qty(const DeducedIceberg&); + +void add_iceberg_annotation( + MDP3Order& order, + const boost::container::flat_map& icebergs, + const boost::container::flat_map& overrides); + +void add_iceberg_annotation( + MDP3Order& order, + const boost::container::flat_map& icebergs); + +void detect_show_qty_changes( + const MDP3Event& ev, + boost::container::flat_map& icebergs, + boost::container::flat_map& show_qty_changes); + +void detect_show_qty_changes( + OrderChange& change, + boost::container::flat_map& icebergs); + +void subtract_iceberg_fills( + const MDP3Event& ev, + boost::container::flat_map& icebergs); + +template +inline void subtract_iceberg_fills( + TIter first, + TIter last, + boost::container::flat_map& icebergs) { + while (first != last) { + auto it = icebergs.find(first->key); + if (it != icebergs.end()) { + auto orig_qty = it->second.estimated_quantity.value(); + it->second.estimated_quantity.value() -= first->quantity.value(); + bts_assert(icebergs[first->key].estimated_quantity.value() < orig_qty); + } + first++; + } +} +} // namespace bts::spartan::mdp3::iceberg_helper \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/matcher_diagnostics.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/matcher_diagnostics.h index a45499112..e7aaa5251 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/matcher_diagnostics.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/matcher_diagnostics.h @@ -1,12 +1,13 @@ #pragma once -#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" -#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" -#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h" -#include "server/sparkd/cmds/spartan/util/table_display.h" -#include "lib/orderman/orderman_utils/Formatter.h" #include "lib/md/IntPrice.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h" #include "server/sparkd/cmds/spartan/util/action_summarizer.h" #include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" #include #include #include @@ -108,8 +109,8 @@ inline void print_order_changes(TIter first, TIter last, std::ostream& out = std {" N Price ", 12, [](auto ch){ return FMT(ch.order.price.price()); }}, {" O Qty ", 8, [](auto ch){return FMT(ch.previous_order.quantity); }}, {" N Qty ", 8, [](auto ch){ return FMT(ch.order.quantity); }}, - {" O Ice ", 8, [](auto ch){ return FMT(ch.previous_order.iceberg_quantity.has_value() ? ch.previous_order.iceberg_quantity.value().value() : 0); }}, - {" N Ice ", 8, [](auto ch){ return FMT(ch.order.iceberg_quantity.has_value() ? ch.order.iceberg_quantity.value().value() : 0); }}}); + {" O Ice ", 8, [](auto ch){ return FMT(ch.previous_order.iceberg_quantity); }}, + {" N Ice ", 8, [](auto ch){ return FMT(ch.order.iceberg_quantity); }}}); out << std::endl; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp index aab01ad01..c5272e9e7 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp @@ -3,6 +3,7 @@ #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h" #include #include +#include #include #include "../MatchingCommon.h" @@ -31,18 +32,14 @@ public: MDP3Order create_order(uint64_t key, int32_t sec, api::Side side, uint64_t priority, double price, uint32_t qty, - MatcherIB ib = {false, MDP3Quantity(0), - MDP3Quantity(0)}) { + std::optional ib = { }) { auto rv = MDP3Order{MDP3OrderKey(key), MDP3SecurityID(sec), side, MDP3OrderPriority(priority), md::IntPrice_from_double<7>(price), MDP3Quantity(qty), - ib.is_ib ? std::make_optional(ib.total_qty) - : std::nullopt}; - // if(ib.is_ib) - // rv.iceberg_quantity = std::make_optional(ib.total_qty); + ib}; return rv; } @@ -155,7 +152,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher]") { // iceberg, show qty=2, total_qty=100 o = create_order( k++, sec_key, api::Side::SIDE_BUY, 0, price, 2, - MatcherIB{true, MDP3Quantity(2), MDP3Quantity(100)}); + IcebergQuantity{MDP3Quantity(2), MDP3Quantity(100)}); price_level.on_new_same_side_order(o); // non ib, to get the visible qty to 15 o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 7); @@ -437,8 +434,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { // send a new sell at 15.07 to take the first 3 levels auto cross_ib = create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.07, 13, - MatcherIB{true, MDP3Quantity(13), - MDP3Quantity(12500)}); + IcebergQuantity{MDP3Quantity(13), MDP3Quantity(12500)}); sec_me.on_order_change(OrderChange{MDP3Order{},cross_ib,OrderChange::added}); bid_bk = sec_me.book(api::Side::SIDE_BUY); verify_book_level(bid_bk.at(0), 15.06, 2485, 70); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp index 0d0e28381..f981f7187 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp @@ -15,6 +15,7 @@ #include "server/sparkd/cmds/spartan/util/action_summarizer.h" #include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" #include "server/sparkd/cmds/spartan/util/table_display.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h" #include #include #include @@ -26,15 +27,15 @@ // to avoid duplicate, and (perhaps more importantly) unsynchronized output. #define _ECHO_ON #ifdef _ECHO_ON -#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__) +#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__); #define _ERROR_LEVEL Info #else #define _ECHO(...) #define _ERROR_LEVEL Error #endif #define _LOG(level, msg) \ - _ECHO(_log_header(), " ", msg, "\n"); \ - //LOG(OrderMan, level, "{} {}", _log_header(), msg) + _ECHO(_log_header(), " ", msg, "\n") \ + LOG(OrderMan, level, "{} {}", _log_header(), msg) #define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) #define ERROR(...) _LOG(_ERROR_LEVEL, bts::orderman::Formatter::fmt(__VA_ARGS__)) @@ -44,6 +45,7 @@ #define _SUMMARIZE_TRANSACTION_NAMED(name, ...) SUMMARIZE_TRANSACTION_NAMED(name, _log_header(), FMT(__VA_ARGS__)) namespace bts::spartan::mdp3 { +using namespace bts::spartan::mdp3::iceberg_helper; // aliases using TradingStatus = MatcherValidationObserver::TradingStatus; template @@ -71,13 +73,10 @@ void _sync_orders( for (auto ch : s.order_changes()) { switch (ch.change) { case OrderChange::added: - case OrderChange::updated: { - auto it = icebergs.find(ch.order.key); - if (it != icebergs.end()) - ch.order.iceberg_quantity = it->second.estimated_quantity; + case OrderChange::updated: + add_iceberg_annotation(ch.order, icebergs); orders[ch.order.key] = ch.order; break; - } case OrderChange::removed: orders.erase(ch.order.key); break; } } @@ -197,7 +196,8 @@ struct MatcherValidationObserver::PImpl { size_t event_id = NO_EVENT; FifoMatcherContext exch_matcher; Verbosity verbosity = Verbosity::Normal; - EventFlags ignore_flags = EventFlags::IcebergOrders | EventFlags::MultipleSecurityAggressors; + //EventFlags ignore_flags = EventFlags::IcebergOrders | EventFlags::MultipleSecurityAggressors; + EventFlags ignore_flags = EventFlags::MultipleSecurityAggressors; // event_id is used to transition the security's trading status from // Preopen to Uninitialized @@ -226,11 +226,8 @@ struct MatcherValidationObserver::PImpl { data.snapshot = &s; data.last_observed_event = event_id; data.previous_orders = s.orders(); - for (auto& o : data.previous_orders) { - auto it = icebergs.find(o.first); - if (it != icebergs.end()) - o.second.iceberg_quantity = it->second.estimated_quantity; - } + for (auto& o : data.previous_orders) + add_iceberg_annotation(o.second, icebergs); // the first event at the open isn't processed bc // we don't have a clear understanding of how preopen matching works. data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade @@ -277,42 +274,6 @@ MatcherValidationObserver::MatcherValidationObserver( MatcherValidationObserver::~MatcherValidationObserver() {} -/////////////////////// -// iceberg annotation -/////////////////////// -void MatcherValidationObserver::_add_iceberg_annotation(MDP3Order& order) { - auto it = _me->icebergs.find(order.key); - if (it != _me->icebergs.end()) - order.iceberg_quantity = it->second.estimated_quantity; -} - -void MatcherValidationObserver::_add_iceberg_annotation(Orders& orders) { - for (auto& o : orders) - _add_iceberg_annotation(o.second); -} - -void MatcherValidationObserver::_add_iceberg_annotation(OrderChanges& changes) { - for (auto& o : changes) { - _add_iceberg_annotation(o.previous_order); - _add_iceberg_annotation(o.order); - } -} - -void MatcherValidationObserver::_add_iceberg_annotations( - const InstrumentSnapshot& s, std::unordered_map& target) { - // Update our order cache so we can use it to - // populate the matcher on the next event - for (auto ch : s.order_changes()) { - switch (ch.change) { - case OrderChange::added: - case OrderChange::updated: - _add_iceberg_annotation(ch.order); - target[ch.order.key] = ch.order; - break; - case OrderChange::removed: target.erase(ch.order.key); break; - } - } -} /////////////////////// // misc private methods /////////////////////// @@ -361,34 +322,24 @@ void MatcherValidationObserver::_reset_matcher() { void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap) { _SUMMARIZE; for (auto s : snap) { - auto annotated_orders = s->orders(); - _add_iceberg_annotation(annotated_orders); auto& sec_data = _me->get_security_data(*s, NO_EVENT); - if (s->trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade) - sec_data.status = TradingStatus::Uninitialized; + sec_data.current_orders.clear(); + _sync_orders(*s, sec_data.current_orders, _me->icebergs); + sec_data.previous_orders = sec_data.current_orders; } } void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { bts_assert(ev.event_id > _me->event_id); _me->event_id = ev.event_id; - // uncomment code below for a quick way to track an order's state - // on every update - for (auto& kv : _me->interesting_orders) { - for (auto s : ev.affected_securities) { - auto it = s->orders().find(kv.first); - if (it != s->orders().end() && !orders_equal(it->second, kv.second)) { - INFO("Interesting Order (event ", ev.event_id, "): ", it->second); - kv.second = it->second; - } - } - } switch (get_event_ordinal(_me->interesting_event_ranges, ev.event_id, MIN_OBSERVED_EVENT)) { case bts::spartan::EventOrdinal::Waiting: // just need to synchronize the order caches. // the matcher is only validated during 'interesting events.' _sync_current_orders(ev); _sync_previous_orders(ev); + subtract_iceberg_fills(ev, _me->icebergs); + detect_show_qty_changes(ev, _me->icebergs, _me->icebergs); return; case bts::spartan::EventOrdinal::Interesting: _sync_current_orders(ev); @@ -406,6 +357,8 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags)); _reset_matcher(); _sync_previous_orders(ev); + subtract_iceberg_fills(ev, _me->icebergs); + detect_show_qty_changes(ev, _me->icebergs, _me->icebergs); return; } ++_me->interesting_event_count; @@ -453,8 +406,10 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { } // securities must be added to the matcher before the order // changes are processed to ensure the correct implied markets are used. - for (const auto* s : sorted_secs) + for (const auto* s : sorted_secs) { _me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs()); + _me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching()); + } // initialize matcher with previous state (only applies // to securities whose orders the matcher hasn't seen yet). for (const auto* s : sorted_secs) { @@ -462,42 +417,53 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { continue; auto it = _me->security_cache.find(s->security_id()); bts_assert(it != _me->security_cache.end()); - _SUMMARIZE_TRANSACTION( - "restoring ", s->symbol(), " orders(", it->second.previous_orders.size(), ")"); + for (auto& kv : it->second.previous_orders) + add_iceberg_annotation(kv.second, _me->icebergs); auto prev_ords = it->second.previous_orders | - ranges::views::transform([](auto p) { return p.second; }); + ranges::views::transform([](auto& p) { return p.second; }); _me->exch_matcher.reset(s->security_id(), prev_ords.begin(), prev_ords.end()); _me->security_cache[s->security_id()].status = TradingStatus::Initialized; } + // capture iceberg show qty changes (these are a special case) + flat_map show_qty_changes; + detect_show_qty_changes(ev, _me->icebergs, show_qty_changes); // process changes with matcher FillDepot fills; for (const auto* s : sorted_secs) { - auto num_fills_before = fills.fill_count(); switch (_trading_status(*s, ev.event_id)) { - case TradingStatus::Initialized: - case TradingStatus::Uninitialized: break; + case TradingStatus::Initialized: break; default: continue; } auto deduced = s->deduced_order_changes(); if (deduced.size()) { std::vector deduced_sm; - for (auto& ch : deduced) + for (auto& ch : deduced) { + add_iceberg_annotation(ch.order, _me->icebergs, show_qty_changes); + auto it = show_qty_changes.find(ch.order.key); + if (it != show_qty_changes.end()) { + auto exp_ibq = to_iceberg_qty(it->second); + PRINT_LINE("expecting this: ", exp_ibq, ", getting this: ", ch.order.iceberg_quantity.value()); + bts_assert(ch.order.iceberg_quantity.has_value() && ch.order.iceberg_quantity.value() == exp_ibq); + } deduced_sm.emplace_back(ch); - _SUMMARIZE_TRANSACTION( - "processing ", s->symbol(), " order changes(", deduced_sm.size(), ")"); + } _me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills); - if (_me->verbosity > Verbosity::Normal) - INFO("matcher generated ", fills.fill_count() - num_fills_before, " fills for ", s->symbol()); } } - // verify the matcher's orders are identitical to the exchange's + for (auto ib : show_qty_changes) + _me->icebergs[ib.first] = ib.second; + // Verify the matcher's orders are identitical to the exchange's. + // subtract the fills from iceberg orders' total quantities + // to get the correct expected orders for comparison with the matcher's orders. + subtract_iceberg_fills(ev, _me->icebergs); bool orders_match = true; for (const auto* s : sorted_secs) { if (!_is_ready_to_trade(*s, ev.event_id)) continue; - _SUMMARIZE_TRANSACTION_NAMED(_incr_val, "validating order changes for ", s->symbol()); auto& sec_data = _me->security_cache[s->security_id()]; auto& curr_orders = sec_data.current_orders; + for (auto& kv : curr_orders) + add_iceberg_annotation(kv.second, _me->icebergs); auto error = _me->exch_matcher.has_same_orders(s->security_id(), curr_orders); if (error) { ERROR( @@ -509,45 +475,44 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { } bool fills_match = match_fills(fills, sorted_secs); if (!orders_match || !fills_match) { + // todo: add the iceberg fills back, to get to the previous state. // describe current state for diagnostic purposes - if (_me->verbosity > Verbosity::Normal) { - std::stringstream msg; - msg << "Spartan Top of Books:" << std::endl; - _me->exch_matcher.print_all_bbos(msg); - msg << std::endl; - // spartan fills - msg << "Spartan Total Fill Qty: " << fills.total_fill_qty() << std::endl; - auto fill_cmp = [](const auto& lhs, const auto& rhs) { - return lhs.trade_entry_id == rhs.trade_entry_id - ? static_cast(lhs.aggressor) > static_cast(rhs.aggressor) - : lhs.trade_entry_id < rhs.trade_entry_id; - }; - std::vector spartan_fills; - fills.copy_to(spartan_fills); - std::sort(spartan_fills.begin(), spartan_fills.end(), fill_cmp); - msg << "Spartan Fills (ordered by Trade ID):" << std::endl; - diagnostics::print_fills(spartan_fills.begin(), spartan_fills.end(), msg); - // exch top of books - msg << "Exchange Top of Books:" << std::endl; - diagnostics::print_many_bbo( - sorted_secs.begin(), sorted_secs.end(), true, true, msg); - int exch_fill_qty = 0; - for (auto* s : sorted_secs) { - for (auto f : s->fills()) - exch_fill_qty += f.quantity.value(); - } - // exch fills - msg << "Exchange Total Fill Qty: " << exch_fill_qty << std::endl; - std::vector all_exch_fills; - for (auto* s : sorted_secs) { - for (auto f : s->fills()) - all_exch_fills.push_back(f); - } - std::sort(all_exch_fills.begin(), all_exch_fills.end(), fill_cmp); - msg << "Exchange Fills (ordered by Trade ID):" << std::endl; - diagnostics::print_fills(all_exch_fills.begin(), all_exch_fills.end(), msg); - INFO(msg.str()); + std::stringstream msg; + msg << "Spartan Top of Books:" << std::endl; + _me->exch_matcher.print_all_bbos(msg); + msg << std::endl; + // spartan fills + msg << "Spartan Total Fill Qty: " << fills.total_fill_qty() << std::endl; + auto fill_cmp = [](const auto& lhs, const auto& rhs) { + return lhs.trade_entry_id == rhs.trade_entry_id + ? static_cast(lhs.aggressor) > static_cast(rhs.aggressor) + : lhs.trade_entry_id < rhs.trade_entry_id; + }; + std::vector spartan_fills; + fills.copy_to(spartan_fills); + std::sort(spartan_fills.begin(), spartan_fills.end(), fill_cmp); + msg << "Spartan Fills (ordered by Trade ID):" << std::endl; + diagnostics::print_fills(spartan_fills.begin(), spartan_fills.end(), msg); + // exch top of books + msg << "Exchange Top of Books:" << std::endl; + diagnostics::print_many_bbo( + sorted_secs.begin(), sorted_secs.end(), true, true, msg); + int exch_fill_qty = 0; + for (auto* s : sorted_secs) { + for (auto f : s->fills()) + exch_fill_qty += f.quantity.value(); } + // exch fills + msg << "Exchange Total Fill Qty: " << exch_fill_qty << std::endl; + std::vector all_exch_fills; + for (auto* s : sorted_secs) { + for (auto f : s->fills()) + all_exch_fills.push_back(f); + } + std::sort(all_exch_fills.begin(), all_exch_fills.end(), fill_cmp); + msg << "Exchange Fills (ordered by Trade ID):" << std::endl; + diagnostics::print_fills(all_exch_fills.begin(), all_exch_fills.end(), msg); + INFO(msg.str()); // reset the matcher to avoid cascading/compounding errors in subsequent events. _reset_matcher(); } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h index f6212938a..34f45ce12 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h @@ -47,12 +47,6 @@ struct MatcherValidationObserver : IChannelObserver { struct PImpl; std::unique_ptr _me; - void _add_iceberg_annotation(MDP3Order& order); - void _add_iceberg_annotation(Orders& orders); - void _add_iceberg_annotation(OrderChanges& changes); - void _add_iceberg_annotations( - const InstrumentSnapshot& s, std::unordered_map& target); - TradingStatus _trading_status(const InstrumentSnapshot& s, size_t event_id); // returns true if the security's TradingStatus is 'Uninitialized', or 'Initialized' // (in other words, if it isn't 'None,' or 'Preopen') diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp index 695168688..bbd35d688 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp @@ -70,7 +70,7 @@ protos::SimInput to_proto(SimInput const &in) { schange->set_quantity(change.order.quantity.value()); if (change.order.iceberg_quantity) schange->set_iceberg_quantity( - change.order.iceberg_quantity->value()); + change.order.iceberg_quantity->total.value()); } } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h index b1e0d48c8..63c60bb74 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h @@ -31,4 +31,4 @@ class GatewayAgent : public sim::AgentBase { std::unique_ptr me; }; -} // namespace bts::spartan::mdp3 +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h index 5fe5e4f85..897153aa6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h @@ -81,6 +81,23 @@ inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) { return s; } +struct IcebergQuantity { + MDP3Quantity show; + MDP3Quantity total; +}; + +inline std::ostream& operator<<(std::ostream& out, const IcebergQuantity& ibq) { + out << ibq.show << "/" << ibq.total; + return out; +} + +inline std::ostream& operator<<(std::ostream& out, const std::optional& ibq) { + if (ibq.has_value()) + out << ibq.value(); + else out << "n/a"; + return out; +} + struct MDP3Order { public: operator bool() const noexcept { return !empty(); } @@ -93,7 +110,7 @@ struct MDP3Order { MDP3OrderPriority priority; md::IntPrice price; MDP3Quantity quantity; - std::optional iceberg_quantity; + std::optional iceberg_quantity; }; struct OrderChange { @@ -136,8 +153,7 @@ inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { s << " k: " << order.key.value() << " sk: " << order.security.value() << " side: " << securities::side_name(order.side) << " price: " << order.price.price() << " qty: " << order.quantity.value() - << " pri: " << order.priority.value() << " ice?: " - << (order.iceberg_quantity ? order.iceberg_quantity.value().value() : 0); + << " pri: " << order.priority.value() << " ice?: " << order.iceberg_quantity; return s; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h index a6f3587ca..b729d0922 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h @@ -2,6 +2,9 @@ #include "mdp3_common.h" namespace bts::spartan::mdp3 { +/////////////////////// +// MDP3Quantity +/////////////////////// // add inline MDP3Quantity operator+(const MDP3Quantity& lhs, const MDP3Quantity& rhs) { return MDP3Quantity(lhs.value() + rhs.value()); @@ -37,6 +40,14 @@ inline MDP3Quantity& operator/=(MDP3Quantity& lhs, const MDP3Quantity& rhs) { lhs = MDP3Quantity(lhs.value() / rhs.value()); return lhs; } +// modulus +inline MDP3Quantity operator%(const MDP3Quantity& lhs, const MDP3Quantity& rhs) { + return MDP3Quantity(lhs.value() % rhs.value()); +} +inline MDP3Quantity& operator%=(MDP3Quantity& lhs, const MDP3Quantity& rhs) { + lhs = MDP3Quantity(lhs.value() % rhs.value()); + return lhs; +} // greater/less than or equal to inline bool operator>=(const MDP3Quantity& lhs, const MDP3Quantity& rhs) { return lhs.value() >= rhs.value(); @@ -49,4 +60,26 @@ inline bool operator<=(const MDP3Quantity& lhs, const MDP3Quantity& rhs) { inline MDP3Quantity abs(MDP3Quantity qty) { return qty.value() < 0 ? MDP3Quantity(-qty.value()) : qty; } + +/////////////////////// +// IcebergQuantity +/////////////////////// +inline bool operator==(const IcebergQuantity& lhs, const IcebergQuantity& rhs) { + return lhs.show.value() == rhs.show.value() && lhs.total.value() == rhs.total.value(); +} +inline bool operator!=(const IcebergQuantity& lhs, const IcebergQuantity& rhs) { + return !(lhs == rhs); +} +inline bool operator==(const IcebergQuantity& lhs, const std::optional& rhs) { + return rhs.has_value() && lhs == rhs.value(); +} +inline bool operator==(const std::optional& lhs, const IcebergQuantity& rhs) { + return rhs == lhs; +} +inline bool operator==(const std::optional& lhs, const std::optional& rhs) { + return lhs.has_value() ? rhs.has_value() && lhs.value() == rhs.value() : !rhs.has_value(); +} +inline bool operator!=(const std::optional& lhs, const std::optional& rhs) { + return !(lhs == rhs); +} } \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.cpp new file mode 100644 index 000000000..d009dfab3 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.cpp @@ -0,0 +1,16 @@ +#include "order_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.h" + +namespace bts::spartan::mdp3 { +MDP3Order to_mdp3_order(const MatcherOrder& mo) { + MDP3Order rv; + rv.key = mo.key; + rv.side = mo.side; + rv.quantity = mo.quantity; + // MatcherOrder.priority doesn't map back to the exchange's MDP3Order.priority. + // Set it to zero so nobody thinks it does. + rv.priority = MDP3OrderPriority(0); + rv.iceberg_quantity = mo.ib; + return rv; +} +} \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h index 55ce92e5e..f2e1d4807 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h @@ -2,7 +2,9 @@ #include "lib/md/IntPrice.h" #include "lib/orderman/orderman_utils/PriceComp.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" #include +#include #include #include #include @@ -16,15 +18,51 @@ inline md::IntPrice get_price(const std::tuple& pric inline MDP3Quantity get_qty(const std::tuple& price_level) { return std::get<1>(price_level); } +inline MDP3Quantity get_total_qty(const MDP3Order& order) { + return order.iceberg_quantity.has_value() ? order.iceberg_quantity.value().total + : order.quantity; +} +// all order quantity props should contain the original values (before subtracting fills, or any other qty updates, etc.) +inline void set_total_qty(MDP3Order& order, MDP3Quantity total_qty, bool reset_show_qty = false) { +// bool verbose = order.key.value() = 647354912656; + bts_assert(total_qty.value() >= 0); + if (order.iceberg_quantity.has_value()) { + auto ibq = order.iceberg_quantity.value(); + int ordinal0 = ibq.total.value() + (ibq.show.value() - order.quantity.value()); + int mod_offset = ordinal0 % ibq.show.value(); + bts_assert(mod_offset >= 0); + int adj = 0; + if (mod_offset > 0) { + adj = ibq.show.value() - mod_offset; + ordinal0 += (ibq.show.value() - mod_offset); + } + ibq.total = total_qty; + if (reset_show_qty) { + order.quantity = std::min(order.iceberg_quantity.value().show, total_qty); + } else { + order.quantity = order.iceberg_quantity.value().show < total_qty + ? (total_qty + MDP3Quantity(adj)) % order.iceberg_quantity.value().show + : total_qty; + if (order.quantity.value() == 0) + order.quantity = std::min(total_qty, order.iceberg_quantity.value().show); + } + order.iceberg_quantity.value().total = total_qty; + } else + order.quantity = total_qty; +} inline MDP3Quantity book_volume(const std::vector>& book) { auto get_qty = [](const auto& bl) { return std::get<1>(bl).value(); }; return MDP3Quantity(ranges::accumulate(book | ranges::views::transform(get_qty), 0)); } +struct MatcherOrder; +MDP3Order to_mdp3_order(const MatcherOrder&); + inline bool orders_equal(const MDP3Order& lhs, const MDP3Order& rhs) { return lhs.key == rhs.key && lhs.side == rhs.side && - lhs.price == rhs.price && lhs.quantity == rhs.quantity; + lhs.price == rhs.price && lhs.quantity == rhs.quantity && + lhs.iceberg_quantity == rhs.iceberg_quantity; } inline std::string diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp index 22f338006..0811c5618 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp @@ -8,7 +8,16 @@ IcebergFinder::~IcebergFinder() {} struct IcebergFinder::OrderData { bool is_iceberg = false; + // "Original quantity" is synonomous with "show quantity." + // The exchange does not allow traders to adjust this property; + // any quantity updates to an iceberg order apply to the total quantity + int show_qty = 0; size_t accumulated_fills = 0; + // count qty changes in events where the order has no fills. + // this is needed to correctly estimated the qty in cases where + // the order is canceled before being filled to completion + size_t accumulated_qty_changes = 0; + size_t last_observed_qty = 0; bool filled = false; }; @@ -28,12 +37,13 @@ void IcebergFinder::finit() { if (od.is_iceberg) { auto &di = result[k]; di.key = k; - di.estimated_quantity.value() = od.accumulated_fills; + di.estimated_quantity.value() = (od.accumulated_fills + od.last_observed_qty); + di.show_quantity = MDP3Quantity(od.show_qty); di.filled = od.filled; // If we have not seen an iceberg order fill completely then // all we can say is that its hidden size is at least one // contract bigger than the observed fills. - if (!od.filled) + if (!od.filled && od.last_observed_qty == 0) di.estimated_quantity.value() += 1; } } @@ -64,14 +74,49 @@ void IcebergFinder::on_real_time_event(MDP3Event const &ev) { // all fills, we also aggregate passive fills, // which we use below to identify icebergs. for (auto s : ev.affected_securities) { - for (auto const &o : s->order_changes()) { + for (auto const& o : s->order_changes()) { me->changes_in_event[o.order.key] = o; - if (o.change == OrderChange::removed) { - for (auto const &f : s->fills()) - if (f.key == o.order.key) { - me->order_data[o.order.key].filled = true; - break; + switch (o.change) { + case OrderChange::ChangeType::added: + me->order_data[o.order.key].show_qty = o.order.quantity.value(); + me->order_data[o.order.key].last_observed_qty = o.order.quantity.value(); + break; + case OrderChange::ChangeType::updated: { + // While this isn't perfect, it should be correct + // for the vast majority of ibs: assume the largest observed + // is quantity is the show quantity. + auto curr_qty = me->order_data[o.order.key].show_qty; + if (o.order.quantity.value() > curr_qty) + me->order_data[o.order.key].show_qty = o.order.quantity.value(); + if (o.order.quantity.value() != o.previous_order.quantity.value()) { + bool has_fills = false; + for (const auto& f : s->fills()) { + if (f.key == o.order.key) { + has_fills = true; + break; + } + } + if (!has_fills) { + // must be an actual qty change by the trader + // these quantities should be excluded from the + // [initial]estimatd quantity, the caller observing + // the MDP3Events can adjust the quantity according + // to those changes as they arrive. + me->order_data[o.order.key].accumulated_qty_changes -= (o.order.quantity.value() - o.previous_order.quantity.value()); + } } + me->order_data[o.order.key].last_observed_qty = o.order.quantity.value(); + break; + } + case OrderChange::ChangeType::removed: + for (auto const& f : s->fills()) + if (f.key == o.order.key) { + me->order_data[o.order.key].filled = true; + me->order_data[o.order.key].last_observed_qty = 0; + break; + } + break; + default: break; } } for (auto const &f : s->fills()) { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h index 0416b21fa..0692d60c4 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h @@ -9,6 +9,7 @@ namespace bts::spartan::mdp3 { struct DeducedIceberg { MDP3OrderKey key; + MDP3Quantity show_quantity; MDP3Quantity estimated_quantity; bool filled; }; @@ -28,4 +29,10 @@ struct IcebergFinder : IChannelObserver { std::unique_ptr me; }; +inline std::ostream& operator <<(std::ostream& out, const DeducedIceberg& ib) { + out << "key " << ib.key << ", show_qty " << ib.show_quantity << ", est_qty " + << ib.estimated_quantity << ", filled " << (ib.filled ? "Y" : "N"); + return out; +} + } // namespace bts::spartan::mdp3