From cbf444d5edae6ebfae1e8db01bafffe92dbbdf7e Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 3 Nov 2020 17:05:32 -0500 Subject: [PATCH] added IcebergStateMapper to determine exact sequence of iceberg state per event id --- src/server/sparkd/cmds/Spartan.cpp | 9 +- .../md/mdp3/back_testing/matcher/Matcher.cpp | 85 ++----------------- .../md/mdp3/back_testing/matcher/Matcher.h | 3 - .../back_testing/matcher/iceberg_helper.cpp | 32 +++++++ .../back_testing/matcher/iceberg_helper.h | 27 ++++++ .../matcher/validation/iceberg_file_cache.cpp | 75 ++++++++-------- .../matcher/validation/iceberg_file_cache.h | 2 +- .../matcher_validation_observer.cpp | 53 +++++++----- .../validation/matcher_validation_observer.h | 6 +- .../cmds/spartan/md/mdp3/utils/order_utils.h | 21 +++++ .../md/mdp3/utils/queries/IcebergFinder.cpp | 82 ++++++++++++++++++ .../md/mdp3/utils/queries/IcebergFinder.h | 22 ++++- .../cmds/spartan/util/file_stream_utils.h | 12 +++ 13 files changed, 281 insertions(+), 148 deletions(-) diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 575ad0a08..48e06faa9 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -35,6 +35,7 @@ #include "server/sparkd/cmds/spartan/util/sql/sql_utils.h" #include #include +#include #include #include #include @@ -47,6 +48,8 @@ namespace cmd { #define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__) using namespace spartan::mdp3; +template +using flat_map = boost::container::flat_map; auto drop_extensions = [](auto const &str) { size_t lastindex = str.find_last_of("."); @@ -334,9 +337,9 @@ std::map const &)>> }}, {"validate_fifo", [](auto const &args) { - return run(args[1], - MatcherValidationObserver( - iceberg_file_cache::find_icebergs(args[1]))); + flat_map ibs; + iceberg_file_cache::find_icebergs(args[1], ibs); + return run(args[1], MatcherValidationObserver(ibs)); }}, {"load_sim_input", [](auto const &args) { return load_sim_input(args[1]); }}, diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp index fbeb1dfa9..1b231d4f9 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp @@ -2,7 +2,6 @@ #include "MatchingCommon.h" #include #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" -#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" #include #include @@ -13,13 +12,12 @@ using namespace google::protobuf::util; namespace bts::spartan::mdp3{ MergedMatcher::MergedMatcher() - : _matcher_map(), icebergs(), + : _matcher_map(), _trace(), _match_count(0), _match_count_this_per(0), _per_start_time(TimeUtil::GetCurrentTime()) { } MergedMatcher::MergedMatcher(MergedMatcher&& src) { _matcher_map = std::move(src._matcher_map); - icebergs = std::move(src.icebergs); } void MergedMatcher::add_def(const InstrumentDefinition& def) { @@ -128,48 +126,7 @@ void MergedMatcher::match(ChangeSrc, // todo: make the fills param optional, as // it will often be more info than we care abt. FillDepot fills; - auto new_ch = c; - switch (c.change) { - case OrderChange::ChangeType::added: - bts_assert(_used_order_keys.emplace(c.order.key).second); - if (new_ch.order.iceberg_quantity.has_value()) - icebergs.insert(std::make_pair(new_ch.order.key, new_ch.order.iceberg_quantity.value())); - break; - case OrderChange::ChangeType::updated: { - bts_assert(_used_order_keys.count(c.order.key)); - auto* old_ord = std::get<1>(ass_matcher.find_order(c.order.security, c.order.key)); - if (old_ord == nullptr) { - // the matcher may not have this order anymore; - // it may have been filled already due to an algo shot. - continue; - } - OrderChange big_ch; - big_ch.change = c.change; - big_ch.previous_order = to_mdp3_order(*old_ord); - big_ch.order = c.order; - if (big_ch.order.iceberg_quantity.has_value()) { - iceberg_helper::detect_show_qty_changes(big_ch, icebergs); - auto it = icebergs.find(big_ch.order.key); - bts_assert(it != icebergs.end()); - big_ch.order.iceberg_quantity = it->second; - } - new_ch.order = big_ch.order; - break; - } - case OrderChange::ChangeType::removed: - bts_assert(_used_order_keys.count(c.order.key)); - break; - } - ass_matcher.on_order_change(new_ch, res, fills); - // the DeducedIceberg's estimated quantity is essentially a fill count - // (obtained by sweeping the entire pcap). Since the order's iceberg - // props (total/show qty) are assigned in this method (anytime the order changes), - // the fills must be incrementally subtracted from the remaining quantity - // in our icebergs cache. That way, the iceberg total qty it will be correct - // the next time the order changes. - std::vector fills_vect; - fills.copy_to(fills_vect); - iceberg_helper::subtract_iceberg_fills(fills_vect.begin(), fills_vect.end(), icebergs); + ass_matcher.on_order_change(c, res, fills); } // log/validate order updates for(const auto& u : *res) @@ -249,41 +206,13 @@ void DirectMatcher::match(ChangeSrc, dirty.reserve(_matcher_map.size()); for (const auto& c : changes) { - auto new_ch = c; - auto it = _matcher_map.find(new_ch.order.security); + auto it = _matcher_map.find(c.order.security); bts_assert (it != _matcher_map.end()); auto& sec_matcher = it->second.matcher; - switch (c.change) { - case OrderChange::ChangeType::added: - if (new_ch.order.iceberg_quantity.has_value()) - icebergs.insert(std::make_pair(new_ch.order.key, new_ch.order.iceberg_quantity.value())); - break; - case OrderChange::ChangeType::updated: { - auto* old_ord = std::get<1>(sec_matcher.find_order(c.order.key)); - if (old_ord == nullptr) { - // the matcher may not have this order anymore; - // it may have been filled already due to an algo shot. - continue; - } - OrderChange big_ch; - big_ch.change = c.change; - big_ch.previous_order = to_mdp3_order(*old_ord); - big_ch.order = c.order; - if (big_ch.order.iceberg_quantity.has_value()) { - iceberg_helper::detect_show_qty_changes(big_ch, icebergs); - auto it = icebergs.find(big_ch.order.key); - bts_assert(it != icebergs.end()); - big_ch.order.iceberg_quantity = it->second; - } - new_ch.order = big_ch.order; - break; - } - default: break; - } - _trace.log_incoming_change(new_ch); - MatcherUtils::validate_incoming_order_change(new_ch); - sec_matcher.on_order_change(new_ch, res); - dirty.emplace_back(new_ch.order.security,&it->second); + _trace.log_incoming_change(c); + MatcherUtils::validate_incoming_order_change(c); + sec_matcher.on_order_change(c, res); + dirty.emplace_back(c.order.security, &it->second); } for(const auto& u : *res) _trace.log_outgoing_update(std::get(u)); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h index bad7689a8..ea7e833c9 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.h @@ -6,7 +6,6 @@ #include "FifoSecurityMatcher.h" #include "FifoMatcherContext.h" #include "MatchingCommon.h" -#include #include #include #include @@ -26,7 +25,6 @@ private: std::unordered_set _used_order_keys; std::map _asset_map; std::map _matcher_map; - boost::container::flat_map icebergs; Trace _trace; size_t _match_count; size_t _match_count_this_per; @@ -71,7 +69,6 @@ private: bool dirty = false; }; std::map _matcher_map; - boost::container::flat_map icebergs; Trace _trace; size_t _match_count; size_t _match_count_this_per; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp index ef79c4077..bd3bfdb07 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.cpp @@ -3,6 +3,7 @@ namespace bts::spartan::mdp3::iceberg_helper { template using flat_map = boost::container::flat_map; +using IcebergStates = boost::container::flat_map; IcebergQuantity to_iceberg_qty(const DeducedIceberg& dib) { // initializer lists with diff fields of the same type seem dangerous @@ -39,6 +40,20 @@ void add_iceberg_annotation( add_iceberg_annotation(order, icebergs, overrides); } +size_t add_iceberg_annotation( + size_t event_id, + MDP3Order& order, + const boost::container::flat_map< + MDP3OrderKey, + boost::container::flat_map>& icebergs) { + auto tup = get_iceberg_qty(event_id, order.key, icebergs); + if (tup.has_value()) { + order.iceberg_quantity = std::get<0>(tup.value()); + return std::get<1>(tup.value()); + } + return 0; +} + void detect_show_qty_changes( const MDP3Event& ev, flat_map& icebergs, @@ -133,4 +148,21 @@ void add_iceberg_annotation( } } } + +std::optional> get_iceberg_qty( + size_t event_id, + MDP3OrderKey k, + const flat_map& icebergs) { + // + auto it = icebergs.find(k); + if (it != icebergs.end()) { + bts_assert(it->second.size() > 0); + auto ub = it->second.upper_bound(event_id); + if (ub != it->second.begin()) + ub--; + if (ub->first <= event_id || event_id == 0) + return std::make_tuple(ub->second, ub->first); + } + return {}; +} } // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h index 3bb68d238..e768088de 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h @@ -17,6 +17,8 @@ #include #include #include +#include +#include namespace bts::spartan::mdp3::iceberg_helper { IcebergQuantity to_iceberg_qty(const DeducedIceberg&); @@ -30,6 +32,15 @@ void add_iceberg_annotation( MDP3Order& order, const boost::container::flat_map& icebergs); +// Assigns the order's IcebergQuantity field with the result of the 'get_iceberg_qty' +// method (see the description of the 'get_iceberg_qty' method below for details). +size_t add_iceberg_annotation( + size_t event_id, + MDP3Order& order, + const boost::container::flat_map< + MDP3OrderKey, + boost::container::flat_map>& icebergs); + void detect_show_qty_changes( const MDP3Event& ev, boost::container::flat_map& icebergs, @@ -39,6 +50,22 @@ void detect_show_qty_changes( OrderChange& change, boost::container::flat_map& icebergs); +// Finds the IcebergQuantity for the specified order at or before the specified event. +// If found, the event id associated with the resultant IcebergQuantity's state is +// returned along with the IcebergQuantity itself. Fills that occurred during that event +// will already be subtracted from the iceberg's total. +// +// Note: event_id == 0 is treated as a special case (presumably the caller does not +// have the event id handy, as would be the case if this call was made from an +// IChannelObserver::on_recovery_complete implementation). If event_id == 0, the earliest +// event's IcebergQuantity is returned. +std::optional> get_iceberg_qty( + size_t event_id, + MDP3OrderKey k, + const boost::container::flat_map< + MDP3OrderKey, + boost::container::flat_map>& icebergs); + void subtract_iceberg_fills( const MDP3Event& ev, boost::container::flat_map& icebergs); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp index 06d84c59a..f49f7d474 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.cpp @@ -1,69 +1,66 @@ #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h" #include "lib/logging/logging.h" +#include "lib/orderman/orderman_utils/Formatter.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" #include "server/sparkd/cmds/spartan/util/file_stream_utils.h" #include #include +#include +#include #include +#define _LOG(level, msg) LOG(OrderMan, Info, "{} {}.", _log_header(), msg) +#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) + namespace bts::spartan::mdp3::iceberg_file_cache { // increment this anytime the deduced icebergs change // so old .iceberg files will be regenerated -const ushort VERSION = 1; +const ushort VERSION = 2; template using flat_map = boost::container::flat_map; inline std::string _log_header() { return std::string("IcebergFileCache"); } -inline std::string _iceberg_file_path(const std::string& name) { - return bts::spartan::file::current_working_dir() + - R"(/)" + name + ".icebergs"; +inline std::string _iceberg_file_path(const std::string& pcap) { + auto dir = spartan::file::dir_from_path(pcap); + auto name = bts::spartan::file::file_name_from_path(pcap); + return dir + R"(/)" + name + ".icebergs"; } -static flat_map _find_icebergs(const std::string& pcap) { - LOG(OrderMan, Info, "{} sweeping '{}' for icebergs", _log_header(), pcap); - IcebergFinder find_ice; - observe_channel(pcap, find_ice); - boost::container::flat_map rv; - for (const auto& kv : find_ice.result) - rv.insert(std::make_pair(kv.first, kv.second)); - return rv; -} - -flat_map find_icebergs(const std::string& pcap) { - auto file_name = bts::spartan::file::file_name_from_path(pcap); - auto path = _iceberg_file_path(file_name); - std::vector ibs_vect; - flat_map rv; +void find_icebergs(const std::string& pcap, flat_map& dest) { + std::string path = _iceberg_file_path(pcap); // goes without saying: read_op and write_op must // agree with one another, wrt the sequence of fields - auto read_op = [](std::istream& in, DeducedIceberg& d) -> std::istream& { - d.key.value() = file::read_from_bin(in); - d.show_quantity.value() = file::read_from_bin(in); - d.estimated_quantity.value() = file::read_from_bin(in); - d.filled = file::read_from_bin(in); + auto read_op = [&](std::istream& in) -> std::istream& { + auto ok = file::read_from_bin(in); + auto& p = *dest.emplace(MDP3OrderKey(ok), IcebergStates()).first; + for (auto num_states = file::read_from_bin(in); num_states > 0; num_states--) { + size_t event_id = file::read_from_bin(in); + auto& state = *p.second.emplace(event_id, IcebergQuantity()).first; + state.second.show.value() = file::read_from_bin(in); + state.second.total.value() = file::read_from_bin(in); + } return in; }; - if (!bts::spartan::file::read_from_file(ibs_vect, path, read_op, VERSION)) { - rv = _find_icebergs(pcap); + if (!bts::spartan::file::read_from_file(path, read_op, VERSION)) { + IcebergStateMapper::map_states(pcap, dest); auto write_op = [](std::ostream& out, - const std::pair& p) -> std::ostream& { - auto& d = p.second; - file::write_to_bin(out, d.key.value()); - file::write_to_bin(out, d.show_quantity.value()); - file::write_to_bin(out, d.estimated_quantity.value()); - file::write_to_bin(out, d.filled); + const std::pair& p) -> std::ostream& { + file::write_to_bin(out, p.first.value()); + file::write_to_bin(out, static_cast(p.second.size())); + for (const auto& ibsp : p.second) { + file::write_to_bin(out, ibsp.first); + file::write_to_bin(out, ibsp.second.show.value()); + file::write_to_bin(out, ibsp.second.total.value()); + } return out; }; - bts::spartan::file::write_to_file(rv.begin(), rv.end(), path, write_op, VERSION); + bts::spartan::file::write_to_file(dest.begin(), dest.end(), path, write_op, VERSION); } else { - LOG( - OrderMan, Info, "{} read {} icebergs from file '{}'", _log_header(), - ibs_vect.size(), path); - for (auto ib : ibs_vect) - rv.insert(std::make_pair(ib.key, ib)); + INFO("read ", dest.size(), " icebergs from '", path, "'"); } - return rv; } } // namespace bts::spartan::mdp3::iceberg_file_cache \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h index ab06ca37b..298c8ee8f 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h @@ -10,5 +10,5 @@ namespace bts::spartan::mdp3::iceberg_file_cache { // If the file doesn't exist, the pcap is swept for icebergs, // and the results are written to the file .icebergs, // for faster retrieval on subsequent runs with the specified pcap. -boost::container::flat_map find_icebergs(const std::string& pcap); +void find_icebergs(const std::string& pcap, boost::container::flat_map>& dest); } // namespace bts::spartan::mdp3::iceberg_file_cache \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp index e45cbc5a8..8da4ec683 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.cpp @@ -50,6 +50,7 @@ using namespace bts::spartan::mdp3::iceberg_helper; using TradingStatus = MatcherValidationObserver::TradingStatus; template using flat_map = boost::container::flat_map; +using IcebergStates = flat_map; // consts const size_t NO_EVENT = 0, MIN_OBSERVED_EVENT = 100; @@ -66,15 +67,16 @@ inline bool _is_interesting_sec(const InstrumentSnapshot& s) { } void _sync_orders( + size_t event_id, const InstrumentSnapshot& s, Orders& orders, - const flat_map& icebergs) { + const flat_map& icebergs) { // for (auto ch : s.order_changes()) { switch (ch.change) { case OrderChange::added: case OrderChange::updated: - add_iceberg_annotation(ch.order, icebergs); + add_iceberg_annotation(event_id, ch.order, icebergs); orders[ch.order.key] = ch.order; break; case OrderChange::removed: orders.erase(ch.order.key); break; @@ -181,11 +183,11 @@ struct _SecurityData { struct MatcherValidationObserver::PImpl { PImpl() {} - PImpl(flat_map icebergs, flat_map interesting_event_ranges) + PImpl(flat_map icebergs, flat_map interesting_event_ranges) : icebergs(icebergs), interesting_event_ranges(interesting_event_ranges) {} flat_map security_cache; - flat_map icebergs; + flat_map icebergs; flat_map interesting_event_ranges; flat_map interesting_orders; @@ -225,7 +227,7 @@ struct MatcherValidationObserver::PImpl { data.last_observed_event = event_id; data.previous_orders = s.orders(); for (auto& o : data.previous_orders) - add_iceberg_annotation(o.second, icebergs); + add_iceberg_annotation(event_id, o.second, icebergs); // the first event at the open isn't processed bc // we don't have a clear understanding of how preopen matching works. data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade @@ -262,11 +264,11 @@ struct MatcherValidationObserver::PImpl { // constructors /////////////////////// MatcherValidationObserver::MatcherValidationObserver( - flat_map icebergs) + flat_map icebergs) : MatcherValidationObserver(icebergs, flat_map{ }) {} MatcherValidationObserver::MatcherValidationObserver( - flat_map icebergs, + flat_map icebergs, flat_map interesting_event_ranges) : _me(new MatcherValidationObserver::PImpl(icebergs, interesting_event_ranges)) { } @@ -290,14 +292,14 @@ bool MatcherValidationObserver::_is_ready_to_trade(const InstrumentSnapshot& s, void MatcherValidationObserver::_sync_current_orders(MDP3Event const& ev) { for (auto s : ev.affected_securities) { auto& new_orders = _me->get_security_data(*s, ev.event_id).current_orders; - _sync_orders(*s, new_orders, _me->icebergs); + _sync_orders(ev.event_id, *s, new_orders, _me->icebergs); } } void MatcherValidationObserver::_sync_previous_orders(const MDP3Event& ev) { for (auto s : ev.affected_securities) { auto& old_orders = _me->get_security_data(*s, ev.event_id).previous_orders; - _sync_orders(*s, old_orders, _me->icebergs); + _sync_orders(ev.event_id, *s, old_orders, _me->icebergs); } } @@ -322,7 +324,7 @@ void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap for (auto s : snap) { auto& sec_data = _me->get_security_data(*s, NO_EVENT); sec_data.current_orders.clear(); - _sync_orders(*s, sec_data.current_orders, _me->icebergs); + _sync_orders(NO_EVENT, *s, sec_data.current_orders, _me->icebergs); sec_data.previous_orders = sec_data.current_orders; } } @@ -334,8 +336,6 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { case bts::spartan::EventOrdinal::Waiting: // just need to synchronize the order caches. // the matcher is only validated during 'interesting events.' - subtract_iceberg_fills(ev, _me->icebergs); - detect_show_qty_changes(ev, _me->icebergs, _me->icebergs); _sync_current_orders(ev); _sync_previous_orders(ev); return; @@ -352,8 +352,6 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { if ((flags & _me->ignore_flags) != EventFlags::None) { if (_me->verbosity > Verbosity::Minimal) INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags)); - subtract_iceberg_fills(ev, _me->icebergs); - detect_show_qty_changes(ev, _me->icebergs, _me->icebergs); _sync_current_orders(ev); _sync_previous_orders(ev); _reset_matcher(); @@ -375,13 +373,13 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { std::sort(sorted_secs.begin(), sorted_secs.end(), security_comp::SymbolComp()); // securities must be added to the matcher before the order // changes are processed to ensure the correct implied markets are used. + // Fills are also indexed in this pass, so they can be added to the iceberg qty + std::unordered_map fill_map; for (const auto* s : sorted_secs) { + accumulate_fills(s->fills(), fill_map); _me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs()); _me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching()); } - // capture iceberg show qty changes (these are a special case) - flat_map show_qty_changes; - detect_show_qty_changes(ev, _me->icebergs, show_qty_changes); // initialize matcher with previous state (only applies // to securities whose orders the matcher hasn't seen yet). for (const auto* s : sorted_secs) { @@ -405,16 +403,29 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { if (deduced.size()) { std::vector deduced_sm; for (auto& ch : deduced) { - add_iceberg_annotation(ch.order, _me->icebergs); + auto ib_event_src = add_iceberg_annotation(ev.event_id, ch.order, _me->icebergs); + if (ch.order.iceberg_quantity.has_value()) { + // the fills would have already been subtracted from the ib total qty; + // they need to be added back so the matcher can work the ib itself. + auto fit = fill_map.find(ch.order.key); + if (fit != fill_map.end()) { + // only subtract fills from the total if the iceberg quantity's + // state came from THIS event. This is because the iceberg quantity + // annotated to the order is a snapshot of the iceberg quantity + // AFTER the event was processed (i.e., fills will have already been subtracted). + if (ib_event_src == ev.event_id) + ch.order.iceberg_quantity.value().total += fit->second; + ch.order.quantity -= fit->second; + } + if (ch.change == OrderChange::ChangeType::added) + ch.order.quantity = ch.order.iceberg_quantity.value().show; + } deduced_sm.emplace_back(ch); } _me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills); } } // Verify the matcher's orders are identitical to the exchange's. - // subtract the fills from iceberg orders' total quantities - // to get the correct expected orders for comparison with the matcher's orders. - subtract_iceberg_fills(ev, _me->icebergs); _sync_current_orders(ev); bool orders_match = true; for (const auto* s : sorted_secs) { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h index 34f45ce12..ac55b136b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/matcher_validation_observer.h @@ -13,12 +13,14 @@ namespace bts::spartan::mdp3 { struct MatcherValidationObserver : IChannelObserver { MatcherValidationObserver(const std::string& pcap); MatcherValidationObserver( - boost::container::flat_map icebergs); + boost::container::flat_map> icebergs); // interesting_event_ranges: // - keys indicate the first event in an interesting range // - values indicate the number of events in the range (i.e., the range's length). MatcherValidationObserver( - boost::container::flat_map icebergs, + boost::container::flat_map> icebergs, boost::container::flat_map interesting_event_ranges); ~MatcherValidationObserver(); void on_recovery_complete(InitialSnapshot const& snap) override; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h index 205e00c3b..b079d3ef2 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h @@ -73,4 +73,25 @@ fmt_order(MDP3OrderKey key, api::Side side, md::IntPrice price, MDP3Quantity qty << " @ " << price.price(); return str.str(); } + +template +inline void accumulate_fills(TIter first, TIter last, std::unordered_map& dest) { + while (first != last) { + auto it = dest.find(first->key); + if (it == dest.end()) + dest.emplace(first->key, first->quantity); + else it->second.value() += first->quantity.value(); + first++; + } +} + +inline void accumulate_fills(const std::vector& fills, std::unordered_map& dest) { + accumulate_fills(fills.begin(), fills.end(), dest); +} + +inline std::unordered_map accumulate_fills(const std::vector& fills) { + std::unordered_map rv; + accumulate_fills(fills.begin(), fills.end(), rv); + return rv; +} } // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp index 6eceb26a0..01d307143 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp @@ -1,7 +1,11 @@ #include "IcebergFinder.h" #include "../../common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h" namespace bts::spartan::mdp3 { +template +using flat_map = boost::container::flat_map; IcebergFinder::IcebergFinder() {} IcebergFinder::~IcebergFinder() {} @@ -174,4 +178,82 @@ void IcebergFinder::on_real_time_event(MDP3Event const &ev) { } } +void IcebergStateMapper::on_real_time_event(MDP3Event const& ev) { + std::unordered_map fills; + std::unordered_map updates; + for (const auto* s : ev.affected_securities) { + for (const auto& f : s->fills()) { + auto fit = fills.find(f.key); + if (fit == fills.end()) + fills.emplace(f.key, f.quantity); + else + fit->second += f.quantity; + } + for (const auto& ch : s->order_changes()) { + auto it = _initial_ibs.find(ch.order.key); + if (it == _initial_ibs.end()) + continue; + switch (ch.change) { + case OrderChange::ChangeType::added: + case OrderChange::ChangeType::updated: { + auto fit = fills.find(ch.order.key); + auto ord_states_it = _states.find(ch.order.key); + if (ord_states_it == _states.end()) { + auto ib = iceberg_helper::to_iceberg_qty(it->second); + if (fit != fills.end()) + ib.total -= fit->second; + _states.emplace(ch.order.key, IcebergStates()) + .first->second[ev.event_id] = ib; + break; + } + bts_assert(ord_states_it->second.size() > 0); + auto prev_ib = (--ord_states_it->second.end())->second; + auto& ib = + ord_states_it->second.emplace(ev.event_id, prev_ib).first->second; + if (fit == fills.end()) { + // ONLY set the show quantity to the order's current quantity + // if the order's quantity has actually changed. + // + // Consider this scenario: + // 1) an iceberg order is launched with a show qty of 5. + // 2) the order is partially filled by qty of 2. Now the "visible" + // qty is 3, + // meaning that which is "visible" to the market, but the "show" + // qty is still 5. + // 3) the trader decides to change the order's price, without + // changing its qty. + // Conclusion: if the "show" qty is blindly updated to the order's + // "visible" qty when the trader only wanted to change its price, the + // "show" qty may be incorrectly decreased. In the scenario above, it + // would have been decreased from 5 -> 3. + auto qty_diff = ch.order.quantity - ch.previous_order.quantity; + if (qty_diff.value() != 0) { + ib.total += qty_diff; + ib.show = ch.order.quantity; + } + } else { + ib.total -= fit->second; + } + break; + } + case OrderChange::ChangeType::removed: break; + } + } + } +} + +void IcebergStateMapper::map_states( + const std::string& pcap, flat_map& states) { + LOG(OrderMan, Info, "sweeping '{}' for icebergs (1/2): determine iceberg orders", pcap); + IcebergFinder ib_finder; + observe_channel(pcap, ib_finder); + flat_map ibs; + for (const auto& kv : ib_finder.result) + ibs.insert(std::make_pair(kv.first, kv.second)); + // Sweep the pcap again to capture all other states for each iceberg. + LOG(OrderMan, Info, "found {} iceberg orders in '{}.'", ibs.size(), pcap); + LOG(OrderMan, Info, "sweeping '{}' for icebergs (2/2): capture iceberg state changes.", pcap); + IcebergStateMapper ib_state_mapper(ibs, states); + observe_channel(pcap, ib_state_mapper); +} } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h index 0692d60c4..d7e00d3c0 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h @@ -2,8 +2,10 @@ #include "../../common/mdp3_common.h" #include "../../handler/IChannelObserver.h" +#include namespace bts::spartan::mdp3 { +using IcebergStates = boost::container::flat_map; // Find all the icebegers in the pcap, estimate their size. @@ -29,10 +31,28 @@ struct IcebergFinder : IChannelObserver { std::unique_ptr me; }; +struct IcebergStateMapper : IChannelObserver { + static void map_states( + const std::string& pcap, + boost::container::flat_map& states); + + void init() override {} + void finit() override {} + void on_real_time_event(MDP3Event const& ev) override; + + private: + boost::container::flat_map& _initial_ibs; + boost::container::flat_map& _states; + + IcebergStateMapper( + boost::container::flat_map& initial_ibs, + boost::container::flat_map& states) + : _initial_ibs(initial_ibs), _states(states) {} +}; + inline std::ostream& operator <<(std::ostream& out, const DeducedIceberg& ib) { out << "key " << ib.key << ", show_qty " << ib.show_quantity << ", est_qty " << ib.estimated_quantity << ", filled " << (ib.filled ? "Y" : "N"); return out; } - } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/util/file_stream_utils.h b/src/server/sparkd/cmds/spartan/util/file_stream_utils.h index 6712d7ec4..d8b964ad2 100644 --- a/src/server/sparkd/cmds/spartan/util/file_stream_utils.h +++ b/src/server/sparkd/cmds/spartan/util/file_stream_utils.h @@ -117,6 +117,18 @@ inline bool read_from_file(std::vector& values, const std::string& path, TOpe return false; } +template +inline bool read_from_file(const std::string& path, TOper op, ushort version) { + std::ifstream istr(path, std::ios::binary); + if (istr) { + if (version != read_from_bin(istr)) + return false; + while (op(istr)) {} + return true; + } + return false; +} + template inline void write_to_file(Iterator first, Iterator last, const std::string& path, TOper op) { std::cout << "opening file '" << path << "'" << std::endl;