added IcebergStateMapper to determine exact sequence of iceberg state per event id

This commit is contained in:
John Colvin committed 2020-12-01 13:41:14 -06:00
1 parent 25e52df136
commit cbf444d5ed
13 files changed
+281 -148

No files matched your search

+6 -3
View File
@@ -35,6 +35,7 @@
#include "server/sparkd/cmds/spartan/util/sql/sql_utils.h"
#include <Pcap++/PcapFileDevice.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/container/flat_map.hpp>
#include <fmt/format.h>
#include <fstream>
#include <iostream>
@@ -47,6 +48,8 @@ namespace cmd {
#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__)
using namespace spartan::mdp3;
template<class K, class V>
using flat_map = boost::container::flat_map<K, V>;
auto drop_extensions = [](auto const &str) {
size_t lastindex = str.find_last_of(".");
@@ -334,9 +337,9 @@ std::map<std::string, std::function<int(std::vector<std::string> const &)>>
}},
{"validate_fifo",
[](auto const &args) {
return run(args[1],
MatcherValidationObserver(
iceberg_file_cache::find_icebergs(args[1])));
flat_map<MDP3OrderKey, IcebergStates> ibs;
iceberg_file_cache::find_icebergs(args[1], ibs);
return run(args[1], MatcherValidationObserver(ibs));
}},
{"load_sim_input",
[](auto const &args) { return load_sim_input(args[1]); }},
@@ -2,7 +2,6 @@
#include "MatchingCommon.h"
#include <google/protobuf/util/time_util.h>
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include <boost/container/flat_map.hpp>
#include <functional>
@@ -13,13 +12,12 @@ using namespace google::protobuf::util;
namespace bts::spartan::mdp3{
MergedMatcher::MergedMatcher()
: _matcher_map(), icebergs(),
: _matcher_map(),
_trace(), _match_count(0), _match_count_this_per(0),
_per_start_time(TimeUtil::GetCurrentTime()) { }
MergedMatcher::MergedMatcher(MergedMatcher&& src) {
_matcher_map = std::move(src._matcher_map);
icebergs = std::move(src.icebergs);
}
void MergedMatcher::add_def(const InstrumentDefinition& def) {
@@ -128,48 +126,7 @@ void MergedMatcher::match(ChangeSrc,
// todo: make the fills param optional, as
// it will often be more info than we care abt.
FillDepot fills;
auto new_ch = c;
switch (c.change) {
case OrderChange::ChangeType::added:
bts_assert(_used_order_keys.emplace(c.order.key).second);
if (new_ch.order.iceberg_quantity.has_value())
icebergs.insert(std::make_pair(new_ch.order.key, new_ch.order.iceberg_quantity.value()));
break;
case OrderChange::ChangeType::updated: {
bts_assert(_used_order_keys.count(c.order.key));
auto* old_ord = std::get<1>(ass_matcher.find_order(c.order.security, c.order.key));
if (old_ord == nullptr) {
// the matcher may not have this order anymore;
// it may have been filled already due to an algo shot.
continue;
}
OrderChange big_ch;
big_ch.change = c.change;
big_ch.previous_order = to_mdp3_order(*old_ord);
big_ch.order = c.order;
if (big_ch.order.iceberg_quantity.has_value()) {
iceberg_helper::detect_show_qty_changes(big_ch, icebergs);
auto it = icebergs.find(big_ch.order.key);
bts_assert(it != icebergs.end());
big_ch.order.iceberg_quantity = it->second;
}
new_ch.order = big_ch.order;
break;
}
case OrderChange::ChangeType::removed:
bts_assert(_used_order_keys.count(c.order.key));
break;
}
ass_matcher.on_order_change(new_ch, res, fills);
// the DeducedIceberg's estimated quantity is essentially a fill count
// (obtained by sweeping the entire pcap). Since the order's iceberg
// props (total/show qty) are assigned in this method (anytime the order changes),
// the fills must be incrementally subtracted from the remaining quantity
// in our icebergs cache. That way, the iceberg total qty it will be correct
// the next time the order changes.
std::vector<MDP3Fill> fills_vect;
fills.copy_to(fills_vect);
iceberg_helper::subtract_iceberg_fills(fills_vect.begin(), fills_vect.end(), icebergs);
ass_matcher.on_order_change(c, res, fills);
}
// log/validate order updates
for(const auto& u : *res)
@@ -249,41 +206,13 @@ void DirectMatcher::match(ChangeSrc,
dirty.reserve(_matcher_map.size());
for (const auto& c : changes) {
auto new_ch = c;
auto it = _matcher_map.find(new_ch.order.security);
auto it = _matcher_map.find(c.order.security);
bts_assert (it != _matcher_map.end());
auto& sec_matcher = it->second.matcher;
switch (c.change) {
case OrderChange::ChangeType::added:
if (new_ch.order.iceberg_quantity.has_value())
icebergs.insert(std::make_pair(new_ch.order.key, new_ch.order.iceberg_quantity.value()));
break;
case OrderChange::ChangeType::updated: {
auto* old_ord = std::get<1>(sec_matcher.find_order(c.order.key));
if (old_ord == nullptr) {
// the matcher may not have this order anymore;
// it may have been filled already due to an algo shot.
continue;
}
OrderChange big_ch;
big_ch.change = c.change;
big_ch.previous_order = to_mdp3_order(*old_ord);
big_ch.order = c.order;
if (big_ch.order.iceberg_quantity.has_value()) {
iceberg_helper::detect_show_qty_changes(big_ch, icebergs);
auto it = icebergs.find(big_ch.order.key);
bts_assert(it != icebergs.end());
big_ch.order.iceberg_quantity = it->second;
}
new_ch.order = big_ch.order;
break;
}
default: break;
}
_trace.log_incoming_change(new_ch);
MatcherUtils::validate_incoming_order_change(new_ch);
sec_matcher.on_order_change(new_ch, res);
dirty.emplace_back(new_ch.order.security,&it->second);
_trace.log_incoming_change(c);
MatcherUtils::validate_incoming_order_change(c);
sec_matcher.on_order_change(c, res);
dirty.emplace_back(c.order.security, &it->second);
}
for(const auto& u : *res)
_trace.log_outgoing_update(std::get<OrderUpdate>(u));
@@ -6,7 +6,6 @@
#include "FifoSecurityMatcher.h"
#include "FifoMatcherContext.h"
#include "MatchingCommon.h"
#include <boost/container/flat_map.hpp>
#include <google/protobuf/timestamp.pb.h>
#include <map>
#include <unordered_set>
@@ -26,7 +25,6 @@ private:
std::unordered_set<MDP3OrderKey> _used_order_keys;
std::map<MDP3SecurityID, std::string> _asset_map;
std::map<std::string, CheckedMatcher> _matcher_map;
boost::container::flat_map<MDP3OrderKey, IcebergQuantity> icebergs;
Trace _trace;
size_t _match_count;
size_t _match_count_this_per;
@@ -71,7 +69,6 @@ private:
bool dirty = false;
};
std::map<MDP3SecurityID,CheckedMatcher> _matcher_map;
boost::container::flat_map<MDP3OrderKey, IcebergQuantity> icebergs;
Trace _trace;
size_t _match_count;
size_t _match_count_this_per;
@@ -3,6 +3,7 @@
namespace bts::spartan::mdp3::iceberg_helper {
template <typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
using IcebergStates = boost::container::flat_map<size_t, IcebergQuantity>;
IcebergQuantity to_iceberg_qty(const DeducedIceberg& dib) {
// initializer lists with diff fields of the same type seem dangerous
@@ -39,6 +40,20 @@ void add_iceberg_annotation(
add_iceberg_annotation(order, icebergs, overrides);
}
size_t add_iceberg_annotation(
size_t event_id,
MDP3Order& order,
const boost::container::flat_map<
MDP3OrderKey,
boost::container::flat_map<size_t, IcebergQuantity>>& icebergs) {
auto tup = get_iceberg_qty(event_id, order.key, icebergs);
if (tup.has_value()) {
order.iceberg_quantity = std::get<0>(tup.value());
return std::get<1>(tup.value());
}
return 0;
}
void detect_show_qty_changes(
const MDP3Event& ev,
flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
@@ -133,4 +148,21 @@ void add_iceberg_annotation(
}
}
}
std::optional<std::tuple<IcebergQuantity, size_t>> get_iceberg_qty(
size_t event_id,
MDP3OrderKey k,
const flat_map<MDP3OrderKey, IcebergStates>& icebergs) {
//
auto it = icebergs.find(k);
if (it != icebergs.end()) {
bts_assert(it->second.size() > 0);
auto ub = it->second.upper_bound(event_id);
if (ub != it->second.begin())
ub--;
if (ub->first <= event_id || event_id == 0)
return std::make_tuple(ub->second, ub->first);
}
return {};
}
} // namespace bts::spartan::mdp3
@@ -17,6 +17,8 @@
#include <boost/container/flat_map.hpp>
#include <functional>
#include <iostream>
#include <optional>
#include <tuple>
namespace bts::spartan::mdp3::iceberg_helper {
IcebergQuantity to_iceberg_qty(const DeducedIceberg&);
@@ -30,6 +32,15 @@ void add_iceberg_annotation(
MDP3Order& order,
const boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
// Assigns the order's IcebergQuantity field with the result of the 'get_iceberg_qty'
// method (see the description of the 'get_iceberg_qty' method below for details).
size_t add_iceberg_annotation(
size_t event_id,
MDP3Order& order,
const boost::container::flat_map<
MDP3OrderKey,
boost::container::flat_map<size_t, IcebergQuantity>>& icebergs);
void detect_show_qty_changes(
const MDP3Event& ev,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
@@ -39,6 +50,22 @@ void detect_show_qty_changes(
OrderChange& change,
boost::container::flat_map<MDP3OrderKey, IcebergQuantity>& icebergs);
// Finds the IcebergQuantity for the specified order at or before the specified event.
// If found, the event id associated with the resultant IcebergQuantity's state is
// returned along with the IcebergQuantity itself. Fills that occurred during that event
// will already be subtracted from the iceberg's total.
//
// Note: event_id == 0 is treated as a special case (presumably the caller does not
// have the event id handy, as would be the case if this call was made from an
// IChannelObserver::on_recovery_complete implementation). If event_id == 0, the earliest
// event's IcebergQuantity is returned.
std::optional<std::tuple<IcebergQuantity, size_t>> get_iceberg_qty(
size_t event_id,
MDP3OrderKey k,
const boost::container::flat_map<
MDP3OrderKey,
boost::container::flat_map<size_t, IcebergQuantity>>& icebergs);
void subtract_iceberg_fills(
const MDP3Event& ev,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
@@ -1,69 +1,66 @@
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
#include "lib/logging/logging.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <functional>
#include <iostream>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#include <vector>
#define _LOG(level, msg) LOG(OrderMan, Info, "{} {}.", _log_header(), msg)
#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
namespace bts::spartan::mdp3::iceberg_file_cache {
// increment this anytime the deduced icebergs change
// so old .iceberg files will be regenerated
const ushort VERSION = 1;
const ushort VERSION = 2;
template <typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
inline std::string _log_header() { return std::string("IcebergFileCache"); }
inline std::string _iceberg_file_path(const std::string& name) {
return bts::spartan::file::current_working_dir() +
R"(/)" + name + ".icebergs";
inline std::string _iceberg_file_path(const std::string& pcap) {
auto dir = spartan::file::dir_from_path(pcap);
auto name = bts::spartan::file::file_name_from_path(pcap);
return dir + R"(/)" + name + ".icebergs";
}
static flat_map<MDP3OrderKey, DeducedIceberg> _find_icebergs(const std::string& pcap) {
LOG(OrderMan, Info, "{} sweeping '{}' for icebergs", _log_header(), pcap);
IcebergFinder find_ice;
observe_channel(pcap, find_ice);
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> rv;
for (const auto& kv : find_ice.result)
rv.insert(std::make_pair(kv.first, kv.second));
return rv;
}
flat_map<MDP3OrderKey, DeducedIceberg> find_icebergs(const std::string& pcap) {
auto file_name = bts::spartan::file::file_name_from_path(pcap);
auto path = _iceberg_file_path(file_name);
std::vector<DeducedIceberg> ibs_vect;
flat_map<MDP3OrderKey, DeducedIceberg> rv;
void find_icebergs(const std::string& pcap, flat_map<MDP3OrderKey, IcebergStates>& dest) {
std::string path = _iceberg_file_path(pcap);
// goes without saying: read_op and write_op must
// agree with one another, wrt the sequence of fields
auto read_op = [](std::istream& in, DeducedIceberg& d) -> std::istream& {
d.key.value() = file::read_from_bin<uint64_t>(in);
d.show_quantity.value() = file::read_from_bin<int>(in);
d.estimated_quantity.value() = file::read_from_bin<int>(in);
d.filled = file::read_from_bin<bool>(in);
auto read_op = [&](std::istream& in) -> std::istream& {
auto ok = file::read_from_bin<uint64_t>(in);
auto& p = *dest.emplace(MDP3OrderKey(ok), IcebergStates()).first;
for (auto num_states = file::read_from_bin<ushort>(in); num_states > 0; num_states--) {
size_t event_id = file::read_from_bin<size_t>(in);
auto& state = *p.second.emplace(event_id, IcebergQuantity()).first;
state.second.show.value() = file::read_from_bin<int>(in);
state.second.total.value() = file::read_from_bin<int>(in);
}
return in;
};
if (!bts::spartan::file::read_from_file(ibs_vect, path, read_op, VERSION)) {
rv = _find_icebergs(pcap);
if (!bts::spartan::file::read_from_file(path, read_op, VERSION)) {
IcebergStateMapper::map_states(pcap, dest);
auto write_op =
[](std::ostream& out,
const std::pair<MDP3OrderKey, DeducedIceberg>& p) -> std::ostream& {
auto& d = p.second;
file::write_to_bin(out, d.key.value());
file::write_to_bin(out, d.show_quantity.value());
file::write_to_bin(out, d.estimated_quantity.value());
file::write_to_bin(out, d.filled);
const std::pair<MDP3OrderKey, IcebergStates>& p) -> std::ostream& {
file::write_to_bin(out, p.first.value());
file::write_to_bin(out, static_cast<ushort>(p.second.size()));
for (const auto& ibsp : p.second) {
file::write_to_bin(out, ibsp.first);
file::write_to_bin(out, ibsp.second.show.value());
file::write_to_bin(out, ibsp.second.total.value());
}
return out;
};
bts::spartan::file::write_to_file(rv.begin(), rv.end(), path, write_op, VERSION);
bts::spartan::file::write_to_file(dest.begin(), dest.end(), path, write_op, VERSION);
} else {
LOG(
OrderMan, Info, "{} read {} icebergs from file '{}'", _log_header(),
ibs_vect.size(), path);
for (auto ib : ibs_vect)
rv.insert(std::make_pair(ib.key, ib));
INFO("read ", dest.size(), " icebergs from '", path, "'");
}
return rv;
}
} // namespace bts::spartan::mdp3::iceberg_file_cache
@@ -10,5 +10,5 @@ namespace bts::spartan::mdp3::iceberg_file_cache {
// If the file doesn't exist, the pcap is swept for icebergs,
// and the results are written to the file <pcap>.icebergs,
// for faster retrieval on subsequent runs with the specified pcap.
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> find_icebergs(const std::string& pcap);
void find_icebergs(const std::string& pcap, boost::container::flat_map<MDP3OrderKey, boost::container::flat_map<size_t, IcebergQuantity>>& dest);
} // namespace bts::spartan::mdp3::iceberg_file_cache
@@ -50,6 +50,7 @@ using namespace bts::spartan::mdp3::iceberg_helper;
using TradingStatus = MatcherValidationObserver::TradingStatus;
template<typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
using IcebergStates = flat_map<size_t, IcebergQuantity>;
// consts
const size_t NO_EVENT = 0, MIN_OBSERVED_EVENT = 100;
@@ -66,15 +67,16 @@ inline bool _is_interesting_sec(const InstrumentSnapshot& s) {
}
void _sync_orders(
size_t event_id,
const InstrumentSnapshot& s,
Orders& orders,
const flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
const flat_map<MDP3OrderKey, IcebergStates>& icebergs) {
//
for (auto ch : s.order_changes()) {
switch (ch.change) {
case OrderChange::added:
case OrderChange::updated:
add_iceberg_annotation(ch.order, icebergs);
add_iceberg_annotation(event_id, ch.order, icebergs);
orders[ch.order.key] = ch.order;
break;
case OrderChange::removed: orders.erase(ch.order.key); break;
@@ -181,11 +183,11 @@ struct _SecurityData {
struct MatcherValidationObserver::PImpl {
PImpl() {}
PImpl(flat_map<MDP3OrderKey, DeducedIceberg> icebergs, flat_map<size_t, size_t> interesting_event_ranges)
PImpl(flat_map<MDP3OrderKey, IcebergStates> icebergs, flat_map<size_t, size_t> interesting_event_ranges)
: icebergs(icebergs), interesting_event_ranges(interesting_event_ranges) {}
flat_map<MDP3SecurityID, _SecurityData> security_cache;
flat_map<MDP3OrderKey, DeducedIceberg> icebergs;
flat_map<MDP3OrderKey, IcebergStates> icebergs;
flat_map<size_t, size_t> interesting_event_ranges;
flat_map<MDP3OrderKey, MDP3Order> interesting_orders;
@@ -225,7 +227,7 @@ struct MatcherValidationObserver::PImpl {
data.last_observed_event = event_id;
data.previous_orders = s.orders();
for (auto& o : data.previous_orders)
add_iceberg_annotation(o.second, icebergs);
add_iceberg_annotation(event_id, o.second, icebergs);
// the first event at the open isn't processed bc
// we don't have a clear understanding of how preopen matching works.
data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade
@@ -262,11 +264,11 @@ struct MatcherValidationObserver::PImpl {
// constructors
///////////////////////
MatcherValidationObserver::MatcherValidationObserver(
flat_map<MDP3OrderKey, DeducedIceberg> icebergs)
flat_map<MDP3OrderKey, IcebergStates> icebergs)
: MatcherValidationObserver(icebergs, flat_map<size_t, size_t>{ }) {}
MatcherValidationObserver::MatcherValidationObserver(
flat_map<MDP3OrderKey, DeducedIceberg> icebergs,
flat_map<MDP3OrderKey, IcebergStates> icebergs,
flat_map<size_t, size_t> interesting_event_ranges)
: _me(new MatcherValidationObserver::PImpl(icebergs, interesting_event_ranges)) { }
@@ -290,14 +292,14 @@ bool MatcherValidationObserver::_is_ready_to_trade(const InstrumentSnapshot& s,
void MatcherValidationObserver::_sync_current_orders(MDP3Event const& ev) {
for (auto s : ev.affected_securities) {
auto& new_orders = _me->get_security_data(*s, ev.event_id).current_orders;
_sync_orders(*s, new_orders, _me->icebergs);
_sync_orders(ev.event_id, *s, new_orders, _me->icebergs);
}
}
void MatcherValidationObserver::_sync_previous_orders(const MDP3Event& ev) {
for (auto s : ev.affected_securities) {
auto& old_orders = _me->get_security_data(*s, ev.event_id).previous_orders;
_sync_orders(*s, old_orders, _me->icebergs);
_sync_orders(ev.event_id, *s, old_orders, _me->icebergs);
}
}
@@ -322,7 +324,7 @@ void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap
for (auto s : snap) {
auto& sec_data = _me->get_security_data(*s, NO_EVENT);
sec_data.current_orders.clear();
_sync_orders(*s, sec_data.current_orders, _me->icebergs);
_sync_orders(NO_EVENT, *s, sec_data.current_orders, _me->icebergs);
sec_data.previous_orders = sec_data.current_orders;
}
}
@@ -334,8 +336,6 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
case bts::spartan::EventOrdinal::Waiting:
// just need to synchronize the order caches.
// the matcher is only validated during 'interesting events.'
subtract_iceberg_fills(ev, _me->icebergs);
detect_show_qty_changes(ev, _me->icebergs, _me->icebergs);
_sync_current_orders(ev);
_sync_previous_orders(ev);
return;
@@ -352,8 +352,6 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
if ((flags & _me->ignore_flags) != EventFlags::None) {
if (_me->verbosity > Verbosity::Minimal)
INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags));
subtract_iceberg_fills(ev, _me->icebergs);
detect_show_qty_changes(ev, _me->icebergs, _me->icebergs);
_sync_current_orders(ev);
_sync_previous_orders(ev);
_reset_matcher();
@@ -375,13 +373,13 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
std::sort(sorted_secs.begin(), sorted_secs.end(), security_comp::SymbolComp());
// securities must be added to the matcher before the order
// changes are processed to ensure the correct implied markets are used.
// Fills are also indexed in this pass, so they can be added to the iceberg qty
std::unordered_map<MDP3OrderKey, MDP3Quantity> fill_map;
for (const auto* s : sorted_secs) {
accumulate_fills(s->fills(), fill_map);
_me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs());
_me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching());
}
// capture iceberg show qty changes (these are a special case)
flat_map<MDP3OrderKey, DeducedIceberg> show_qty_changes;
detect_show_qty_changes(ev, _me->icebergs, show_qty_changes);
// initialize matcher with previous state (only applies
// to securities whose orders the matcher hasn't seen yet).
for (const auto* s : sorted_secs) {
@@ -405,16 +403,29 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
if (deduced.size()) {
std::vector<SmallOrderChange> deduced_sm;
for (auto& ch : deduced) {
add_iceberg_annotation(ch.order, _me->icebergs);
auto ib_event_src = add_iceberg_annotation(ev.event_id, ch.order, _me->icebergs);
if (ch.order.iceberg_quantity.has_value()) {
// the fills would have already been subtracted from the ib total qty;
// they need to be added back so the matcher can work the ib itself.
auto fit = fill_map.find(ch.order.key);
if (fit != fill_map.end()) {
// only subtract fills from the total if the iceberg quantity's
// state came from THIS event. This is because the iceberg quantity
// annotated to the order is a snapshot of the iceberg quantity
// AFTER the event was processed (i.e., fills will have already been subtracted).
if (ib_event_src == ev.event_id)
ch.order.iceberg_quantity.value().total += fit->second;
ch.order.quantity -= fit->second;
}
if (ch.change == OrderChange::ChangeType::added)
ch.order.quantity = ch.order.iceberg_quantity.value().show;
}
deduced_sm.emplace_back(ch);
}
_me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills);
}
}
// Verify the matcher's orders are identitical to the exchange's.
// subtract the fills from iceberg orders' total quantities
// to get the correct expected orders for comparison with the matcher's orders.
subtract_iceberg_fills(ev, _me->icebergs);
_sync_current_orders(ev);
bool orders_match = true;
for (const auto* s : sorted_secs) {
@@ -13,12 +13,14 @@ namespace bts::spartan::mdp3 {
struct MatcherValidationObserver : IChannelObserver {
MatcherValidationObserver(const std::string& pcap);
MatcherValidationObserver(
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> icebergs);
boost::container::flat_map<MDP3OrderKey,
boost::container::flat_map<size_t, IcebergQuantity>> icebergs);
// interesting_event_ranges:
// - keys indicate the first event in an interesting range
// - values indicate the number of events in the range (i.e., the range's length).
MatcherValidationObserver(
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> icebergs,
boost::container::flat_map<MDP3OrderKey,
boost::container::flat_map<size_t, IcebergQuantity>> icebergs,
boost::container::flat_map<size_t, size_t> interesting_event_ranges);
~MatcherValidationObserver();
void on_recovery_complete(InitialSnapshot const& snap) override;
@@ -73,4 +73,25 @@ fmt_order(MDP3OrderKey key, api::Side side, md::IntPrice price, MDP3Quantity qty
<< " @ " << price.price();
return str.str();
}
template<class TIter>
inline void accumulate_fills(TIter first, TIter last, std::unordered_map<MDP3OrderKey, MDP3Quantity>& dest) {
while (first != last) {
auto it = dest.find(first->key);
if (it == dest.end())
dest.emplace(first->key, first->quantity);
else it->second.value() += first->quantity.value();
first++;
}
}
inline void accumulate_fills(const std::vector<MDP3Fill>& fills, std::unordered_map<MDP3OrderKey, MDP3Quantity>& dest) {
accumulate_fills(fills.begin(), fills.end(), dest);
}
inline std::unordered_map<MDP3OrderKey, MDP3Quantity> accumulate_fills(const std::vector<MDP3Fill>& fills) {
std::unordered_map<MDP3OrderKey, MDP3Quantity> rv;
accumulate_fills(fills.begin(), fills.end(), rv);
return rv;
}
} // namespace bts::spartan::mdp3
@@ -1,7 +1,11 @@
#include "IcebergFinder.h"
#include "../../common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
namespace bts::spartan::mdp3 {
template<class K, class V>
using flat_map = boost::container::flat_map<K, V>;
IcebergFinder::IcebergFinder() {}
IcebergFinder::~IcebergFinder() {}
@@ -174,4 +178,82 @@ void IcebergFinder::on_real_time_event(MDP3Event const &ev) {
}
}
void IcebergStateMapper::on_real_time_event(MDP3Event const& ev) {
std::unordered_map<MDP3OrderKey, MDP3Quantity> fills;
std::unordered_map<MDP3OrderKey, MDP3Order> updates;
for (const auto* s : ev.affected_securities) {
for (const auto& f : s->fills()) {
auto fit = fills.find(f.key);
if (fit == fills.end())
fills.emplace(f.key, f.quantity);
else
fit->second += f.quantity;
}
for (const auto& ch : s->order_changes()) {
auto it = _initial_ibs.find(ch.order.key);
if (it == _initial_ibs.end())
continue;
switch (ch.change) {
case OrderChange::ChangeType::added:
case OrderChange::ChangeType::updated: {
auto fit = fills.find(ch.order.key);
auto ord_states_it = _states.find(ch.order.key);
if (ord_states_it == _states.end()) {
auto ib = iceberg_helper::to_iceberg_qty(it->second);
if (fit != fills.end())
ib.total -= fit->second;
_states.emplace(ch.order.key, IcebergStates())
.first->second[ev.event_id] = ib;
break;
}
bts_assert(ord_states_it->second.size() > 0);
auto prev_ib = (--ord_states_it->second.end())->second;
auto& ib =
ord_states_it->second.emplace(ev.event_id, prev_ib).first->second;
if (fit == fills.end()) {
// ONLY set the show quantity to the order's current quantity
// if the order's quantity has actually changed.
//
// Consider this scenario:
// 1) an iceberg order is launched with a show qty of 5.
// 2) the order is partially filled by qty of 2. Now the "visible"
// qty is 3,
// meaning that which is "visible" to the market, but the "show"
// qty is still 5.
// 3) the trader decides to change the order's price, without
// changing its qty.
// Conclusion: if the "show" qty is blindly updated to the order's
// "visible" qty when the trader only wanted to change its price, the
// "show" qty may be incorrectly decreased. In the scenario above, it
// would have been decreased from 5 -> 3.
auto qty_diff = ch.order.quantity - ch.previous_order.quantity;
if (qty_diff.value() != 0) {
ib.total += qty_diff;
ib.show = ch.order.quantity;
}
} else {
ib.total -= fit->second;
}
break;
}
case OrderChange::ChangeType::removed: break;
}
}
}
}
void IcebergStateMapper::map_states(
const std::string& pcap, flat_map<MDP3OrderKey, IcebergStates>& states) {
LOG(OrderMan, Info, "sweeping '{}' for icebergs (1/2): determine iceberg orders", pcap);
IcebergFinder ib_finder;
observe_channel(pcap, ib_finder);
flat_map<MDP3OrderKey, DeducedIceberg> ibs;
for (const auto& kv : ib_finder.result)
ibs.insert(std::make_pair(kv.first, kv.second));
// Sweep the pcap again to capture all other states for each iceberg.
LOG(OrderMan, Info, "found {} iceberg orders in '{}.'", ibs.size(), pcap);
LOG(OrderMan, Info, "sweeping '{}' for icebergs (2/2): capture iceberg state changes.", pcap);
IcebergStateMapper ib_state_mapper(ibs, states);
observe_channel(pcap, ib_state_mapper);
}
} // namespace bts::spartan::mdp3
@@ -2,8 +2,10 @@
#include "../../common/mdp3_common.h"
#include "../../handler/IChannelObserver.h"
#include <boost/container/flat_map.hpp>
namespace bts::spartan::mdp3 {
using IcebergStates = boost::container::flat_map<size_t, IcebergQuantity>;
// Find all the icebegers in the pcap, estimate their size.
@@ -29,10 +31,28 @@ struct IcebergFinder : IChannelObserver {
std::unique_ptr<PImpl> me;
};
struct IcebergStateMapper : IChannelObserver {
static void map_states(
const std::string& pcap,
boost::container::flat_map<MDP3OrderKey, IcebergStates>& states);
void init() override {}
void finit() override {}
void on_real_time_event(MDP3Event const& ev) override;
private:
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& _initial_ibs;
boost::container::flat_map<MDP3OrderKey, IcebergStates>& _states;
IcebergStateMapper(
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& initial_ibs,
boost::container::flat_map<MDP3OrderKey, IcebergStates>& states)
: _initial_ibs(initial_ibs), _states(states) {}
};
inline std::ostream& operator <<(std::ostream& out, const DeducedIceberg& ib) {
out << "key " << ib.key << ", show_qty " << ib.show_quantity << ", est_qty "
<< ib.estimated_quantity << ", filled " << (ib.filled ? "Y" : "N");
return out;
}
} // namespace bts::spartan::mdp3
@@ -117,6 +117,18 @@ inline bool read_from_file(std::vector<T>& values, const std::string& path, TOpe
return false;
}
template<class TOper>
inline bool read_from_file(const std::string& path, TOper op, ushort version) {
std::ifstream istr(path, std::ios::binary);
if (istr) {
if (version != read_from_bin<ushort>(istr))
return false;
while (op(istr)) {}
return true;
}
return false;
}
template<class Iterator, class TOper>
inline void write_to_file(Iterator first, Iterator last, const std::string& path, TOper op) {
std::cout << "opening file '" << path << "'" << std::endl;