added IcebergStateMapper to determine exact sequence of iceberg state per event id
This commit is contained in:
1 parent
25e52df136
commit
cbf444d5ed
13 files changed
+281
-148
No files matched your search
@@ -35,6 +35,7 @@
|
||||
#include "server/sparkd/cmds/spartan/util/sql/sql_utils.h"
|
||||
#include <Pcap++/PcapFileDevice.h>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <fmt/format.h>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
@@ -47,6 +48,8 @@ namespace cmd {
|
||||
#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__)
|
||||
|
||||
using namespace spartan::mdp3;
|
||||
template<class K, class V>
|
||||
using flat_map = boost::container::flat_map<K, V>;
|
||||
|
||||
auto drop_extensions = [](auto const &str) {
|
||||
size_t lastindex = str.find_last_of(".");
|
||||
@@ -334,9 +337,9 @@ std::map<std::string, std::function<int(std::vector<std::string> const &)>>
|
||||
}},
|
||||
{"validate_fifo",
|
||||
[](auto const &args) {
|
||||
return run(args[1],
|
||||
MatcherValidationObserver(
|
||||
iceberg_file_cache::find_icebergs(args[1])));
|
||||
flat_map<MDP3OrderKey, IcebergStates> ibs;
|
||||
iceberg_file_cache::find_icebergs(args[1], ibs);
|
||||
return run(args[1], MatcherValidationObserver(ibs));
|
||||
}},
|
||||
{"load_sim_input",
|
||||
[](auto const &args) { return load_sim_input(args[1]); }},
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
#include "MatchingCommon.h"
|
||||
#include <google/protobuf/util/time_util.h>
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <functional>
|
||||
@@ -13,13 +12,12 @@ using namespace google::protobuf::util;
|
||||
namespace bts::spartan::mdp3{
|
||||
|
||||
MergedMatcher::MergedMatcher()
|
||||
: _matcher_map(), icebergs(),
|
||||
: _matcher_map(),
|
||||
_trace(), _match_count(0), _match_count_this_per(0),
|
||||
_per_start_time(TimeUtil::GetCurrentTime()) { }
|
||||
|
||||
MergedMatcher::MergedMatcher(MergedMatcher&& src) {
|
||||
_matcher_map = std::move(src._matcher_map);
|
||||
icebergs = std::move(src.icebergs);
|
||||
}
|
||||
|
||||
void MergedMatcher::add_def(const InstrumentDefinition& def) {
|
||||
@@ -128,48 +126,7 @@ void MergedMatcher::match(ChangeSrc,
|
||||
// todo: make the fills param optional, as
|
||||
// it will often be more info than we care abt.
|
||||
FillDepot fills;
|
||||
auto new_ch = c;
|
||||
switch (c.change) {
|
||||
case OrderChange::ChangeType::added:
|
||||
bts_assert(_used_order_keys.emplace(c.order.key).second);
|
||||
if (new_ch.order.iceberg_quantity.has_value())
|
||||
icebergs.insert(std::make_pair(new_ch.order.key, new_ch.order.iceberg_quantity.value()));
|
||||
break;
|
||||
case OrderChange::ChangeType::updated: {
|
||||
bts_assert(_used_order_keys.count(c.order.key));
|
||||
auto* old_ord = std::get<1>(ass_matcher.find_order(c.order.security, c.order.key));
|
||||
if (old_ord == nullptr) {
|
||||
// the matcher may not have this order anymore;
|
||||
// it may have been filled already due to an algo shot.
|
||||
continue;
|
||||
}
|
||||
OrderChange big_ch;
|
||||
big_ch.change = c.change;
|
||||
big_ch.previous_order = to_mdp3_order(*old_ord);
|
||||
big_ch.order = c.order;
|
||||
if (big_ch.order.iceberg_quantity.has_value()) {
|
||||
iceberg_helper::detect_show_qty_changes(big_ch, icebergs);
|
||||
auto it = icebergs.find(big_ch.order.key);
|
||||
bts_assert(it != icebergs.end());
|
||||
big_ch.order.iceberg_quantity = it->second;
|
||||
}
|
||||
new_ch.order = big_ch.order;
|
||||
break;
|
||||
}
|
||||
case OrderChange::ChangeType::removed:
|
||||
bts_assert(_used_order_keys.count(c.order.key));
|
||||
break;
|
||||
}
|
||||
ass_matcher.on_order_change(new_ch, res, fills);
|
||||
// the DeducedIceberg's estimated quantity is essentially a fill count
|
||||
// (obtained by sweeping the entire pcap). Since the order's iceberg
|
||||
// props (total/show qty) are assigned in this method (anytime the order changes),
|
||||
// the fills must be incrementally subtracted from the remaining quantity
|
||||
// in our icebergs cache. That way, the iceberg total qty it will be correct
|
||||
// the next time the order changes.
|
||||
std::vector<MDP3Fill> fills_vect;
|
||||
fills.copy_to(fills_vect);
|
||||
iceberg_helper::subtract_iceberg_fills(fills_vect.begin(), fills_vect.end(), icebergs);
|
||||
ass_matcher.on_order_change(c, res, fills);
|
||||
}
|
||||
// log/validate order updates
|
||||
for(const auto& u : *res)
|
||||
@@ -249,41 +206,13 @@ void DirectMatcher::match(ChangeSrc,
|
||||
dirty.reserve(_matcher_map.size());
|
||||
|
||||
for (const auto& c : changes) {
|
||||
auto new_ch = c;
|
||||
auto it = _matcher_map.find(new_ch.order.security);
|
||||
auto it = _matcher_map.find(c.order.security);
|
||||
bts_assert (it != _matcher_map.end());
|
||||
auto& sec_matcher = it->second.matcher;
|
||||
switch (c.change) {
|
||||
case OrderChange::ChangeType::added:
|
||||
if (new_ch.order.iceberg_quantity.has_value())
|
||||
icebergs.insert(std::make_pair(new_ch.order.key, new_ch.order.iceberg_quantity.value()));
|
||||
break;
|
||||
case OrderChange::ChangeType::updated: {
|
||||
auto* old_ord = std::get<1>(sec_matcher.find_order(c.order.key));
|
||||
if (old_ord == nullptr) {
|
||||
// the matcher may not have this order anymore;
|
||||
// it may have been filled already due to an algo shot.
|
||||
continue;
|
||||
}
|
||||
OrderChange big_ch;
|
||||
big_ch.change = c.change;
|
||||
big_ch.previous_order = to_mdp3_order(*old_ord);
|
||||
big_ch.order = c.order;
|
||||
if (big_ch.order.iceberg_quantity.has_value()) {
|
||||
iceberg_helper::detect_show_qty_changes(big_ch, icebergs);
|
||||
auto it = icebergs.find(big_ch.order.key);
|
||||
bts_assert(it != icebergs.end());
|
||||
big_ch.order.iceberg_quantity = it->second;
|
||||
}
|
||||
new_ch.order = big_ch.order;
|
||||
break;
|
||||
}
|
||||
default: break;
|
||||
}
|
||||
_trace.log_incoming_change(new_ch);
|
||||
MatcherUtils::validate_incoming_order_change(new_ch);
|
||||
sec_matcher.on_order_change(new_ch, res);
|
||||
dirty.emplace_back(new_ch.order.security,&it->second);
|
||||
_trace.log_incoming_change(c);
|
||||
MatcherUtils::validate_incoming_order_change(c);
|
||||
sec_matcher.on_order_change(c, res);
|
||||
dirty.emplace_back(c.order.security, &it->second);
|
||||
}
|
||||
for(const auto& u : *res)
|
||||
_trace.log_outgoing_update(std::get<OrderUpdate>(u));
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include "FifoSecurityMatcher.h"
|
||||
#include "FifoMatcherContext.h"
|
||||
#include "MatchingCommon.h"
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <google/protobuf/timestamp.pb.h>
|
||||
#include <map>
|
||||
#include <unordered_set>
|
||||
@@ -26,7 +25,6 @@ private:
|
||||
std::unordered_set<MDP3OrderKey> _used_order_keys;
|
||||
std::map<MDP3SecurityID, std::string> _asset_map;
|
||||
std::map<std::string, CheckedMatcher> _matcher_map;
|
||||
boost::container::flat_map<MDP3OrderKey, IcebergQuantity> icebergs;
|
||||
Trace _trace;
|
||||
size_t _match_count;
|
||||
size_t _match_count_this_per;
|
||||
@@ -71,7 +69,6 @@ private:
|
||||
bool dirty = false;
|
||||
};
|
||||
std::map<MDP3SecurityID,CheckedMatcher> _matcher_map;
|
||||
boost::container::flat_map<MDP3OrderKey, IcebergQuantity> icebergs;
|
||||
Trace _trace;
|
||||
size_t _match_count;
|
||||
size_t _match_count_this_per;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace bts::spartan::mdp3::iceberg_helper {
|
||||
template <typename K, typename V>
|
||||
using flat_map = boost::container::flat_map<K, V>;
|
||||
using IcebergStates = boost::container::flat_map<size_t, IcebergQuantity>;
|
||||
|
||||
IcebergQuantity to_iceberg_qty(const DeducedIceberg& dib) {
|
||||
// initializer lists with diff fields of the same type seem dangerous
|
||||
@@ -39,6 +40,20 @@ void add_iceberg_annotation(
|
||||
add_iceberg_annotation(order, icebergs, overrides);
|
||||
}
|
||||
|
||||
size_t add_iceberg_annotation(
|
||||
size_t event_id,
|
||||
MDP3Order& order,
|
||||
const boost::container::flat_map<
|
||||
MDP3OrderKey,
|
||||
boost::container::flat_map<size_t, IcebergQuantity>>& icebergs) {
|
||||
auto tup = get_iceberg_qty(event_id, order.key, icebergs);
|
||||
if (tup.has_value()) {
|
||||
order.iceberg_quantity = std::get<0>(tup.value());
|
||||
return std::get<1>(tup.value());
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void detect_show_qty_changes(
|
||||
const MDP3Event& ev,
|
||||
flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
|
||||
@@ -133,4 +148,21 @@ void add_iceberg_annotation(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<std::tuple<IcebergQuantity, size_t>> get_iceberg_qty(
|
||||
size_t event_id,
|
||||
MDP3OrderKey k,
|
||||
const flat_map<MDP3OrderKey, IcebergStates>& icebergs) {
|
||||
//
|
||||
auto it = icebergs.find(k);
|
||||
if (it != icebergs.end()) {
|
||||
bts_assert(it->second.size() > 0);
|
||||
auto ub = it->second.upper_bound(event_id);
|
||||
if (ub != it->second.begin())
|
||||
ub--;
|
||||
if (ub->first <= event_id || event_id == 0)
|
||||
return std::make_tuple(ub->second, ub->first);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -17,6 +17,8 @@
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <tuple>
|
||||
|
||||
namespace bts::spartan::mdp3::iceberg_helper {
|
||||
IcebergQuantity to_iceberg_qty(const DeducedIceberg&);
|
||||
@@ -30,6 +32,15 @@ void add_iceberg_annotation(
|
||||
MDP3Order& order,
|
||||
const boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
|
||||
|
||||
// Assigns the order's IcebergQuantity field with the result of the 'get_iceberg_qty'
|
||||
// method (see the description of the 'get_iceberg_qty' method below for details).
|
||||
size_t add_iceberg_annotation(
|
||||
size_t event_id,
|
||||
MDP3Order& order,
|
||||
const boost::container::flat_map<
|
||||
MDP3OrderKey,
|
||||
boost::container::flat_map<size_t, IcebergQuantity>>& icebergs);
|
||||
|
||||
void detect_show_qty_changes(
|
||||
const MDP3Event& ev,
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
|
||||
@@ -39,6 +50,22 @@ void detect_show_qty_changes(
|
||||
OrderChange& change,
|
||||
boost::container::flat_map<MDP3OrderKey, IcebergQuantity>& icebergs);
|
||||
|
||||
// Finds the IcebergQuantity for the specified order at or before the specified event.
|
||||
// If found, the event id associated with the resultant IcebergQuantity's state is
|
||||
// returned along with the IcebergQuantity itself. Fills that occurred during that event
|
||||
// will already be subtracted from the iceberg's total.
|
||||
//
|
||||
// Note: event_id == 0 is treated as a special case (presumably the caller does not
|
||||
// have the event id handy, as would be the case if this call was made from an
|
||||
// IChannelObserver::on_recovery_complete implementation). If event_id == 0, the earliest
|
||||
// event's IcebergQuantity is returned.
|
||||
std::optional<std::tuple<IcebergQuantity, size_t>> get_iceberg_qty(
|
||||
size_t event_id,
|
||||
MDP3OrderKey k,
|
||||
const boost::container::flat_map<
|
||||
MDP3OrderKey,
|
||||
boost::container::flat_map<size_t, IcebergQuantity>>& icebergs);
|
||||
|
||||
void subtract_iceberg_fills(
|
||||
const MDP3Event& ev,
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
|
||||
|
||||
+36
-39
@@ -1,69 +1,66 @@
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/validation/iceberg_file_cache.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/orderman/orderman_utils/Formatter.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
|
||||
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <range/v3/numeric.hpp>
|
||||
#include <range/v3/view.hpp>
|
||||
#include <vector>
|
||||
|
||||
#define _LOG(level, msg) LOG(OrderMan, Info, "{} {}.", _log_header(), msg)
|
||||
#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__))
|
||||
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
|
||||
|
||||
namespace bts::spartan::mdp3::iceberg_file_cache {
|
||||
// increment this anytime the deduced icebergs change
|
||||
// so old .iceberg files will be regenerated
|
||||
const ushort VERSION = 1;
|
||||
const ushort VERSION = 2;
|
||||
template <typename K, typename V>
|
||||
using flat_map = boost::container::flat_map<K, V>;
|
||||
|
||||
inline std::string _log_header() { return std::string("IcebergFileCache"); }
|
||||
|
||||
inline std::string _iceberg_file_path(const std::string& name) {
|
||||
return bts::spartan::file::current_working_dir() +
|
||||
R"(/)" + name + ".icebergs";
|
||||
inline std::string _iceberg_file_path(const std::string& pcap) {
|
||||
auto dir = spartan::file::dir_from_path(pcap);
|
||||
auto name = bts::spartan::file::file_name_from_path(pcap);
|
||||
return dir + R"(/)" + name + ".icebergs";
|
||||
}
|
||||
|
||||
static flat_map<MDP3OrderKey, DeducedIceberg> _find_icebergs(const std::string& pcap) {
|
||||
LOG(OrderMan, Info, "{} sweeping '{}' for icebergs", _log_header(), pcap);
|
||||
IcebergFinder find_ice;
|
||||
observe_channel(pcap, find_ice);
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> rv;
|
||||
for (const auto& kv : find_ice.result)
|
||||
rv.insert(std::make_pair(kv.first, kv.second));
|
||||
return rv;
|
||||
}
|
||||
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> find_icebergs(const std::string& pcap) {
|
||||
auto file_name = bts::spartan::file::file_name_from_path(pcap);
|
||||
auto path = _iceberg_file_path(file_name);
|
||||
std::vector<DeducedIceberg> ibs_vect;
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> rv;
|
||||
void find_icebergs(const std::string& pcap, flat_map<MDP3OrderKey, IcebergStates>& dest) {
|
||||
std::string path = _iceberg_file_path(pcap);
|
||||
// goes without saying: read_op and write_op must
|
||||
// agree with one another, wrt the sequence of fields
|
||||
auto read_op = [](std::istream& in, DeducedIceberg& d) -> std::istream& {
|
||||
d.key.value() = file::read_from_bin<uint64_t>(in);
|
||||
d.show_quantity.value() = file::read_from_bin<int>(in);
|
||||
d.estimated_quantity.value() = file::read_from_bin<int>(in);
|
||||
d.filled = file::read_from_bin<bool>(in);
|
||||
auto read_op = [&](std::istream& in) -> std::istream& {
|
||||
auto ok = file::read_from_bin<uint64_t>(in);
|
||||
auto& p = *dest.emplace(MDP3OrderKey(ok), IcebergStates()).first;
|
||||
for (auto num_states = file::read_from_bin<ushort>(in); num_states > 0; num_states--) {
|
||||
size_t event_id = file::read_from_bin<size_t>(in);
|
||||
auto& state = *p.second.emplace(event_id, IcebergQuantity()).first;
|
||||
state.second.show.value() = file::read_from_bin<int>(in);
|
||||
state.second.total.value() = file::read_from_bin<int>(in);
|
||||
}
|
||||
return in;
|
||||
};
|
||||
if (!bts::spartan::file::read_from_file(ibs_vect, path, read_op, VERSION)) {
|
||||
rv = _find_icebergs(pcap);
|
||||
if (!bts::spartan::file::read_from_file(path, read_op, VERSION)) {
|
||||
IcebergStateMapper::map_states(pcap, dest);
|
||||
auto write_op =
|
||||
[](std::ostream& out,
|
||||
const std::pair<MDP3OrderKey, DeducedIceberg>& p) -> std::ostream& {
|
||||
auto& d = p.second;
|
||||
file::write_to_bin(out, d.key.value());
|
||||
file::write_to_bin(out, d.show_quantity.value());
|
||||
file::write_to_bin(out, d.estimated_quantity.value());
|
||||
file::write_to_bin(out, d.filled);
|
||||
const std::pair<MDP3OrderKey, IcebergStates>& p) -> std::ostream& {
|
||||
file::write_to_bin(out, p.first.value());
|
||||
file::write_to_bin(out, static_cast<ushort>(p.second.size()));
|
||||
for (const auto& ibsp : p.second) {
|
||||
file::write_to_bin(out, ibsp.first);
|
||||
file::write_to_bin(out, ibsp.second.show.value());
|
||||
file::write_to_bin(out, ibsp.second.total.value());
|
||||
}
|
||||
return out;
|
||||
};
|
||||
bts::spartan::file::write_to_file(rv.begin(), rv.end(), path, write_op, VERSION);
|
||||
bts::spartan::file::write_to_file(dest.begin(), dest.end(), path, write_op, VERSION);
|
||||
} else {
|
||||
LOG(
|
||||
OrderMan, Info, "{} read {} icebergs from file '{}'", _log_header(),
|
||||
ibs_vect.size(), path);
|
||||
for (auto ib : ibs_vect)
|
||||
rv.insert(std::make_pair(ib.key, ib));
|
||||
INFO("read ", dest.size(), " icebergs from '", path, "'");
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
} // namespace bts::spartan::mdp3::iceberg_file_cache
|
||||
+1
-1
@@ -10,5 +10,5 @@ namespace bts::spartan::mdp3::iceberg_file_cache {
|
||||
// If the file doesn't exist, the pcap is swept for icebergs,
|
||||
// and the results are written to the file <pcap>.icebergs,
|
||||
// for faster retrieval on subsequent runs with the specified pcap.
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> find_icebergs(const std::string& pcap);
|
||||
void find_icebergs(const std::string& pcap, boost::container::flat_map<MDP3OrderKey, boost::container::flat_map<size_t, IcebergQuantity>>& dest);
|
||||
} // namespace bts::spartan::mdp3::iceberg_file_cache
|
||||
+32
-21
@@ -50,6 +50,7 @@ using namespace bts::spartan::mdp3::iceberg_helper;
|
||||
using TradingStatus = MatcherValidationObserver::TradingStatus;
|
||||
template<typename K, typename V>
|
||||
using flat_map = boost::container::flat_map<K, V>;
|
||||
using IcebergStates = flat_map<size_t, IcebergQuantity>;
|
||||
// consts
|
||||
const size_t NO_EVENT = 0, MIN_OBSERVED_EVENT = 100;
|
||||
|
||||
@@ -66,15 +67,16 @@ inline bool _is_interesting_sec(const InstrumentSnapshot& s) {
|
||||
}
|
||||
|
||||
void _sync_orders(
|
||||
size_t event_id,
|
||||
const InstrumentSnapshot& s,
|
||||
Orders& orders,
|
||||
const flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
|
||||
const flat_map<MDP3OrderKey, IcebergStates>& icebergs) {
|
||||
//
|
||||
for (auto ch : s.order_changes()) {
|
||||
switch (ch.change) {
|
||||
case OrderChange::added:
|
||||
case OrderChange::updated:
|
||||
add_iceberg_annotation(ch.order, icebergs);
|
||||
add_iceberg_annotation(event_id, ch.order, icebergs);
|
||||
orders[ch.order.key] = ch.order;
|
||||
break;
|
||||
case OrderChange::removed: orders.erase(ch.order.key); break;
|
||||
@@ -181,11 +183,11 @@ struct _SecurityData {
|
||||
|
||||
struct MatcherValidationObserver::PImpl {
|
||||
PImpl() {}
|
||||
PImpl(flat_map<MDP3OrderKey, DeducedIceberg> icebergs, flat_map<size_t, size_t> interesting_event_ranges)
|
||||
PImpl(flat_map<MDP3OrderKey, IcebergStates> icebergs, flat_map<size_t, size_t> interesting_event_ranges)
|
||||
: icebergs(icebergs), interesting_event_ranges(interesting_event_ranges) {}
|
||||
|
||||
flat_map<MDP3SecurityID, _SecurityData> security_cache;
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> icebergs;
|
||||
flat_map<MDP3OrderKey, IcebergStates> icebergs;
|
||||
flat_map<size_t, size_t> interesting_event_ranges;
|
||||
flat_map<MDP3OrderKey, MDP3Order> interesting_orders;
|
||||
|
||||
@@ -225,7 +227,7 @@ struct MatcherValidationObserver::PImpl {
|
||||
data.last_observed_event = event_id;
|
||||
data.previous_orders = s.orders();
|
||||
for (auto& o : data.previous_orders)
|
||||
add_iceberg_annotation(o.second, icebergs);
|
||||
add_iceberg_annotation(event_id, o.second, icebergs);
|
||||
// the first event at the open isn't processed bc
|
||||
// we don't have a clear understanding of how preopen matching works.
|
||||
data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade
|
||||
@@ -262,11 +264,11 @@ struct MatcherValidationObserver::PImpl {
|
||||
// constructors
|
||||
///////////////////////
|
||||
MatcherValidationObserver::MatcherValidationObserver(
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> icebergs)
|
||||
flat_map<MDP3OrderKey, IcebergStates> icebergs)
|
||||
: MatcherValidationObserver(icebergs, flat_map<size_t, size_t>{ }) {}
|
||||
|
||||
MatcherValidationObserver::MatcherValidationObserver(
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> icebergs,
|
||||
flat_map<MDP3OrderKey, IcebergStates> icebergs,
|
||||
flat_map<size_t, size_t> interesting_event_ranges)
|
||||
: _me(new MatcherValidationObserver::PImpl(icebergs, interesting_event_ranges)) { }
|
||||
|
||||
@@ -290,14 +292,14 @@ bool MatcherValidationObserver::_is_ready_to_trade(const InstrumentSnapshot& s,
|
||||
void MatcherValidationObserver::_sync_current_orders(MDP3Event const& ev) {
|
||||
for (auto s : ev.affected_securities) {
|
||||
auto& new_orders = _me->get_security_data(*s, ev.event_id).current_orders;
|
||||
_sync_orders(*s, new_orders, _me->icebergs);
|
||||
_sync_orders(ev.event_id, *s, new_orders, _me->icebergs);
|
||||
}
|
||||
}
|
||||
|
||||
void MatcherValidationObserver::_sync_previous_orders(const MDP3Event& ev) {
|
||||
for (auto s : ev.affected_securities) {
|
||||
auto& old_orders = _me->get_security_data(*s, ev.event_id).previous_orders;
|
||||
_sync_orders(*s, old_orders, _me->icebergs);
|
||||
_sync_orders(ev.event_id, *s, old_orders, _me->icebergs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -322,7 +324,7 @@ void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap
|
||||
for (auto s : snap) {
|
||||
auto& sec_data = _me->get_security_data(*s, NO_EVENT);
|
||||
sec_data.current_orders.clear();
|
||||
_sync_orders(*s, sec_data.current_orders, _me->icebergs);
|
||||
_sync_orders(NO_EVENT, *s, sec_data.current_orders, _me->icebergs);
|
||||
sec_data.previous_orders = sec_data.current_orders;
|
||||
}
|
||||
}
|
||||
@@ -334,8 +336,6 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
|
||||
case bts::spartan::EventOrdinal::Waiting:
|
||||
// just need to synchronize the order caches.
|
||||
// the matcher is only validated during 'interesting events.'
|
||||
subtract_iceberg_fills(ev, _me->icebergs);
|
||||
detect_show_qty_changes(ev, _me->icebergs, _me->icebergs);
|
||||
_sync_current_orders(ev);
|
||||
_sync_previous_orders(ev);
|
||||
return;
|
||||
@@ -352,8 +352,6 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
|
||||
if ((flags & _me->ignore_flags) != EventFlags::None) {
|
||||
if (_me->verbosity > Verbosity::Minimal)
|
||||
INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags));
|
||||
subtract_iceberg_fills(ev, _me->icebergs);
|
||||
detect_show_qty_changes(ev, _me->icebergs, _me->icebergs);
|
||||
_sync_current_orders(ev);
|
||||
_sync_previous_orders(ev);
|
||||
_reset_matcher();
|
||||
@@ -375,13 +373,13 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
|
||||
std::sort(sorted_secs.begin(), sorted_secs.end(), security_comp::SymbolComp());
|
||||
// securities must be added to the matcher before the order
|
||||
// changes are processed to ensure the correct implied markets are used.
|
||||
// Fills are also indexed in this pass, so they can be added to the iceberg qty
|
||||
std::unordered_map<MDP3OrderKey, MDP3Quantity> fill_map;
|
||||
for (const auto* s : sorted_secs) {
|
||||
accumulate_fills(s->fills(), fill_map);
|
||||
_me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs());
|
||||
_me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching());
|
||||
}
|
||||
// capture iceberg show qty changes (these are a special case)
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> show_qty_changes;
|
||||
detect_show_qty_changes(ev, _me->icebergs, show_qty_changes);
|
||||
// initialize matcher with previous state (only applies
|
||||
// to securities whose orders the matcher hasn't seen yet).
|
||||
for (const auto* s : sorted_secs) {
|
||||
@@ -405,16 +403,29 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
|
||||
if (deduced.size()) {
|
||||
std::vector<SmallOrderChange> deduced_sm;
|
||||
for (auto& ch : deduced) {
|
||||
add_iceberg_annotation(ch.order, _me->icebergs);
|
||||
auto ib_event_src = add_iceberg_annotation(ev.event_id, ch.order, _me->icebergs);
|
||||
if (ch.order.iceberg_quantity.has_value()) {
|
||||
// the fills would have already been subtracted from the ib total qty;
|
||||
// they need to be added back so the matcher can work the ib itself.
|
||||
auto fit = fill_map.find(ch.order.key);
|
||||
if (fit != fill_map.end()) {
|
||||
// only subtract fills from the total if the iceberg quantity's
|
||||
// state came from THIS event. This is because the iceberg quantity
|
||||
// annotated to the order is a snapshot of the iceberg quantity
|
||||
// AFTER the event was processed (i.e., fills will have already been subtracted).
|
||||
if (ib_event_src == ev.event_id)
|
||||
ch.order.iceberg_quantity.value().total += fit->second;
|
||||
ch.order.quantity -= fit->second;
|
||||
}
|
||||
if (ch.change == OrderChange::ChangeType::added)
|
||||
ch.order.quantity = ch.order.iceberg_quantity.value().show;
|
||||
}
|
||||
deduced_sm.emplace_back(ch);
|
||||
}
|
||||
_me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills);
|
||||
}
|
||||
}
|
||||
// Verify the matcher's orders are identitical to the exchange's.
|
||||
// subtract the fills from iceberg orders' total quantities
|
||||
// to get the correct expected orders for comparison with the matcher's orders.
|
||||
subtract_iceberg_fills(ev, _me->icebergs);
|
||||
_sync_current_orders(ev);
|
||||
bool orders_match = true;
|
||||
for (const auto* s : sorted_secs) {
|
||||
|
||||
+4
-2
@@ -13,12 +13,14 @@ namespace bts::spartan::mdp3 {
|
||||
struct MatcherValidationObserver : IChannelObserver {
|
||||
MatcherValidationObserver(const std::string& pcap);
|
||||
MatcherValidationObserver(
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> icebergs);
|
||||
boost::container::flat_map<MDP3OrderKey,
|
||||
boost::container::flat_map<size_t, IcebergQuantity>> icebergs);
|
||||
// interesting_event_ranges:
|
||||
// - keys indicate the first event in an interesting range
|
||||
// - values indicate the number of events in the range (i.e., the range's length).
|
||||
MatcherValidationObserver(
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> icebergs,
|
||||
boost::container::flat_map<MDP3OrderKey,
|
||||
boost::container::flat_map<size_t, IcebergQuantity>> icebergs,
|
||||
boost::container::flat_map<size_t, size_t> interesting_event_ranges);
|
||||
~MatcherValidationObserver();
|
||||
void on_recovery_complete(InitialSnapshot const& snap) override;
|
||||
|
||||
@@ -73,4 +73,25 @@ fmt_order(MDP3OrderKey key, api::Side side, md::IntPrice price, MDP3Quantity qty
|
||||
<< " @ " << price.price();
|
||||
return str.str();
|
||||
}
|
||||
|
||||
template<class TIter>
|
||||
inline void accumulate_fills(TIter first, TIter last, std::unordered_map<MDP3OrderKey, MDP3Quantity>& dest) {
|
||||
while (first != last) {
|
||||
auto it = dest.find(first->key);
|
||||
if (it == dest.end())
|
||||
dest.emplace(first->key, first->quantity);
|
||||
else it->second.value() += first->quantity.value();
|
||||
first++;
|
||||
}
|
||||
}
|
||||
|
||||
inline void accumulate_fills(const std::vector<MDP3Fill>& fills, std::unordered_map<MDP3OrderKey, MDP3Quantity>& dest) {
|
||||
accumulate_fills(fills.begin(), fills.end(), dest);
|
||||
}
|
||||
|
||||
inline std::unordered_map<MDP3OrderKey, MDP3Quantity> accumulate_fills(const std::vector<MDP3Fill>& fills) {
|
||||
std::unordered_map<MDP3OrderKey, MDP3Quantity> rv;
|
||||
accumulate_fills(fills.begin(), fills.end(), rv);
|
||||
return rv;
|
||||
}
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -1,7 +1,11 @@
|
||||
#include "IcebergFinder.h"
|
||||
#include "../../common/InstrumentSnapshot.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
template<class K, class V>
|
||||
using flat_map = boost::container::flat_map<K, V>;
|
||||
|
||||
IcebergFinder::IcebergFinder() {}
|
||||
IcebergFinder::~IcebergFinder() {}
|
||||
@@ -174,4 +178,82 @@ void IcebergFinder::on_real_time_event(MDP3Event const &ev) {
|
||||
}
|
||||
}
|
||||
|
||||
void IcebergStateMapper::on_real_time_event(MDP3Event const& ev) {
|
||||
std::unordered_map<MDP3OrderKey, MDP3Quantity> fills;
|
||||
std::unordered_map<MDP3OrderKey, MDP3Order> updates;
|
||||
for (const auto* s : ev.affected_securities) {
|
||||
for (const auto& f : s->fills()) {
|
||||
auto fit = fills.find(f.key);
|
||||
if (fit == fills.end())
|
||||
fills.emplace(f.key, f.quantity);
|
||||
else
|
||||
fit->second += f.quantity;
|
||||
}
|
||||
for (const auto& ch : s->order_changes()) {
|
||||
auto it = _initial_ibs.find(ch.order.key);
|
||||
if (it == _initial_ibs.end())
|
||||
continue;
|
||||
switch (ch.change) {
|
||||
case OrderChange::ChangeType::added:
|
||||
case OrderChange::ChangeType::updated: {
|
||||
auto fit = fills.find(ch.order.key);
|
||||
auto ord_states_it = _states.find(ch.order.key);
|
||||
if (ord_states_it == _states.end()) {
|
||||
auto ib = iceberg_helper::to_iceberg_qty(it->second);
|
||||
if (fit != fills.end())
|
||||
ib.total -= fit->second;
|
||||
_states.emplace(ch.order.key, IcebergStates())
|
||||
.first->second[ev.event_id] = ib;
|
||||
break;
|
||||
}
|
||||
bts_assert(ord_states_it->second.size() > 0);
|
||||
auto prev_ib = (--ord_states_it->second.end())->second;
|
||||
auto& ib =
|
||||
ord_states_it->second.emplace(ev.event_id, prev_ib).first->second;
|
||||
if (fit == fills.end()) {
|
||||
// ONLY set the show quantity to the order's current quantity
|
||||
// if the order's quantity has actually changed.
|
||||
//
|
||||
// Consider this scenario:
|
||||
// 1) an iceberg order is launched with a show qty of 5.
|
||||
// 2) the order is partially filled by qty of 2. Now the "visible"
|
||||
// qty is 3,
|
||||
// meaning that which is "visible" to the market, but the "show"
|
||||
// qty is still 5.
|
||||
// 3) the trader decides to change the order's price, without
|
||||
// changing its qty.
|
||||
// Conclusion: if the "show" qty is blindly updated to the order's
|
||||
// "visible" qty when the trader only wanted to change its price, the
|
||||
// "show" qty may be incorrectly decreased. In the scenario above, it
|
||||
// would have been decreased from 5 -> 3.
|
||||
auto qty_diff = ch.order.quantity - ch.previous_order.quantity;
|
||||
if (qty_diff.value() != 0) {
|
||||
ib.total += qty_diff;
|
||||
ib.show = ch.order.quantity;
|
||||
}
|
||||
} else {
|
||||
ib.total -= fit->second;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case OrderChange::ChangeType::removed: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void IcebergStateMapper::map_states(
|
||||
const std::string& pcap, flat_map<MDP3OrderKey, IcebergStates>& states) {
|
||||
LOG(OrderMan, Info, "sweeping '{}' for icebergs (1/2): determine iceberg orders", pcap);
|
||||
IcebergFinder ib_finder;
|
||||
observe_channel(pcap, ib_finder);
|
||||
flat_map<MDP3OrderKey, DeducedIceberg> ibs;
|
||||
for (const auto& kv : ib_finder.result)
|
||||
ibs.insert(std::make_pair(kv.first, kv.second));
|
||||
// Sweep the pcap again to capture all other states for each iceberg.
|
||||
LOG(OrderMan, Info, "found {} iceberg orders in '{}.'", ibs.size(), pcap);
|
||||
LOG(OrderMan, Info, "sweeping '{}' for icebergs (2/2): capture iceberg state changes.", pcap);
|
||||
IcebergStateMapper ib_state_mapper(ibs, states);
|
||||
observe_channel(pcap, ib_state_mapper);
|
||||
}
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
#include "../../common/mdp3_common.h"
|
||||
#include "../../handler/IChannelObserver.h"
|
||||
#include <boost/container/flat_map.hpp>
|
||||
|
||||
namespace bts::spartan::mdp3 {
|
||||
using IcebergStates = boost::container::flat_map<size_t, IcebergQuantity>;
|
||||
|
||||
// Find all the icebegers in the pcap, estimate their size.
|
||||
|
||||
@@ -29,10 +31,28 @@ struct IcebergFinder : IChannelObserver {
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
|
||||
struct IcebergStateMapper : IChannelObserver {
|
||||
static void map_states(
|
||||
const std::string& pcap,
|
||||
boost::container::flat_map<MDP3OrderKey, IcebergStates>& states);
|
||||
|
||||
void init() override {}
|
||||
void finit() override {}
|
||||
void on_real_time_event(MDP3Event const& ev) override;
|
||||
|
||||
private:
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& _initial_ibs;
|
||||
boost::container::flat_map<MDP3OrderKey, IcebergStates>& _states;
|
||||
|
||||
IcebergStateMapper(
|
||||
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& initial_ibs,
|
||||
boost::container::flat_map<MDP3OrderKey, IcebergStates>& states)
|
||||
: _initial_ibs(initial_ibs), _states(states) {}
|
||||
};
|
||||
|
||||
inline std::ostream& operator <<(std::ostream& out, const DeducedIceberg& ib) {
|
||||
out << "key " << ib.key << ", show_qty " << ib.show_quantity << ", est_qty "
|
||||
<< ib.estimated_quantity << ", filled " << (ib.filled ? "Y" : "N");
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace bts::spartan::mdp3
|
||||
@@ -117,6 +117,18 @@ inline bool read_from_file(std::vector<T>& values, const std::string& path, TOpe
|
||||
return false;
|
||||
}
|
||||
|
||||
template<class TOper>
|
||||
inline bool read_from_file(const std::string& path, TOper op, ushort version) {
|
||||
std::ifstream istr(path, std::ios::binary);
|
||||
if (istr) {
|
||||
if (version != read_from_bin<ushort>(istr))
|
||||
return false;
|
||||
while (op(istr)) {}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template<class Iterator, class TOper>
|
||||
inline void write_to_file(Iterator first, Iterator last, const std::string& path, TOper op) {
|
||||
std::cout << "opening file '" << path << "'" << std::endl;
|
||||
|
||||
Reference in new issue
Block a user