Files
spark/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp
T
John Colvin 3efb4da14b diag: Add Trace parameter to matcher constructor
This makes it easy for matcher-validation commands to bind trace
parameters, such as order/event ids, to cli options.
2021-02-01 22:56:21 -06:00

310 lines
11 KiB
C++

#include "Matcher.h"
#include "MatchingCommon.h"
#include "back_testing/common/Trace.h"
#include "market_data/utils/book_utils.h"
#include <boost/container/flat_map.hpp>
#include <functional>
#include <google/protobuf/util/time_util.h>
#include <iostream>
using namespace google::protobuf::util;
namespace bts::spartan::mdp3 {
struct MergedMatcher::PImpl {
PImpl(MergedMatcher::Args args)
: matcher_map(), trace(args.trace), match_count(0),
match_count_this_per(0), per_start_time(TimeUtil::GetCurrentTime()) {}
std::unordered_set<OrderKey> used_order_keys;
std::map<SecurityID, std::string> asset_map;
std::map<std::string, FifoMatcherContext> matcher_map;
Trace trace;
size_t match_count;
size_t match_count_this_per;
google::protobuf::Timestamp per_start_time;
};
MergedMatcher::MergedMatcher() : MergedMatcher(Args{}) {}
MergedMatcher::MergedMatcher(MergedMatcher::Args args)
: _me(new MergedMatcher::PImpl(args)) {}
MergedMatcher::~MergedMatcher() {}
void MergedMatcher::add_def(const SecurityDefinition &def) {
std::string ass(def.asset());
_me->asset_map.emplace(def.security_id(), ass);
auto m = _me->matcher_map.find(ass);
if (m == _me->matcher_map.end())
m = _me->matcher_map.emplace(ass, FifoMatcherContext({true, _me->trace}))
.first;
m->second.add_security(
def.security_id(), std::string(def.symbol()),
std::vector<LegDef>{def.legs().begin(), def.legs().end()});
m->second.implied_matching_status(def.security_id(), def.implied_matching());
}
std::string MergedMatcher::_get_asset(SecurityID sec) const {
auto it = _me->asset_map.find(sec);
return it == _me->asset_map.end() ? std::string() : it->second;
}
template <int L>
using MBP = std::array<BookLevel, L>;
template <int L>
inline bool mbp_equals(const MBP<L> &a, const MBP<L> &b) {
for (uint32_t i = 0; i < a.size(); i++) {
if (a[i].price != b[i].price || a[i].quantity != b[i].quantity)
return false;
}
return true;
}
// using structs rather than std::tuples to avoid
// accidentally mixing up before/after, or bids/asks.
struct MBPData {
MBPData() {}
MBPData(FifoMergedSecurityMatcher &mat)
: bids(mat.book(Side::Buy)), asks(mat.book(Side::Sell)),
ibids(mat.implied_book(Side::Buy)),
iasks(mat.implied_book(Side::Sell)) {}
BookSide bids, asks;
ImpliedBookSide ibids, iasks;
};
inline bool operator==(const MBPData &lhs, const MBPData &rhs) {
auto lev_eq = [](const BookLevel &x, const BookLevel &y) {
return x.price == y.price && x.quantity == y.quantity;
};
return std::equal(
lhs.bids.begin(), lhs.bids.end(), rhs.bids.begin(), lev_eq) &&
std::equal(
lhs.asks.begin(), lhs.asks.end(), rhs.asks.begin(), lev_eq) &&
std::equal(
lhs.ibids.begin(), lhs.ibids.end(), rhs.ibids.begin(), lev_eq) &&
std::equal(
lhs.iasks.begin(), lhs.iasks.end(), rhs.iasks.begin(), lev_eq);
}
struct MBPSnapshot {
MBPSnapshot() : _matcher(nullptr), _initial_mbp() {}
MBPSnapshot(FifoMergedSecurityMatcher *matcher)
: _matcher(matcher), _initial_mbp(*_matcher) {}
std::optional<MBPData> get_new_mbp() {
bts_assert(_matcher != nullptr);
auto curr = MBPData(*_matcher);
if (_initial_mbp == curr)
return {};
return curr;
}
private:
FifoMergedSecurityMatcher *_matcher;
MBPData _initial_mbp;
};
FifoMatcherContext &
MergedMatcher::_asset_matcher(SecurityID sec, bool add_if_missing) {
auto ass_it = _me->asset_map.find(sec);
if (add_if_missing && ass_it == _me->asset_map.end())
ass_it = _me->asset_map.emplace(sec, std::string()).first;
bts_assert(ass_it != _me->asset_map.end());
auto mat_it = _me->matcher_map.find(ass_it->second);
if (add_if_missing && mat_it == _me->matcher_map.end())
mat_it =
_me->matcher_map
.emplace(ass_it->second, FifoMatcherContext({true, _me->trace}))
.first;
bts_assert(mat_it != _me->matcher_map.end());
return mat_it->second;
}
void MergedMatcher::reset(
SecurityID sec,
std::vector<MDP3Order> orders,
std::vector<MatchEffect> *effects) {
for (const auto &o : orders)
_me->used_order_keys.emplace(o.key);
_asset_matcher(sec).reset(sec, orders.begin(), orders.end(), effects);
}
void MergedMatcher::match(
ChangeSrc,
SmallOrderChangeVec const &changes,
std::vector<MatchEffect> *res) {
_me->match_count_this_per++;
_me->match_count++;
// The caller is not obligated to pass an empty vector. Consider an event
// that involves multiple securities; perhaps the caller wants to put the
// effects for each security in the same vector. The elements that were
// already present in the effects vector are skipped/ignored.
uint32_t effects_origin = res == nullptr ? 0 : res->size();
// This could be better optimized, but the idea is to
// 1) take a snapshot of every security's book/implied book,
// 2) process the order changes,
// 3) compare the current books with the snapshots to detect mbp changes.
// The map's hierarchcy is asset->security->books.
std::map<std::string, std::map<SecurityID, MBPSnapshot>> mbps;
for (const auto &c : changes) {
bts_assert(c.order.key.value() != 0);
auto ass = _get_asset(c.order.security);
auto it = mbps.find(ass);
if (it == mbps.end())
it = mbps.emplace(ass, std::map<SecurityID, MBPSnapshot>()).first;
auto mbp_tup_it = it->second.find(c.order.security);
if (mbp_tup_it == it->second.end()) {
auto *mat = _me->matcher_map[ass].get_matcher(c.order.security);
bts_assert(mat != nullptr);
it->second.emplace(c.order.security, MBPSnapshot(mat));
}
}
for (const auto &c : changes) {
_me->trace.log_incoming_change(c);
MatcherUtils::validate_incoming_order_change(c);
switch (c.change) {
case OrderChange::ChangeType::added:
bts_assert(_me->used_order_keys.emplace(c.order.key).second);
break;
case OrderChange::ChangeType::updated:
case OrderChange::ChangeType::updated_price:
case OrderChange::ChangeType::removed:
bts_assert(_me->used_order_keys.count(c.order.key));
break;
}
// this holds individual fills, meaning one aggr fill per price,
// and potentially multiple fills per order.
// todo: make the fills param optional, as
// it will often be more info than we care abt.
FillDepot fills;
_asset_matcher(c.order.security).on_order_change(c, res, fills);
}
// log/validate order updates
if (res != nullptr) {
for (auto it = res->begin() + effects_origin; it != res->end(); it++)
_me->trace.log_outgoing_update(std::get<SimOrderUpdate>(*it));
for (auto it = res->begin() + effects_origin; it != res->end(); it++)
MatcherUtils::validate_outgoing_update(std::get<SimOrderUpdate>(*it));
// add mbp updates
for (auto &ass_kv : mbps) {
for (auto &sec_kv : ass_kv.second) {
auto new_mbp = sec_kv.second.get_new_mbp();
if (new_mbp.has_value()) {
res->emplace_back(SimBookUpdate{
sec_kv.first, new_mbp.value().bids, new_mbp.value().asks,
new_mbp.value().ibids, new_mbp.value().iasks});
}
}
}
}
auto now = TimeUtil::GetCurrentTime();
auto usecs = TimeUtil::TimestampToMicroseconds(now) -
TimeUtil::TimestampToMicroseconds(_me->per_start_time);
if (usecs >= 1000000) {
LOG(OrderMan, Diag,
"usecs:{} events processed:{} tot events processed:{}", usecs,
_me->match_count_this_per, _me->match_count);
_me->match_count_this_per = 0;
_me->per_start_time = now;
}
}
BookSide MergedMatcher::book(SecurityID id, Side s) {
return _asset_matcher(id).book(id, s);
}
ImpliedBookSide MergedMatcher::implied_book(SecurityID id, Side s) {
return _asset_matcher(id).implied_book(id, s);
}
void MergedMatcher::clear() {
for (auto &mp : _me->matcher_map)
mp.second.clear();
}
struct DirectMatcher::PImpl {
PImpl()
: matcher_map(), trace(), match_count(0), match_count_this_per(0),
per_start_time(TimeUtil::GetCurrentTime()) {}
std::map<SecurityID, FifoSecurityMatcher> matcher_map;
Trace trace;
size_t match_count;
size_t match_count_this_per;
google::protobuf::Timestamp per_start_time;
};
/////////////////
// DirectMatcher
////////////////
DirectMatcher::DirectMatcher() : _me(new DirectMatcher::PImpl()) {}
DirectMatcher::DirectMatcher(const std::vector<SecurityID> &inst_defs)
: DirectMatcher() {
for (const auto &sec_id : inst_defs)
_me->matcher_map.insert(std::pair<SecurityID, FifoSecurityMatcher>(
sec_id, FifoSecurityMatcher(sec_id)));
}
DirectMatcher::~DirectMatcher() {}
void DirectMatcher::match(
ChangeSrc,
SmallOrderChangeVec const &changes,
std::vector<MatchEffect> *res) {
_me->match_count_this_per++;
_me->match_count++;
std::vector<std::pair<SecurityID, FifoSecurityMatcher *>> dirty;
dirty.reserve(_me->matcher_map.size());
for (const auto &c : changes) {
auto it = _me->matcher_map.find(c.order.security);
bts_assert(it != _me->matcher_map.end());
auto &sec_matcher = it->second;
_me->trace.log_incoming_change(c);
MatcherUtils::validate_incoming_order_change(c);
sec_matcher.on_order_change(c, res);
dirty.emplace_back(c.order.security, &it->second);
}
for (const auto &u : *res)
_me->trace.log_outgoing_update(std::get<SimOrderUpdate>(u));
for (const auto &u : *res)
MatcherUtils::validate_outgoing_update(std::get<SimOrderUpdate>(u));
// all order activity is not reflecting in the
// books
for (auto cm : dirty)
res->emplace_back(SimBookUpdate{
cm.first, cm.second->book(Side::Buy), cm.second->book(Side::Sell),
ImpliedBookSide(), ImpliedBookSide()});
auto now = TimeUtil::GetCurrentTime();
auto usecs = TimeUtil::TimestampToMicroseconds(now) -
TimeUtil::TimestampToMicroseconds(_me->per_start_time);
if (usecs >= 1000000) {
LOG(OrderMan, Diag,
"usecs:{} events processed:{} tot events processed:{}", usecs,
_me->match_count_this_per, _me->match_count);
_me->match_count_this_per = 0;
_me->per_start_time = now;
}
}
BookSide DirectMatcher::book(SecurityID id, Side s) {
auto it = _me->matcher_map.find(id);
bts_assert(it != _me->matcher_map.end());
return it->second.book(s);
}
void DirectMatcher::clear() {
for (auto &p : _me->matcher_map)
p.second.clear();
}
std::map<SecurityID, FifoSecurityMatcher> &DirectMatcher::_matcher_map() {
return _me->matcher_map;
}
} // namespace bts::spartan::mdp3