better iceberg deduction; integrate deduced icebergs into matcher

This commit is contained in:
John Colvin committed 2020-10-23 13:29:37 -04:00
1 parent 4f493aaf29
commit bef915fd56
26 files changed
+596 -282

No files matched your search

@@ -88,7 +88,7 @@ void SimOrderGateway:: place_order(SecurityKey skey,
MDP3OrderPriority{},
md::IntPrice_from_double<7>(p.value()),
MDP3Quantity{static_cast<int>(q.value())},
MDP3Quantity(0)
{}
},
OrderChange::added
});
@@ -153,6 +153,14 @@ FifoMergedSecurityMatcher* FifoMatcherContext::_get_or_add_matcher(MDP3SecurityI
return m->second;
}
std::tuple<FifoPriceLevelMatcher*, MatcherOrder*>
FifoMatcherContext::find_order(MDP3SecurityID sec_id, MDP3OrderKey ok) {
auto* m = _get_matcher(sec_id);
if (m != nullptr)
return m->matcher()->find_order(ok);
return { nullptr, nullptr };
}
///////////////////////
// order callbacks
///////////////////////
@@ -38,6 +38,7 @@ struct FifoMatcherContext {
void on_order_change(const SmallOrderChange& change, std::vector<MatchEffect>* effects, FillDepot& fills);
void print_all_bbos(std::ostream& out);
std::optional<std::string> has_same_orders(MDP3SecurityID, const Orders&);
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *> find_order(MDP3SecurityID, MDP3OrderKey);
private:
struct PImpl;
@@ -82,6 +83,7 @@ struct FifoMatcherContext {
OrderChange ch;
ch.change = OrderChange::ChangeType::added;
ch.order = kv.second;
bts_assert(ch.order.quantity.value() > 0);
matcher->matcher()->on_order_change(ch);
}
}
@@ -131,16 +131,15 @@ void FifoMergedSecurityMatcher::on_order_change(const SmallOrderChange& ch, std:
auto prev_ord_tup = matcher()->find_order(ch.order.key);
auto prev_ord = std::get<1>(prev_ord_tup);
if (prev_ord == nullptr) {
bts_assert_msg(_me->ignore_ops_on_deleted_orders, "unable to process order-remove event: failed to find order.");
bts_assert_msg(_me->ignore_ops_on_deleted_orders, "unable to process order-update event: failed to find order.");
break;
}
if (orderman::price_comp::is_better(ch.order.side, ch.order.price, prev_ord->price)) {
// price improved, try to fill the order
auto new_ch = ch;
_fill(new_ch.order, effects, fills, prev_ord->ib.is_ib ? std::max(prev_ord->ib.show_qty, new_ch.order.quantity) : new_ch.order.quantity);
auto rem = new_ch.order.quantity;
PRINT_LINE(_log_header(), "updated order rem qty: ", rem, ", orig state: ", ch.order);
if (rem.value() == 0) {
_fill(new_ch.order, effects, fills, false);
// PRINT_LINE(_log_header(), " updated order rem qty: ", new_ch.order.quantity, ", orig state: ", ch.order);
if (new_ch.order.quantity.value() == 0) {
matcher()->on_order_completed(ch.order.key);
} else {
matcher()->on_order_change(new_ch, effects);
@@ -157,8 +156,8 @@ void FifoMergedSecurityMatcher::on_order_change(const SmallOrderChange& ch, std:
}
case OrderChange::ChangeType::added: {
auto new_ch = ch;
_fill(new_ch.order, effects, fills, ch.order.quantity);
if (new_ch.order.quantity.value() > 0)
_fill(new_ch.order, effects, fills, false);
if (get_total_qty(new_ch.order).value() > 0)
matcher()->on_order_change(new_ch);
break;
}
@@ -284,21 +283,14 @@ void FifoMergedSecurityMatcher::merged_book(
}
}
void FifoMergedSecurityMatcher::_fill(MDP3Order& order, std::vector<MatchEffect>* effects, FillDepot& fills, MDP3Quantity show_qty) {
auto total_qty = order.iceberg_quantity.has_value() ? order.iceberg_quantity.value()
: order.quantity;
void FifoMergedSecurityMatcher::_fill(MDP3Order& order, std::vector<MatchEffect>* effects, FillDepot& fills, bool reset_show_qty) {
if (order.iceberg_quantity.has_value()) {
// the order's quantity cannot exceed the show qty (but it can be smaller).
if (show_qty < order.quantity) {
//bts_assert(show_qty >= order.quantity);
bts_assert_msg(false, FMT("ord: ", order, " ***SHOW QTY: ", show_qty));
}
bts_assert(order.quantity <= order.iceberg_quantity.value().show);
}
auto rem = _fill(order.key, order.side, order.price, total_qty, effects, fills);
if (order.iceberg_quantity.has_value()) {
order.iceberg_quantity = rem;
order.quantity = std::min(show_qty, rem);
} else order.quantity = rem;
auto rem = _fill(order.key, order.side, order.price, get_total_qty(order), effects, fills);
reset_show_qty |= rem == get_total_qty(order);
set_total_qty(order, rem, reset_show_qty);
}
MDP3Quantity FifoMergedSecurityMatcher::_fill(
@@ -311,7 +303,7 @@ MDP3Quantity FifoMergedSecurityMatcher::_fill(
//
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp oside_comp(oside);
// SUMMARIZE_TRANSACTION_NAMED(_order_fill_log, _log_header(), FMT("filling ", fmt_order(key, side, quantity, price)));
SUMMARIZE_TRANSACTION_NAMED(_order_fill_log, _log_header(), FMT("filling ", fmt_order(key, side, quantity, price)));
while (quantity.value() > 0) {
// Fill one price level at a time.
// While one market may have the best 1st price level,
@@ -39,7 +39,9 @@ struct FifoMergedSecurityMatcher {
FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher&);
FifoImpliedSecurityMatcher* _best_implied_matcher(api::Side);
void _fill(MDP3Order&, std::vector<MatchEffect>*, FillDepot&, MDP3Quantity ib_show_qty);
// the order's params (quantity, and iceberg_quantity if applicable)
// will be appropriately adjusted according to the number of fills.
void _fill(MDP3Order&, std::vector<MatchEffect>*, FillDepot&, bool reset_show_qty);
MDP3Quantity _fill(
MDP3OrderKey,
api::Side,
@@ -1,5 +1,7 @@
#include "FifoPriceLevelMatcher.h"
#include "../../common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
namespace bts::spartan::mdp3 {
@@ -31,13 +33,8 @@ void FifoPriceLevelMatcher::_logically_delete(MatcherOrder &o) {
o.is_active = false;
}
void FifoPriceLevelMatcher::_add_order(const MDP3Order &ord) {
auto ib =
MatcherIB{ord.iceberg_quantity.has_value(), ord.quantity,
ord.iceberg_quantity.has_value()
? ord.iceberg_quantity.value()
: ord.quantity};
MatcherOrder new_ord (ord.key,ord.side,ord.price,ord.quantity,
_priority++,ib);
_priority++,ord.iceberg_quantity);
auto[p,res] = _master_order_map->try_emplace(new_ord.key,new_ord);
MatcherOrder* new_ord_ptr = &p->second;
_orders.push_back(new_ord_ptr);
@@ -129,7 +126,7 @@ void FifoPriceLevelMatcher::_log_order(const MatcherOrder& ord,
LOG(OrderMan, Diag,
"{} key:{} side:{} qty:{} ib_qty:{} price:{}",
msg, ord.key, static_cast<int>(side()), ord.quantity,
ord.ib.is_ib? ord.ib.total_qty : MDP3Quantity(0),
ord.ib.has_value() ? ord.ib.value().total : MDP3Quantity(0),
price());
}
void FifoPriceLevelMatcher::_log_order(const MDP3Order& ord,
@@ -138,8 +135,8 @@ void FifoPriceLevelMatcher::_log_order(const MDP3Order& ord,
LOG(OrderMan, Diag,
"{} key:{} side:{} qty:{} ib_qty:{} price:{}",
msg, ord.key, static_cast<int>(ord.side), ord.quantity,
ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value()
: MDP3Quantity(0),
ord.iceberg_quantity.has_value() ? FMT(ord.iceberg_quantity.value())
: FMT(MDP3Quantity(0)),
ord.price.price());
}
// constructors ///////////////////////////////////////////////////////////////
@@ -248,8 +245,11 @@ void FifoPriceLevelMatcher::on_order_update(
_orders.push_back(&ord);
// udpate value
ord.quantity = upd_ord.quantity;
} else
} else {
ord.quantity = upd_ord.quantity;
if (ord.ib.has_value() && upd_ord.iceberg_quantity.has_value() && ord.ib.value() != upd_ord.iceberg_quantity.value())
ord.ib = upd_ord.iceberg_quantity;
}
}
void FifoPriceLevelMatcher::on_order_cancel(MatcherOrder& order,
std::vector<MatchEffect>* res){
@@ -1,5 +1,7 @@
#include "FifoSecurityMatcher.h"
#include "../../common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include <experimental/map>
namespace bts::spartan::mdp3 {
@@ -16,7 +18,7 @@ inline bool _is_cross(const MDP3Order &ord,
}
inline MDP3Quantity total_quantity(const MDP3Order &ord) {
return ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value()
return ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value().total
: ord.quantity;
}
@@ -107,21 +109,6 @@ void FifoSecurityMatcher::_on_new_order(const MDP3Order &ord,
price_level = &*std::get<0>(p);
}
MDP3Order ord_corrected = ord;
if (ord.iceberg_quantity.has_value()) {
uint32_t tot =
static_cast<uint32_t>(ord.iceberg_quantity.value().value());
// this is not exactly correct
uint32_t show = static_cast<uint32_t>(ord.quantity.value());
uint32_t filled = tot-new_ord_rem;
ord_corrected.iceberg_quantity = MDP3Quantity(new_ord_rem);
uint32_t working;
if(filled < show)
working = show - filled;
else
working = std::min(new_ord_rem, show);
ord_corrected.quantity = MDP3Quantity(working);
} else
ord_corrected.quantity = MDP3Quantity(new_ord_rem);
price_level->on_new_same_side_order(ord_corrected);
}
else{
@@ -331,7 +318,8 @@ FifoSecurityMatcher::has_same_orders(const Orders &ord_map) {
auto match = plevel != nullptr && ord != nullptr &&
plevel->price() == exp_ord.price &&
plevel->side() == exp_ord.side &&
ord->quantity == exp_ord.quantity;
ord->quantity == exp_ord.quantity &&
ord->ib == exp_ord.iceberg_quantity;
if (!match) {
std::ostringstream buff;
buff << "eid:" << _event_id << " n_ords:" << ord_map.size()
@@ -343,10 +331,7 @@ FifoSecurityMatcher::has_same_orders(const Orders &ord_map) {
buff << "Expected price:" << exp_ord.price << std::endl;
buff << "Expected qty:" << exp_ord.quantity << std::endl;
buff << "Expected side:" << exp_ord.side << std::endl;
uint32_t exp_ib_val = exp_ord.iceberg_quantity.has_value()
? exp_ord.iceberg_quantity.value().value()
: 0;
buff << "Expected IB state:" << exp_ib_val << std::endl;
buff << "Expected IB state:" << exp_ord.iceberg_quantity << std::endl;
}
else {
buff << "Order with Key:" << exp_ord.key.value()
@@ -357,13 +342,8 @@ FifoSecurityMatcher::has_same_orders(const Orders &ord_map) {
<< " matcher:" << ord->quantity << std::endl;
buff << "Expected side:" << exp_ord.side
<< " matcher:" << plevel->side() << std::endl;
uint32_t exp_ib_val = exp_ord.iceberg_quantity.has_value()
? exp_ord.iceberg_quantity.value().value()
: 0,
matcher_ib_val =
ord->ib.is_ib ? ord->ib.total_qty.value() : 0;
buff << "Expected IB state:" << exp_ib_val
<< " matcher:" << matcher_ib_val << std::endl;
buff << "Expected IB state:" << exp_ord.iceberg_quantity
<< " matcher:" << ord->ib << std::endl;
}
LOG(OrderMan, Error, "FifoSecurityMatcher::has_same_orders failed:{}",
buff.str());
@@ -418,10 +398,8 @@ void FifoSecurityMatcher::_log_trace_order_state(const std::string& msg){
auto[pl,o] = _find_order(k);
if(pl != nullptr && o != nullptr)
LOG(OrderMan, Diag,
"Order state msg:{} eid:{} key:{} qty:{} ib_show_qty:{} ib_total_qty:{} "
"price:{}",
msg, _event_id, o->key, o->quantity, o->ib.show_qty, o->ib.total_qty,
pl->price());
"Order state msg:{} eid:{} key:{} qty:{} ib:{} price:{}",
msg, _event_id, o->key, o->quantity, o->ib, pl->price());
}
}
void FifoSecurityMatcher::_log_trace_order(const MDP3Order& ord,
@@ -430,9 +408,7 @@ void FifoSecurityMatcher::_log_trace_order(const MDP3Order& ord,
LOG(OrderMan, Diag,
"{} eid:{} key:{} side:{} qty:{} ib_qty:{} price:{} book:{}",
msg, _event_id, ord.key, static_cast<int>(ord.side), ord.quantity,
ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value()
: MDP3Quantity(0),
ord.price.price(), print_book());
ord.iceberg_quantity, ord.price.price(), print_book());
_log_trace_order_state(msg);
}
}
@@ -62,6 +62,7 @@ public:
void _on_order_update(const MDP3Order&, std::vector<MatchEffect>* = nullptr);
void _on_order_remove(MDP3OrderKey,
std::vector<MatchEffect>* = nullptr);
std::string _log_header() const;
public:
// cons/dest ///////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
@@ -1,7 +1,10 @@
#include "Matcher.h"
#include "MatchingCommon.h"
#include <google/protobuf/util/time_util.h>
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include <boost/container/flat_map.hpp>
#include <functional>
#include <iostream>
@@ -10,20 +13,22 @@ using namespace google::protobuf::util;
namespace bts::spartan::mdp3{
MergedMatcher::MergedMatcher()
:
_matcher_map(),
_trace(),
_match_count(0),
_match_count_this_per(0),
_per_start_time(TimeUtil::GetCurrentTime()){
: MergedMatcher(std::unordered_map<MDP3OrderKey, DeducedIceberg>{}) {}
MergedMatcher::MergedMatcher(
const std::unordered_map<MDP3OrderKey, DeducedIceberg>& icebergs)
: _matcher_map(), icebergs(),
_trace(), _match_count(0), _match_count_this_per(0),
_per_start_time(TimeUtil::GetCurrentTime()) {
for (const auto& ib : icebergs)
this->icebergs.emplace(ib);
}
MergedMatcher::MergedMatcher(MergedMatcher&& src) {
_matcher_map = std::move(src._matcher_map);
icebergs = std::move(src.icebergs);
}
MergedMatcher::MergedMatcher (std::vector<const InstrumentDefinition&> const&) { }
void MergedMatcher::add_def(const InstrumentDefinition& def) {
std::string ass(def.asset());
_asset_map.emplace(def.security_id(), ass);
@@ -129,7 +134,38 @@ void MergedMatcher::match(ChangeSrc,
// todo: make the fills param optional, as
// it will often be more info than we care abt.
FillDepot fills;
ass_matcher.on_order_change(c, res, fills);
auto new_ch = c;
switch (c.change) {
case OrderChange::ChangeType::added:
iceberg_helper::add_iceberg_annotation(new_ch.order, icebergs);
break;
case OrderChange::ChangeType::updated: {
auto* old_ord = std::get<1>(ass_matcher.find_order(c.order.security, c.order.key));
if (old_ord == nullptr) {
// the matcher may not have this order anymore;
// it may have been filled already due to an algo shot.
continue;
}
OrderChange big_ch;
big_ch.change = c.change;
big_ch.previous_order = to_mdp3_order(*old_ord);
big_ch.order = c.order;
iceberg_helper::detect_show_qty_changes(big_ch, icebergs);
new_ch.order = big_ch.order;
break;
}
default: break;
}
ass_matcher.on_order_change(new_ch, res, fills);
// the DeducedIceberg's estimated quantity is essentially a fill count
// (obtained by sweeping the entire pcap). Since the order's iceberg
// props (total/show qty) are assigned in this method (anytime the order changes),
// the fills must be incrementally subtracted from the remaining quantity
// in our icebergs cache. That way, the iceberg total qty it will be correct
// the next time the order changes.
std::vector<MDP3Fill> fills_vect;
fills.copy_to(fills_vect);
iceberg_helper::subtract_iceberg_fills(fills_vect.begin(), fills_vect.end(), icebergs);
}
// log/validate order updates
for(const auto& u : *res)
@@ -171,7 +207,6 @@ Book MergedMatcher::book(MDP3SecurityID id, api::Side s) {
return it->second.matcher.book(id, s);
}
/////////////////
// DirectMatcher
////////////////
@@ -244,6 +279,5 @@ Book DirectMatcher::book(MDP3SecurityID id, api::Side s) {
bts_assert(it != _matcher_map.end());
return it->second.matcher.book(s);
}
}
@@ -6,6 +6,7 @@
#include "FifoSecurityMatcher.h"
#include "FifoMatcherContext.h"
#include "MatchingCommon.h"
#include <boost/container/flat_map.hpp>
#include <google/protobuf/timestamp.pb.h>
#include <map>
#include <string>
@@ -13,6 +14,7 @@
namespace bts::spartan::mdp3 {
class InstrumentDefinition;
class DeducedIceberg;
class MergedMatcher {
private:
@@ -23,6 +25,7 @@ private:
std::map<MDP3SecurityID, std::string> _asset_map;
std::map<std::string, CheckedMatcher> _matcher_map;
boost::container::flat_map<MDP3OrderKey, DeducedIceberg> icebergs;
Trace _trace;
size_t _match_count;
size_t _match_count_this_per;
@@ -31,7 +34,7 @@ private:
public:
MergedMatcher ();
MergedMatcher (std::vector<const InstrumentDefinition&> const&);
MergedMatcher(const std::unordered_map<MDP3OrderKey, DeducedIceberg>&);
MergedMatcher (MergedMatcher&&);
MergedMatcher &operator=(MergedMatcher&&) {return *this;}
@@ -48,6 +51,7 @@ public:
std::vector<MatchEffect> *);
Book book(MDP3SecurityID, api::Side);
template<class TIter>
void init(TIter first, TIter last) {
while (first != last) {
@@ -89,6 +93,7 @@ public:
std::vector<MatchEffect> *);
Book book(MDP3SecurityID, api::Side);
ImpliedBook implied_book(MDP3SecurityID, api::Side);
};
}
@@ -0,0 +1,26 @@
#include "MatchingCommon.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
namespace bts::spartan::mdp3 {
std::tuple<uint32_t, uint32_t, uint32_t> MatcherOrder::cross(uint32_t cross_qty) {
auto [r, c] = compute_cross(quantity.value(), cross_qty);
bts_assert(quantity.value() - r == cross_qty - c);
auto filled = cross_qty - c;
quantity = MDP3Quantity(r);
if (ib.has_value()) {
ib.value().total -= MDP3Quantity(filled);
is_active = ib.value().total.value() > 0;
} else
is_active = quantity.value() > 0;
return std::make_tuple(r, c, filled);
}
void MatcherOrder::reload_ib() {
if (ready_to_load_ib()) {
is_active = ib.value().total.value() > 0;
if (quantity.value() == 0)
quantity = std::min(ib.value().show, ib.value().total);
is_active = quantity.value() > 0;
}
}
} // namespace bts::spartan::mdp3
@@ -4,6 +4,7 @@
// the templated classes (will fix later)
#include "../../common/InstrumentSnapshot.h"
#include "../simulator/AlgoEvent.h"
#include <optional>
namespace bts::spartan::mdp3 {
@@ -34,8 +35,7 @@ public:
"Matcher incoming change:{} ok:{} side:{} price:{} qty:{} ib:{}",
c.change, c.order.key, _side(c.order.side), c.order.price,
c.order.quantity,
c.order.iceberg_quantity.has_value() ?
c.order.iceberg_quantity.value().value() : 0);
c.order.iceberg_quantity);
}
void log_outgoing_update(const OrderUpdate& u){
if(traced_orders.find(u.key) != traced_orders.end())
@@ -61,12 +61,6 @@ public:
}
};
struct MatcherIB {
bool is_ib = false;
MDP3Quantity show_qty = MDP3Quantity(0);
MDP3Quantity total_qty = MDP3Quantity(0);
};
struct MatcherOrder {
static std::tuple<uint32_t, uint32_t> compute_cross(uint32_t resting_qty,
uint32_t cross_qty) {
@@ -85,14 +79,14 @@ struct MatcherOrder {
md::IntPrice price;
MDP3Quantity quantity;
uint32_t priority = 0;
MatcherIB ib;
std::optional<IcebergQuantity> ib;
bool is_active = true;
MatcherOrder(MDP3OrderKey key,
api::Side side,
md::IntPrice price,
MDP3Quantity qty, // working qty out there
uint32_t priority,
MatcherIB ib)
std::optional<IcebergQuantity> ib = {})
:
key(key),
side(side),
@@ -100,58 +94,23 @@ struct MatcherOrder {
quantity(qty),
priority(priority),
ib(ib) {}
MatcherOrder(MDP3OrderKey key,
api::Side side,
md::IntPrice price,
MDP3Quantity qty, // working qty out there
uint32_t priority)
:
MatcherOrder(key,
side,
price,
qty,
priority,
MatcherIB()) {
}
// returns the remainder of this order, remainder of opp order
// (ie the new qty NOT the filled qty, for that subtract from
// original)
std::tuple<uint32_t, uint32_t, uint32_t> cross(uint32_t cross_qty) {
auto [r, c] =
compute_cross(quantity.value(), cross_qty);
bts_assert(quantity.value() - r == cross_qty - c);
auto filled = cross_qty - c;
quantity = MDP3Quantity(r);
if (ib.is_ib){
ib.total_qty.value() = ib.total_qty.value() - filled;
is_active = ib.total_qty.value() > 0;
}
else
is_active = quantity.value() > 0;
return std::make_tuple(r, c, filled);
}
std::tuple<uint32_t, uint32_t, uint32_t> cross(uint32_t cross_qty);
MDP3Quantity total_qty() const {
return ib.is_ib ? ib.total_qty : quantity;
return ib.has_value() ? ib.value().total : quantity;
}
MDP3Quantity working_qty() const { return quantity; }
bool ready_to_load_ib() const {
return ib.is_ib && quantity.value() == 0 && ib.total_qty.value() > 0;
}
void reload_ib() {
if (ready_to_load_ib()) {
auto load_qty = std::min(ib.show_qty.value(), ib.total_qty.value());
quantity = MDP3Quantity(load_qty);
is_active = true;
}
return ib.has_value() && quantity.value() == 0 && ib.value().total.value() > 0;
}
void reload_ib();
};
inline std::ostream& operator <<(std::ostream& out, const MatcherOrder& ord) {
out << "[" << ord.key << "] " << ord.side << " " << ord.quantity << " @ " << ord.price << " (ice ";
if (ord.ib.is_ib)
out << "qty " << ord.ib.total_qty;
else out << "n/a";
out << " active " << (ord.is_active ? "Y" : "N");
out << "[" << ord.key << "] " << ord.side << " " << ord.quantity << " @ " << ord.price
<< " (ice " << ord.ib << " active " << (ord.is_active ? "Y" : "N") << ")";
return out;
}
@@ -0,0 +1,137 @@
#include "iceberg_helper.h"
namespace bts::spartan::mdp3::iceberg_helper {
template <typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
IcebergQuantity to_iceberg_qty(const DeducedIceberg& dib) {
// initializer lists with diff fields of the same type seem dangerous
// ...what if the declaration order changes? accidents happen.
IcebergQuantity ib_qty;
ib_qty.show = dib.show_quantity;
ib_qty.total = dib.estimated_quantity;
return ib_qty;
}
void add_iceberg_annotation(
MDP3Order& order,
const flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
const flat_map<MDP3OrderKey, DeducedIceberg>& overrides) {
auto it = icebergs.find(order.key);
if (it != icebergs.end()) {
auto over_it = overrides.find(order.key);
auto ibq = to_iceberg_qty(it->second);
if (over_it != overrides.end())
ibq = to_iceberg_qty(over_it->second);
if (order.iceberg_quantity != ibq)
order.iceberg_quantity = ibq;
// adjust order's actual quantity in accordance
// wih the new iceberg quantity
order.quantity = order.quantity.value() == 0
? std::min(ibq.show, ibq.total)
: std::min(order.quantity, std::min(ibq.show, ibq.total));
}
}
void add_iceberg_annotation(
MDP3Order& order, const flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
flat_map<MDP3OrderKey, DeducedIceberg> overrides;
add_iceberg_annotation(order, icebergs, overrides);
}
void detect_show_qty_changes(
const MDP3Event& ev,
flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
flat_map<MDP3OrderKey, DeducedIceberg>& show_qty_changes) {
std::set<MDP3OrderKey> ibs_with_fills;
for (const auto* s : ev.affected_securities) {
for (auto f : s->fills()) {
auto ib_it = icebergs.find(f.key);
if (ib_it != icebergs.end())
ibs_with_fills.insert(f.key);
}
}
// the show qty can change, most of these changes
// can be detected simply by looking for order updates
// that change the qty, when there are no fills
for (auto s : ev.affected_securities) {
for (auto ch : s->order_changes()) {
switch (ch.change) {
case OrderChange::ChangeType::updated: {
// if the qty changed without any fills,
// then the show qty must have changed.
if (ch.order.quantity == ch.previous_order.quantity)
continue;
auto ib_it = icebergs.find(ch.order.key);
if (ib_it == icebergs.end())
continue;
if (ibs_with_fills.find(ch.order.key) != ibs_with_fills.end())
continue;
auto new_dib = ib_it->second;
new_dib.estimated_quantity -= (new_dib.show_quantity - ch.order.quantity);
new_dib.show_quantity = ch.order.quantity;
show_qty_changes.insert_or_assign(new_dib.key, new_dib);
auto tmp = ib_it->second;
ib_it->second.estimated_quantity +=
(ch.order.quantity - ch.previous_order.quantity);
PRINT_LINE("adjusting est qty: ", tmp, " --> ", ib_it->second);
break;
}
default: break;
}
}
}
}
void detect_show_qty_changes(
OrderChange& ch,
flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
// the show qty can change, most of these changes
// can be detected simply by looking for order updates
// that change the qty, when there are no fills
switch (ch.change) {
case OrderChange::ChangeType::updated: {
// if the qty changed without any fills,
// then the show qty must have changed.
if (ch.order.quantity == ch.previous_order.quantity)
break;
auto ib_it = icebergs.find(ch.order.key);
if (ib_it == icebergs.end())
break;
auto qty_diff = ch.order.quantity - ch.previous_order.quantity;
ib_it->second.estimated_quantity += qty_diff;
ib_it->second.show_quantity = ch.order.quantity;
break;
}
default: break;
}
}
void subtract_iceberg_fills(
const MDP3Event& ev, flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
for (const auto* s : ev.affected_securities) {
for (auto f : s->fills()) {
auto ib_it = icebergs.find(f.key);
if (ib_it != icebergs.end())
ib_it->second.estimated_quantity -= f.quantity;
}
}
}
void add_iceberg_annotation(
const InstrumentSnapshot& s,
Orders& orders,
const flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
//
for (auto ch : s.order_changes()) {
switch (ch.change) {
case OrderChange::added:
case OrderChange::updated:
add_iceberg_annotation(ch.order, icebergs);
orders[ch.order.key] = ch.order;
break;
case OrderChange::removed: orders.erase(ch.order.key); break;
}
}
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,61 @@
#pragma once
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoMatcherContext.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/matcher_diagnostics.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/security_comp.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include <boost/container/flat_map.hpp>
#include <functional>
#include <iostream>
namespace bts::spartan::mdp3::iceberg_helper {
IcebergQuantity to_iceberg_qty(const DeducedIceberg&);
void add_iceberg_annotation(
MDP3Order& order,
const boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
const boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& overrides);
void add_iceberg_annotation(
MDP3Order& order,
const boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
void detect_show_qty_changes(
const MDP3Event& ev,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& show_qty_changes);
void detect_show_qty_changes(
OrderChange& change,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
void subtract_iceberg_fills(
const MDP3Event& ev,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs);
template <class TIter>
inline void subtract_iceberg_fills(
TIter first,
TIter last,
boost::container::flat_map<MDP3OrderKey, DeducedIceberg>& icebergs) {
while (first != last) {
auto it = icebergs.find(first->key);
if (it != icebergs.end()) {
auto orig_qty = it->second.estimated_quantity.value();
it->second.estimated_quantity.value() -= first->quantity.value();
bts_assert(icebergs[first->key].estimated_quantity.value() < orig_qty);
}
first++;
}
}
} // namespace bts::spartan::mdp3::iceberg_helper
@@ -1,12 +1,13 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/md/IntPrice.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include <functional>
#include <iostream>
#include <range/v3/numeric.hpp>
@@ -108,8 +109,8 @@ inline void print_order_changes(TIter first, TIter last, std::ostream& out = std
{" N Price ", 12, [](auto ch){ return FMT(ch.order.price.price()); }},
{" O Qty ", 8, [](auto ch){return FMT(ch.previous_order.quantity); }},
{" N Qty ", 8, [](auto ch){ return FMT(ch.order.quantity); }},
{" O Ice ", 8, [](auto ch){ return FMT(ch.previous_order.iceberg_quantity.has_value() ? ch.previous_order.iceberg_quantity.value().value() : 0); }},
{" N Ice ", 8, [](auto ch){ return FMT(ch.order.iceberg_quantity.has_value() ? ch.order.iceberg_quantity.value().value() : 0); }}});
{" O Ice ", 8, [](auto ch){ return FMT(ch.previous_order.iceberg_quantity); }},
{" N Ice ", 8, [](auto ch){ return FMT(ch.order.iceberg_quantity); }}});
out << std::endl;
}
@@ -3,6 +3,7 @@
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h"
#include <google/protobuf/timestamp.pb.h>
#include <google/protobuf/util/time_util.h>
#include <optional>
#include <random>
#include "../MatchingCommon.h"
@@ -31,18 +32,14 @@ public:
MDP3Order create_order(uint64_t key, int32_t sec, api::Side side,
uint64_t priority, double price, uint32_t qty,
MatcherIB ib = {false, MDP3Quantity(0),
MDP3Quantity(0)}) {
std::optional<IcebergQuantity> ib = { }) {
auto rv = MDP3Order{MDP3OrderKey(key),
MDP3SecurityID(sec),
side,
MDP3OrderPriority(priority),
md::IntPrice_from_double<7>(price),
MDP3Quantity(qty),
ib.is_ib ? std::make_optional<MDP3Quantity>(ib.total_qty)
: std::nullopt};
// if(ib.is_ib)
// rv.iceberg_quantity = std::make_optional<MDP3Quantity>(ib.total_qty);
ib};
return rv;
}
@@ -155,7 +152,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher]") {
// iceberg, show qty=2, total_qty=100
o = create_order(
k++, sec_key, api::Side::SIDE_BUY, 0, price, 2,
MatcherIB{true, MDP3Quantity(2), MDP3Quantity(100)});
IcebergQuantity{MDP3Quantity(2), MDP3Quantity(100)});
price_level.on_new_same_side_order(o);
// non ib, to get the visible qty to 15
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 7);
@@ -437,8 +434,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") {
// send a new sell at 15.07 to take the first 3 levels
auto cross_ib =
create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.07, 13,
MatcherIB{true, MDP3Quantity(13),
MDP3Quantity(12500)});
IcebergQuantity{MDP3Quantity(13), MDP3Quantity(12500)});
sec_me.on_order_change(OrderChange{MDP3Order{},cross_ib,OrderChange::added});
bid_bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bid_bk.at(0), 15.06, 2485, 70);
@@ -15,6 +15,7 @@
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/iceberg_helper.h"
#include <boost/container/flat_map.hpp>
#include <functional>
#include <iostream>
@@ -26,15 +27,15 @@
// to avoid duplicate, and (perhaps more importantly) unsynchronized output.
#define _ECHO_ON
#ifdef _ECHO_ON
#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__)
#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__);
#define _ERROR_LEVEL Info
#else
#define _ECHO(...)
#define _ERROR_LEVEL Error
#endif
#define _LOG(level, msg) \
_ECHO(_log_header(), " ", msg, "\n"); \
//LOG(OrderMan, level, "{} {}", _log_header(), msg)
_ECHO(_log_header(), " ", msg, "\n") \
LOG(OrderMan, level, "{} {}", _log_header(), msg)
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define ERROR(...) _LOG(_ERROR_LEVEL, bts::orderman::Formatter::fmt(__VA_ARGS__))
@@ -44,6 +45,7 @@
#define _SUMMARIZE_TRANSACTION_NAMED(name, ...) SUMMARIZE_TRANSACTION_NAMED(name, _log_header(), FMT(__VA_ARGS__))
namespace bts::spartan::mdp3 {
using namespace bts::spartan::mdp3::iceberg_helper;
// aliases
using TradingStatus = MatcherValidationObserver::TradingStatus;
template<typename K, typename V>
@@ -71,13 +73,10 @@ void _sync_orders(
for (auto ch : s.order_changes()) {
switch (ch.change) {
case OrderChange::added:
case OrderChange::updated: {
auto it = icebergs.find(ch.order.key);
if (it != icebergs.end())
ch.order.iceberg_quantity = it->second.estimated_quantity;
case OrderChange::updated:
add_iceberg_annotation(ch.order, icebergs);
orders[ch.order.key] = ch.order;
break;
}
case OrderChange::removed: orders.erase(ch.order.key); break;
}
}
@@ -197,7 +196,8 @@ struct MatcherValidationObserver::PImpl {
size_t event_id = NO_EVENT;
FifoMatcherContext exch_matcher;
Verbosity verbosity = Verbosity::Normal;
EventFlags ignore_flags = EventFlags::IcebergOrders | EventFlags::MultipleSecurityAggressors;
//EventFlags ignore_flags = EventFlags::IcebergOrders | EventFlags::MultipleSecurityAggressors;
EventFlags ignore_flags = EventFlags::MultipleSecurityAggressors;
// event_id is used to transition the security's trading status from
// Preopen to Uninitialized
@@ -226,11 +226,8 @@ struct MatcherValidationObserver::PImpl {
data.snapshot = &s;
data.last_observed_event = event_id;
data.previous_orders = s.orders();
for (auto& o : data.previous_orders) {
auto it = icebergs.find(o.first);
if (it != icebergs.end())
o.second.iceberg_quantity = it->second.estimated_quantity;
}
for (auto& o : data.previous_orders)
add_iceberg_annotation(o.second, icebergs);
// the first event at the open isn't processed bc
// we don't have a clear understanding of how preopen matching works.
data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade
@@ -277,42 +274,6 @@ MatcherValidationObserver::MatcherValidationObserver(
MatcherValidationObserver::~MatcherValidationObserver() {}
///////////////////////
// iceberg annotation
///////////////////////
void MatcherValidationObserver::_add_iceberg_annotation(MDP3Order& order) {
auto it = _me->icebergs.find(order.key);
if (it != _me->icebergs.end())
order.iceberg_quantity = it->second.estimated_quantity;
}
void MatcherValidationObserver::_add_iceberg_annotation(Orders& orders) {
for (auto& o : orders)
_add_iceberg_annotation(o.second);
}
void MatcherValidationObserver::_add_iceberg_annotation(OrderChanges& changes) {
for (auto& o : changes) {
_add_iceberg_annotation(o.previous_order);
_add_iceberg_annotation(o.order);
}
}
void MatcherValidationObserver::_add_iceberg_annotations(
const InstrumentSnapshot& s, std::unordered_map<MDP3OrderKey, MDP3Order>& target) {
// Update our order cache so we can use it to
// populate the matcher on the next event
for (auto ch : s.order_changes()) {
switch (ch.change) {
case OrderChange::added:
case OrderChange::updated:
_add_iceberg_annotation(ch.order);
target[ch.order.key] = ch.order;
break;
case OrderChange::removed: target.erase(ch.order.key); break;
}
}
}
///////////////////////
// misc private methods
///////////////////////
@@ -361,34 +322,24 @@ void MatcherValidationObserver::_reset_matcher() {
void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap) {
_SUMMARIZE;
for (auto s : snap) {
auto annotated_orders = s->orders();
_add_iceberg_annotation(annotated_orders);
auto& sec_data = _me->get_security_data(*s, NO_EVENT);
if (s->trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade)
sec_data.status = TradingStatus::Uninitialized;
sec_data.current_orders.clear();
_sync_orders(*s, sec_data.current_orders, _me->icebergs);
sec_data.previous_orders = sec_data.current_orders;
}
}
void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
bts_assert(ev.event_id > _me->event_id);
_me->event_id = ev.event_id;
// uncomment code below for a quick way to track an order's state
// on every update
for (auto& kv : _me->interesting_orders) {
for (auto s : ev.affected_securities) {
auto it = s->orders().find(kv.first);
if (it != s->orders().end() && !orders_equal(it->second, kv.second)) {
INFO("Interesting Order (event ", ev.event_id, "): ", it->second);
kv.second = it->second;
}
}
}
switch (get_event_ordinal(_me->interesting_event_ranges, ev.event_id, MIN_OBSERVED_EVENT)) {
case bts::spartan::EventOrdinal::Waiting:
// just need to synchronize the order caches.
// the matcher is only validated during 'interesting events.'
_sync_current_orders(ev);
_sync_previous_orders(ev);
subtract_iceberg_fills(ev, _me->icebergs);
detect_show_qty_changes(ev, _me->icebergs, _me->icebergs);
return;
case bts::spartan::EventOrdinal::Interesting:
_sync_current_orders(ev);
@@ -406,6 +357,8 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags));
_reset_matcher();
_sync_previous_orders(ev);
subtract_iceberg_fills(ev, _me->icebergs);
detect_show_qty_changes(ev, _me->icebergs, _me->icebergs);
return;
}
++_me->interesting_event_count;
@@ -453,8 +406,10 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
}
// securities must be added to the matcher before the order
// changes are processed to ensure the correct implied markets are used.
for (const auto* s : sorted_secs)
for (const auto* s : sorted_secs) {
_me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs());
_me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching());
}
// initialize matcher with previous state (only applies
// to securities whose orders the matcher hasn't seen yet).
for (const auto* s : sorted_secs) {
@@ -462,42 +417,53 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
continue;
auto it = _me->security_cache.find(s->security_id());
bts_assert(it != _me->security_cache.end());
_SUMMARIZE_TRANSACTION(
"restoring ", s->symbol(), " orders(", it->second.previous_orders.size(), ")");
for (auto& kv : it->second.previous_orders)
add_iceberg_annotation(kv.second, _me->icebergs);
auto prev_ords = it->second.previous_orders |
ranges::views::transform([](auto p) { return p.second; });
ranges::views::transform([](auto& p) { return p.second; });
_me->exch_matcher.reset(s->security_id(), prev_ords.begin(), prev_ords.end());
_me->security_cache[s->security_id()].status = TradingStatus::Initialized;
}
// capture iceberg show qty changes (these are a special case)
flat_map<MDP3OrderKey, DeducedIceberg> show_qty_changes;
detect_show_qty_changes(ev, _me->icebergs, show_qty_changes);
// process changes with matcher
FillDepot fills;
for (const auto* s : sorted_secs) {
auto num_fills_before = fills.fill_count();
switch (_trading_status(*s, ev.event_id)) {
case TradingStatus::Initialized:
case TradingStatus::Uninitialized: break;
case TradingStatus::Initialized: break;
default: continue;
}
auto deduced = s->deduced_order_changes();
if (deduced.size()) {
std::vector<SmallOrderChange> deduced_sm;
for (auto& ch : deduced)
for (auto& ch : deduced) {
add_iceberg_annotation(ch.order, _me->icebergs, show_qty_changes);
auto it = show_qty_changes.find(ch.order.key);
if (it != show_qty_changes.end()) {
auto exp_ibq = to_iceberg_qty(it->second);
PRINT_LINE("expecting this: ", exp_ibq, ", getting this: ", ch.order.iceberg_quantity.value());
bts_assert(ch.order.iceberg_quantity.has_value() && ch.order.iceberg_quantity.value() == exp_ibq);
}
deduced_sm.emplace_back(ch);
_SUMMARIZE_TRANSACTION(
"processing ", s->symbol(), " order changes(", deduced_sm.size(), ")");
}
_me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills);
if (_me->verbosity > Verbosity::Normal)
INFO("matcher generated ", fills.fill_count() - num_fills_before, " fills for ", s->symbol());
}
}
// verify the matcher's orders are identitical to the exchange's
for (auto ib : show_qty_changes)
_me->icebergs[ib.first] = ib.second;
// Verify the matcher's orders are identitical to the exchange's.
// subtract the fills from iceberg orders' total quantities
// to get the correct expected orders for comparison with the matcher's orders.
subtract_iceberg_fills(ev, _me->icebergs);
bool orders_match = true;
for (const auto* s : sorted_secs) {
if (!_is_ready_to_trade(*s, ev.event_id))
continue;
_SUMMARIZE_TRANSACTION_NAMED(_incr_val, "validating order changes for ", s->symbol());
auto& sec_data = _me->security_cache[s->security_id()];
auto& curr_orders = sec_data.current_orders;
for (auto& kv : curr_orders)
add_iceberg_annotation(kv.second, _me->icebergs);
auto error = _me->exch_matcher.has_same_orders(s->security_id(), curr_orders);
if (error) {
ERROR(
@@ -509,45 +475,44 @@ void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
}
bool fills_match = match_fills(fills, sorted_secs);
if (!orders_match || !fills_match) {
// todo: add the iceberg fills back, to get to the previous state.
// describe current state for diagnostic purposes
if (_me->verbosity > Verbosity::Normal) {
std::stringstream msg;
msg << "Spartan Top of Books:" << std::endl;
_me->exch_matcher.print_all_bbos(msg);
msg << std::endl;
// spartan fills
msg << "Spartan Total Fill Qty: " << fills.total_fill_qty() << std::endl;
auto fill_cmp = [](const auto& lhs, const auto& rhs) {
return lhs.trade_entry_id == rhs.trade_entry_id
? static_cast<int>(lhs.aggressor) > static_cast<int>(rhs.aggressor)
: lhs.trade_entry_id < rhs.trade_entry_id;
};
std::vector<MDP3Fill> spartan_fills;
fills.copy_to(spartan_fills);
std::sort(spartan_fills.begin(), spartan_fills.end(), fill_cmp);
msg << "Spartan Fills (ordered by Trade ID):" << std::endl;
diagnostics::print_fills(spartan_fills.begin(), spartan_fills.end(), msg);
// exch top of books
msg << "Exchange Top of Books:" << std::endl;
diagnostics::print_many_bbo(
sorted_secs.begin(), sorted_secs.end(), true, true, msg);
int exch_fill_qty = 0;
for (auto* s : sorted_secs) {
for (auto f : s->fills())
exch_fill_qty += f.quantity.value();
}
// exch fills
msg << "Exchange Total Fill Qty: " << exch_fill_qty << std::endl;
std::vector<MDP3Fill> all_exch_fills;
for (auto* s : sorted_secs) {
for (auto f : s->fills())
all_exch_fills.push_back(f);
}
std::sort(all_exch_fills.begin(), all_exch_fills.end(), fill_cmp);
msg << "Exchange Fills (ordered by Trade ID):" << std::endl;
diagnostics::print_fills(all_exch_fills.begin(), all_exch_fills.end(), msg);
INFO(msg.str());
std::stringstream msg;
msg << "Spartan Top of Books:" << std::endl;
_me->exch_matcher.print_all_bbos(msg);
msg << std::endl;
// spartan fills
msg << "Spartan Total Fill Qty: " << fills.total_fill_qty() << std::endl;
auto fill_cmp = [](const auto& lhs, const auto& rhs) {
return lhs.trade_entry_id == rhs.trade_entry_id
? static_cast<int>(lhs.aggressor) > static_cast<int>(rhs.aggressor)
: lhs.trade_entry_id < rhs.trade_entry_id;
};
std::vector<MDP3Fill> spartan_fills;
fills.copy_to(spartan_fills);
std::sort(spartan_fills.begin(), spartan_fills.end(), fill_cmp);
msg << "Spartan Fills (ordered by Trade ID):" << std::endl;
diagnostics::print_fills(spartan_fills.begin(), spartan_fills.end(), msg);
// exch top of books
msg << "Exchange Top of Books:" << std::endl;
diagnostics::print_many_bbo(
sorted_secs.begin(), sorted_secs.end(), true, true, msg);
int exch_fill_qty = 0;
for (auto* s : sorted_secs) {
for (auto f : s->fills())
exch_fill_qty += f.quantity.value();
}
// exch fills
msg << "Exchange Total Fill Qty: " << exch_fill_qty << std::endl;
std::vector<MDP3Fill> all_exch_fills;
for (auto* s : sorted_secs) {
for (auto f : s->fills())
all_exch_fills.push_back(f);
}
std::sort(all_exch_fills.begin(), all_exch_fills.end(), fill_cmp);
msg << "Exchange Fills (ordered by Trade ID):" << std::endl;
diagnostics::print_fills(all_exch_fills.begin(), all_exch_fills.end(), msg);
INFO(msg.str());
// reset the matcher to avoid cascading/compounding errors in subsequent events.
_reset_matcher();
}
@@ -47,12 +47,6 @@ struct MatcherValidationObserver : IChannelObserver {
struct PImpl;
std::unique_ptr<PImpl> _me;
void _add_iceberg_annotation(MDP3Order& order);
void _add_iceberg_annotation(Orders& orders);
void _add_iceberg_annotation(OrderChanges& changes);
void _add_iceberg_annotations(
const InstrumentSnapshot& s, std::unordered_map<MDP3OrderKey, MDP3Order>& target);
TradingStatus _trading_status(const InstrumentSnapshot& s, size_t event_id);
// returns true if the security's TradingStatus is 'Uninitialized', or 'Initialized'
// (in other words, if it isn't 'None,' or 'Preopen')
@@ -70,7 +70,7 @@ protos::SimInput to_proto(SimInput const &in) {
schange->set_quantity(change.order.quantity.value());
if (change.order.iceberg_quantity)
schange->set_iceberg_quantity(
change.order.iceberg_quantity->value());
change.order.iceberg_quantity->total.value());
}
}
@@ -31,4 +31,4 @@ class GatewayAgent : public sim::AgentBase {
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
} // namespace bts::spartan::mdp3
@@ -81,6 +81,23 @@ inline std::ostream &operator<<(std::ostream &s, BookLevel const &l) {
return s;
}
struct IcebergQuantity {
MDP3Quantity show;
MDP3Quantity total;
};
inline std::ostream& operator<<(std::ostream& out, const IcebergQuantity& ibq) {
out << ibq.show << "/" << ibq.total;
return out;
}
inline std::ostream& operator<<(std::ostream& out, const std::optional<IcebergQuantity>& ibq) {
if (ibq.has_value())
out << ibq.value();
else out << "n/a";
return out;
}
struct MDP3Order {
public:
operator bool() const noexcept { return !empty(); }
@@ -93,7 +110,7 @@ struct MDP3Order {
MDP3OrderPriority priority;
md::IntPrice price;
MDP3Quantity quantity;
std::optional<MDP3Quantity> iceberg_quantity;
std::optional<IcebergQuantity> iceberg_quantity;
};
struct OrderChange {
@@ -136,8 +153,7 @@ inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) {
s << " k: " << order.key.value() << " sk: " << order.security.value()
<< " side: " << securities::side_name(order.side)
<< " price: " << order.price.price() << " qty: " << order.quantity.value()
<< " pri: " << order.priority.value() << " ice?: "
<< (order.iceberg_quantity ? order.iceberg_quantity.value().value() : 0);
<< " pri: " << order.priority.value() << " ice?: " << order.iceberg_quantity;
return s;
}
@@ -2,6 +2,9 @@
#include "mdp3_common.h"
namespace bts::spartan::mdp3 {
///////////////////////
// MDP3Quantity
///////////////////////
// add
inline MDP3Quantity operator+(const MDP3Quantity& lhs, const MDP3Quantity& rhs) {
return MDP3Quantity(lhs.value() + rhs.value());
@@ -37,6 +40,14 @@ inline MDP3Quantity& operator/=(MDP3Quantity& lhs, const MDP3Quantity& rhs) {
lhs = MDP3Quantity(lhs.value() / rhs.value());
return lhs;
}
// modulus
inline MDP3Quantity operator%(const MDP3Quantity& lhs, const MDP3Quantity& rhs) {
return MDP3Quantity(lhs.value() % rhs.value());
}
inline MDP3Quantity& operator%=(MDP3Quantity& lhs, const MDP3Quantity& rhs) {
lhs = MDP3Quantity(lhs.value() % rhs.value());
return lhs;
}
// greater/less than or equal to
inline bool operator>=(const MDP3Quantity& lhs, const MDP3Quantity& rhs) {
return lhs.value() >= rhs.value();
@@ -49,4 +60,26 @@ inline bool operator<=(const MDP3Quantity& lhs, const MDP3Quantity& rhs) {
inline MDP3Quantity abs(MDP3Quantity qty) {
return qty.value() < 0 ? MDP3Quantity(-qty.value()) : qty;
}
///////////////////////
// IcebergQuantity
///////////////////////
inline bool operator==(const IcebergQuantity& lhs, const IcebergQuantity& rhs) {
return lhs.show.value() == rhs.show.value() && lhs.total.value() == rhs.total.value();
}
inline bool operator!=(const IcebergQuantity& lhs, const IcebergQuantity& rhs) {
return !(lhs == rhs);
}
inline bool operator==(const IcebergQuantity& lhs, const std::optional<IcebergQuantity>& rhs) {
return rhs.has_value() && lhs == rhs.value();
}
inline bool operator==(const std::optional<IcebergQuantity>& lhs, const IcebergQuantity& rhs) {
return rhs == lhs;
}
inline bool operator==(const std::optional<IcebergQuantity>& lhs, const std::optional<IcebergQuantity>& rhs) {
return lhs.has_value() ? rhs.has_value() && lhs.value() == rhs.value() : !rhs.has_value();
}
inline bool operator!=(const std::optional<IcebergQuantity>& lhs, const std::optional<IcebergQuantity>& rhs) {
return !(lhs == rhs);
}
}
@@ -0,0 +1,16 @@
#include "order_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/MatchingCommon.h"
namespace bts::spartan::mdp3 {
MDP3Order to_mdp3_order(const MatcherOrder& mo) {
MDP3Order rv;
rv.key = mo.key;
rv.side = mo.side;
rv.quantity = mo.quantity;
// MatcherOrder.priority doesn't map back to the exchange's MDP3Order.priority.
// Set it to zero so nobody thinks it does.
rv.priority = MDP3OrderPriority(0);
rv.iceberg_quantity = mo.ib;
return rv;
}
}
@@ -2,7 +2,9 @@
#include "lib/md/IntPrice.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include <iostream>
#include <optional>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#include <sstream>
@@ -16,15 +18,51 @@ inline md::IntPrice get_price(const std::tuple<md::IntPrice, MDP3Quantity>& pric
inline MDP3Quantity get_qty(const std::tuple<md::IntPrice, MDP3Quantity>& price_level) {
return std::get<1>(price_level);
}
inline MDP3Quantity get_total_qty(const MDP3Order& order) {
return order.iceberg_quantity.has_value() ? order.iceberg_quantity.value().total
: order.quantity;
}
// all order quantity props should contain the original values (before subtracting fills, or any other qty updates, etc.)
inline void set_total_qty(MDP3Order& order, MDP3Quantity total_qty, bool reset_show_qty = false) {
// bool verbose = order.key.value() = 647354912656;
bts_assert(total_qty.value() >= 0);
if (order.iceberg_quantity.has_value()) {
auto ibq = order.iceberg_quantity.value();
int ordinal0 = ibq.total.value() + (ibq.show.value() - order.quantity.value());
int mod_offset = ordinal0 % ibq.show.value();
bts_assert(mod_offset >= 0);
int adj = 0;
if (mod_offset > 0) {
adj = ibq.show.value() - mod_offset;
ordinal0 += (ibq.show.value() - mod_offset);
}
ibq.total = total_qty;
if (reset_show_qty) {
order.quantity = std::min(order.iceberg_quantity.value().show, total_qty);
} else {
order.quantity = order.iceberg_quantity.value().show < total_qty
? (total_qty + MDP3Quantity(adj)) % order.iceberg_quantity.value().show
: total_qty;
if (order.quantity.value() == 0)
order.quantity = std::min(total_qty, order.iceberg_quantity.value().show);
}
order.iceberg_quantity.value().total = total_qty;
} else
order.quantity = total_qty;
}
inline MDP3Quantity book_volume(const std::vector<std::tuple<md::IntPrice, MDP3Quantity>>& book) {
auto get_qty = [](const auto& bl) { return std::get<1>(bl).value(); };
return MDP3Quantity(ranges::accumulate(book | ranges::views::transform(get_qty), 0));
}
struct MatcherOrder;
MDP3Order to_mdp3_order(const MatcherOrder&);
inline bool orders_equal(const MDP3Order& lhs, const MDP3Order& rhs) {
return lhs.key == rhs.key && lhs.side == rhs.side &&
lhs.price == rhs.price && lhs.quantity == rhs.quantity;
lhs.price == rhs.price && lhs.quantity == rhs.quantity &&
lhs.iceberg_quantity == rhs.iceberg_quantity;
}
inline std::string
@@ -8,7 +8,16 @@ IcebergFinder::~IcebergFinder() {}
struct IcebergFinder::OrderData {
bool is_iceberg = false;
// "Original quantity" is synonomous with "show quantity."
// The exchange does not allow traders to adjust this property;
// any quantity updates to an iceberg order apply to the total quantity
int show_qty = 0;
size_t accumulated_fills = 0;
// count qty changes in events where the order has no fills.
// this is needed to correctly estimated the qty in cases where
// the order is canceled before being filled to completion
size_t accumulated_qty_changes = 0;
size_t last_observed_qty = 0;
bool filled = false;
};
@@ -28,12 +37,13 @@ void IcebergFinder::finit() {
if (od.is_iceberg) {
auto &di = result[k];
di.key = k;
di.estimated_quantity.value() = od.accumulated_fills;
di.estimated_quantity.value() = (od.accumulated_fills + od.last_observed_qty);
di.show_quantity = MDP3Quantity(od.show_qty);
di.filled = od.filled;
// If we have not seen an iceberg order fill completely then
// all we can say is that its hidden size is at least one
// contract bigger than the observed fills.
if (!od.filled)
if (!od.filled && od.last_observed_qty == 0)
di.estimated_quantity.value() += 1;
}
}
@@ -64,14 +74,49 @@ void IcebergFinder::on_real_time_event(MDP3Event const &ev) {
// all fills, we also aggregate passive fills,
// which we use below to identify icebergs.
for (auto s : ev.affected_securities) {
for (auto const &o : s->order_changes()) {
for (auto const& o : s->order_changes()) {
me->changes_in_event[o.order.key] = o;
if (o.change == OrderChange::removed) {
for (auto const &f : s->fills())
if (f.key == o.order.key) {
me->order_data[o.order.key].filled = true;
break;
switch (o.change) {
case OrderChange::ChangeType::added:
me->order_data[o.order.key].show_qty = o.order.quantity.value();
me->order_data[o.order.key].last_observed_qty = o.order.quantity.value();
break;
case OrderChange::ChangeType::updated: {
// While this isn't perfect, it should be correct
// for the vast majority of ibs: assume the largest observed
// is quantity is the show quantity.
auto curr_qty = me->order_data[o.order.key].show_qty;
if (o.order.quantity.value() > curr_qty)
me->order_data[o.order.key].show_qty = o.order.quantity.value();
if (o.order.quantity.value() != o.previous_order.quantity.value()) {
bool has_fills = false;
for (const auto& f : s->fills()) {
if (f.key == o.order.key) {
has_fills = true;
break;
}
}
if (!has_fills) {
// must be an actual qty change by the trader
// these quantities should be excluded from the
// [initial]estimatd quantity, the caller observing
// the MDP3Events can adjust the quantity according
// to those changes as they arrive.
me->order_data[o.order.key].accumulated_qty_changes -= (o.order.quantity.value() - o.previous_order.quantity.value());
}
}
me->order_data[o.order.key].last_observed_qty = o.order.quantity.value();
break;
}
case OrderChange::ChangeType::removed:
for (auto const& f : s->fills())
if (f.key == o.order.key) {
me->order_data[o.order.key].filled = true;
me->order_data[o.order.key].last_observed_qty = 0;
break;
}
break;
default: break;
}
}
for (auto const &f : s->fills()) {
@@ -9,6 +9,7 @@ namespace bts::spartan::mdp3 {
struct DeducedIceberg {
MDP3OrderKey key;
MDP3Quantity show_quantity;
MDP3Quantity estimated_quantity;
bool filled;
};
@@ -28,4 +29,10 @@ struct IcebergFinder : IChannelObserver {
std::unique_ptr<PImpl> me;
};
inline std::ostream& operator <<(std::ostream& out, const DeducedIceberg& ib) {
out << "key " << ib.key << ", show_qty " << ib.show_quantity << ", est_qty "
<< ib.estimated_quantity << ", filled " << (ib.filled ? "Y" : "N");
return out;
}
} // namespace bts::spartan::mdp3