Refactor.

This commit is contained in:
Pedro Pinto committed 2020-12-29 20:01:48 -05:00
1 parent bb1e7df7a9
commit ae7b25e120
62 files changed
+9255

No files matched your search

@@ -0,0 +1,172 @@
#include "PolyObserver.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
namespace bts::spartan::mdp3 {
static auto constexpr Worthless = -1.0;
struct PolyObserver::Target {
size_t polygon_id;
Polygon const polygon;
std::string const descriptor;
api::Side side;
std::string const product;
double value = Worthless;
MDP3Quantity quantity = MDP3Quantity{0};
void mark_as_worthless() { value = Worthless; }
bool has_value() const { return value > securities::market_epsilon(); }
};
struct PolyObserver::Opportunity {
Target *target;
double value;
MDP3Quantity quantity;
EventHeader start;
};
struct PolyObserver::PImpl {
std::vector<std::unique_ptr<Target>> targets;
std::optional<Opportunity> current_opportunity;
std::unordered_map<MDP3SecurityID, MBPUpdate> books;
EventHeader last_event;
ISim *sim = nullptr;
};
PolyObserver::PolyObserver() : me(new PImpl()) {}
PolyObserver::~PolyObserver() {}
void PolyObserver::init(ISim &s, protos::SimRunOptions const&) {
me->sim = &s;
for (auto &p : s.polygons()) {
auto descriptor = polygon_descriptor(p.second, s.instruments());
auto buy_t = Target{p.first, p.second, descriptor, api::SIDE_BUY,
polygon_product(p.second, s.instruments())};
auto sell_t = Target{p.first, flip(p.second), descriptor, api::SIDE_SELL,
polygon_product(p.second, s.instruments())};
me->targets.emplace_back(std::make_unique<Target>(buy_t));
me->targets.emplace_back(std::make_unique<Target>(sell_t));
}
}
void PolyObserver::finit() {}
void PolyObserver::on_replay_start(
EventHeader const &current_event,
std::vector<MBPUpdate> const &current_books) {
me->sim->log("replay_start", current_event.event_id.first);
me->books.clear();
for (auto const &b : current_books)
me->books[b.security_id] = b;
}
void PolyObserver::eval() {
for (auto &t : me->targets) {
t->value = 0;
int qty = std::numeric_limits<int>::max();
for (auto &l : t->polygon.parts) {
auto mkt_side = securities::other_side(side_from_ratio(l.ratio));
auto &mkt = me->books[l.sec_id].mbp_side(mkt_side);
auto this_qty = mkt[0].quantity.value();
auto this_ratio = this_qty / std::abs(l.ratio);
if (this_ratio < qty)
qty = this_ratio;
if (qty == 0) {
t->mark_as_worthless();
break;
} else {
auto cash = me->sim->get_sec(l.sec_id).to_cash(mkt[0].price);
t->quantity.value() = qty;
t->value += -l.ratio * cash;
}
}
}
}
void PolyObserver::rank() {
std::sort(
me->targets.begin(), me->targets.end(),
[](std::unique_ptr<Target> const &ptr1,
std::unique_ptr<Target> const &ptr2) {
auto val1 = ptr1->value; // * ptr1->quantity.value();
auto val2 = ptr2->value; // * ptr2->quantity.value();
if (std::abs(val1 - val2) > securities::market_epsilon())
return val1 > val2;
else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size())
return ptr1->polygon.parts.size() < ptr2->polygon.parts.size();
else
return ptr1->polygon_id < ptr2->polygon_id;
});
}
void PolyObserver::try_start_new_opportunity() {
bts_assert(!me->current_opportunity);
auto best = me->targets.begin()->get();
if (best->has_value()) {
// We have an opportunity lets record it
me->current_opportunity =
Opportunity{best, best->value, best->quantity, me->last_event};
me->sim->log(
"opportunity_begin", best->polygon_id, best->descriptor,
api::Side_Name(best->side), best->value, best->quantity.value(), 0, 0,
us_diff(me->last_event.observable_time, me->last_event.transact_time),
me->last_event.event_id.first, me->last_event.event_id.first);
}
}
void PolyObserver::analyze() {
if (me->current_opportunity) {
auto &opportunity = *me->current_opportunity;
auto &target = *opportunity.target;
if (target.has_value()) {
// Add value and still has value, opportunity is still ongoing
me->sim->log("opportunity_continue", target.polygon_id,
target.descriptor, api::Side_Name(target.side),
target.value, target.quantity.value(),
us_diff(me->last_event.observable_time,
opportunity.start.observable_time),
us_diff(me->last_event.transact_time,
opportunity.start.transact_time),
us_diff(opportunity.start.observable_time,
opportunity.start.transact_time),
opportunity.start.event_id.first);
} else {
// Had value but not anymore, opportunity is concluded
me->sim->log("opportunity_end", target.polygon_id, target.descriptor,
api::Side_Name(target.side), opportunity.value,
opportunity.quantity.value(),
us_diff(me->last_event.observable_time,
opportunity.start.observable_time),
us_diff(me->last_event.transact_time,
opportunity.start.transact_time),
us_diff(opportunity.start.observable_time,
opportunity.start.transact_time),
opportunity.start.event_id.first,
me->last_event.event_id.first - 1);
me->current_opportunity.reset();
try_start_new_opportunity();
}
} else {
// We did not have an opportunity before. Maybe we do now?
try_start_new_opportunity();
}
}
void PolyObserver::on_order_update(EventHeader const &, OrderUpdate const &) {bts_assert(false);}
void PolyObserver::on_mbp(EventHeader const &h, MBPUpdate const &mbp) {
me->last_event = h;
me->books[mbp.security_id] = mbp;
eval();
rank();
analyze();
}
void PolyObserver::on_time() {}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,33 @@
#pragma once
#include "../sim/IAlgoModel.h"
#include <memory>
#include <optional>
namespace bts::spartan::mdp3 {
struct Polygon;
class PolyObserver : public IAlgoModel {
public:
PolyObserver();
~PolyObserver();
void init(ISim &, protos::SimRunOptions const&) override;
void finit() override;
void on_replay_start(EventHeader const &, std::vector<MBPUpdate> const &) override;
void on_order_update(EventHeader const &, OrderUpdate const &) override;
void on_mbp(EventHeader const &, MBPUpdate const &) override;
void on_time() override;
private:
struct Target;
struct Opportunity;
struct PImpl;
void eval();
void rank();
void analyze();
void try_start_new_opportunity();
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,569 @@
#include "SniperModel.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include <boost/algorithm/string.hpp>
#include <boost/container/small_vector.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
namespace bts::spartan::mdp3 {
enum class SniperModel::State { W_OPPORTUNITY, SHOOTING, PAUSED };
struct SniperModel::Target {
constexpr static double Worthless = -1.0;
size_t polygon_id;
Polygon const polygon;
boost::container::small_vector<DependentTargets const *, 3> leg_sec_info;
std::string const descriptor;
api::Side side;
double value = Worthless;
MDP3Quantity quantity = MDP3Quantity{0};
void mark_as_worthless() { value = Worthless; }
bool has_value(double v) const { return value > v; }
};
struct SniperModel::WorkingOrder {
SmallOrderChange order;
std::vector<OrderUpdate> fills;
};
struct SniperModel::Shot {
Target &target;
EventHeader observed;
EventHeader concluded;
double value_at_observation_ts;
MDP3Quantity quantity;
bool forced_completion;
std::unordered_map<MDP3OrderKey, WorkingOrder> working;
size_t transaction_duration;
};
struct SniperModel::DependentTargets {
InstrumentDefinition definition;
MBPUpdate book;
std::vector<Target *> dependencies;
};
struct SniperModel::PImpl {
ISim *sim = nullptr;
protos::SimRunOptions options;
int min_orders_at_top;
int max_qty;
int min_depth;
double min_value;
size_t max_transaction_latency_us;
bool double_dip;
size_t ms_to_work_shot;
size_t min_cumm_qty;
double min_cumm_edge;
size_t pay_up_ticks;
size_t throttle_ns;
double max_shot_loss;
std::set<std::string> polys;
EventHeader current_event;
size_t streak_id = 0;
State state = State::W_OPPORTUNITY;
std::unordered_map<MDP3SecurityID, DependentTargets> books = {};
std::vector<std::unique_ptr<Target>> targets = {};
std::optional<Shot> current_shot{};
};
SniperModel::SniperModel() : me(new PImpl()) {}
SniperModel::~SniperModel() {}
void SniperModel::init(ISim &s, protos::SimRunOptions const &options) {
me->sim = &s;
me->options = options;
}
void SniperModel::on_replay_start(EventHeader const &h,
std::vector<MBPUpdate> const &books) {
auto parse_poly_filter = [] (std::string const&pf) {
std::vector<std::string> polys;
boost::split(polys,pf,boost::is_any_of(","));
std::set<std::string> result;
for (std::string &s : polys) {
boost::trim(s);
if (s.size())
result.insert(s);
}
return result;
};
auto sim = me->sim;
auto options = me->options;
auto streak_id = me->streak_id;
me.reset(
new PImpl{sim, options, get_param<int>(me->options, "min_orders_at_top"),
get_param<int>(me->options, "max_qty"),
get_param<int>(me->options, "min_depth"),
get_param<double>(me->options, "min_value"),
get_param<size_t>(me->options, "max_transaction_latency_us"),
get_param<bool>(me->options, "double_dip"),
get_param<size_t>(me->options, "ms_to_work_shot"),
get_param<size_t>(me->options, "min_cumm_qty"),
get_param<double>(me->options, "min_cumm_edge"),
get_param<size_t>(me->options, "pay_up_ticks"),
get_param<size_t>(me->options, "throttle_ns"),
get_param<double>(me->options, "max_shot_loss"),
parse_poly_filter(get_param(me->options,"polys")),
h, streak_id});
for (auto &p : me->sim->polygons()) {
auto desc = polygon_descriptor(p.second, me->sim->instruments());
if (me->polys.size() && !me->polys.count(desc))
continue;
auto buy_t = std::make_unique<Target>(
Target{p.first,
p.second,
{},
desc,
api::SIDE_BUY});
auto sell_t = std::make_unique<Target>(
Target{p.first,
flip(p.second),
{},
desc,
api::SIDE_SELL});
me->targets.emplace_back(std::move(buy_t));
me->targets.emplace_back(std::move(sell_t));
}
for (auto &t : me->targets) {
for (auto &pp : t->polygon.parts) {
auto &dt = me->books[pp.sec_id];
dt.book.security_id = pp.sec_id;
dt.definition = me->sim->get_sec(pp.sec_id);
t->leg_sec_info.push_back(&dt);
auto &dependencies = dt.dependencies;
dependencies.push_back(t.get());
}
}
me->sim->log("replay_start");
for (auto const &mbp : books) {
auto it = me->books.find(mbp.security_id);
if (it != me->books.end()) {
me->sim->log(
"book_at_replay_start", me->sim->get_sec(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value());
it->second.book = mbp;
}
}
}
void SniperModel::on_mbp(EventHeader const &h, const MBPUpdate &mbp) {
me->current_event = h;
auto it = me->books.find(mbp.security_id);
if (it == me->books.end())
return;
auto &book = it->second;
book.book = mbp;
eval(book.dependencies);
switch (me->state) {
case State::W_OPPORTUNITY: {
auto best_shot = pick();
if (best_shot) {
me->streak_id++;
me->sim->start_simulation();
transition(State::SHOOTING);
fire(best_shot.value());
}
break;
}
case State::SHOOTING:
break;
case State::PAUSED:
check_if_pause_over();
break;
}
}
void SniperModel::on_order_update(EventHeader const &h,
const OrderUpdate &update) {
me->current_event = h;
bts_assert(me->state == State::SHOOTING);
bts_assert(me->current_shot);
auto &in_flight = me->current_shot.value();
bts_assert(in_flight.working.count(update.key));
in_flight.working[update.key].fills.push_back(update);
auto pending_orders = in_flight.working.size();
bool clean_fills = true;
for (auto const &p : in_flight.working) {
auto &these_fills = p.second.fills;
for (auto &f : these_fills) {
clean_fills = clean_fills && (f.fill_type == OrderUpdate::AGGRESSIVE);
if (f.status == OrderUpdate::CANCELED ||
f.status == OrderUpdate::FILLED) {
pending_orders--;
break;
}
}
}
if (pending_orders == 0) {
std::vector<OrderUpdate> all_fills;
for (auto const &p : in_flight.working) {
auto &these_fills = p.second.fills;
all_fills.insert(all_fills.end(), these_fills.begin(),
these_fills.end());
}
PandL pandl(me->sim->instruments());
auto cash = pandl.net_cash(all_fills);
auto commission = pandl.commission_cost(all_fills);
auto net = cash - commission;
in_flight.concluded = h;
me->sim->log(
"shot_outcome", in_flight.target.descriptor,
api::Side_Name(in_flight.target.side),
in_flight.observed.event_id.first, in_flight.observed.event_id.second,
in_flight.value_at_observation_ts * in_flight.quantity.value(), net,
commission, in_flight.quantity.value(),
clean_fills && !me->current_shot->forced_completion,
me->current_shot->forced_completion,
in_flight.transaction_duration);
me->sim->log_pandl();
if (!me->double_dip)
transition(State::PAUSED);
else {
auto best_shot = pick();
if (best_shot) {
me->sim->log("double_dip", me->streak_id);
fire(best_shot.value());
} else
transition(State::PAUSED);
}
}
}
void SniperModel::on_time() {
switch (me->state) {
case State::W_OPPORTUNITY: {
break;
}
case State::SHOOTING: {
maybe_force_fills();
break;
}
case State::PAUSED:
check_if_pause_over();
break;
}
}
void SniperModel::maybe_force_fills() {
bts_assert(me->state == State::SHOOTING);
bts_assert(me->current_shot);
auto ts = me->sim->time();
auto shot_observed_ts = me->current_shot->observed.observable_time;
auto shot_duration = ts.value() - shot_observed_ts.value();
if (me->current_shot->forced_completion)
return;
if (me->ms_to_work_shot) {
if (shot_duration > 1000 * 1000 * me->ms_to_work_shot) {
me->sim->log("forcing_fills_timeout", shot_observed_ts.value());
me->current_shot->forced_completion = true;
} else if (me->current_shot->target.value <
-me->max_shot_loss) {
me->sim->log("forcing_fills_maxloss", shot_observed_ts.value());
me->current_shot->forced_completion = true;
}
}
if (!me->current_shot->forced_completion)
return;
auto &in_flight = me->current_shot.value();
for (auto &p : in_flight.working) {
auto &order = p.second.order;
auto &these_fills = p.second.fills;
auto total_fills = 0;
for (auto &f : these_fills) {
bts_assert(f.status != OrderUpdate::CANCELED);
total_fills += f.last_fills.value();
}
if (order.order.quantity.value() > total_fills) {
md::IntPrice worst_price = order.order.price;
if (order.order.side == api::SIDE_BUY)
worst_price += me->sim->get_sec(order.order.security)
.min_price_increment_int_price() *
100;
else
worst_price -= me->sim->get_sec(order.order.security)
.min_price_increment_int_price() *
100;
order.change = OrderChange::updated_price;
order.order.price = worst_price;
me->sim->send(order);
}
}
}
void SniperModel::eval(std::vector<Target *> const &targets) {
for (auto &t : targets) {
t->value = 0;
int qty = me->max_qty;
size_t p_idx = 0;
for (auto &l : t->polygon.parts) {
auto mkt_side = securities::other_side(side_from_ratio(l.ratio));
auto &leg_sec_info = t->leg_sec_info[p_idx++];
auto &mkt = leg_sec_info->book.mbp_side(mkt_side);
auto this_qty = mkt[0].quantity.value();
auto this_ratio = this_qty / std::abs(l.ratio);
if (this_ratio < qty)
qty = this_ratio;
if (qty == 0 || mkt[0].n_orders < me->min_orders_at_top ||
depth(mkt).value() < me->min_depth) {
t->mark_as_worthless();
break;
} else {
auto cash = leg_sec_info->definition.to_cash(mkt[0].price.price());
t->quantity.value() = qty;
t->value += -l.ratio * cash;
}
}
}
}
bool SniperModel::within_safety_margin(Target const &t) const {
Book synth_book;
using LegInfo = std::tuple<MDP3Quantity, md::IntPrice, Book>;
std::vector<LegInfo> leg_mkts;
for (auto &leg : t.polygon.parts) {
auto &leg_mkt = leg_mkts.emplace_back();
std::get<0>(leg_mkt) = MDP3Quantity{leg.ratio} ;
std::get<1>(leg_mkt) = {};
auto leg_book_side = securities::other_side(side_from_ratio(leg.ratio));
std::get<2>(leg_mkt) = me->books[leg.sec_id].book.mbp_side(leg_book_side);
}
book_utils::compute_synthetic_book(synth_book, leg_mkts,
std::tuple_size<Book>::value);
auto cumm_qty = 0ul;
auto min_price = me->min_cumm_edge;
// This is not right, exactly, we are assuming all the legs are
// expressed in the same terms as the first.
auto &first_leg_def = t.leg_sec_info[0]->definition;
auto p_value = t.value ;
// The synth book calculation uses a different convention. if a
// polygon has value the synth book will have a < 0 price.
auto s_value = - (first_leg_def.to_cash(synth_book[0].price));
// The synth book should match the first level of the polygon value.
bts_assert(std::abs(p_value - s_value) < securities::market_epsilon());
for (auto &level : synth_book) {
// Again, we need to flip the sign because the convention of the
// synth book calculation is to return <0 for polygons with
// value.
auto level_cash_price = first_leg_def.to_cash(-level.price);
// We crossed the threshold that allows us to ignore the quantity.
if (level_cash_price >= min_price)
return true;
else if (level_cash_price >= me->min_value)
cumm_qty += level.quantity.value();
else
break;
}
return cumm_qty >= me->min_cumm_qty;
}
SniperModel::MaybeShot SniperModel::pick() {
std::sort(
me->targets.begin(), me->targets.end(),
[](std::unique_ptr<Target> const &ptr1,
std::unique_ptr<Target> const &ptr2) {
auto val1 = ptr1->value * ptr1->quantity.value();
auto val2 = ptr2->value * ptr2->quantity.value();
if (std::abs(val1 - val2) > securities::market_epsilon())
return val1 > val2;
else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size())
return ptr1->polygon.parts.size() < ptr2->polygon.parts.size();
else
return ptr1->polygon_id < ptr2->polygon_id;
});
auto &best_target = *me->targets[0];
if (best_target.has_value(me->min_value)) {
bool has_any_qty = me->max_qty;
auto transaction_duration =
me->current_event.observable_time.value() -
me->current_event.transact_time.value();
bool transaction_is_fresh = transaction_duration <
1000 * me->max_transaction_latency_us;
if (has_any_qty && transaction_is_fresh &&
within_safety_margin(best_target))
return Shot{best_target,
me->current_event,
{},
best_target.value,
best_target.quantity,
false,
{},
transaction_duration};
else
return {};
} else
return {};
}
void SniperModel::fire(Shot const &s) {
auto &shot = me->current_shot.emplace(s);
me->sim->log("shot_begin", s.target.descriptor,
api::Side_Name(s.target.side), s.value_at_observation_ts,
s.quantity.value(), me->streak_id);
for (auto const &b : me->books) {
auto const &mbp = b.second.book;
me->sim->log(
"book_at_shot_start", me->sim->get_sec(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value());
}
if (me->ms_to_work_shot == 0) {
shot.forced_completion = true;
for (auto &p : shot.target.polygon.parts) {
auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL;
auto mkt_side = securities::other_side(part_side);
auto &book = me->books[p.sec_id].book.mbp_side(mkt_side);
md::IntPrice worst_price;
size_t available_qty = 0;
for (int i = book.size() - 1; i >= 0; i--) {
if (book[i].quantity.value()) {
worst_price = book[i].price;
for (int n = 0; n <= i; n++)
available_qty += book[n].quantity.value();
break;
}
}
if (part_side == api::SIDE_BUY)
worst_price +=
me->sim->get_sec(p.sec_id).min_price_increment_int_price() *
100;
else
worst_price -=
me->sim->get_sec(p.sec_id).min_price_increment_int_price() *
100;
auto order_change = SmallOrderChange{};
order_change.order.security = p.sec_id;
order_change.order.price = worst_price;
order_change.order.quantity =
MDP3Quantity{s.quantity.value() * std::abs(p.ratio)};
order_change.order.side = part_side;
order_change.change = OrderChange::added;
order_change.order.key = me->sim->send(order_change);
shot.working[order_change.order.key].order = order_change;
order_change.change = OrderChange::removed;
me->sim->send(order_change);
}
} else {
for (auto &p : shot.target.polygon.parts) {
auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL;
auto mkt_side = securities::other_side(part_side);
auto &book = me->books[p.sec_id].book.mbp_side(mkt_side);
auto limit_price = book[0].price;
auto pay_up_adj =
me->sim->get_sec(p.sec_id).min_price_increment_int_price() *
me->pay_up_ticks;
if (part_side == api::SIDE_BUY)
limit_price += pay_up_adj;
else
limit_price -= pay_up_adj;
auto order_change = SmallOrderChange{};
order_change.order.security = p.sec_id;
order_change.order.price = limit_price;
order_change.order.quantity =
MDP3Quantity{s.quantity.value() * std::abs(p.ratio)};
order_change.order.side = part_side;
order_change.change = OrderChange::added;
order_change.order.key = me->sim->send(order_change);
me->sim->log("sent_order", api::Side_Name(part_side),
me->sim->get_sec(p.sec_id).symbol(),
book[0].price.price(), book[0].price.price(),
book[0].quantity.value());
shot.working[order_change.order.key].order = order_change;
}
}
}
void SniperModel::check_if_pause_over() {
bts_assert(me->state == State::PAUSED);
bts_assert(me->current_shot);
bts_assert(me->throttle_ns);
auto ns_since_shot_concluded =
me->sim->time().value() -
me->current_shot->concluded.observable_time.value();
if (ns_since_shot_concluded > me->throttle_ns)
me->sim->stop_simulation();
else
me->sim->log("paused");
}
const char *SniperModel::state_name(State s) {
switch (s) {
case State::SHOOTING:
return "WAIT_FILLS";
case State::W_OPPORTUNITY:
return "WAIT_OPPORTUNITY";
case State::PAUSED:
return "WAIT_OPPORTUNITY_CLEAR";
}
return nullptr;
}
void SniperModel::transition(State new_state) {
me->sim->log("transition", state_name(me->state), state_name(new_state));
me->state = new_state;
}
protos::SimRunOptions
SniperModel::fill_defaults(protos::SimRunOptions const &in) {
protos::SimRunOptions result = in;
set_param_if_empty(&result, "min_orders_at_top", 1);
set_param_if_empty(&result, "max_qty", 1);
set_param_if_empty(&result, "min_depth", 0);
set_param_if_empty(&result, "min_value", 2.0);
set_param_if_empty(&result, "max_transaction_latency_us", 10000);
set_param_if_empty(&result, "double_dip", false);
set_param_if_empty(&result, "ms_to_work_shot", 0);
set_param_if_empty(&result, "min_cumm_edge", 17.5);
set_param_if_empty(&result, "min_cumm_qty", 10);
set_param_if_empty(&result, "pay_up_ticks", 10);
set_param_if_empty(&result, "throttle_ns", 1 * 1000 * 1000 * 1000);
set_param_if_empty(&result, "max_shot_loss", 200);
set_param_if_empty(&result, "polys", "");
return result;
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,47 @@
#pragma once
#include <memory>
#include <optional>
#include "server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h"
namespace bts::spartan::mdp3 {
struct Polygon;
}
namespace bts::spartan::mdp3 {
class SniperModel : public IAlgoModel {
public:
SniperModel();
~SniperModel();
void init(ISim &, protos::SimRunOptions const&) override;
protos::SimRunOptions fill_defaults (protos::SimRunOptions const&) override;
void on_replay_start(EventHeader const &, std::vector<MBPUpdate> const &) override;
void on_mbp(EventHeader const &, MBPUpdate const &) override;
void on_order_update(EventHeader const &, OrderUpdate const &) override;
void on_time() override;
private:
enum class State;
struct PImpl;
struct Target;
struct Shot;
struct WorkingOrder;
struct DependentTargets;
using MaybeShot = std::optional<Shot>;
void eval(std::vector<Target*> const&);
MaybeShot pick();
void fire(Shot const&);
bool within_safety_margin(Target const&) const;
void check_if_pause_over();
void maybe_force_fills();
void transition(State);
const char *state_name(State);
std::unique_ptr<PImpl> me;
};
}
@@ -0,0 +1,166 @@
// #include "SimOrderGateway.h"
// #include "SimSpreader.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/ISimServices.h"
// #include "lib/api/services/OrderService.pb.h"
// #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
// using namespace bts::orderman;
// using namespace bts::api::orderman;
// namespace bts::spartan::mdp3 {
// struct OrderTuple{
// const OrderKey key;
// OrderChange ochange;
// uint32_t total_fills = 0;
// double avg_filled_price = 0;
// bool order_send_acked = false;
// OrderTuple (OrderKey key,
// OrderChange ochange)
// :
// key(key),
// ochange(ochange)
// {}
// };
// OrderStatus translate_status(const OrderUpdate& ord){
// switch(ord.status){
// case(OrderUpdate::Status::CANCELED):
// return OrderStatus::ORDER_STATUS_CANCELED;
// case(OrderUpdate::Status::FILLED):
// return OrderStatus::ORDER_STATUS_COMPLETED;
// case(OrderUpdate::Status::PARTIALLY_FILLED):
// return OrderStatus::ORDER_STATUS_WORKING;
// }
// bts_assert(false);
// return OrderStatus::ORDER_STATUS_WORKING;
// }
// OrderTuple* find_ord(OrderKey key,
// std::vector<OrderTuple>& orders){
// for(auto& o : orders)
// if(o.key == key)
// return &o;
// return nullptr;
// }
// OrderTuple* find_ord(MDP3OrderKey key,
// std::vector<OrderTuple>& orders){
// for(auto& o : orders)
// if(o.ochange.order.key == key)
// return &o;
// return nullptr;
// }
// struct SimOrderGateway::PImpl {
// ISimServices* sim_services = nullptr;
// SpreaderOrder* spreader = nullptr;
// std::vector<OrderTuple> orders;
// };
// SimOrderGateway::SimOrderGateway()
// :
// me(new PImpl()){
// }
// void SimOrderGateway::init(ISimServices& sim_services,
// SpreaderOrder& spreader){
// me->sim_services = &sim_services;
// me->spreader = &spreader;
// }
// void SimOrderGateway:: place_order(SecurityKey skey,
// OrderKey okey,
// api::Side side,
// Price p,
// Quantity q){
// bts_assert(me->sim_services!=nullptr);
// const auto& t =
// me->orders.emplace_back(
// okey,
// OrderChange{
// MDP3Order(),
// MDP3Order{
// me->sim_services->generate_order_key(),
// MDP3SecurityID{static_cast<int>(skey.value())},
// side,
// MDP3OrderPriority{},
// md::IntPrice_from_double<7>(p.value()),
// MDP3Quantity{static_cast<int>(q.value())},
// {}
// },
// OrderChange::added
// });
// me->sim_services->send(t.ochange);
// }
// void SimOrderGateway::modify_order(OrderKey okey,
// Price p,
// Quantity q){
// bts_assert(me->sim_services!=nullptr);
// OrderTuple* t = find_ord(okey,me->orders);
// bts_assert(t != nullptr);
// bts_assert(t->order_send_acked);
// auto& ochange = t->ochange;
// ochange.previous_order = ochange.order;
// ochange.order.price = md::IntPrice_from_double<7>(p.value());
// ochange.order.quantity = MDP3Quantity{static_cast<int>(q.value())};
// ochange.change = OrderChange::updated;
// me->sim_services->send(ochange);
// }
// void SimOrderGateway::cancel_order(OrderKey okey){
// bts_assert(me->sim_services!=nullptr);
// OrderTuple* t = find_ord(okey,me->orders);
// bts_assert(t != nullptr);
// bts_assert(t->order_send_acked);
// auto& ochange = t->ochange;
// ochange.change = OrderChange::removed;
// me->sim_services->send(ochange);
// }
// OrderKey SimOrderGateway::generate_key(){
// auto rv =
// static_cast<uint32_t>(bts::Timer::now().time_since_epoch().count());
// return OrderKey(rv);
// }
// void SimOrderGateway::poll(){
// for(auto& t : me->orders){
// if(!t.order_send_acked){
// t.order_send_acked=true;
// route_update_to_spreader(
// OrderUpdate{
// t.ochange.order.key,
// OrderUpdate::Status::PARTIALLY_FILLED,
// t.ochange.order.security,
// t.ochange.order.side,
// MDP3Quantity{},
// md::IntPrice()
// });
// }
// }
// }
// void SimOrderGateway::route_update_to_spreader(const OrderUpdate& u){
// OrderTuple* t = find_ord(u.key,me->orders);
// bts_assert(t!=nullptr);
// uint32_t total_fills = t->total_fills +
// static_cast<uint32_t>(u.last_fills.value());
// double avg_filled_price = 0;
// if(total_fills > 0){
// avg_filled_price = (t->total_fills*t->avg_filled_price) +
// (u.last_fills.value() * u.last_price.price());
// avg_filled_price = avg_filled_price / total_fills;
// }
// t->total_fills = total_fills;
// t->avg_filled_price = avg_filled_price;
// me->spreader->test_only_on_child_order_update(
// t->key,
// translate_status(u),
// Quantity(t->total_fills),
// Price(t->avg_filled_price)
// );
// }
// }
@@ -0,0 +1,40 @@
// #pragma once
// #include "lib/orderman/orderman_utils/orderman_common.h"
// #include "lib/domain_model/Security.pb.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h"
// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
// #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/IAlternateOrderGateway.h"
// namespace bts { namespace orderman {class SpreaderOrder;}}
// namespace bts::spartan::mdp3 {
// class ISimServices;
// class SimOrderGateway : public bts::orderman::IAlternateOrderGateway{
// private:
// struct PImpl;
// PImpl* me;
// public:
// SimOrderGateway();
// ~SimOrderGateway(){}
// void init(ISimServices&,
// bts::orderman::SpreaderOrder&);
// // IAlternateOrderGateway functions
// // (SpreaderChildOrderState's calls need to end up here)
// void place_order(SecurityKey,
// bts::orderman::OrderKey,
// api::Side,
// Price,
// Quantity) override;
// void modify_order(bts::orderman::OrderKey, Price, Quantity) override;
// void cancel_order(bts::orderman::OrderKey) override;
// bts::orderman::OrderKey generate_key() override;
// // Sends ISimServices's updates to spreader
// void poll();
// void route_update_to_spreader(const OrderUpdate&);
// };
// }
@@ -0,0 +1,114 @@
// #include "SimSpreader.h"
// #include "lib/domain_model/Security.pb.h"
// #include "lib/api/services/OrderService.pb.h"
// #include "lib/sparkproto/SparkProtos.pb.h"
// #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
// #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.h"
// #include "lib/orderman/orderman_utils/Book.h"
// #include "SimOrderGateway.h"
// using namespace bts::orderman;
// using namespace bts::api::orderman;
// namespace bts::spartan::mdp3 {
// void translate_book(const MBPUpdate& u_in,
// bts::orderman::Book& b_out){
// size_t i=-1;
// for (auto const& bl : u_in.mbp_buy)
// if(++i<5)
// b_out.side(api::Side::SIDE_BUY)[i] =
// md::PriceSize (bl.price,bl.quantity.value());
// i=-1;
// for (auto const& bl : u_in.mbp_sell)
// if(++i<5)
// b_out.side(api::Side::SIDE_SELL)[i] =
// md::PriceSize (bl.price,bl.quantity.value());
// }
// bool compare_keys (SecurityKey spark_k,
// MDP3SecurityID spartan_k){
// return static_cast<int>(spark_k.value()) == spartan_k.value();
// }
// bts::orderman::Book* find_book(MDP3SecurityID id,
// std::vector<bts::orderman::Book>& books){
// for (auto& b : books)
// if(compare_keys(b.security_key(),id))
// return &b;
// return nullptr;
// }
// struct SimSpreader::PImpl {
// const OrderDefinition order_def;
// const SpreaderPreActivationOrderServices pao_services;
// SpreaderOrderServices so_services;
// ISimServices* sim_services = nullptr;
// SimOrderGateway sim_order_gateway;
// SpreaderOrder spreader;
// Price price;
// Quantity quantity;
// std::vector<bts::orderman::Book> books;
// PImpl(const OrderDefinition& order_def,
// const bts::api::Security& sec,
// Price price,
// Quantity quantity)
// :
// order_def(order_def),
// pao_services(sec),
// so_services(nullptr),
// sim_order_gateway(),
// spreader(&sim_order_gateway),
// price(price),
// quantity(quantity),
// books(){
// for (const auto& l : sec.synthetic().legs())
// books.emplace_back(SecurityKey(l.security().id()));
// }
// };
// SimSpreader::SimSpreader(const OrderDefinition& order_def,
// const bts::api::Security& sec,
// Price price,
// Quantity qty)
// :
// me(new PImpl(order_def,sec,price,qty)){
// }
// const SpreaderOrder& SimSpreader::spreader() const{
// return me->spreader;
// }
// // IAlgoModel /////////////////////////////////////////////////////////////////
// ///////////////////////////////////////////////////////////////////////////////
// void SimSpreader::init(ISimServices& sim_services, protos::SimRunOptions const&){
// me->sim_services = &sim_services;
// me->sim_order_gateway.init(sim_services,me->spreader);
// // TODO (no test_ blah)
// me->spreader.test_only_on_init_new (me->order_def,
// me->pao_services);
// me->spreader.test_only_on_activate(me->price,
// me->quantity,
// me->so_services);
// }
// void SimSpreader::on_time(MDP3TimeStamp){
// me->sim_order_gateway.poll();
// }
// void SimSpreader::on_notification(AlgoNotification const& ev) {
// if(std::holds_alternative<MBPUpdate>(ev.event)){
// // book update
// const MBPUpdate& bk = std::get<MBPUpdate>(ev.event);
// // copy book
// bts::orderman::Book* leg_bk = find_book(bk.security_id, me->books);
// if(leg_bk != nullptr){
// auto& leg_bk_r = *leg_bk;
// translate_book(bk, leg_bk_r);
// me->spreader.test_only_on_book_update(leg_bk_r);
// }
// }
// else{
// const OrderUpdate& u = std::get<OrderUpdate>(ev.event);
// me->sim_order_gateway.route_update_to_spreader(u);
// }
// }
// }
@@ -0,0 +1,33 @@
// #pragma once
// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/IAlgoModel.h"
// #include "SimOrderGateway.h"
// namespace bts { namespace api { namespace orderman {class OrderDefinition;}}}
// namespace bts { namespace api { class OrderDefinition;}}
// namespace bts { namespace orderman { class SpreaderOrder;}}
// namespace bts::spartan::mdp3 {
// class ISimServices;
// class SimSpreader : public IAlgoModel {
// private:
// struct PImpl;
// //std::unique_ptr<PImpl> me;
// PImpl* me;
// public:
// SimSpreader(const bts::api::orderman::OrderDefinition&,
// const bts::api::Security&,
// Price, Quantity);
// const bts::orderman::SpreaderOrder& spreader() const;
// // IAlgoModel
// void init(ISimServices&, protos::SimRunOptions const&) override;
// void on_time(MDP3TimeStamp) override;
// void on_notification(AlgoNotification const&) override;
// ~SimSpreader(){}
// };
// }
@@ -0,0 +1,359 @@
// #include <catch/catch.hpp>
// #include "lib/domain_model/Security.pb.h"
// #include "server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.h"
// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h"
// #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/ISimServices.h"
// #include "lib/orderman/orders/spreader_orders/tests/tests_common.h"
// #include "server/sparkd/cmds/spartan/back_testing/orchestrator/SimSessionOptions.pb.h"
// using namespace bts::spartan::mdp3;
// namespace bts {
// namespace orderman {
// using namespace api;
// using namespace spark;
// class TestSimServices : public bts::spartan::mdp3::ISimServices{
// private:
// InstrumentMap _instruments;
// std::unordered_map<size_t,Polygon> _polygons;
// protected:
// void _log(std::string const &,
// std::initializer_list<std::string> const &) override {}
// public:
// SmallOrderChange last_change;
// size_t n_updates = 0;
// size_t key;
// public:
// TestSimServices ()
// :
// key(0){
// }
// InstrumentMap const &instruments() const override{
// bts_assert(false);
// return _instruments;
// }
// void terminate_simulation() override {}
// void log_pandl() override {}
// std::unordered_map<size_t,Polygon> const&polygons() const override {
// return _polygons;
// }
// spartan::mdp3::Book book(MDP3SecurityID,api::Side) const override {
// return {};
// }
// MDP3OrderKey generate_order_key() override{
// return MDP3OrderKey(key++);
// }
// MDP3TimeStamp now() const override{
// return MDP3TimeStamp(0);
// }
// void wait(uint64_t) override{
// }
// void send(const SmallOrderChange& c, uint64_t) override{
// last_change = c;
// n_updates++;
// }
// ~TestSimServices(){}
// };
// bts::spartan::mdp3::MBPUpdate get_book_update(
// const std::vector<std::tuple<double,uint32_t>>& bids,
// const std::vector<std::tuple<double,uint32_t>>& asks,
// uint32_t key){
// bts::spartan::mdp3::MBPUpdate rv;
// size_t i=0;
// for(auto[p,q] : bids)
// rv.mbp_buy[i++] = BookLevel{
// md::IntPrice_from_double<7>(p),
// bts::spartan::mdp3::MDP3Quantity(q),
// 0
// };
// i=0;
// for(auto[p,q] : asks)
// rv.mbp_sell[i++] = BookLevel{
// md::IntPrice_from_double<7>(p),
// bts::spartan::mdp3::MDP3Quantity(q),
// 0
// };
// rv.security_id = MDP3SecurityID(key);
// return rv;
// }
// TEST_CASE ("sim_spreader", "[spreader]"){
// bts::orderman::OrderKey order_key(1);
// // security
// ContractLeg leg_1, leg_2;
// fill_future_contract_leg (leg_1, 1, "CL",
// bts::api::Venue::VENUE_GLBX,
// 2020, 11, 15,
// 1.0, 1.0,
// api::Side::SIDE_BUY,
// 1.0, 0.01);
// fill_future_contract_leg (leg_2, 2, "CL",
// bts::api::Venue::VENUE_GLBX,
// 2020, 12, 15,
// -1.0, -1.0,
// api::Side::SIDE_SELL,
// 1.0, 0.01);
// std::vector<ContractLeg> legs;
// legs.push_back(leg_1);
// legs.push_back(leg_2);
// Security sec;
// fill_synthetic_security (sec, 2000, "simspreader_2legs", legs);
// // spreader params
// LegSpreaderParams lsp_1, lsp_2, lsp_3, lsp_4;
// fill_leg_spreader_params(lsp_1, 1.0, false, 1.0);
// fill_leg_spreader_params(lsp_2, -1.0, true, 1.0);
// // post hedge time out here
// std::vector<LegSpreaderParams> params;
// params.push_back(lsp_1);
// params.push_back(lsp_2);
// // putting it together in the order def
// bts::api::orderman::OrderDefinition order_def;
// fill_spreader_order_definition(order_def, Side::SIDE_BUY, sec, params);
// bts::spartan::mdp3::SimSpreader sim_spreader(order_def,sec,Price(-10),Quantity(1));
// //ISimServices* sim_services = new TestSimServices();
// TestSimServices sim_services;
// spartan::protos::SimRunOptions options;
// sim_spreader.init(*(ISimServices*)&sim_services,options);
// SECTION("requoting"){
// std::vector<std::tuple<double,uint32_t>> bids =
// {
// {100,5},
// {101,5},
// {102,5}
// },
// asks =
// {
// {103,5},
// {104,5},
// {105,5}
// };
// auto md_udpate_leg_1 = get_book_update(bids,asks,leg_1.security().id());
// // send leg_1 book update => leg_2 should quote
// sim_spreader.on_notification(
// {
// {},
// md_udpate_leg_1
// }
// );
// sim_spreader.on_time(bts::spartan::mdp3::MDP3TimeStamp());
// REQUIRE(sim_services.n_updates==1);
// REQUIRE(sim_services.last_change.order.side == api::Side::SIDE_SELL);
// REQUIRE(sim_services.last_change.order.quantity == bts::spartan::mdp3::MDP3Quantity(1));
// REQUIRE(
// bts::securities::is_same(Price(sim_services.last_change.order.price.price()),
// Price(113)));
// // send multiple book updates
// double p = 103;
// for(int i=0; i<100; i++){
// asks =
// {
// {p,5},
// {p+1,5},
// {p+2,5}
// };
// auto md_udpate_leg_1 = get_book_update(bids,asks,leg_1.security().id());
// sim_spreader.on_notification(
// {
// {},
// md_udpate_leg_1
// });
// REQUIRE(sim_services.n_updates==i+1); // +1 because order_place is also counted
// REQUIRE(sim_services.last_change.order.security ==
// bts::spartan::mdp3::MDP3SecurityID(2));
// REQUIRE(sim_services.last_change.order.side == api::Side::SIDE_SELL);
// REQUIRE(sim_services.last_change.order.quantity == bts::spartan::mdp3::MDP3Quantity(1));
// REQUIRE(
// bts::securities::is_same(Price(sim_services.last_change.order.price.price()),
// Price(p+10)));
// p++;
// }
// //force a quoting leg fill
// auto q_ord = sim_services.last_change.order;
// sim_spreader.on_notification(
// {
// {},
// bts::spartan::mdp3::OrderUpdate{
// q_ord.key,
// bts::spartan::mdp3::OrderUpdate::FILLED,
// q_ord.security,
// q_ord.side,
// bts::spartan::mdp3::MDP3Quantity(1),
// md::IntPrice_from_double<7>(202)
// }
// });
// // the above should have sent a hedge order, verify that
// auto hedge_ord = sim_services.last_change.order;
// REQUIRE(hedge_ord.security == bts::spartan::mdp3::MDP3SecurityID(1));
// REQUIRE(hedge_ord.side == api::Side::SIDE_BUY);
// REQUIRE(hedge_ord.quantity == bts::spartan::mdp3::MDP3Quantity(1));
// REQUIRE(
// bts::securities::is_same(Price(hedge_ord.price.price()),
// Price(192)));
// REQUIRE(sim_spreader.spreader().test_only_func_spreader_state() !=
// SpreaderState::END);
// sim_spreader.on_time(bts::spartan::mdp3::MDP3TimeStamp());
// // send the hedge fill and it should go to end state
// sim_spreader.on_notification(
// {
// {},
// bts::spartan::mdp3::OrderUpdate{
// hedge_ord.key,
// bts::spartan::mdp3::OrderUpdate::FILLED,
// hedge_ord.security,
// hedge_ord.side,
// bts::spartan::mdp3::MDP3Quantity(1),
// md::IntPrice_from_double<7>(192)
// }
// });
// REQUIRE(sim_spreader.spreader().test_only_func_spreader_state() ==
// SpreaderState::END);
// }
// }
// class RV
// {
// public:
// int i;
// RV(int i_arg)
// :
// i(i_arg){
// }
// RV()=delete;
// RV(const RV&&)=delete;
// RV (RV&& other){
// i=other.i;
// other.i=0;
// }
// RV& operator=(const RV&)=delete;
// RV& operator=(RV &&src){
// i=src.i;
// src.i=0;
// return *this;
// }
// int foo (double d1, double d2){
// return static_cast<int>(d1+d2);
// }
// };
// class TTemp{
// };
// class TInterface{
// virtual void foo() = 0;
// };
// template <class T, size_t N>
// class TestContainer : public TInterface{
// private:
// T _t;
// std::array<T,N> _at;
// public:
// void add (T &t){
// _at[0] = t;
// }
// T get() {
// return _t;
// }
// void foo () override{
// }
// };
// class RVUser{
// public:
// RV rv;
// RVUser(RV& rv_arg)
// : rv(std::move(rv_arg))
// {
// }
// };
// TEST_CASE ("scratch_pad", "[spreader]"){
// int (RV::*f_ptr)(double, double) = &RV::foo;
// char* f_addr = (char*&)f_ptr;
// TInterface* r = nullptr;
// int n = gen_rand(0.0, 10.0);
// if (n <= 5)
// r = new TestContainer<TTemp,5>();
// else
// r = new TestContainer<TTemp,10>();
// REQUIRE(r != nullptr);
// RV rv_1(100);
// RVUser rvuser_1(rv_1);
// REQUIRE(rv_1.i==0);
// REQUIRE(rvuser_1.rv.i==100);
// }
// TEST_CASE ("scratch_pad_2", "[spreader]"){
// time_t rawtime;
// time (&rawtime);
// struct tm* ptm = gmtime (&rawtime);
// LOG(OrderMan, Info, "hrs:{} mins:{} sec:{}", ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
// }
// // vector insert //////////////////////////////////////
// ///////////////////////////////////////////////////////
// class Elem{
// public:
// int i;
// Elem()
// :
// i(-1)
// {}
// Elem (int i_arg)
// :
// i(i_arg)
// {}
// Elem(const Elem& other){
// i = other.i;
// }
// Elem (Elem&& other){
// i=other.i;
// }
// Elem& operator=(const Elem& other){
// i=other.i;
// return *this;
// }
// Elem& operator=(Elem &&src){
// i = src.i;
// return *this;
// }
// bool operator==(const Elem& e) const {return i == e.i;}
// bool operator!=(const Elem& e) const {return i != e.i;}
// bool operator<(const Elem& e) const {return i < e.i;}
// bool operator>(const Elem& e) const {return i > e.i;}
// };
// TEST_CASE ("scratch_pad_3", "[spreader]"){
// std::vector<Elem> v;
// for(int i=1;i<10;i++)
// v.emplace_back(i);
// Elem e(0);
// // move op= is called when inserting or erasing
// v.insert(v.begin()+2,e);
// v.erase(v.begin()+2);
// std::random_shuffle(v.begin(),v.end());
// // flat set
// boost::container::flat_set<Elem> fs;
// for(int i=0;i<20;i+=2){
// Elem e(i);
// fs.insert(e);
// }
// Elem e9(9);
// // move op is called when inserting
// fs.insert(e9);
// for (const auto& e : fs)
// std::cout << e.i << std::endl;
// // vector assignment vs move assignment
// std::vector<Elem> vs;
// int n_elems = 10;
// for(int i=1;i<n_elems;i++)
// vs.emplace_back(i);
// std::vector<Elem> vt;
// // new elems are cresated in vt from the elem's in vs,
// // with their copy cons
// vt = vs;
// std::vector<Elem> vt_2;
// vt_2 = std::move(vs);
// }
// }}
@@ -0,0 +1,299 @@
#include "SimDB.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/datetime.h"
#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include <SQLiteCpp/SQLiteCpp.h>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem.hpp>
#include <mutex>
namespace bts::spartan::mdp3 {
struct SimDB::PendingLog {
RunID rid;
size_t sim_seq;
std::string src;
MDP3TimeStamp ts;
EventHeader h;
std::string log_entry_type;
std::vector<std::string> args;
};
struct SimDB::PImpl {
PImpl(std::string const &loc)
: db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE), db_lock() {}
SQLite::Database db;
std::mutex db_lock;
std::mutex pending_log_lock;
std::vector<PendingLog> pending;
std::unordered_map<RunID, size_t> log_entry_order;
std::thread logger_thread;
std::atomic<bool> done = false;
};
SimDB::SimDB(std::string const &loc) {
auto parent = boost::filesystem::path(loc).parent_path();
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
if (!boost::filesystem::is_regular_file(loc)) {
auto sql_path = os::sparkd_wd() + "/data_files/spartan/sim_db.sql";
std::ifstream f(sql_path);
std::string str((std::istreambuf_iterator<char>(f)),
(std::istreambuf_iterator<char>()));
SQLite::Database db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE);
db.exec(str);
}
me = std::make_unique<PImpl>(loc);
std::lock_guard<std::mutex> l(me->db_lock);
SQLite::Statement sync_off(me->db, "PRAGMA synchronous = OFF;");
sync_off.executeStep();
SQLite::Statement journal_mem(me->db, "PRAGMA journal_mode = MEMORY;");
journal_mem.executeStep();
me->logger_thread = std::thread{[this]() { do_work(); }};
}
SimDB::~SimDB() {
me->done = true;
me->logger_thread.join();
}
SimDB::SessionID SimDB::create_session(std::string const &algo,
bool clear_all) {
std::lock_guard<std::mutex> l(me->db_lock);
if (clear_all) {
SQLite::Statement dl(me->db, "delete from log");
dl.exec();
SQLite::Statement dr(me->db, "delete from run");
dr.exec();
}
auto max_session_query = "select max(rowid) from run";
SQLite::Statement statement(me->db, max_session_query);
size_t session_id = 0;
while (statement.executeStep()) {
session_id = (long long)statement.getColumn(0);
}
return SessionID{session_id, algo};
}
SimDB::RunID SimDB::begin_run(SessionID sid,
protos::SimRunOptions const &options) {
std::lock_guard<std::mutex> l(me->db_lock);
auto insert_run = "insert into run("
"session_id,"
"algo,"
"begin_run_time,"
"end_run_time,"
"market_date,"
"channel,"
"param_set_id,"
"sim_start_stop_time,"
"base_algo_latency_ns,"
"transaction_latency_ns,"
"use_first_packet_st,"
"simulate_icebergs,"
"realized_pandl,"
"commission,"
"position_pandl,"
"param1,"
"param2,"
"param3,"
"param4,"
"param5,"
"param6,"
"param7,"
"param8,"
"param9,"
"param10,"
"param11,"
"param12,"
"param13,"
"param14,"
"param15"
") values ("
":session_id,"
":algo,"
":begin_run_time,"
":end_run_time,"
":market_date,"
":channel,"
":param_set_id,"
":sim_start_stop_time,"
":base_algo_latency_ns,"
":transaction_latency_ns,"
":use_first_packet_st,"
":simulate_icebergs,"
":realized_pandl,"
":commission,"
":position_pandl,"
":param1,"
":param2,"
":param3,"
":param4,"
":param5,"
":param6,"
":param7,"
":param8,"
":param9,"
":param10,"
":param11,"
":param12,"
":param13,"
":param14,"
":param15"
")";
SQLite::Statement statement(me->db, insert_run);
statement.bind(":session_id", (long long)sid.id);
statement.bind(":algo", sid.algo);
statement.bind(":begin_run_time", datetime::time_string(datetime::now()));
statement.bind(":end_run_time", "");
statement.bind(":market_date", options.market_date());
statement.bind(":channel", options.channel());
statement.bind(":param_set_id", options.param_set_id());
statement.bind(":sim_start_stop_time", options.sim_start_stop_time());
statement.bind(":base_algo_latency_ns",
(long long)options.base_algo_latency_ns());
statement.bind(":transaction_latency_ns",
(long long)options.transaction_latency_ns());
statement.bind(":use_first_packet_st",
(long long)options.use_first_packet_st());
statement.bind(":simulate_icebergs",
(long long)options.simulate_icebergs());
statement.bind(":realized_pandl", 0);
statement.bind(":commission", 0);
statement.bind(":position_pandl", 0);
std::map<std::string, std::string> sorted_params(options.param().begin(),
options.param().end());
size_t p_idx = 1;
for (auto const &[name, val] : sorted_params)
statement.bind(fmt::format(":param{}", p_idx++),
fmt::format("{}: {}", name, val));
for (auto i = p_idx; i <= 15; i++)
statement.bind(fmt::format(":param{}", i), "");
statement.exec();
auto run_id = RunID{last_insert_rowid()};
me->log_entry_order[run_id] = 1;
return run_id;
}
void SimDB::end_run(RunID run_id, double realized_pandl, double commission,
double position_pandl) {
std::lock_guard<std::mutex> l(me->db_lock);
auto update_run = "update run "
"set "
"end_run_time=:end_run_time, "
"realized_pandl=:realized_pandl,"
"commission=:commission,"
"position_pandl=:position_pandl "
"where "
"run_id=:run_id";
SQLite::Statement statement(me->db, update_run);
statement.bind(":end_run_time", datetime::time_string(datetime::now()));
statement.bind(":run_id", run_id.value());
statement.bind(":realized_pandl", realized_pandl);
statement.bind(":commission", commission);
statement.bind(":position_pandl", position_pandl);
statement.exec();
}
void SimDB::log(RunID rid, size_t sim_seq, std::string const &src,
MDP3TimeStamp now, EventHeader const &h,
std::string const &entry_type,
std::vector<std::string> const &log_args) {
std::lock_guard<std::mutex> l(me->pending_log_lock);
me->pending.emplace_back(
PendingLog{rid, sim_seq, src, now, h, entry_type, log_args});
}
void SimDB::_log(std::vector<PendingLog> const &log) {
std::lock_guard<std::mutex> l(me->db_lock);
me->db.exec("begin");
for (auto &l : log) {
auto rid = l.rid;
auto now = l.ts;
auto &entry_type = l.log_entry_type;
auto &log_args = l.args;
auto &log_entry_order = me->log_entry_order[rid];
std::ostringstream insert_log_entry;
insert_log_entry << "insert into log("
"run_id,"
"log_entry_order,"
"sim_seq,"
"src,"
"sim_time,"
"h_sim_time,"
"entry_type,"
"ev_eid1,"
"ev_eid2,"
"ev_t_time,"
"ev_o_time";
for (size_t i = 0; i < log_args.size(); i++)
insert_log_entry << ",val" << i + 1;
insert_log_entry << ") values "
"(:run_id,:log_entry_order,:sim_seq,:src,:sim_time,:h_sim_"
"time,:entry_type,"
":ev_eid1,:ev_eid2,:ev_t_time,:ev_o_time";
for (size_t i = 0; i < log_args.size(); i++)
insert_log_entry << ",:val" << i + 1;
insert_log_entry << ")";
SQLite::Statement statement(me->db, insert_log_entry.str());
statement.bind(":run_id", rid.value());
statement.bind(":log_entry_order", (long long)log_entry_order++);
statement.bind(":sim_seq", (long long)l.sim_seq);
statement.bind(":src", l.src);
statement.bind(":sim_time", (long long)now.value());
statement.bind(":h_sim_time", to_time_string(now));
statement.bind(":entry_type", entry_type);
statement.bind(":ev_eid1", (long long)l.h.event_id.first);
statement.bind(":ev_eid2", (long long)l.h.event_id.second);
statement.bind(":ev_t_time", (long long)l.h.transact_time.value());
statement.bind(":ev_o_time", (long long)l.h.observable_time.value());
size_t i = 1;
for (auto &arg : log_args) {
statement.bind(fmt::format(":val{}", i++), arg);
}
statement.exec();
}
me->db.exec("commit");
}
void SimDB::do_work() {
while (!me->done) {
std::vector<PendingLog> to_store;
{
std::lock_guard<std::mutex> l(me->pending_log_lock);
to_store = std::move(me->pending);
}
if (to_store.size())
_log(to_store);
sleep_for(100ms);
}
_log(me->pending);
}
int SimDB::last_insert_rowid() {
SQLite::Statement statement(me->db, "select last_insert_rowid()");
statement.executeStep();
return statement.getColumn("last_insert_rowid()");
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,48 @@
#pragma once
#include <memory>
#include "lib/persistence/persistence_utils.h"
#include "lib/utils/ValueWrapper.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
namespace bts::spartan::protos {
class SimRunOptions;
}
namespace bts::spartan::mdp3 {
class SimDB {
public:
SimDB(std::string const &location = persistence::storage_location("sim_db"));
~SimDB();
struct SessionID {
size_t id;
std::string algo;
};
SessionID create_session(std::string const &algo, bool clear_all = false);
using RunID = utils::ValueWrapper<struct RunIDTag, int>;
RunID begin_run(SessionID, protos::SimRunOptions const &);
void end_run(RunID,double realized_pandl, double commission,double position_pandl);
void log(RunID,
size_t sim_seq,
std::string const&src,
MDP3TimeStamp,
EventHeader const&h,
std::string const &log_entry_type,
std::vector<std::string> const &log_args);
private:
struct PendingLog;
struct PImpl;
void do_work();
void _log(std::vector<PendingLog> const& );
int last_insert_rowid();
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,16 @@
#pragma once
#include <functional>
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
namespace bts::spartan::mdp3 {
using SimLogger =
std::function<void(size_t sim_seq,
std::string const&src,
MDP3TimeStamp,
EventHeader const &last_event,
std::string const &log_entry_type,
std::vector<std::string> const &log_args)>;
}
@@ -0,0 +1,207 @@
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h"
#include "lib/domain_model/Security.h"
#include "lib/md/IntPrice.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include <limits>
#include <variant>
// Error logs are automatically logged to the console
// (w/ default logging configuration), but they lag behind
// immediate console output (i.e., calls to std::cout).
// If _ECHO_ON is defined, the _ERROR_LEVEL macro is set to Info
// to avoid duplicate, and (perhaps more importantly) unsynchronized output.
//#define _ECHO_ON
#ifdef _ECHO_ON
#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__)
#define _ERROR_LEVEL Info
#else
#define _ECHO(...)
#define _ERROR_LEVEL Error
#endif
#define _LOG(level, msg) \
_ECHO(_log_header(), " ", msg, "\n"); \
LOG(OrderMan, level, "{} {}", _log_header(), msg)
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define ERROR(...) _LOG(_ERROR_LEVEL, bts::orderman::Formatter::fmt(__VA_ARGS__))
// SUMMARIZE macro wrappers
#define _SUMMARIZE SUMMARIZE(_log_header())
#define _SUMMARIZE_TRANSACTION(...) SUMMARIZE_TRANSACTION(_log_header(), FMT(__VA_ARGS__))
#define _SUMMARIZE_TRANSACTION_NAMED(name, ...) SUMMARIZE_TRANSACTION_NAMED(name, _log_header(), FMT(__VA_ARGS__))
namespace bts::spartan::mdp3 {
using _BookLevel = std::tuple<bts::md::IntPrice, MDP3Quantity>;
struct FifoImpliedSecurityMatcher::PImpl {
PImpl(MDP3SecurityID security_id, std::string symbol)
: security_id(security_id), implied_mkt(), symbol(symbol), leg_symbols() {}
MDP3SecurityID security_id;
std::map<MDP3SecurityID, std::tuple<FifoSecurityMatcher*, MDP3Quantity>> implied_mkt;
// symbol/leg_symbols serve diagnostic purposes
// (easier to read logs w/ actual symbols than sec ids)
std::string symbol;
std::map<MDP3SecurityID, std::string> leg_symbols;
};
///////////////////////
// constructors
///////////////////////
FifoImpliedSecurityMatcher::FifoImpliedSecurityMatcher(MDP3SecurityID security_id)
: FifoImpliedSecurityMatcher(security_id, std::string()) {}
FifoImpliedSecurityMatcher::FifoImpliedSecurityMatcher(
MDP3SecurityID security_id, std::string symbol)
: _me(new FifoImpliedSecurityMatcher::PImpl(security_id, symbol)) {}
FifoImpliedSecurityMatcher::~FifoImpliedSecurityMatcher() {}
///////////////////////
// diagnostic
///////////////////////
MDP3SecurityID FifoImpliedSecurityMatcher::security_id() const {
return _me->security_id;
}
std::string FifoImpliedSecurityMatcher::symbol() const { return _me->symbol; }
std::string FifoImpliedSecurityMatcher::_log_header() const {
return FMT("FifoImpliedSecurityMatcher (", security_id(), ", ", symbol(), ")");
}
void FifoImpliedSecurityMatcher::print_legs(std::ostream& out) const {
bool append_bbo = true;
for (const auto& kv : _me->implied_mkt) {
auto leg_size = std::get<1>(kv.second);
std::string leg_sym;
auto it = _me->leg_symbols.find(kv.first);
if (it != _me->leg_symbols.end()) {
leg_sym = it->second;
}
out << std::setw(12) << kv.first << " " << std::setw(12) << leg_sym << " " << leg_size;
if (append_bbo) {
append_bbo = false;
auto bid = top_of_book(api::Side::SIDE_BUY);
auto ask = top_of_book(api::Side::SIDE_SELL);
out << std::setw(8) << FMT("(", bid.quantity, ") ") << std::setw(6) << bid.price << std::setw(6) << ask.price <<
FMT("(", ask.quantity, ")");
}
out << std::endl;
}
}
///////////////////////
// book related
///////////////////////
ImpliedBook FifoImpliedSecurityMatcher::book(api::Side side) const {
// this isn't the best way to do this (copying),
// but the ImpliedBook only has 2 levels in practice,
// so it should be fine.
SmallBook tuple_ib;
ImpliedBook ib;
_book(side, tuple_ib, ib.size());
for (uint32_t i = 0; i < tuple_ib.size() && i < ib.size(); i++) {
ib[i].price = tuple_ib[i].price;
ib[i].quantity = tuple_ib[i].quantity;
}
return ib;
}
SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(api::Side side) const {
SmallBook b;
_book(side, b, 1);
if (b.empty())
return {md::IntPrice(), MDP3Quantity(0)};
return b[0];
}
void FifoImpliedSecurityMatcher::_book(api::Side side, SmallBook& dest, uint32_t depth) const {
std::vector<std::tuple<MDP3Quantity, md::IntPrice, SmallBook>> im_vec;
for (const auto& p : _me->implied_mkt) {
SmallBook bk;
auto [sec_matcher, leg_qty] = p.second;
bts_assert(leg_qty.value() != 0);
auto leg_side = leg_qty.value() > 0 ? securities::other_side(side) : side;
for (auto bl : sec_matcher->book(leg_side))
bk.emplace_back(bl.price, bl.quantity);
im_vec.emplace_back(leg_qty, md::INT_NAN, bk);
}
return book_utils::compute_synthetic_book(dest, im_vec, depth);
}
void FifoImpliedSecurityMatcher::add_leg(MDP3SecurityID sec_id, FifoSecurityMatcher* m, MDP3Quantity unit_qty, std::string leg_sym) {
_me->implied_mkt.insert({sec_id, {m, unit_qty}});
_me->leg_symbols.insert_or_assign(sec_id, leg_sym);
}
MDP3Quantity FifoImpliedSecurityMatcher::fill(MDP3OrderKey key, api::Side side, MDP3Quantity qty, std::vector<MatchEffect>* effects, FillDepot& fills) {
SmallBook b;
auto oside = securities::other_side(side);
_book(oside, b, MDP3_MAX_IMPLIED_BOOK_DEPTH);
auto orig_qty = qty;
// can't fill more than the volume in the book
qty = std::min(book_volume(b), qty);
md::IntPrice impl_fill_price = top_of_book(oside).price;
bool has_added_aggr_fills = false;
int fill_qty = 0;
bool local_fills = effects == nullptr;
for (auto kv : _me->implied_mkt) {
auto leg_size = std::get<1>(kv.second);
// todo: support other leg sizes (keeping it simple for now)
bts_assert(leg_size.value() == 1 || leg_size.value() == -1);
api::Side leg_side = leg_size.value() > 0 ? side : securities::other_side(side);
FifoSecurityMatcher& m = *std::get<0>(kv.second);
if (local_fills)
effects = new std::vector<MatchEffect>();
uint32_t leg_fills_pos = effects->size();
auto raw_leg_fill_qty = qty * abs(leg_size);
// INFO("expecting to fill ", raw_leg_fill_qty);
auto raw_leg_rem_qty = m.fill(key, leg_side, md::IntPrice(), raw_leg_fill_qty, effects, false);
// INFO("remaining (unfilled) qty: ", raw_leg_rem_qty);
bts_assert(raw_leg_rem_qty.value() == 0);
for (auto it = effects->begin() + leg_fills_pos; it != effects->end(); it++) {
auto& passive = *it;
bts_assert(std::holds_alternative<OrderUpdate>(passive));
fills.add_fill(kv.first, key, _to_passive_fill(std::get<OrderUpdate>(passive), true));
}
// the implied matchers should not produce partial leg fills (modulus leg_size should be 0)
bts_assert(raw_leg_fill_qty.value() == 0 || raw_leg_fill_qty.value() % abs(leg_size).value() == 0);
uint32_t leg_fill_qty = (raw_leg_fill_qty / abs(leg_size)).value();
bts_assert(leg_fill_qty == static_cast<uint32_t>(qty.value()));
if (!has_added_aggr_fills) {
// This only needs to be done once.
// Any leg's passive fills can serve as a reference for
// the fill qty wrt the implied market: impl_fill_qty = leg_fill_qty / abs(leg_size),
// as the ratio raw_leg_fill_qty:abs(leg_size)
// should be the same for all legs.
// It might not be a bad idea to assert this is the case:
// that all legs produce the exact same aggressor fills.
for (auto it = effects->begin() + leg_fills_pos; it != effects->end(); it++) {
auto& f = *it;
bts_assert(std::holds_alternative<OrderUpdate>(f));
MDP3Fill aggr;
aggr.key = key;
aggr.aggressor = _to_aggressor(side);
aggr.price = impl_fill_price;
aggr.quantity = std::get<OrderUpdate>(f).last_fills / abs(leg_size);
fills.add_fill(security_id(), key, aggr);
fill_qty += aggr.quantity.value();
}
has_added_aggr_fills = true;
}
if (local_fills) {
delete effects;
effects = nullptr;
}
}
bts_assert(fill_qty <= orig_qty.value());
return MDP3Quantity(orig_qty.value() - fill_qty);
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,48 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <tuple>
#include <vector>
namespace bts::spartan::mdp3 {
struct FifoImpliedSecurityMatcher {
FifoImpliedSecurityMatcher(MDP3SecurityID security_id);
// Symbol/sec id serve diagnostic purposes, but aren't absolutely neccessary.
// Logs made by this class begin with a header that includes the symbol/sec id.
FifoImpliedSecurityMatcher(MDP3SecurityID security_id, std::string symbol);
~FifoImpliedSecurityMatcher();
ImpliedBook book(api::Side) const;
MDP3Quantity fill(MDP3OrderKey, api::Side, MDP3Quantity, std::vector<MatchEffect>*, FillDepot&);
// Note: this class is NOT responsible for deleting any FifoSecurityMatcher pointers.
// The FifoMatcherContext creates one FifoMergedSecurityMatcher per each security,
// and each FifoMergedSecurityMatcher creates/owns a FifoSecurityMatcher pointer
// (and is responsible for deleting it).
// The FifoSecurityMatcher pointer passed to this method should be owned by a
// FifoMergedSecurityMatcher.
// Note the absense of a 'side' parameter: unit_qty should be positive for Buys,
// negative for Sells.
void add_leg(MDP3SecurityID sec_id, FifoSecurityMatcher* m, MDP3Quantity unit_qty, std::string leg_sym);
MDP3SecurityID security_id() const;
std::string symbol() const;
SmallBookLevel top_of_book(api::Side side) const;
void print_legs(std::ostream& out = std::cout) const;
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
// hide copy ctr
FifoImpliedSecurityMatcher(const FifoImpliedSecurityMatcher&);
std::string _log_header() const;
void _book(api::Side side, SmallBook& dest, uint32_t depth) const;
uint32_t _fill_order(const MDP3Order& order);
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,242 @@
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/implied/implied_market.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/security_comp.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include <algorithm>
#include <functional>
#include <iostream>
#include <string>
#include <vector>
using namespace bts::spartan::diagnostics;
template<typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
namespace bts::spartan::mdp3 {
///////////////////////
// stateless helpers
///////////////////////
//static inline std::string _log_header() { return "FifoMatcherContext"; }
static inline MDP3Quantity _norm_qty(api::Side side, MDP3Quantity qty) {
switch (side) {
case api::Side::SIDE_BUY: return qty;
case api::Side::SIDE_SELL: return -qty;
default:
bts_assert_msg(false, FMT("unexpected side '", side, "'"));
return qty;
}
}
struct FifoMatcherContext::PImpl {
const bool ignore_ops_on_deleted_orders;
// _implied_candidates_dirty should be set to true anytime a security is added/removed.
// This signals that implied market candidates must be re-evaluated
// before any susbequent order-matching (re-evaluation is performed lazily).
bool implied_candidates_dirty;
ImpliedMarket contract_matrix;
flat_map<MDP3SecurityID, FifoMergedSecurityMatcher*> matchers;
flat_map<MDP3SecurityID, std::string> symbols;
PImpl() : ignore_ops_on_deleted_orders(false) {}
PImpl(const InitArgs& args)
: ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders) {}
~PImpl() {
for (auto& p : matchers)
delete p.second;
matchers.clear();
}
};
///////////////////////
// constructors
///////////////////////
FifoMatcherContext::FifoMatcherContext() : FifoMatcherContext({false}) {}
FifoMatcherContext::FifoMatcherContext(const InitArgs& args) : _me(new PImpl(args)) {}
FifoMatcherContext::~FifoMatcherContext() { }
///////////////////////
// security accessors
///////////////////////
bool FifoMatcherContext::add_security(MDP3SecurityID sec_id, const std::string& symbol, const std::vector<LegDef>& legs) {
auto curr = _me->matchers.find(sec_id);
if (curr == _me->matchers.end()) {
_me->symbols.insert(std::make_pair(sec_id, symbol));
auto& ic = _me->contract_matrix.add_contract(sec_id);
// Add the legs explicitly (ImpliedMarketBuilder doesn't know about
// 'InstrumentDefinition,' or its leg structure 'LegDef').
if (legs.empty()) {
// must be an outright (to normalize outrights and spreads,
// the former is treated like a 'single leg spread')
ic.add_leg(sec_id, MDP3Quantity(1));
} else {
for (auto l : legs)
ic.add_leg(l.security_id, _norm_qty(l.side, l.ratio));
}
_me->matchers.insert(std::make_pair(sec_id, new FifoMergedSecurityMatcher({sec_id, _me->ignore_ops_on_deleted_orders})));
} else {
// Not sure why, but [John]I've seen _me->symbols contain empty strings.
// Overwrite the existing value to ensure a valid symbol.
_me->symbols[sec_id] = symbol;
return false;
}
_me->implied_candidates_dirty = true;
return true;
}
bool FifoMatcherContext::remove_security(MDP3SecurityID sec_id) {
auto curr = _me->symbols.find(sec_id);
if (curr == _me->symbols.end())
return false;
bts_assert(_me->symbols[sec_id] == curr->second);
_me->contract_matrix.remove_contract(sec_id);
_me->symbols.erase(sec_id);
_me->implied_candidates_dirty = true;
return true;
}
bool FifoMatcherContext::implied_matching_status(MDP3SecurityID sec_id) const {
auto* m = _get_matcheri(sec_id);
return m != nullptr && m->implied_matching_status();
}
void FifoMatcherContext::implied_matching_status(MDP3SecurityID sec_id, bool value) {
auto* m = _get_or_add_matcher(sec_id);
if (m->implied_matching_status() != value) {
m->implied_matching_status(value);
_me->implied_candidates_dirty = true;
}
}
///////////////////////
// book
///////////////////////
Book FifoMatcherContext::book(MDP3SecurityID sec_id, api::Side side) {
return _get_matcher(sec_id)->book(side);
}
ImpliedBook FifoMatcherContext::implied_book(MDP3SecurityID sec_id, api::Side side) {
return _get_matcher(sec_id)->implied_book(side);
}
///////////////////////
// matcher accessors
///////////////////////
FifoMergedSecurityMatcher* FifoMatcherContext::_get_matcher(MDP3SecurityID sec_id) {
auto m = _me->matchers.find(sec_id);
return m == _me->matchers.end() ? nullptr : m->second;
}
const FifoMergedSecurityMatcher* FifoMatcherContext::_get_matcheri(MDP3SecurityID sec_id) const {
auto m = _me->matchers.find(sec_id);
return m == _me->matchers.end() ? nullptr : m->second;
}
FifoMergedSecurityMatcher* FifoMatcherContext::_get_or_add_matcher(MDP3SecurityID sec_id) {
auto m = _me->matchers.find(sec_id);
if (m == _me->matchers.end()) {
auto* new_m = new FifoMergedSecurityMatcher({sec_id, _me->ignore_ops_on_deleted_orders});
_me->matchers.insert(std::make_pair(sec_id, new_m));
return new_m;
}
return m->second;
}
std::tuple<FifoPriceLevelMatcher*, MatcherOrder*>
FifoMatcherContext::find_order(MDP3SecurityID sec_id, MDP3OrderKey ok) {
auto* m = _get_matcher(sec_id);
if (m != nullptr)
return m->matcher()->find_order(ok);
return { nullptr, nullptr };
}
///////////////////////
// order callbacks
///////////////////////
void FifoMatcherContext::on_order_change(
MDP3SecurityID sec_id,
const std::vector<SmallOrderChange>& changes,
std::vector<MatchEffect>* effects,
FillDepot& fills) {
//
// SUMMARIZE(_log_header());
_ensure_sync_implied_candidates();
auto* matcher = _get_or_add_matcher(sec_id);
bts_assert(matcher != nullptr);
for (auto ch : changes)
matcher->on_order_change(ch, effects, fills);
}
void FifoMatcherContext::on_order_change(
const SmallOrderChange& change, std::vector<MatchEffect>* effects, FillDepot& fills) {
//
on_order_change(change.order.security, {change}, effects, fills);
}
///////////////////////
// diagnostic
///////////////////////
std::optional<std::string> FifoMatcherContext::has_same_orders(MDP3SecurityID sec, const Orders& orders) {
return _get_matcher(sec)->matcher()->has_same_orders(orders);
}
void FifoMatcherContext::clear() {
for (auto kv : _me->matchers)
kv.second->matcher()->clear();
}
using BBORow = std::tuple<std::string, bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice, MDP3Quantity>;
void FifoMatcherContext::print_all_bbos(std::ostream& out) {
std::vector<BBORow> rows;
for (auto kv : _me->matchers) {
auto bid = kv.second->matcher()->book_level(api::Side::SIDE_BUY, 0);
auto ask = kv.second->matcher()->book_level(api::Side::SIDE_SELL, 0);
rows.emplace_back(
std::make_tuple(_me->symbols[kv.first], std::get<0>(bid), std::get<1>(bid),
std::get<0>(ask), std::get<1>(ask)));
}
std::sort(rows.begin(), rows.end(), [](const BBORow& lhs, const BBORow& rhs) {
return security_comp::SymbolComp()({std::get<0>(lhs)}, {std::get<0>(rhs)});
});
bts::spartan::table_display::print_table(out, rows.begin(), rows.end(),
std::vector<std::tuple<std::string, int, std::function<std::string(BBORow)>>> {
{ " Security ", 12, [](auto r) { return std::get<0>(r); } },
{ " Qty ", 10, [](auto r) { return FMT(std::get<2>(r)); } },
{ " Bid ", 10, [](auto r) { return FMT(std::get<1>(r)); } },
{ " Ask ", 10, [](auto r) { return FMT(std::get<3>(r)); } },
{ " Qty ", 10, [](auto r) { return FMT(std::get<4>(r)); } }
});
}
void FifoMatcherContext::_ensure_sync_implied_candidates() {
if (!_me->implied_candidates_dirty)
return;
for (auto p : _me->matchers) {
std::vector<LegData> candidates = _me->contract_matrix.implied_mkts(p.first);
for (auto can : candidates) {
auto* imatcher = p.second->add_implied_matcher(p.first);
for (auto& leg : can) {
std::string sym;
auto it = _me->symbols.find(leg.first);
if (it != _me->symbols.end())
sym = it->second;
imatcher->add_leg(
leg.first, _get_or_add_matcher(leg.first)->matcher(),
std::get<1>(leg.second), sym);
}
}
}
_me->implied_candidates_dirty = false;
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,94 @@
#pragma once
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <vector>
namespace bts::spartan::mdp3 {
struct FifoMatcherContext {
struct InitArgs {
bool ignore_ops_on_deleted_orders;
};
FifoMatcherContext();
FifoMatcherContext(const InitArgs& args);
~FifoMatcherContext();
bool add_security(MDP3SecurityID sec_id, const std::string& symbol, const std::vector<LegDef>& legs);
bool remove_security(MDP3SecurityID sec_id);
void implied_matching_status(MDP3SecurityID sec_id, bool value);
bool implied_matching_status(MDP3SecurityID sec_id) const;
void clear();
Book book(MDP3SecurityID sec_id, api::Side side);
ImpliedBook implied_book(MDP3SecurityID sec_id, api::Side side);
// the Fills param records individual fills (potentially more than one per order)
void on_order_change(MDP3SecurityID sec_id, const std::vector<SmallOrderChange>& changes, std::vector<MatchEffect>* effects, FillDepot& fills);
void on_order_change(const SmallOrderChange& change, std::vector<MatchEffect>* effects, FillDepot& fills);
void print_all_bbos(std::ostream& out);
std::optional<std::string> has_same_orders(MDP3SecurityID, const Orders&);
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *> find_order(MDP3SecurityID, MDP3OrderKey);
private:
struct PImpl;
std::shared_ptr<PImpl> _me;
//FifoMatcherContext(const FifoMatcherContext& other);
FifoMergedSecurityMatcher* _get_matcher(MDP3SecurityID sec_id);
const FifoMergedSecurityMatcher* _get_matcheri(MDP3SecurityID sec_id) const;
FifoMergedSecurityMatcher* _get_or_add_matcher(MDP3SecurityID sec_id);
void _ensure_sync_implied_candidates();
public:
FifoMergedSecurityMatcher* get_matcher(MDP3SecurityID sec_id) {
return _get_matcher(sec_id);
}
// This method doesn't check for crosses, as it assumes all of the orders
// are resting. The expectation is that this method will be called
// to initialize/reset the books (i.e., in response to the first callback,
// or after reset events).
template<class TIter>
inline void reset(MDP3SecurityID sec_id, TIter first, TIter last, std::vector<MatchEffect>* effects = nullptr) {
_ensure_sync_implied_candidates();
auto* matcher = _get_or_add_matcher(sec_id);
// The 'order' in which orders are added to the matcher matters.
// The matcher uses an internal order priority counter
// so it can integrate custom orders (i.e., generated by our algos)
// into its books during simulations.
// The scope of an order's priority does not extend beyond its price level.
struct Comp {
bool operator()(const MDP3Order& a, const MDP3Order& b) {
if (a.side == b.side) {
return bts::orderman::price_comp::is_better(a.side, a.price, b.price) ||
(a.price == b.price && a.priority < b.priority);
}
return a.side == api::Side::SIDE_BUY;
}
};
std::set<MDP3Order, Comp> sorted_orders;
while (first != last) {
sorted_orders.emplace(*first);
first++;
}
matcher->matcher()->reset(sorted_orders.begin(), sorted_orders.end());
if (effects != nullptr) {
effects->emplace_back(MBPUpdate());
MBPUpdate& mbp = std::get<MBPUpdate>(effects->back());
mbp.security_id = sec_id;
mbp.mbp_buy = matcher->book(api::Side::SIDE_BUY);
mbp.mbp_sell = matcher->book(api::Side::SIDE_SELL);
mbp.implied_mbp_buy = matcher->implied_book(api::Side::SIDE_BUY);
mbp.implied_mbp_sell = matcher->implied_book(api::Side::SIDE_SELL);
}
}
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,369 @@
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h"
#include "lib/md/IntPrice.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include <range/v3/algorithm.hpp>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#define _LOG(level, msg) LOG(OrderMan, level, "{} {}", _log_header(), msg)
#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define WARN(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define ERROR(...) _LOG(Error, bts::orderman::Formatter::fmt(__VA_ARGS__))
using namespace ranges;
using namespace bts::spartan::mdp3::book_utils;
namespace bts::spartan::mdp3 {
struct FifoMergedSecurityMatcher::PImpl {
PImpl(const FifoMergedSecurityMatcher::InitArgs& args)
: security_id(args.security_id),
ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders),
implied_matching_status(true),
matcher(new FifoSecurityMatcher({args.security_id, 0, args.ignore_ops_on_deleted_orders, nullptr})),
implied_matchers() {}
~PImpl() {
for (uint32_t i = 0; i < implied_matchers.size(); i++)
delete implied_matchers[i];
implied_matchers.clear();
}
const MDP3SecurityID security_id;
const bool ignore_ops_on_deleted_orders;
bool implied_matching_status;
// the Security matcher is kept in a shared_ptr
// so it can be passed around to one or more ImpliedMatchers.
// Note: if a signficant performance penalty is attributed
// to the use of shared_ptrs here, it should be fine if this
// class deletes them in its destructor.
// It seems inappropriate for an ImpliedMatcher to outlive
// the SecurityMatchers that make up its legs anyways.
std::shared_ptr<FifoSecurityMatcher> matcher;
std::vector<FifoImpliedSecurityMatcher*> implied_matchers;
};
///////////////////////
// constructors
///////////////////////
FifoMergedSecurityMatcher::FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher::InitArgs& args)
: _me(new FifoMergedSecurityMatcher::PImpl(args)) {}
FifoMergedSecurityMatcher::~FifoMergedSecurityMatcher() { }
///////////////////////
// diagnostic related
///////////////////////
std::string FifoMergedSecurityMatcher::_log_header() const {
return FMT("[", security_id(), "]", "FifoMergedSecurityMatcher");
}
MDP3SecurityID FifoMergedSecurityMatcher::security_id() const { return _me->security_id; }
///////////////////////
// matcher accessors
///////////////////////
FifoSecurityMatcher* FifoMergedSecurityMatcher::matcher() { return _me->matcher.get(); }
const FifoSecurityMatcher* FifoMergedSecurityMatcher::matcheri() const {
return _me->matcher.get();
}
const std::vector<FifoImpliedSecurityMatcher*>& FifoMergedSecurityMatcher::implied_matchers() {
return _me->implied_matchers;
}
FifoImpliedSecurityMatcher* FifoMergedSecurityMatcher::add_implied_matcher(MDP3SecurityID sec) {
auto* m = new FifoImpliedSecurityMatcher(sec);
_me->implied_matchers.push_back(m);
return m;
}
FifoImpliedSecurityMatcher*
FifoMergedSecurityMatcher::_best_implied_matcher(api::Side side) {
if (!_me->implied_matching_status)
return nullptr;
bts::orderman::SidePriceComp cmp(side);
FifoImpliedSecurityMatcher* rv = nullptr;
md::IntPrice best_price;
for (auto& im : _me->implied_matchers) {
auto itop = im->top_of_book(side);
if (itop.quantity.value() > 0 && cmp(itop.price, best_price)) {
rv = im;
best_price = itop.price;
}
}
return rv;
}
bool FifoMergedSecurityMatcher::implied_matching_status() const {
return _me->implied_matching_status;
}
void FifoMergedSecurityMatcher::implied_matching_status(bool value) {
_me->implied_matching_status = value;
}
void FifoMergedSecurityMatcher::on_order_change(const SmallOrderChange& ch, std::vector<MatchEffect>* effects, FillDepot& fills) {
if (!implied_matching_status()) {
// there is no "merged matching" when implied matching is off,
// just pass the order changes the direct matcher.
matcher()->on_order_change(ch, effects);
return;
}
bts_assert_msg(ch.affect_book, "affect-free matching is not supported in markets with implied matching.");
switch (ch.change) {
case OrderChange::ChangeType::updated_price:
case OrderChange::ChangeType::updated:
{
auto prev_ord_tup = matcher()->find_order(ch.order.key);
auto prev_ord = std::get<1>(prev_ord_tup);
if (prev_ord == nullptr) {
bts_assert_msg(_me->ignore_ops_on_deleted_orders, "unable to process order-update event: failed to find order.");
break;
}
if (orderman::price_comp::is_better(ch.order.side, ch.order.price, prev_ord->price)) {
// price improved, try to fill the order
auto new_ch = ch;
_fill(new_ch.order, effects, fills, false);
if (new_ch.order.quantity.value() == 0) {
matcher()->on_order_completed(ch.order.key);
} else {
matcher()->on_order_change(new_ch, effects);
}
} else {
// As far as we know, the only type of update that could trigger fills is a
// price improvement. It is worth noting, however, that there is the
// potential for this assumption to be invalid. In a very strange market, we
// may observe down the road: an implied market with a leg for this security's
// same side market, that also has a leg size greater than 1.
matcher()->on_order_change(ch);
}
break;
}
case OrderChange::ChangeType::added: {
auto new_ch = ch;
_fill(new_ch.order, effects, fills, false);
if (get_total_qty(new_ch.order).value() > 0)
matcher()->on_order_change(new_ch);
break;
}
case OrderChange::ChangeType::removed: {
matcher()->on_order_change(ch);
break;
}
}
}
/////////////////////////
// books
/////////////////////////
Book FifoMergedSecurityMatcher::book(api::Side side) {
return matcher()->book(side);
}
ImpliedBook FifoMergedSecurityMatcher::implied_book(api::Side side) {
if (_me->implied_matchers.empty())
return ImpliedBook();
// In theory, computing the quantity of any price level beyond the top of the book
// should be done in a hypothetical scenario, where an order with the specified price
// and infinite qty is partially filled.
//
// This accounts for the possibility that implied markets may have securities in
// common, and therefore filling an order in one implied market may impact another
// implied market.
//
// However, in practice, the implied market candidates the exchange uses are independent.
//
// TODO: this should be a linked list, as
// the only functionality needed is forward iteration,
// and erase (the latter for optimization).
std::vector<std::tuple<ImpliedBook, int>> ibooks;
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp cmp(oside);
for (auto it = _me->implied_matchers.begin(); it != _me->implied_matchers.end(); it++) {
auto res = ibooks.emplace_back((*it)->book(oside), 0);
auto& ib = std::get<0>(res);
if (ib[0].price.is_nan())
ibooks.pop_back();
}
auto is_exhausted = [&](const std::tuple<ImpliedBook, int>& ib_tup) {
const auto& ib = std::get<0>(ib_tup);
auto i = std::get<1>(ib_tup);
return static_cast<uint32_t>(i) >= ib.size() || ib[i].price.is_nan();
};
auto rem_exhausted = [&]() {
ibooks.erase(std::remove_if(ibooks.begin(), ibooks.end(), is_exhausted));
};
ImpliedBook dest;
for (auto& curr : dest) {
// find best price
curr.price = std::get<0>(*max_element(ibooks, cmp,
[](const auto& ibt) { return std::get<0>(ibt)[std::get<1>(ibt)].price.price(); })
)[0].price;
if (curr.price.is_nan())
break;
curr.quantity = MDP3Quantity(0);
// combine quantities of implied books with the same top price, as there could be multiple.
for (auto& ib_tup : ibooks) {
auto& ib = std::get<0>(ib_tup);
auto& index = std::get<1>(ib_tup);
if (ib[index].price == curr.price)
curr.quantity = MDP3Quantity(curr.quantity.value() + ib[index++].quantity.value());
}
rem_exhausted();
}
return dest;
}
void FifoMergedSecurityMatcher::merged_book(
api::Side side, std::vector<BookLevel>& dest) {
//
std::vector<std::tuple<ImpliedBook, int>> ibooks;
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp cmp(oside);
for (auto it = _me->implied_matchers.begin(); it != _me->implied_matchers.end(); it++) {
auto res = ibooks.emplace_back((*it)->book(oside), 0);
auto& ib = std::get<0>(res);
if (ib[0].price.is_nan())
ibooks.pop_back();
}
auto dbook = matcher()->book(side);
int dindex = 0;
auto is_exhausted = [&](const std::tuple<ImpliedBook, int>& ib_tup) {
const auto& ib = std::get<0>(ib_tup);
auto i = std::get<1>(ib_tup);
return static_cast<uint32_t>(i) >= ib.size() || ib[i].price.is_nan();
};
while (true) {
// erase empties
ibooks.erase(std::remove_if(ibooks.begin(), ibooks.end(), is_exhausted), ibooks.end());
// dir top price/qty
bts::md::IntPrice dprice(
static_cast<uint32_t>(dindex) < dbook.size() ? dbook[dindex].price : bts::md::IntPrice());
MDP3Quantity dqty(dprice.is_nan() ? 0 : dbook[dindex].quantity.value());
// init best price/qty
bts::md::IntPrice best_price(dprice);
MDP3Quantity curr_qty(0);
// find best impl price
for (auto& ib_tup : ibooks) {
auto& ib = std::get<0>(ib_tup);
auto index = std::get<1>(ib_tup);
if (cmp(ib[index].price, best_price))
best_price = ib[index].price;
}
// combine quantities of books with the same top price
for (auto& ib_tup : ibooks) {
auto& ib = std::get<0>(ib_tup);
auto& index = std::get<1>(ib_tup);
auto& bl = ib[index];
if (bl.price == best_price) {
curr_qty = MDP3Quantity(curr_qty.value() + bl.quantity.value());
index++;
}
}
if (dprice == best_price) {
curr_qty = MDP3Quantity(curr_qty.value() + dqty.value());
dindex++;
}
if (curr_qty.value() == 0)
break;
dest.push_back({best_price, curr_qty});
}
}
void FifoMergedSecurityMatcher::_fill(MDP3Order& order, std::vector<MatchEffect>* effects, FillDepot& fills, bool reset_show_qty) {
if (order.iceberg_quantity.has_value()) {
// the order's quantity cannot exceed the show qty (but it can be smaller).
bts_assert(order.quantity <= order.iceberg_quantity.value().show);
}
auto rem = _fill(order.key, order.side, order.price, get_total_qty(order), effects, fills);
reset_show_qty |= rem == get_total_qty(order);
set_total_qty(order, rem, reset_show_qty);
}
inline SmallBookLevel tuple_to_small_book_level(std::tuple<md::IntPrice, MDP3Quantity> t) {
return SmallBookLevel(std::get<0>(t), std::get<1>(t));
}
MDP3Quantity FifoMergedSecurityMatcher::_fill(
MDP3OrderKey key,
api::Side side,
bts::md::IntPrice price,
MDP3Quantity quantity,
std::vector<MatchEffect>* effects,
FillDepot& fills) {
//
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp oside_comp(oside);
while (quantity.value() > 0) {
// Fill one price level at a time.
// While one market may have the best 1st price level,
// another may have the best 2nd price level.
auto* imatcher = _best_implied_matcher(oside);
auto itop = imatcher == nullptr ?
SmallBookLevel() :
imatcher->top_of_book(oside);
auto dtop = tuple_to_small_book_level(matcher()->book_level(oside, 0));
bool is_dcross = !dtop.price.is_nan() && !oside_comp(price, dtop.price);
bool is_icross = !itop.price.is_nan() && !oside_comp(price, itop.price);
if (!is_dcross && !is_icross)
break;
// the order crossed at least one of the markets
auto ord_desc = [&]() { return fmt_order(key, side, price, quantity); };
if (!is_dcross || (is_icross && oside_comp(itop.price, dtop.price))) {
// fill with the implied market
DIAG("filling ", ord_desc(), " in implied market ", fmt_price_level(oside, itop));
auto target_fill_qty = std::min(quantity, itop.quantity);
auto rem = imatcher->fill(key, side, target_fill_qty, effects, fills);
auto fill_qty = target_fill_qty - rem;
quantity -= fill_qty;
DIAG("filled ", fill_qty, ". Order now: ", ord_desc());
} else {
// fill with the direct market
DIAG("filling ", ord_desc(), " in direct market ", fmt_price_level(oside, dtop));
bool local_effects = effects == nullptr;
if (local_effects)
effects = new std::vector<MatchEffect>();
uint32_t fills_pos = effects->size();
auto target_fill_qty = std::min(quantity, dtop.quantity);
auto rem = matcher()->fill(key, side, dtop.price, target_fill_qty, effects, true);
auto fill_qty = target_fill_qty - rem;
quantity -= fill_qty;
DIAG("filled ", fill_qty, ". Order now: ", ord_desc());
MDP3Fill aggr_fill;
bts_assert(aggr_fill.price.is_nan());
aggr_fill.key = key;
aggr_fill.aggressor = _to_aggressor(side);
for (auto it = effects->begin() + fills_pos; it != effects->end(); it++) {
auto& df = *it;
bts_assert(std::holds_alternative<OrderUpdate>(df));
auto& upd = std::get<OrderUpdate>(df);
if (upd.key == key) {
if (aggr_fill.price.is_nan()) {
aggr_fill.price = upd.last_price;
aggr_fill.quantity = upd.last_fills;
} else if (aggr_fill.price == upd.last_price) {
aggr_fill.quantity = MDP3Quantity(aggr_fill.quantity.value() + upd.last_fills.value());
} else {
fills.add_fill(security_id(), key, aggr_fill);
aggr_fill.quantity = MDP3Quantity(0);
aggr_fill.price = bts::md::IntPrice();
bts_assert(aggr_fill.price.is_nan());
}
} else {
fills.add_fill(security_id(), key, _to_passive_fill(upd, false));
}
}
if (local_effects) {
delete effects;
effects = nullptr;
}
if (!aggr_fill.price.is_nan()) {
fills.add_fill(security_id(), key, aggr_fill);
}
}
}
bts_assert(quantity.value() >= 0);
return quantity;
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,54 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include <memory>
#include <string>
#include <vector>
namespace bts::spartan::mdp3 {
struct FifoMergedSecurityMatcher {
struct InitArgs {
MDP3SecurityID security_id;
bool ignore_ops_on_deleted_orders;
};
FifoMergedSecurityMatcher(const InitArgs& args);
~FifoMergedSecurityMatcher();
MDP3SecurityID security_id() const;
void implied_matching_status(bool value);
bool implied_matching_status() const;
void on_order_change(const SmallOrderChange&, std::vector<MatchEffect>*, FillDepot&);
FifoImpliedSecurityMatcher* add_implied_matcher(MDP3SecurityID);
FifoSecurityMatcher* matcher();
const FifoSecurityMatcher* matcheri() const;
const std::vector<FifoImpliedSecurityMatcher*>& implied_matchers();
// books
Book book(api::Side);
ImpliedBook implied_book(api::Side);
void merged_book(api::Side, std::vector<BookLevel>&);
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
// hide copy constr
FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher&);
FifoImpliedSecurityMatcher* _best_implied_matcher(api::Side);
// the order's params (quantity, and iceberg_quantity if applicable)
// will be appropriately adjusted according to the number of fills.
void _fill(MDP3Order&, std::vector<MatchEffect>*, FillDepot&, bool reset_show_qty);
MDP3Quantity _fill(
MDP3OrderKey,
api::Side,
bts::md::IntPrice,
MDP3Quantity,
std::vector<MatchEffect>*,
FillDepot&);
std::string _log_header() const;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,319 @@
#include "FifoPriceLevelMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
namespace bts::spartan::mdp3 {
// private helpers ////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
// orders can cease to be part of this level if there is
// a null slot (ex when the order is physcially moved to another
// price level or at the end of the queue) or the order is cxl'ed
// with the is_active marker
inline bool in_this_level(MatcherOrder const* o_ptr){
return o_ptr!=nullptr && o_ptr->is_active;
}
api::Side _opp_side(api::Side side){
return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL :
api::Side::SIDE_BUY;
}
// O(n) linear search
MatcherOrder *FifoPriceLevelMatcher::_find(MDP3OrderKey key) {
for (MatcherOrder* op : _orders)
if (in_this_level(op) && op->key == key)
return op;
return nullptr;
}
void FifoPriceLevelMatcher::_logically_delete(MatcherOrder &o) {
_log_order(o,"FifoPriceLevelMatcher::_logically_delete");
o.is_active = false;
}
void FifoPriceLevelMatcher::_add_order(const MDP3Order &ord) {
MatcherOrder new_ord (ord.key,ord.side,ord.price,ord.quantity,
_priority++,ord.iceberg_quantity);
auto[p,res] = _master_order_map->try_emplace(new_ord.key,new_ord);
MatcherOrder* new_ord_ptr = &p->second;
_orders.push_back(new_ord_ptr);
if(!res)
// it exists, so this order is moved from another price level
// to this price level
*new_ord_ptr = new_ord;
}
// O(n) iteration through all orders to detect loadable ib's
void FifoPriceLevelMatcher::_reload_icebergs() {
_aux_usage.clear();
for (auto it=_orders.begin();it!=_orders.end();it++){
MatcherOrder* o_ptr = *it;
if(in_this_level(o_ptr) && o_ptr->ready_to_load_ib()){
_aux_usage.push_back(o_ptr);
*it = nullptr;
}
}
// add it back
for (auto o : _aux_usage) {
o->reload_ib();
o->priority = _priority++;
_orders.push_back(o);
}
}
std::tuple<MDP3Quantity, MDP3Quantity>
FifoPriceLevelMatcher::_one_pass_cross_order(MDP3OrderKey cross_ord_key,
MDP3Quantity cross_ord_qty,
bool add_agg_side_to_match_effect,
std::vector<MatchEffect>* res) {
// go through all the orders and see which ones get filled
// (orders are sorted by priority)
for (MatcherOrder* op : _orders) {
if (in_this_level(op)) {
auto& o = *op;
auto [o_rem, cross_rem, filled] = o.cross(cross_ord_qty.value());
cross_ord_qty = MDP3Quantity(cross_rem);
if(res != nullptr){
OrderUpdate passive_upd =
{
o.key,
o.total_qty().value() > 0 ?
OrderUpdate::Status::PARTIALLY_FILLED :
OrderUpdate::Status::FILLED,
_sec,
_side,
MDP3Quantity(filled),
_price,
OrderUpdate::FillType::PASSIVE
};
MatcherUtils::validate_outgoing_update(passive_upd);
res->push_back(passive_upd);
if(add_agg_side_to_match_effect){
OrderUpdate agg_upd =
{
cross_ord_key,
cross_ord_qty.value() > 0 ?
OrderUpdate::Status::PARTIALLY_FILLED :
OrderUpdate::Status::FILLED,
_sec,
_opp_side(_side),
MDP3Quantity(filled),
_price,
OrderUpdate::FillType::AGGRESSIVE
};
MatcherUtils::validate_outgoing_update(agg_upd);
res->push_back(agg_upd);
}
}
if(_trace.log(o.key) || _trace.log(cross_ord_key))
LOG(OrderMan, Diag,
"FifoPriceLevelMatcher::_one_pass_cross_order "
"{} (new_qty:{}) crossed by {} (new_qty:{})",
o.key, o_rem, cross_ord_key, cross_rem);
if (cross_rem > 0)
bts_assert(o_rem == 0); // visible qty should be completely crossed
else
break; // crossed order is completely filled
}
}
if (cross_ord_qty.value() > 0)
bts_assert(quantity().value() == 0);
// reload the filled ib's and put them at the back
_reload_icebergs();
return std::make_tuple(total_quantity(), cross_ord_qty);
}
void FifoPriceLevelMatcher::_log_order(const MatcherOrder& ord,
const std::string& msg){
if(_trace.log(ord.key))
LOG(OrderMan, Diag,
"{} key:{} side:{} qty:{} ib_qty:{} price:{}",
msg, ord.key, static_cast<int>(side()), ord.quantity,
ord.ib.has_value() ? ord.ib.value().total : MDP3Quantity(0),
price());
}
void FifoPriceLevelMatcher::_log_order(const MDP3Order& ord,
const std::string& msg){
if(_trace.log(ord.key))
LOG(OrderMan, Diag,
"{} key:{} side:{} qty:{} ib_qty:{} price:{}",
msg, ord.key, static_cast<int>(ord.side), ord.quantity,
ord.iceberg_quantity.has_value() ? FMT(ord.iceberg_quantity.value())
: FMT(MDP3Quantity(0)),
ord.price.price());
}
// constructors ///////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
FifoPriceLevelMatcher::FifoPriceLevelMatcher(
std::map<MDP3OrderKey,MatcherOrder>& master_order_map,
MDP3SecurityID sec,
md::IntPrice price,
api::Side side,
const Trace& trace)
: _master_order_map(&master_order_map),
_sec(sec), _price(price), _side(side), _orders(), _aux_usage(),
_priority(0), _trace(trace) {}
FifoPriceLevelMatcher::FifoPriceLevelMatcher(const FifoPriceLevelMatcher &o) {
*this = o;
}
FifoPriceLevelMatcher::FifoPriceLevelMatcher(FifoPriceLevelMatcher &&o) {
*this = std::move(o);
}
// operator overloads /////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
FifoPriceLevelMatcher &
FifoPriceLevelMatcher::operator=(const FifoPriceLevelMatcher &o) {
_master_order_map = o._master_order_map;
_sec = o._sec;
_price = o._price;
_side = o._side;
_orders = o._orders;
_aux_usage = o._aux_usage;
_priority = o._priority;
return *this;
}
FifoPriceLevelMatcher &
FifoPriceLevelMatcher::operator=(FifoPriceLevelMatcher &&o) {
_master_order_map = std::move(o._master_order_map);
_sec = std::move(o._sec);
_price = std::move(o._price);
_side = std::move(o._side);
_orders = std::move(o._orders);
_aux_usage = std::move(o._aux_usage);
_priority = std::move(o._priority);
return *this;
}
bool FifoPriceLevelMatcher::operator<(const FifoPriceLevelMatcher &o) const {
bts_assert(_side == o._side);
// bid book is sorted high to low (since highest is the top of the book)
return _side == api::Side::SIDE_SELL ? _price < o._price : o._price < _price;
}
// simple accessors ///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
std::tuple<uint32_t, uint32_t> FifoPriceLevelMatcher::n_orders_qty(
bool visible_only) const{
auto [c, q] = std::make_tuple(0, 0);
for (MatcherOrder* o : _orders)
if(in_this_level(o)){
c++;
q += visible_only ? o->working_qty().value() : o->total_qty().value();
}
return std::make_tuple(c, q);
}
bool FifoPriceLevelMatcher::populated() const {
for (auto* o : _orders)
if (in_this_level(o) && o->working_qty().value() > 0)
return true;
return false;
}
void FifoPriceLevelMatcher::copy_values_to(std::vector<MatcherOrder>& out_v) const{
out_v.clear();
for(MatcherOrder* o : _orders)
if(in_this_level(o))
out_v.push_back(*o);
}
// order interactions /////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void FifoPriceLevelMatcher::on_new_same_side_order(const MDP3Order &new_ord) {
bts_assert(new_ord.price == _price);
bts_assert(new_ord.side == _side);
_add_order(new_ord);
}
std::tuple<MDP3Quantity,MDP3Quantity> FifoPriceLevelMatcher::on_new_opp_order(
MDP3OrderKey opp_ord_key,
MDP3Quantity cross_qty,
bool add_agg_side_to_match_effect,
std::vector<MatchEffect>* res) {
auto rv = _one_pass_cross_order(opp_ord_key, cross_qty,
add_agg_side_to_match_effect, res);
// continue until either the crossing order is consumed
// or the price level is consumed
while (std::get<0>(rv).value() > 0 && std::get<1>(rv).value() > 0)
rv = _one_pass_cross_order(opp_ord_key, std::get<1>(rv),
add_agg_side_to_match_effect, res);
return rv;
}
void FifoPriceLevelMatcher::on_order_update(
const MDP3Order &upd_ord,
MatcherOrder* ord_ptr_hint) {
// ord_ptr_hint is a hint, if its populated use, else find
bts_assert(upd_ord.side == _side);
bts_assert(upd_ord.price == _price);
if(ord_ptr_hint == nullptr){
auto map_it = _master_order_map->find(upd_ord.key);
bts_assert(map_it != _master_order_map->end());
ord_ptr_hint = &map_it->second;
}
MatcherOrder& ord = *ord_ptr_hint;
if (upd_ord.quantity.value() > ord.quantity.value()) {
// remove it from the current location O(N) and add it to
// the end. unlike qty decreases this is a lot more expensive
remove_from_price_level(ord.key);
_orders.push_back(&ord);
// udpate value
ord.quantity = upd_ord.quantity;
} else {
ord.quantity = upd_ord.quantity;
if (ord.ib.has_value() && upd_ord.iceberg_quantity.has_value() && ord.ib.value() != upd_ord.iceberg_quantity.value())
ord.ib = upd_ord.iceberg_quantity;
}
}
void FifoPriceLevelMatcher::on_order_cancel(MatcherOrder& order,
std::vector<MatchEffect>* res){
_logically_delete(order);
OrderUpdate o = {
order.key,
OrderUpdate::Status::CANCELED,
_sec,
_side,
MDP3Quantity(0),
_price,
std::nullopt
};
if(res != nullptr)
res->push_back(o);
}
void FifoPriceLevelMatcher::on_order_remove(MDP3OrderKey k,
std::vector<MatchEffect>* res) {
MatcherOrder *ord_p = _find(k);
bts_assert(ord_p != nullptr);
on_order_cancel(*ord_p,res);
}
void FifoPriceLevelMatcher::remove_from_price_level(MDP3OrderKey k) {
// puts a nullptr in place of where this order was
// (perioric cleanup will physically delete the slot)
auto it = std::find_if(_orders.begin(), _orders.end(),
[&](MatcherOrder* o) {
return in_this_level(o) && o->key == k;
});
bts_assert(it != _orders.end());
bts_assert(*it != nullptr);
*it = nullptr;
}
void FifoPriceLevelMatcher::cleanup() {
_orders.erase(std::remove_if(_orders.begin(), _orders.end(),
[](const MatcherOrder* o) {
return !in_this_level(o);
}),
_orders.end());
}
std::string FifoPriceLevelMatcher::print_mbo(size_t n_orders) const{
std::ostringstream buff;
auto[n_ords,qty] = n_orders_qty();
buff << (side()==api::Side::SIDE_BUY ? "B" : "S") << "," << price();
buff << "," << qty << "," << n_ords << ":";
size_t i=0;
for (MatcherOrder* o : _orders){
if(in_this_level(o)){
if(++i>n_orders)
break;
buff << o->key << "," << o->quantity << ":";
}
}
return buff.str();
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,116 @@
#pragma once
// did not want to fwd decl everything specially
// the templated classes (will fix later)
#include "MatchingCommon.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
namespace bts::spartan::mdp3 {
// Builds the order state for all orders correponding to one
// price level for one security
class FifoPriceLevelMatcher {
private:
// this is the order_map that the sec level matcher owns
// for all the orders for the security. orders are created
// here but added to the map mantained by the
// sec level matcher
std::map<MDP3OrderKey,MatcherOrder>* _master_order_map;
MDP3SecurityID _sec;
md::IntPrice _price;
api::Side _side;
std::vector<MatcherOrder*> _orders;
std::vector<MatcherOrder*> _aux_usage;
uint32_t _priority;
Trace _trace;
// helpers ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
private:
MatcherOrder *_find(MDP3OrderKey key);
void _logically_delete(MatcherOrder &o);
void _add_order(const MDP3Order &ord);
// crosses the cross_order with existing orders. returns
// the remaining qty in the level and the cross order
void _reload_icebergs();
// crosses the order in the args with the orders at this level.
// returns the tuple of rem qty at this level and the rem qty
// of the cross order
std::tuple<MDP3Quantity, MDP3Quantity>
_one_pass_cross_order(MDP3OrderKey cross_ord_key,
MDP3Quantity cross_ord_qty,
bool add_agg_side_to_match_effect,
std::vector<MatchEffect>* res);
void _log_order(const MatcherOrder& ord,
const std::string& msg);
void _log_order(const MDP3Order& ord, const std::string& msg);
public:
// constructors ////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
FifoPriceLevelMatcher(std::map<MDP3OrderKey,MatcherOrder>&,
MDP3SecurityID sec, md::IntPrice price,
api::Side side, const Trace&);
FifoPriceLevelMatcher(const FifoPriceLevelMatcher &);
FifoPriceLevelMatcher(FifoPriceLevelMatcher &&);
// operator overloads //////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
FifoPriceLevelMatcher &operator=(const FifoPriceLevelMatcher &o);
FifoPriceLevelMatcher &operator=(FifoPriceLevelMatcher &&o);
bool operator==(const FifoPriceLevelMatcher &o) const {
bts_assert(_side == o._side);
return _price == o._price;
}
bool operator!=(const FifoPriceLevelMatcher &o) const {
bts_assert(_side == o._side);
return _price != o._price;
}
bool operator<(const FifoPriceLevelMatcher &o) const;
// simple accessors ////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
std::tuple<uint32_t, uint32_t> n_orders_qty(bool visible_only = true) const;
MDP3Quantity quantity() const {
return MDP3Quantity(std::get<1>(n_orders_qty()));
}
MDP3Quantity total_quantity() const {
return MDP3Quantity(std::get<1>(n_orders_qty(false)));
}
uint32_t n_orders() const { return std::get<0>(n_orders_qty()); }
bool populated() const;
md::IntPrice price() const { return _price; }
double price_double() const { return _price.price(); }
api::Side side() const { return _side; }
MatcherOrder *find_order(MDP3OrderKey key) { return _find(key); }
BookLevel book_level() const {
return BookLevel{_price, quantity(), static_cast<int>(n_orders())};
}
// used for testing
void copy_values_to(std::vector<MatcherOrder>& out_v) const;
// mutators (silly reason) /////////////////////////////////////////////////
void set_price_side(md::IntPrice p, api::Side s) {
_price = p;
_side = s;
}
// order interactions //////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// order with the same side and price as this price level
void on_new_same_side_order(const MDP3Order &new_ord);
// order update such that only the qty changed (other changes
// are handled at a higher level)
// new order such that it crosses this price level
std::tuple<MDP3Quantity,MDP3Quantity> on_new_opp_order(
MDP3OrderKey opp_ord_key,
MDP3Quantity opp_ord_qty,
bool add_agg_side_to_match_effect,
std::vector<MatchEffect>* = nullptr);
void on_order_update(const MDP3Order& upd_ord,
MatcherOrder* ord_hint = nullptr);
void on_order_cancel(MatcherOrder& order,
std::vector<MatchEffect>* = nullptr);
void on_order_remove(MDP3OrderKey k,
std::vector<MatchEffect>* = nullptr);
void remove_from_price_level(MDP3OrderKey k);
void cleanup();
std::string print_mbo(size_t n_orders) const;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,552 @@
#include "FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include <experimental/map>
namespace bts::spartan::mdp3 {
// helpers ////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
inline bool _is_cross(const MDP3Order &ord,
const FifoPriceLevelMatcher &price_level) {
return ord.side != price_level.side() &&
((ord.side == api::Side::SIDE_BUY &&
ord.price >= price_level.price()) ||
(ord.side == api::Side::SIDE_SELL &&
ord.price <= price_level.price()));
}
inline MDP3Quantity total_quantity(const MDP3Order &ord) {
return ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value().total
: ord.quantity;
}
inline api::Side _opp_side(api::Side side) {
return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL
: api::Side::SIDE_BUY;
}
// O(lg(N)) on the price levels
FifoPriceLevelMatcher *
FifoSecurityMatcher::_find_price_level(api::Side side, md::IntPrice price) {
auto &bk = _book(side);
_search_tok.set_price_side(price, side);
auto it = bk.find(_search_tok);
return it != bk.end() ? &*it : nullptr;
}
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *>
FifoSecurityMatcher::_find_order(MDP3OrderKey key) {
auto map_it = _order_map.find(key);
if(map_it != _order_map.end()){
auto& ord = map_it->second;
if(ord.is_active){
FifoPriceLevelMatcher* p_level = _find_price_level(ord.side,ord.price);
return std::make_tuple(p_level,&ord);
}
}
return std::tuple<FifoPriceLevelMatcher *, MatcherOrder *>(
nullptr, nullptr);
}
FifoPriceLevelMatcher* FifoSecurityMatcher::_top(api::Side side){
for(auto& plevel : _book(side))
if(plevel.populated())
return &plevel;
return nullptr;
}
// this is expensive
void FifoSecurityMatcher::_cleanup() {
// remove orders from the price levels
for (auto& p : _bids.book())
p.cleanup();
for (auto& p : _asks.book())
p.cleanup();
// remove empty levels
auto rem_if = [](const auto &p_lev) { return !p_lev.populated(); };
_bids.book().erase(std::remove_if(_bids.book().begin(), _bids.book().end(), rem_if),
_bids.book().end());
_asks.book().erase(std::remove_if(_asks.book().begin(), _asks.book().end(), rem_if),
_asks.book().end());
// remove inactive orders from map
std::experimental::erase_if(_order_map, [](const auto& p) {
auto& ord = std::get<1>(p);
return !ord.is_active;
});
}
FifoPriceLevelMatcher* FifoSecurityMatcher::_get_or_add_price_level(api::Side side, md::IntPrice price) {
auto* price_level = _find_price_level(side, price);
if (price_level == nullptr) {
auto& same_side_book = _book(side);
// this is the only place where a new price level is created
auto p = same_side_book.emplace(_order_map, _sec, price, side, _trace);
bts_assert(std::get<1>(p));
price_level = &*std::get<0>(p);
}
return price_level;
}
void FifoSecurityMatcher::_add_resting_order(const MDP3Order& ord) {
_get_or_add_price_level(ord.side, ord.price)->on_new_same_side_order(ord);
}
void FifoSecurityMatcher::_on_new_order(const MDP3Order &ord,
std::vector<MatchEffect>* res) {
_log_trace_order(ord,"FifoSecurityMatcher::_on_new_order begin");
auto& opp_book = _book(_opp_side(ord.side));
uint32_t new_ord_rem = total_quantity(ord).value();
for (FifoPriceLevelMatcher &pl : opp_book) {
if (_is_cross(ord, pl)){
auto t = pl.on_new_opp_order(ord.key, MDP3Quantity(new_ord_rem),
true, res);
new_ord_rem = std::get<1>(t).value();
}
else
break;
if (new_ord_rem == 0)
break;
bts_assert(pl.quantity().value() == 0);
// stops crossing when the price level is not a cross or there
// no qty left
}
if (new_ord_rem != 0) {
// either the price did not cross into the opp market
// or it cleared out the entire opposite side
// in either case, it becomes a resting order
MDP3Order ord_corrected = ord;
if (ord_corrected.iceberg_quantity.has_value()) {
auto ibq = ord.iceberg_quantity.value();
auto filled_qty = ibq.total - MDP3Quantity(new_ord_rem);
// important: don't subtract filled_qty from the total
// of the IcebergQuantity (ibq) param passed to 'get_reload_qty.'
ord_corrected.quantity = iceberg_helper::get_reload_qty(ord.quantity, filled_qty, ibq);
ord_corrected.iceberg_quantity.value().total.value() = new_ord_rem;
} else
ord_corrected.quantity.value() = new_ord_rem;
_add_resting_order(ord_corrected);
}
else{
// since _on_new_order is also the same code path
// thats called when an order changes price levels
auto it = _order_map.find(ord.key);
if(it != _order_map.end()){
MatcherOrder& ex_ord = it->second;
ex_ord.quantity = MDP3Quantity(0);
ex_ord.is_active = false;
}
}
}
void FifoSecurityMatcher::_on_order_update(const MDP3Order &ord,
bool price_only,
std::vector<MatchEffect>* res) {
_log_trace_order(ord,"FifoSecurityMatcher::_on_order_update begin");
auto[price_level,curr_ord] = _find_order(ord.key);
if(price_level == nullptr || curr_ord == nullptr){
if(_ignore_ops_on_deleted_orders)
return;
}
bts_assert(price_level != nullptr);
bts_assert(curr_ord != nullptr);
auto new_ord = ord;
// ib total qty is considered immutable, and show qty
// changes are intepreted through regular quantity chagnes.
// Once created, the iceberg_quantity field carries no new
// information about the order, so it is expected to be null.
bts_assert(!new_ord.iceberg_quantity.has_value());
// side can not be changed (or can it?)
bts_assert(price_level->side() == new_ord.side);
if (curr_ord->ib.has_value()) {
// check if show qty changed
if (new_ord.quantity.value() > 0) {
// For iceberg orders, quantity and show quantity
// changes are one in the same.
// Note that quantity updates are bounded by the iceberg's
// total qty, which is considered immutable.
if (new_ord.quantity > curr_ord->ib.value().total)
new_ord.quantity = curr_ord->ib.value().total;
curr_ord->ib.value().show = new_ord.quantity;
} else {
new_ord.quantity = curr_ord->quantity;
}
new_ord.iceberg_quantity = curr_ord->ib;
}
if (price_only) {
new_ord.quantity = curr_ord->quantity;
}
// check for price change
if (price_level->price() != new_ord.price) {
// remove from current level and add to another
price_level->remove_from_price_level(curr_ord->key);
_on_new_order(new_ord, res);
} else {
price_level->on_order_update(new_ord, curr_ord);
}
_log_trace_order(ord,"FifoSecurityMatcher::on_order_update end");
}
void FifoSecurityMatcher::_on_order_remove(MDP3OrderKey key,
std::vector<MatchEffect>* res) {
if(_trace.log(key))
LOG(OrderMan, Diag,
"FifoSecurityMatcher::_on_order_remove "
"eid:{} key:{}",
_event_id, key);
auto[price_level,curr_ord] = _find_order(key);
if(price_level == nullptr || curr_ord == nullptr)
if(_ignore_ops_on_deleted_orders)
return;
bts_assert(price_level != nullptr);
bts_assert(curr_ord != nullptr);
price_level->on_order_cancel(*curr_ord,res);
// we can short circuit this by setting order's is_active
// marker to false here, but its more consistent to let
// the price level handlers take care of this
//(which does make it O(lg(M*N)) instead of lg(N), for the
// extra price level lookup)
}
void FifoSecurityMatcher::_on_order_not_affecting_books(
const SmallOrderChange& o,
std::vector<MatchEffect>* res){
auto& ord = o.order;
switch (o.change) {
case OrderChange::added:
case OrderChange::updated:
case OrderChange::updated_price:
{
// since this does not affect the market all we need to do
// is deal with the cross
auto& opp_book = _book(_opp_side(ord.side));
uint32_t new_ord_rem = total_quantity(ord).value();
for (const FifoPriceLevelMatcher &pl : opp_book) {
if(!pl.populated())
continue;
if (_is_cross(ord, pl)){
uint32_t level_qty =
static_cast<uint32_t>(pl.total_quantity().value());
auto[new_level_qty_ret,new_ord_rem_ret] =
MatcherOrder::compute_cross(level_qty,new_ord_rem);
auto fill_size = level_qty - new_level_qty_ret;
new_ord_rem = new_ord_rem_ret;
if(res != nullptr)
res->push_back(OrderUpdate{
ord.key,
new_ord_rem > 0 ?
OrderUpdate::Status::PARTIALLY_FILLED :
OrderUpdate::Status::FILLED,
ord.security,
ord.side,
MDP3Quantity(fill_size),
pl.price(),
OrderUpdate::FillType::AGGRESSIVE
});
}
else
break;
if (new_ord_rem == 0)
break;
}
break;
}
case OrderChange::removed:
if(res != nullptr)
res->push_back(OrderUpdate{
ord.key,
OrderUpdate::Status::CANCELED,
ord.security,
ord.side,
MDP3Quantity(0),
md::IntPrice(),
std::nullopt
});
// nothing to do since this order was never made a part of the book
break;
}
}
// constructors ///////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
FifoSecurityMatcher::FifoSecurityMatcher(MDP3SecurityID security,
size_t event_id)
:
FifoSecurityMatcher(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
security,event_id,true,nullptr
})
{}
FifoSecurityMatcher::FifoSecurityMatcher(MDP3SecurityID security,
size_t event_id,
const Orders& orders)
:
FifoSecurityMatcher(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
security,event_id,true,&orders
})
{}
FifoSecurityMatcher::FifoSecurityMatcher(
FifoSecurityMatcher::FifoSecurityMatcherArgs args)
:
_order_map(),
_bids(),
_asks(),
_search_tok(_order_map, MDP3SecurityID{},
md::IntPrice(), api::Side::SIDE_BUY, Trace()),
_sec(args.security_id),
_event_id(args.event_id),
_trace(),
_ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders),
_clean_up_op_threshold(2500),
_running_op_count(0) {
if(args.orders != nullptr){
const auto& orders = *args.orders;
std::vector<MDP3Order> sorted_orders;
sorted_orders.reserve(orders.size());
for (const auto &kv : orders)
sorted_orders.emplace_back(kv.second);
std::sort(sorted_orders.begin(), sorted_orders.end(),
[](const auto &o1, const auto &o2) {
return o1.priority < o2.priority;
});
LOG(OrderMan, Diag, "FifoSecurityMatcher::cons begin eid:{} n_orders:{}",
_event_id, orders.size());
for (auto const &o : sorted_orders)
_on_new_order(o);
LOG(OrderMan, Diag, "FifoSecurityMatcher::cons end eid:{}", _event_id);
_log_mbo(10,100);
}
_running_op_count = 0;
}
FifoSecurityMatcher::~FifoSecurityMatcher(){
_log_trace_order_state("FifoSecurityMatcher::dest");
}
// SecurityMatcher implementation /////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
const Book& FifoSecurityMatcher::BookKeeper::mbp() {
if (_is_dirty) {
int i = 0;
for (const auto& p_level : _book) {
if (p_level.populated()) {
if (i < MDP3_MAX_BOOK_DEPTH)
_mbp[i++] = p_level.book_level();
else
break;
}
}
// Explicitly set the remaining entries to nan/0
// (they could have stale values since the book is recycled).
while (i < MDP3_MAX_BOOK_DEPTH) {
_mbp[i].price = md::IntPrice();
_mbp[i++].quantity.value() = 0;
}
_is_dirty = false;
}
return _mbp;
}
Book FifoSecurityMatcher::book(api::Side side) {
return (side == api::Side::SIDE_BUY ? _bids : _asks).mbp();
}
size_t FifoSecurityMatcher::total_n_orders(api::Side side) const{
auto rv=0;
for (const auto& p_level : _book(side))
rv+=p_level.n_orders();
return rv;
}
MDP3Quantity FifoSecurityMatcher::total_qty(api::Side side) const{
auto rv=0;
for (const auto& p_level : _book(side))
rv+=p_level.total_quantity().value();
return MDP3Quantity(rv);
}
std::tuple<md::IntPrice, MDP3Quantity> FifoSecurityMatcher::book_level (
api::Side side, size_t idx) const {
size_t i = 0;
for (const auto& p_level : _book(side))
if (p_level.populated() && i++ == idx)
return std::make_tuple(p_level.price(), p_level.total_quantity());
return std::make_tuple(md::INT_NAN, MDP3Quantity(0));
}
void FifoSecurityMatcher::on_order_change(SmallOrderChange const &o,
std::vector<MatchEffect>* res) {
if(!o.affect_book){
_on_order_not_affecting_books(o,res);
return;
}
switch (o.change) {
case OrderChange::added:
_on_new_order(o.order, res);
break;
case OrderChange::updated:
_on_order_update(o.order, false,res);
break;
case OrderChange::updated_price:
_on_order_update(o.order, true,res);
break;
case OrderChange::removed:
_on_order_remove(o.order.key, res);
break;
}
_invalidate_mbp();
if(_running_op_count++ > _clean_up_op_threshold){
_running_op_count = 0;
_cleanup();
}
}
void FifoSecurityMatcher::on_order_completed(MDP3OrderKey key) {
// todo: actual implementation
// (this is a hack that treats completed orders like cancels)
auto [pl, ord] = _find_order(key);
bts_assert(pl != nullptr);
bts_assert_msg(ord != nullptr, "did not find order that was supposedely 'completed.'");
MDP3Order order;
order.key = key;
order.side = api::SIDE_UNDEFINED;
SmallOrderChange ch (order,OrderChange::ChangeType::removed);
on_order_change(ch);
}
MDP3Quantity FifoSecurityMatcher::fill(MDP3OrderKey aggr_key,
api::Side side,
md::IntPrice agg_price,
MDP3Quantity qty,
std::vector<MatchEffect>* res,
bool add_aggr_fill){
FifoPriceLevelMatcher* top = _top(_opp_side(side));
if(top != nullptr){
bts_assert(top->price()==agg_price || agg_price.is_nan());
auto t = top->on_new_opp_order(aggr_key,qty,add_aggr_fill,res);
_invalidate_mbp();
return std::get<1>(t);
}
return MDP3Quantity(0);
}
std::optional<std::string>
FifoSecurityMatcher::has_same_orders(const Orders &ord_map) {
if(_trace.is_ignored_event(_event_id))
return std::nullopt;
for (const auto &kv : ord_map) {
const auto &exp_ord = kv.second;
auto [plevel, ord] = _find_order(exp_ord.key);
auto match = plevel != nullptr && ord != nullptr &&
plevel->price() == exp_ord.price &&
plevel->side() == exp_ord.side &&
ord->quantity == exp_ord.quantity &&
((!exp_ord.iceberg_quantity.has_value() ||
exp_ord.iceberg_quantity.value().total.value() == 0) ||
ord->ib == exp_ord.iceberg_quantity);
if (!match) {
std::ostringstream buff;
buff << "eid:" << _event_id << " n_ords:" << ord_map.size()
<< std::endl;
buff << "Book:" << print_book(10) << std::endl;
if (ord == nullptr){
buff << "Order with Key:" << exp_ord.key.value() << " not found"
<< std::endl;
buff << "Expected price:" << exp_ord.price << std::endl;
buff << "Expected qty:" << exp_ord.quantity << std::endl;
buff << "Expected side:" << exp_ord.side << std::endl;
buff << "Expected IB state:" << exp_ord.iceberg_quantity << std::endl;
}
else {
buff << "Order with Key:" << exp_ord.key.value()
<< " does not match" << std::endl;
buff << "Expected price:" << exp_ord.price
<< " matcher:" << plevel->price() << std::endl;
buff << "Expected qty:" << exp_ord.quantity
<< " matcher:" << ord->quantity << std::endl;
buff << "Expected side:" << exp_ord.side
<< " matcher:" << plevel->side() << std::endl;
buff << "Expected IB state:" << exp_ord.iceberg_quantity
<< " matcher:" << ord->ib << std::endl;
}
return buff.str();
}
}
LOG(OrderMan, Diag, "FifoSecurityMatcher::has_same_orders passed");
return std::nullopt;
}
void FifoSecurityMatcher::clear() {
_bids.book().clear();
_asks.book().clear();
_order_map.clear();
_invalidate_mbp();
_running_op_count = 0;
}
// logging and printing ///////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
std::string FifoSecurityMatcher::print_book(size_t n_levels) {
// auto rem_if = [](const auto& p_lev) { return !p_lev.populated(); };
std::ostringstream buff;
auto add_to_buff = [&](const auto &pl) {
std::string side = pl.side() == api::Side::SIDE_BUY ? "B" : "S";
buff << side << ":" << pl.price_double() << ":" << pl.quantity().value()
<< ":" << pl.n_orders() << std::endl;
};
// asks are stored lowest to highest
size_t i=0;
for (const auto &pl : _asks.book()){
if(++i>n_levels)
break;
add_to_buff(pl);
}
// bids are scored highest to lowest
i=0;
for (const auto &pl : _bids.book()){
if(++i>n_levels)
break;
add_to_buff(pl);
}
return buff.str();
}
void FifoSecurityMatcher::_log_trace_order_state(const std::string& msg){
for (const auto& kv : _trace.traced_orders){
MDP3OrderKey k = kv.first;
auto[pl,o] = _find_order(k);
if(pl != nullptr && o != nullptr)
LOG(OrderMan, Diag,
"Order state msg:{} eid:{} key:{} qty:{} ib:{} price:{}",
msg, _event_id, o->key, o->quantity, o->ib, pl->price());
}
}
void FifoSecurityMatcher::_log_trace_order(const MDP3Order& ord,
const std::string& msg){
if(_trace.log(ord.key,_event_id)){
LOG(OrderMan, Diag,
"{} eid:{} key:{} side:{} qty:{} ib_qty:{} price:{} book:{}",
msg, _event_id, ord.key, static_cast<int>(ord.side), ord.quantity,
ord.iceberg_quantity, ord.price.price(), print_book());
_log_trace_order_state(msg);
}
}
void FifoSecurityMatcher::_log_mbo(size_t n_levels, size_t n_orders){
if(_trace.log(_event_id)){
std::ostringstream buff;
buff << "eid:" << _event_id << std::endl;
size_t i=0;
for (const auto &pl : _asks.book()){
if(++i>n_levels)
break;
buff << pl.print_mbo(n_orders) << std::endl;
}
i=0;
for (const auto &pl : _bids.book()){
if(++i>n_levels)
break;
buff << pl.print_mbo(n_orders) << std::endl;
}
auto msg = buff.str();
LOG(OrderMan, Diag, "{}", msg);
}
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,159 @@
#pragma once
// did not want to fwd decl everything specially
// the templated classes (will fix later)
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "FifoPriceLevelMatcher.h"
#include <boost/container/flat_set.hpp>
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "SecurityMatcher.h"
#include <optional>
namespace bts::spartan::mdp3 {
class FifoSecurityMatcher : public SecurityMatcher {
public:
struct FifoSecurityMatcherArgs{
MDP3SecurityID security_id;
const size_t event_id = 0;
const bool ignore_ops_on_deleted_orders = true;
Orders const* orders = nullptr;
};
private:
struct BookKeeper {
boost::container::flat_set<FifoPriceLevelMatcher>& book() { return _book; }
const boost::container::flat_set<FifoPriceLevelMatcher>& booki() const { return _book; }
const Book& mbp();
void invalidate() { _is_dirty = true; }
private:
Book _mbp;
bool _is_dirty = true;
boost::container::flat_set<FifoPriceLevelMatcher> _book;
};
// for orders we need a structure to provide fast access
// and where adding and removing elements does not invalidate
// iterators (or cached pointers)
// https://en.cppreference.com/w/cpp/container#Iterator_invalidation
// while map (trees) are slower than unordered_map (pure hashtables),
// pointers get invalidated when they are rehashed (since we dont
// have a key collision, would it be rehashed?)
std::map<MDP3OrderKey,MatcherOrder> _order_map;
// ask prices are sorted lowest to highest and and bid prices
// are sorted highest to lowest (see BookKeeper::_book).
BookKeeper _bids, _asks;
FifoPriceLevelMatcher _search_tok;
const MDP3SecurityID _sec;
const size_t _event_id;
const Trace _trace;
const bool _ignore_ops_on_deleted_orders;
const size_t _clean_up_op_threshold;
size_t _running_op_count;
private:
// helpers /////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// complexity: O(lg(n))
FifoPriceLevelMatcher *_find_price_level(api::Side, md::IntPrice price);
boost::container::flat_set<FifoPriceLevelMatcher> &_book(api::Side side) {
return (side == api::Side::SIDE_BUY ? _bids : _asks).book();
}
const boost::container::flat_set<FifoPriceLevelMatcher> &
_book(api::Side side) const {
return (side == api::Side::SIDE_BUY ? _bids : _asks).booki();
}
void _invalidate_mbp() {
_bids.invalidate();
_asks.invalidate();
}
// the complexity is O(lgN+lgM == lg(M*N)), since first it first searchess
// for the order, from the order it gets the price, side and searches
// for price level in the book (flat_set)
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *>
_find_order(MDP3OrderKey key);
FifoPriceLevelMatcher* _top(api::Side side);
void _cleanup();
// order interaction
void _on_new_order(const MDP3Order&, std::vector<MatchEffect>* = nullptr);
void _on_order_update(const MDP3Order&, bool price_only = false,std::vector<MatchEffect>* = nullptr);
void _on_order_remove(MDP3OrderKey,
std::vector<MatchEffect>* = nullptr);
void _on_order_not_affecting_books(const SmallOrderChange& o,
std::vector<MatchEffect>* res);
std::string _log_header() const;
public:
// cons/dest ///////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
FifoSecurityMatcher(MDP3SecurityID security, size_t event_id = 0);
FifoSecurityMatcher(MDP3SecurityID security,
size_t event_id,
Orders const &);
FifoSecurityMatcher(FifoSecurityMatcher::FifoSecurityMatcherArgs);
~FifoSecurityMatcher();
// SecurityMatcher implementation //////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
Book book(api::Side) override;
// The qty, and add_aggr_fill params address a couple of issues wrt 'merged matching:'
// Particular quantities of passive fills need to be generated when
// an aggressor order is filled against an implied market, while the aggressor
// fills must be generated independently (there are several passive legs,
// but only one aggressor security). The 'add_aggr_fill' param would be false in
// this case, and the 'qty' param would be limited by either a thinner market in
// another leg, by the volume in this market's top price level, or by the aggressor
// order's quantity itself.
//
// In the case of direct market fills, both aggressor and passive fills
// should be generated (of course, so 'add_aggr_fill' would be true).
// However, the fill qty may also be constricted to a certain number of
// price levels. This is because the market with the best price doesn't
// necessarily have the 2nd best price as well.
MDP3Quantity fill(MDP3OrderKey aggr_key, api::Side side, md::IntPrice price,
MDP3Quantity qty,
std::vector<MatchEffect>* res, bool add_aggr_fill);
std::tuple<md::IntPrice, MDP3Quantity> book_level(
api::Side side, size_t idx) const override;
void on_order_change(SmallOrderChange const &,
std::vector<MatchEffect>* = nullptr) override ;
void on_order_completed(MDP3OrderKey key);
// For testing
std::optional<std::string> has_same_orders(Orders const &);
size_t total_n_orders(api::Side) const;
MDP3Quantity total_qty(api::Side) const;
// Removes existing orders.
void clear();
// Removes existing orders, then populates the book with the
// specified sequence of MDP3Orders.
template <class Iter>
void reset(Iter first, Iter last) {
clear();
while (first != last) {
_add_resting_order(*first);
first++;
}
}
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *> find_order(MDP3OrderKey key) { return _find_order(key); }
// debugging logging //////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
public:
std::string print_book(size_t n_levels=3);
private:
void _log_trace_order_state(const std::string& msg);
void _log_trace_order(const MDP3Order& ord, const std::string& msg);
void _log_mbo(size_t n_levels,size_t n_orders);
// Inserts the specified order at the appropriate price level without
// attempting to fill it. While this won't matter in most cases
// (resting orders don't cross the opposite market anyways) 'on_new_order'
// imposes a side effect on iceberg orders which '_add_resting_order' does not:
//
// after attempting to fill the new iceberg order,
// '_on_new_order' resets the order's [visible]quantity
// to the minimum of its show, and total quantity.
//
// This is obviously problematic if the aim is to inject an exact state
// into the matcher, as the order's visible quantity is often less than
// the show quantity (e.g., after partial fills, before the next reload).
void _add_resting_order(const MDP3Order&);
FifoPriceLevelMatcher* _get_or_add_price_level(api::Side side, md::IntPrice);
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,46 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include <functional>
#include <iostream>
#include <map>
#include <string>
#include <tuple>
#include <vector>
namespace bts::spartan::mdp3 {
struct ImpliedFifoSecurityMatcher {
ImpliedFifoSecurityMatcher(MDP3SecurityID security_id)
: _security_id(security_id), _implied_mkt() {}
ImpliedBook book(api::Side side) const;
MDP3Quantity
fill(MDP3OrderKey key, api::Side side, MDP3Quantity qty, FillDepot& fills);
// Note: this class is NOT responsible for deleting any FifoSecurityMatcher pointers.
// The FifoMatcherContext creates one FifoMergedSecurityMatcher per each security,
// and each FifoMergedSecurityMatcher creates/owns a FifoSecurityMatcher pointer
// (and is responsible for deleting it).
// The FifoSecurityMatcher pointer passed to this method should be owned by a
// FifoMergedSecurityMatcher.
// Note the absense of a 'side' parameter: unit_qty should be positive for Buys,
// negative for Sells.
void add_leg(MDP3SecurityID sec_id, FifoSecurityMatcher* m, MDP3Quantity unit_qty) {
_implied_mkt.insert({sec_id, {m, unit_qty}});
}
MDP3SecurityID security_id() const { return _security_id; }
std::tuple<md::IntPrice, MDP3Quantity> top_of_book(api::Side side) const;
private:
MDP3SecurityID _security_id;
std::map<MDP3SecurityID, std::tuple<FifoSecurityMatcher*, MDP3Quantity>> _implied_mkt;
// hide copy ctr
ImpliedFifoSecurityMatcher(const ImpliedFifoSecurityMatcher&);
void _book(api::Side side, std::vector<std::tuple<md::IntPrice, MDP3Quantity>>& dest) const;
uint32_t _fill_order(const MDP3Order& order);
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,266 @@
#include "Matcher.h"
#include "MatchingCommon.h"
#include <google/protobuf/util/time_util.h>
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h"
#include <boost/container/flat_map.hpp>
#include <functional>
#include <iostream>
using namespace google::protobuf::util;
namespace bts::spartan::mdp3{
MergedMatcher::MergedMatcher()
: _matcher_map(),
_trace(), _match_count(0), _match_count_this_per(0),
_per_start_time(TimeUtil::GetCurrentTime()) { }
MergedMatcher::MergedMatcher(MergedMatcher&& src) {
_matcher_map = std::move(src._matcher_map);
}
void MergedMatcher::add_def(const InstrumentDefinition& def) {
std::string ass(def.asset());
_asset_map.emplace(def.security_id(), ass);
auto m = _matcher_map.find(ass);
if (m == _matcher_map.end())
m = _matcher_map.emplace(ass, FifoMatcherContext({true})).first;
m->second.add_security(def.security_id(), std::string(def.symbol()), def.legs());
m->second.implied_matching_status(def.security_id(), def.implied_matching());
}
std::string MergedMatcher::_get_asset(MDP3SecurityID sec) const {
auto it = _asset_map.find(sec);
return it == _asset_map.end() ? std::string() : it->second;
}
template<int L>
using MBP = std::array<BookLevel, L>;
template<int L>
inline bool mbp_equals(const MBP<L>& a, const MBP<L>& b) {
for (uint32_t i = 0; i < a.size(); i++) {
if (a[i].price != b[i].price || a[i].quantity != b[i].quantity)
return false;
}
return true;
}
// using structs rather than std::tuples to avoid
// accidentally mixing up before/after, or bids/asks.
struct MBPData {
MBPData() {}
MBPData(FifoMergedSecurityMatcher& mat)
: bids(mat.book(api::Side::SIDE_BUY)), asks(mat.book(api::Side::SIDE_SELL)),
ibids(mat.implied_book(api::Side::SIDE_BUY)), iasks(mat.implied_book(api::Side::SIDE_SELL)) {}
Book bids, asks;
ImpliedBook ibids, iasks;
};
inline bool operator==(const MBPData& lhs, const MBPData& rhs) {
auto lev_eq = [](const BookLevel& x, const BookLevel& y) {
return x.price == y.price && x.quantity == y.quantity;
};
return std::equal(lhs.bids.begin(), lhs.bids.end(), rhs.bids.begin(), lev_eq) &&
std::equal(lhs.asks.begin(), lhs.asks.end(), rhs.asks.begin(), lev_eq) &&
std::equal(lhs.ibids.begin(), lhs.ibids.end(), rhs.ibids.begin(), lev_eq) &&
std::equal(lhs.iasks.begin(), lhs.iasks.end(), rhs.iasks.begin(), lev_eq);
}
struct MBPSnapshot {
MBPSnapshot() : _matcher(nullptr), _initial_mbp() {}
MBPSnapshot(FifoMergedSecurityMatcher* matcher)
: _matcher(matcher), _initial_mbp(*_matcher) {}
std::optional<MBPData> get_new_mbp() {
bts_assert(_matcher != nullptr);
auto curr = MBPData(*_matcher);
if (_initial_mbp == curr)
return {};
return curr;
}
private:
FifoMergedSecurityMatcher* _matcher;
MBPData _initial_mbp;
};
FifoMatcherContext& MergedMatcher::_asset_matcher(MDP3SecurityID sec, bool add_if_missing) {
auto ass_it = _asset_map.find(sec);
if (add_if_missing && ass_it == _asset_map.end())
ass_it = _asset_map.emplace(sec, std::string()).first;
bts_assert(ass_it != _asset_map.end());
auto mat_it = _matcher_map.find(ass_it->second);
if (add_if_missing && mat_it == _matcher_map.end())
mat_it = _matcher_map.emplace(ass_it->second, FifoMatcherContext({true})).first;
bts_assert(mat_it != _matcher_map.end());
return mat_it->second;
}
void MergedMatcher::match(ChangeSrc,
SmallOrderChangeVec const&changes,
std::vector<MatchEffect>* res) {
_match_count_this_per++;
_match_count++;
// This could be better optimized, but the idea is to
// 1) take a snapshot of every security's book/implied book,
// 2) process the order changes,
// 3) compare the current books with the snapshots to detect mbp changes.
// The map's hierarchcy is asset->security->books.
std::map<std::string, std::map<MDP3SecurityID, MBPSnapshot>> mbps;
for (const auto& c : changes) {
bts_assert(c.order.key.value() != 0);
auto ass = _get_asset(c.order.security);
auto it = mbps.find(ass);
if (it == mbps.end())
it = mbps.emplace(ass, std::map<MDP3SecurityID, MBPSnapshot>()).first;
auto mbp_tup_it = it->second.find(c.order.security);
if (mbp_tup_it == it->second.end()) {
auto* mat = _matcher_map[ass].get_matcher(c.order.security);
bts_assert(mat != nullptr);
it->second.emplace(c.order.security, MBPSnapshot(mat));
}
}
for (const auto& c : changes){
_trace.log_incoming_change(c);
MatcherUtils::validate_incoming_order_change(c);
switch (c.change) {
case OrderChange::ChangeType::added:
bts_assert(_used_order_keys.emplace(c.order.key).second);
break;
case OrderChange::ChangeType::updated:
case OrderChange::ChangeType::updated_price:
case OrderChange::ChangeType::removed:
bts_assert(_used_order_keys.count(c.order.key));
break;
}
// this holds individual fills, meaning one aggr fill per price,
// and potentially multiple fills per order.
// todo: make the fills param optional, as
// it will often be more info than we care abt.
FillDepot fills;
_asset_matcher(c.order.security).on_order_change(c, res, fills);
}
// log/validate order updates
for(const auto& u : *res)
_trace.log_outgoing_update(std::get<OrderUpdate>(u));
for(const auto& u : *res)
MatcherUtils::validate_outgoing_update(std::get<OrderUpdate>(u));
// add mbp updates
for (auto& ass_kv : mbps) {
for (auto& sec_kv : ass_kv.second) {
auto new_mbp = sec_kv.second.get_new_mbp();
if (new_mbp.has_value()) {
res->emplace_back(MBPUpdate{
sec_kv.first,
new_mbp.value().bids,
new_mbp.value().asks,
new_mbp.value().ibids,
new_mbp.value().iasks
});
}
}
}
auto now = TimeUtil::GetCurrentTime();
auto usecs = TimeUtil::TimestampToMicroseconds(now) -
TimeUtil::TimestampToMicroseconds(_per_start_time);
if(usecs >= 1000000){
LOG(OrderMan, Diag, "usecs:{} events processed:{} tot events processed:{}",
usecs, _match_count_this_per, _match_count);
_match_count_this_per = 0;
_per_start_time = now;
}
}
Book MergedMatcher::book(MDP3SecurityID id, api::Side s) {
return _asset_matcher(id).book(id, s);
}
ImpliedBook MergedMatcher::implied_book(MDP3SecurityID id, api::Side s) {
return _asset_matcher(id).implied_book(id, s);
}
void MergedMatcher::clear() {
for (auto& mp : _matcher_map)
mp.second.clear();
}
/////////////////
// DirectMatcher
////////////////
DirectMatcher::DirectMatcher()
:
_matcher_map(),
_trace(),
_match_count(0),
_match_count_this_per(0),
_per_start_time(TimeUtil::GetCurrentTime()){
}
DirectMatcher::DirectMatcher(DirectMatcher&& src) {
_matcher_map = std::move(src._matcher_map);
}
DirectMatcher::DirectMatcher(const std::vector<MDP3SecurityID>& inst_defs) {
for (const auto& sec_id : inst_defs)
_matcher_map.insert(std::pair<MDP3SecurityID, FifoSecurityMatcher>(
sec_id, FifoSecurityMatcher(sec_id)));
}
void DirectMatcher::match(ChangeSrc,
SmallOrderChangeVec const&changes,
std::vector<MatchEffect>* res) {
_match_count_this_per++;
_match_count++;
std::vector<std::pair<MDP3SecurityID, FifoSecurityMatcher*>> dirty;
dirty.reserve(_matcher_map.size());
for (const auto& c : changes) {
auto it = _matcher_map.find(c.order.security);
bts_assert (it != _matcher_map.end());
auto& sec_matcher = it->second;
_trace.log_incoming_change(c);
MatcherUtils::validate_incoming_order_change(c);
sec_matcher.on_order_change(c, res);
dirty.emplace_back(c.order.security, &it->second);
}
for(const auto& u : *res)
_trace.log_outgoing_update(std::get<OrderUpdate>(u));
for(const auto& u : *res)
MatcherUtils::validate_outgoing_update(std::get<OrderUpdate>(u));
// all order activity is not reflecting in the
// books
for (auto cm : dirty)
res->emplace_back(
MBPUpdate
{
cm.first,
cm.second->book(api::Side::SIDE_BUY),
cm.second->book(api::Side::SIDE_SELL),
ImpliedBook(),
ImpliedBook()
});
auto now = TimeUtil::GetCurrentTime();
auto usecs = TimeUtil::TimestampToMicroseconds(now) -
TimeUtil::TimestampToMicroseconds(_per_start_time);
if(usecs >= 1000000){
LOG(OrderMan, Diag, "usecs:{} events processed:{} tot events processed:{}",
usecs, _match_count_this_per, _match_count);
_match_count_this_per = 0;
_per_start_time = now;
}
}
Book DirectMatcher::book(MDP3SecurityID id, api::Side s) {
auto it = _matcher_map.find(id);
bts_assert(it != _matcher_map.end());
return it->second.book(s);
}
void DirectMatcher::clear() {
for (auto& p : _matcher_map)
p.second.clear();
}
}
@@ -0,0 +1,102 @@
#pragma once
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "FifoSecurityMatcher.h"
#include "FifoMatcherContext.h"
#include "MatchingCommon.h"
#include <google/protobuf/timestamp.pb.h>
#include <map>
#include <unordered_set>
#include <string>
namespace bts::spartan::mdp3 {
class InstrumentDefinition;
class MergedMatcher {
private:
std::unordered_set<MDP3OrderKey> _used_order_keys;
std::map<MDP3SecurityID, std::string> _asset_map;
std::map<std::string, FifoMatcherContext> _matcher_map;
Trace _trace;
size_t _match_count;
size_t _match_count_this_per;
google::protobuf::Timestamp _per_start_time;
std::string _get_asset(MDP3SecurityID sec) const;
FifoMatcherContext& _asset_matcher(MDP3SecurityID, bool add_if_missing = true);
public:
MergedMatcher ();
MergedMatcher (MergedMatcher&&);
MergedMatcher &operator=(MergedMatcher&&) {return *this;}
using MatchEffect =
std::variant<MBPUpdate,OrderUpdate>;
enum ChangeSrc {
ALGO, SIMULATOR_INPUT
};
void add_def(const InstrumentDefinition& def);
void match(ChangeSrc,
SmallOrderChangeVec const&,
std::vector<MatchEffect> *);
Book book(MDP3SecurityID, api::Side);
ImpliedBook implied_book(MDP3SecurityID, api::Side);
template<class TIter>
void init(TIter first, TIter last) {
while (first != last) {
add_def(*first);
first++;
}
}
void reset(MDP3SecurityID sec, std::vector<MDP3Order> orders, std::vector<MatchEffect>* effects = nullptr) {
for (const auto& o : orders)
_used_order_keys.emplace(o.key);
_asset_matcher(sec).reset(sec, orders.begin(), orders.end(), effects);
}
void clear();
};
// no implied matching (implied integration perf isn't great,
// need to investigate before DirectMatcher can be retired).
class DirectMatcher {
private:
std::map<MDP3SecurityID, FifoSecurityMatcher> _matcher_map;
Trace _trace;
size_t _match_count;
size_t _match_count_this_per;
google::protobuf::Timestamp _per_start_time;
public:
DirectMatcher ();
DirectMatcher (std::vector<MDP3SecurityID> const&);
DirectMatcher (DirectMatcher&&);
DirectMatcher &operator=(DirectMatcher&&) {return *this;}
using MatchEffect =
std::variant<MBPUpdate,OrderUpdate>;
enum ChangeSrc {
ALGO, SIMULATOR_INPUT
};
void match(ChangeSrc,
SmallOrderChangeVec const&,
std::vector<MatchEffect> *);
Book book(MDP3SecurityID, api::Side);
ImpliedBook implied_book(MDP3SecurityID, api::Side);
template<class Iter>
void reset(MDP3SecurityID sec, Iter first, Iter last) {
_matcher_map.try_emplace(sec).first->second.reset(first, last);
}
void clear();
};
}
@@ -0,0 +1,26 @@
#include "MatchingCommon.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
namespace bts::spartan::mdp3 {
std::tuple<uint32_t, uint32_t, uint32_t> MatcherOrder::cross(uint32_t cross_qty) {
auto [r, c] = compute_cross(quantity.value(), cross_qty);
bts_assert(quantity.value() - r == cross_qty - c);
auto filled = cross_qty - c;
quantity = MDP3Quantity(r);
if (ib.has_value()) {
ib.value().total -= MDP3Quantity(filled);
is_active = ib.value().total.value() > 0;
} else
is_active = quantity.value() > 0;
return std::make_tuple(r, c, filled);
}
void MatcherOrder::reload_ib() {
if (ready_to_load_ib()) {
is_active = ib.value().total.value() > 0;
if (quantity.value() == 0)
quantity = std::min(ib.value().show, ib.value().total);
is_active = quantity.value() > 0;
}
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,131 @@
#pragma once
// did not want to fwd decl everything specially
// the templated classes (will fix later)
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include <optional>
namespace bts::spartan::mdp3 {
struct Trace{
public:
const std::map<MDP3OrderKey,bool> traced_orders =
{
{MDP3OrderKey(765164629193),true}
};
const std::map<size_t,bool> traced_events = {};
const std::map<size_t,bool> ignored_events = {};
bool log(MDP3OrderKey key, size_t eid=0) const{
return traced_orders.find(key) != traced_orders.end() ||
traced_events.find(eid) != traced_events.end();
}
bool log(size_t eid) const{
return traced_events.find(eid) != traced_events.end();
}
bool is_ignored_event(size_t eid) const{
return ignored_events.find(eid) != ignored_events.end();
}
std::string _side(api::Side s){
return s == api::Side::SIDE_BUY ? "BUY" : "SELL";
}
void log_incoming_change(const SmallOrderChange& c){
if(traced_orders.find(c.order.key) != traced_orders.end())
LOG(OrderMan, Diag,
"Matcher incoming change:{} ok:{} side:{} price:{} qty:{} ib:{}",
c.change, c.order.key, _side(c.order.side), c.order.price,
c.order.quantity,
c.order.iceberg_quantity);
}
void log_outgoing_update(const OrderUpdate& u){
if(traced_orders.find(u.key) != traced_orders.end())
LOG(OrderMan, Diag,
"Matcher outgoing update:{} ok:{} side:{} last_fp:{} last_fq:{}",
u.status, u.key, _side(u.side), u.last_price, u.last_fills);
}
};
class MatcherUtils{
public:
static void validate_incoming_order_change(const SmallOrderChange& c){
// In sim input, when an iceberg order is updated
// such that only its price changed, the quantity field
// is set to 0 so the matcher knows not to change the show qty,
// At the cme, it appears that the show qty can only be
// changed by updating the 'visible' qty (i.e., the order's qty
// in the mbo).
// todo: use the OrderChange::ChangeType::updated_price enum
// to indicate this kind of update, then integrate that logic
// into the assert statement below. Another possible remedy
// would be to indicate show qty changes through the show qty
// field (in which case the matcher would need to reflect the
// show qty change in the visible qty, instead of the other way around).
//
// if(c.change == OrderChange::ChangeType::added ||
// c.change == OrderChange::ChangeType::updated)
if (c.change == OrderChange::ChangeType::added)
bts_assert(c.order.quantity.value() > 0);
}
static void validate_outgoing_update(const OrderUpdate& u){
if (u.status == OrderUpdate::Status::PARTIALLY_FILLED ||
u.status == OrderUpdate::Status::FILLED)
bts_assert(u.last_fills.value() > 0);
if (u.status == OrderUpdate::Status::CANCELED)
bts_assert(u.last_fills.value() == 0);
}
};
struct MatcherOrder {
static std::tuple<uint32_t, uint32_t> compute_cross(uint32_t resting_qty,
uint32_t cross_qty) {
if (cross_qty >= resting_qty) {
cross_qty -= resting_qty;
resting_qty = 0;
} else {
bts_assert(resting_qty > cross_qty);
resting_qty -= cross_qty;
cross_qty = 0;
}
return std::make_tuple(resting_qty, cross_qty);
}
MDP3OrderKey key;
api::Side side;
md::IntPrice price;
MDP3Quantity quantity;
uint32_t priority = 0;
std::optional<IcebergQuantity> ib;
bool is_active = true;
MatcherOrder(MDP3OrderKey key,
api::Side side,
md::IntPrice price,
MDP3Quantity qty, // working qty out there
uint32_t priority,
std::optional<IcebergQuantity> ib = {})
:
key(key),
side(side),
price(price),
quantity(qty),
priority(priority),
ib(ib) {}
// returns the remainder of this order, remainder of opp order
// (ie the new qty NOT the filled qty, for that subtract from
// original)
std::tuple<uint32_t, uint32_t, uint32_t> cross(uint32_t cross_qty);
MDP3Quantity total_qty() const {
return ib.has_value() ? ib.value().total : quantity;
}
MDP3Quantity working_qty() const { return quantity; }
bool ready_to_load_ib() const {
return ib.has_value() && quantity.value() == 0 && ib.value().total.value() > 0;
}
void reload_ib();
};
inline std::ostream& operator <<(std::ostream& out, const MatcherOrder& ord) {
out << "[" << ord.key << "] " << ord.side << " " << ord.quantity << " @ " << ord.price
<< " (ice " << ord.ib << " active " << (ord.is_active ? "Y" : "N") << ")";
return out;
}
}
@@ -0,0 +1,23 @@
#pragma once
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
namespace bts::spartan::mdp3 {
class SecurityMatcher {
public:
virtual ~SecurityMatcher() {}
// accessors
// unf the book() method cannot be const right now.
// The FifoSecurityMatcher recomputes books lazily
// for performance reasons. Its internal state will change
// when this method is called if its book is dirty,
// and has to be recomputed.
virtual Book book(api::Side) = 0;
virtual std::tuple<md::IntPrice, MDP3Quantity> book_level(
api::Side, size_t level = 0) const = 0;
// mutators
virtual void on_order_change(SmallOrderChange const &,
std::vector<MatchEffect>* = nullptr) = 0;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,85 @@
#include "fill_depot.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include <range/v3/algorithm.hpp>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#include <range/v3/algorithm/any_of.hpp>
#include <range/v3/algorithm/for_each.hpp>
#include <range/v3/algorithm/none_of.hpp>
using namespace ranges;
namespace bts::spartan::mdp3 {
bool FillDepot::add_fill(MDP3SecurityID sec, MDP3OrderKey aggressor, MDP3Fill f) {
auto sfit = _cache.find(sec);
if (sfit == _cache.end()) {
auto p = _cache.emplace(
sec, std::unordered_map<MDP3OrderKey, std::unordered_map<double, FillNode>>());
sfit = p.first;
}
auto ofit = sfit->second.find(f.key);
if (ofit == sfit->second.end()) {
auto p = sfit->second.emplace(f.key, std::unordered_map<double, FillNode>());
ofit = p.first;
}
auto pfit = ofit->second.find(f.price.price());
if (pfit == ofit->second.end()) {
pfit = ofit->second.emplace(f.price.price(), FillNode()).first;
}
if (pfit->second.merge(aggressor, f)) {
_fill_count++;
return true;
}
return false;
}
MDP3Quantity FillNode::total_fill_qty() const {
auto dfills = accumulate(
direct_passive | views::transform([](auto& f) { return f.quantity.value(); }), 0);
auto ifills = accumulate(
implied_passive | views::transform([](auto& f) { return f.quantity.value(); }), 0);
return MDP3Quantity(
buy_aggressor.quantity.value() + sell_aggressor.quantity.value() + dfills + ifills);
}
MDP3Quantity FillDepot::total_fill_qty() const {
return MDP3Quantity(
accumulate(_cache | views::transform([](auto& kv) {
return accumulate(kv.second | views::transform([](auto& of) {
return accumulate(of.second | views::transform([](auto& pf) { return pf.second.total_fill_qty().value(); }), 0);
}), 0);
}), 0));
}
void FillDepot::copy_to(std::vector<MDP3Fill>& dest) const {
for (auto& sec_fills : _cache) {
for(auto& ord_fills : sec_fills.second) {
for (auto& price_fills : ord_fills.second)
price_fills.second.copy_to(dest);
}
}
}
void FillDepot::copy_to(std::unordered_map<MDP3SecurityID, std::vector<MDP3Fill>>& dest) const {
for (auto& sec_fills : _cache) {
auto& fills = dest.emplace(sec_fills.first, std::vector<MDP3Fill>()).first->second;
for(auto& ord_fills : sec_fills.second) {
for (auto& price_fills : ord_fills.second)
price_fills.second.copy_to(fills);
}
}
}
bool FillNode::match(const MDP3Fill& f) const {
switch (f.aggressor) {
case MDP3Fill::Aggressor::THIS_BUY: return _price_size_eq(buy_aggressor, f);
case MDP3Fill::Aggressor::THIS_SELL: return _price_size_eq(sell_aggressor, f);
case MDP3Fill::Aggressor::OTHER:
return any_of(direct_passive, [&](auto& of) { return _price_size_eq(of, f); });
case MDP3Fill::Aggressor::IMPLIED:
return any_of(implied_passive, [&](auto& of) { return _price_size_eq(of, f); });
}
return false;
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,132 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "lib/utils/bts_assert.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include <iostream>
#include <string>
#include <tuple>
#include <unordered_map>
#include <vector>
namespace bts::spartan::mdp3 {
struct FillNode {
FillNode() : buy_aggressor(), sell_aggressor(), direct_passive(), implied_passive() {}
private:
static inline bool _price_size_eq(const MDP3Fill& lhs, const MDP3Fill& rhs) {
return lhs.quantity == rhs.quantity;
}
static inline bool _merge_fill(MDP3Fill& target, const MDP3Fill& fill) {
bool is_new = target.price.is_nan();
if (is_new) {
target.key = fill.key;
target.price = fill.price;
target.aggressor = fill.aggressor;
} else {
bts_assert(target.key == fill.key);
bts_assert(target.price == fill.price);
bts_assert(target.aggressor == fill.aggressor);
}
target.quantity = MDP3Quantity(fill.quantity.value() + target.quantity.value());
return is_new;
}
static inline bool _merge_fill(std::vector<MDP3Fill>& target, const MDP3Fill& fill, bool combine = false) {
if (combine && target.size()) {
_merge_fill(target[target.size() -1], fill);
return false;
}
target.push_back(fill);
return true;
}
public:
uint32_t fill_count() const { return _fill_count; }
void set_curr_aggressor(MDP3OrderKey k) {
curr_aggressor = k;
}
bool merge(MDP3OrderKey aggressor, MDP3Fill f) {
bool rv = false;
switch (f.aggressor) {
case MDP3Fill::Aggressor::THIS_BUY: rv = _merge_fill(buy_aggressor, f); break;
case MDP3Fill::Aggressor::THIS_SELL: rv = _merge_fill(sell_aggressor, f); break;
case MDP3Fill::Aggressor::OTHER: rv = _merge_fill(direct_passive, f, aggressor == curr_aggressor); break;
case MDP3Fill::Aggressor::IMPLIED: rv = _merge_fill(implied_passive, f, aggressor == curr_aggressor); break;
default: bts_assert(false); return false;
}
if (rv)
_fill_count++;
curr_aggressor = aggressor;
return rv;
}
bool match(const MDP3Fill& f) const;
void copy_to(std::vector<MDP3Fill>& dest) const {
if (buy_aggressor.quantity.value() > 0)
dest.push_back(buy_aggressor);
if (sell_aggressor.quantity.value() > 0)
dest.push_back(sell_aggressor);
for (auto f : direct_passive)
dest.push_back(f);
for (auto f : implied_passive)
dest.push_back(f);
}
MDP3Quantity total_fill_qty() const;
private:
MDP3Fill buy_aggressor;
MDP3Fill sell_aggressor;
MDP3OrderKey curr_aggressor;
std::vector<MDP3Fill> direct_passive, implied_passive;
uint32_t _fill_count = 0;
};
struct FillDepot {
bool add_fill(MDP3SecurityID sec, MDP3OrderKey aggressor, MDP3Fill f);
uint32_t fill_count() const { return _fill_count; }
MDP3Quantity total_fill_qty() const;
MDP3Quantity aggressor_fill_qty(MDP3SecurityID sec, MDP3OrderKey ord, int max_fill_items = 9999999) {
bts_assert(max_fill_items >= 0);
if (max_fill_items == 0)
return MDP3Quantity(0);
auto it = _cache.find(sec);
if (it == _cache.end()) {
return MDP3Quantity(0);
}
auto ord_it = it->second.find(ord);
if (ord_it == it->second.end()) {
return MDP3Quantity(0);
}
int res = 0;
for (auto pfit = ord_it->second.begin(); pfit != ord_it->second.end(); pfit++) {
std::vector<MDP3Fill> ord_fills;
pfit->second.copy_to(ord_fills);
for (auto& f : ord_fills) {
switch (f.aggressor) {
case MDP3Fill::Aggressor::THIS_BUY:
case MDP3Fill::Aggressor::THIS_SELL:
res += f.quantity.value();
if (--max_fill_items == 0)
return MDP3Quantity(res);
break;
default: break;
}
}
}
return MDP3Quantity(res);
}
void copy_to(std::vector<MDP3Fill>& dest) const;
void copy_to(std::unordered_map<MDP3SecurityID, std::vector<MDP3Fill>>& dest) const;
private:
uint32_t _fill_count = 0;
std::unordered_map<MDP3SecurityID,
std::unordered_map<MDP3OrderKey,
std::unordered_map<double,
FillNode>>> _cache;
};
std::ostream& operator<<(std::ostream& out, const FillDepot& fd);
} // namespace bts::spartan::mdp3
@@ -0,0 +1,32 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
namespace bts::spartan::mdp3 {
inline MDP3Fill::Aggressor _to_aggressor(api::Side s) {
switch (s) {
case api::Side::SIDE_BUY: return MDP3Fill::Aggressor::THIS_BUY;
case api::Side::SIDE_SELL: return MDP3Fill::Aggressor::THIS_SELL;
default: return MDP3Fill::Aggressor::OTHER;
}
}
inline MDP3Fill _to_passive_fill(const OrderUpdate& upd, bool implied_aggressor) {
MDP3Fill passive;
passive.key = upd.key;
passive.price = upd.last_price;
passive.quantity = upd.last_fills;
passive.aggressor =
implied_aggressor ? MDP3Fill::Aggressor::IMPLIED : MDP3Fill::Aggressor::OTHER;
return passive;
}
inline MDP3Fill
_to_aggressor_fill(MDP3OrderKey k, api::Side side, const OrderUpdate& upd) {
MDP3Fill aggr;
aggr.key = k;
aggr.price = upd.last_price;
aggr.quantity = upd.last_fills;
aggr.aggressor = _to_aggressor(side);
return aggr;
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,94 @@
#include "iceberg_helper.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/utils/map_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
namespace bts::spartan::mdp3::iceberg_helper {
#define NOT_IMPLEMENTED(...) \
bts_assert_msg(false, orderman::Formatter::fmt(__VA_ARGS__, " not implemented."));
void add_iceberg_annotation(
MDP3Order& o,
OrderChange::ChangeType change_type,
const std::unordered_map<MDP3OrderKey, IcebergInput>& icebergs,
size_t event_id,
bool set_exact_state) {
const auto* ib = find_value(o.key, icebergs);
if (ib != nullptr)
add_iceberg_annotation(o, change_type, *ib, event_id, set_exact_state);
}
void add_iceberg_annotation(
MDP3Order& o,
OrderChange::ChangeType change_type,
const IcebergInput& ib,
size_t event_id,
bool set_exact_state) {
switch (change_type) {
case OrderChange::ChangeType::added: {
o.iceberg_quantity = IcebergQuantity();
auto show = std::get<1>(ib.show_qty_at(event_id));
o.iceberg_quantity.value().total = ib.total_qty();
o.iceberg_quantity.value().show = show;
bts_assert(o.iceberg_quantity.value().show.value() > 0);
//bts_assert(o.iceberg_quantity.value().total >= o.iceberg_quantity.value().show);
if (set_exact_state) {
// the matcher-validation-observer maintains a snapshot
// of the orderbooks (including iceberg total qty)
// to validate the matcher's internal books.
auto filled = ib.count_fills_by(event_id);
bts_assert(filled < ib.total_qty());
o.iceberg_quantity.value().total = ib.total_qty() - filled;
}
break;
}
case OrderChange::ChangeType::updated: {
auto [ib_event_id, show] = ib.show_qty_at(event_id);
if (set_exact_state) {
// the matcher-validation-observer maintains a snapshot
// of the orderbooks (including iceberg total qty)
// to validate the matcher's internal books.
auto filled = ib.count_fills_by(event_id);
bts_assert(filled < ib.total_qty());
o.iceberg_quantity = IcebergQuantity();
o.iceberg_quantity.value().total = ib.total_qty() - filled;
o.iceberg_quantity.value().show = show;
} else {
// When orders are fed to the matcher,
// the ib total is only included when the order is added.
// On subsequent updates, the iceberg_quantity is set to
// null to indicate that it should be ignored.
o.iceberg_quantity = std::nullopt;
if (ib_event_id == event_id)
o.quantity = show;
else {
// Iceberg show quantities are synchronized with regular quantity changes,
// so if this was a quantity-changing event, the iceberg map would have
// a show qty entry for it. Since ib_event_id != event_id,
// this must be a price change event.
//
// This order's current state in the matcher is not neccessarily the
// same as its state during this event on the actual trading day.
// The simulation may have caused it to diverge. Set the
// quantity to 0 so the matcher knows not to change it.
o.quantity.value() = 0;
}
}
break;
}
case OrderChange::ChangeType::removed: break;
default: NOT_IMPLEMENTED("OrderChange::ChangeType::", change_type); break;
}
}
MDP3Quantity get_reload_qty(
MDP3Quantity vis_qty, MDP3Quantity fill_qty, const IcebergQuantity& ib_before_fill) {
vis_qty = std::min(vis_qty, std::min(ib_before_fill.show, ib_before_fill.total)) -
(fill_qty % ib_before_fill.show);
if (vis_qty.value() <= 0)
vis_qty += ib_before_fill.show;
return std::min(vis_qty, ib_before_fill.total - fill_qty);
}
} // namespace bts::spartan::mdp3::iceberg_helper
@@ -0,0 +1,60 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
#include <unordered_map>
namespace bts::spartan::mdp3::iceberg_helper {
// Checks the iceberg-input map for an entry for the specified order.
// If one is found, the order's quantity, and iceberg_quantity members
// are assigned the appropriate values, for the given event
// (see below for details).
//
// Note: when seeding the matcher with a particular exchange state,
// set_exact_state should be true, otherwise probably false
// (see above for details).
void add_iceberg_annotation(
MDP3Order&,
OrderChange::ChangeType,
const std::unordered_map<MDP3OrderKey, IcebergInput>&,
size_t event_id,
bool set_exact_state = false);
// Populates the iceberg_quantity, and [visible]quantity fields of the
// specified order, with appropriate values for the given event id.
//
// Regardless of 'set_exact_state', if the specified event is the first
// event of the order's life (i.e., when it was added):
// - order.iceberg_quantity.value().total = initial_total;
// - order.iceberg_quantity.value().show = initial_show;
// - order.quantity = initial_show;
//
// If 'set_exact_state' is FALSE, and the specified event coincides with
// a [visible]quantity update (note that visible quantity updates, and
// show quantity updates are treated as one in the same):
// - order.iceberg_quantity = std::nullopt;
// - order.quantity = new_visible_and_show;
//
// If 'set_exact_state' is FALSE, and the specified event doesn't coincide
// with either of the cases described above:
// - order.iceberg_quantity = std::nullopt;
// - order.quantity = 0;
// - this signals to the matcher that it only needs to check
// for a price change.
//
// If 'set_exact_state' is TRUE:
// - order.iceberg_quantity.total = orig_total - num_fills_at_or_before_specified_event;
// - order.iceberg_quantity.show = (most recent show quantity
// at or before the specified event);
// - order.quantity is unchanged, unless the event is the first of
// the order's life, in which case it is synchronized with the show quantity.
// - if the specified event is before the order was actually added,
// the show, and visible quantity will be 0, but the total will be the original total.
void add_iceberg_annotation(
MDP3Order&,
OrderChange::ChangeType,
const IcebergInput&,
size_t event_id,
bool set_exact_state);
MDP3Quantity get_reload_qty(
MDP3Quantity vis_qty, MDP3Quantity fill_qty, const IcebergQuantity& ib_before_fill);
} // namespace bts::spartan::mdp3::iceberg_helper
@@ -0,0 +1,258 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include <functional>
#include <iostream>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#include <string>
#include <tuple>
namespace bts::spartan::mdp3::diagnostics {
inline std::ostream&
operator<<(std::ostream& out, const ::mdp3::SecurityTradingStatus::Value& value) {
switch (value) {
case ::mdp3::SecurityTradingStatus::TradingHalt: out << "TradingHalt"; break;
case ::mdp3::SecurityTradingStatus::Close: out << "Close"; break;
case ::mdp3::SecurityTradingStatus::NewPriceIndication: out << "NewPriceIndication"; break;
case ::mdp3::SecurityTradingStatus::ReadyToTrade: out << "ReadyToTrade"; break;
case ::mdp3::SecurityTradingStatus::NotAvailableForTrading: out << "NotTrading"; break;
case ::mdp3::SecurityTradingStatus::UnknownorInvalid: out << "UnknownorInvalid"; break;
case ::mdp3::SecurityTradingStatus::PreOpen: out << "PreOpen"; break;
case ::mdp3::SecurityTradingStatus::PreCross: out << "PreCross"; break;
case ::mdp3::SecurityTradingStatus::Cross: out << "Cross"; break;
case ::mdp3::SecurityTradingStatus::PostClose: out << "PostClose"; break;
case ::mdp3::SecurityTradingStatus::NoChange: out << "NoChange"; break;
case ::mdp3::SecurityTradingStatus::NULL_VALUE: out << "NULL_VALUE"; break;
default: out << "SecurityTradingStatus_" << static_cast<int>(value); break;
}
return out;
}
template <class TIter>
inline void print_fills(TIter first, TIter last, std::ostream& out = std::cout) {
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(MDP3Fill)>>>{
{" Order ", 12, [](MDP3Fill f) { return FMT(f.key); }},
{" Price ", 12, [](MDP3Fill f) { return FMT(f.price); }},
{" Qty ", 8, [](MDP3Fill f) { return FMT(f.quantity); }},
{" S Qty ", 8, [](MDP3Fill f) { return FMT(f.summary_quantity); }},
{" Agg ", 12, [](MDP3Fill f) { return FMT(f.aggressor); }},
{" Trade ", 12, [](MDP3Fill f) { return FMT(f.trade_entry_id); }}});
out << std::endl;
}
template <class TIter>
inline void print_detailed_fills(std::ostream& out, TIter first, TIter last) {
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(std::tuple<std::string, MDP3Order, MDP3Fill>)>>>{
{" Security ", 12, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return std::get<0>(f); }},
{" Order ", 12, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<2>(f).key); }},
{" Side ", 6, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<1>(f).side); }},
{"O Qty", 6, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<1>(f).quantity); }},
{"O Price", 8, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<1>(f).price); }},
{"F Price", 8, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<2>(f).price); }},
{"F Qty", 6, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<2>(f).quantity); }},
{"S Qty", 6, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<2>(f).summary_quantity); }},
{" Agg ", 9, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<2>(f).aggressor); }},
{" Trade ", 10, [](std::tuple<std::string, MDP3Order, MDP3Fill> f) { return FMT(std::get<2>(f).trade_entry_id); }}});
out << std::endl;
}
template <class TIter>
inline void print_orders(std::ostream& out, TIter first, TIter last) {
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(MDP3Order)>>>{
{" Order ", 12, [](MDP3Order o) { return FMT(o.key); }},
{" Side ", 8, [](MDP3Order o) { return FMT(o.side); }},
{" Price ", 12, [](MDP3Order o) { return FMT(o.price); }},
{" Qty ", 8, [](MDP3Order o) { return FMT(o.quantity); }},
{" Priority ", 12, [](MDP3Order o) { return FMT(o.priority); }}});
out << std::endl;
}
template <class TIter>
inline void print_detailed_orders(TIter first, TIter last, std::ostream& out = std::cout) {
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(std::tuple<MDP3SecurityID, std::string, MDP3Order>)>>>{
{" Sec ID ", 12, [](auto o) { return FMT(std::get<0>(o)); }},
{" Symbol ", 12, [](auto o) { return std::get<1>(o); }},
{" Order ", 12, [](auto o) { return FMT(std::get<2>(o).key); }},
{" Side ", 8, [](auto o) { return FMT(std::get<2>(o).side); }},
{" Price ", 12, [](auto o) { return FMT(std::get<2>(o).price); }},
{" Qty ", 8, [](auto o) { return FMT(std::get<2>(o).quantity); }},
{" Priority ", 12, [](auto o) { return FMT(std::get<2>(o).priority); }}});
out << std::endl;
}
template<class TIter>
inline void print_order_changes(TIter first, TIter last, std::ostream& out = std::cout) {
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(OrderChange)>>> {
{ " Order ", 12, [](auto ch) { return FMT(ch.order.key); } },
{ " Ch. ", 8, [](auto ch) { return FMT(ch.change); } },
{" Side ", 8, [](auto ch){ return FMT(ch.order.side); }},
{" O Price ", 12, [](auto ch){ return FMT(ch.previous_order.price.price()); }},
{" N Price ", 12, [](auto ch){ return FMT(ch.order.price.price()); }},
{" O Qty ", 8, [](auto ch){return FMT(ch.previous_order.quantity); }},
{" N Qty ", 8, [](auto ch){ return FMT(ch.order.quantity); }},
{" O Ice ", 8, [](auto ch){ return FMT(ch.previous_order.iceberg_quantity); }},
{" N Ice ", 8, [](auto ch){ return FMT(ch.order.iceberg_quantity); }}});
out << std::endl;
}
template<class TIter>
inline void print_detailed_order_changes(TIter first, TIter last, std::ostream& out = std::cout) {
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(std::tuple<MDP3SecurityID, std::string, OrderChange>)>>> {
{" Sec ID ", 12, [](auto ch) { return FMT(std::get<0>(ch)); }},
{" Symbol ", 12, [](auto ch) { return std::get<1>(ch); }},
{ " Order ", 12, [](auto ch) { return FMT(std::get<2>(ch).order.key); } },
{ " Ch. ", 8, [](auto ch) { return FMT(std::get<2>(ch).change); } },
{" Side ", 8, [](auto ch){ return FMT(std::get<2>(ch).order.side); }},
{" O Price ", 12, [](auto ch){ return FMT(std::get<2>(ch).previous_order.price.price()); }},
{" N Price ", 12, [](auto ch){ return FMT(std::get<2>(ch).order.price.price()); }},
{" O Qty ", 8, [](auto ch){return FMT(std::get<2>(ch).previous_order.quantity); }},
{" N Qty ", 8, [](auto ch){ return FMT(std::get<2>(ch).order.quantity); }}});
out << std::endl;
}
template <class TIter>
inline void print_instrument_snapshots(TIter first, TIter last, std::ostream& out = std::cout) {
auto book_volume = [](const InstrumentSnapshot* snap, api::Side side) {
const auto& b = snap->book(side);
int vol = ranges::accumulate(b | ranges::views::transform([](const BookLevel& l) { return l.quantity.value(); }), 0);
return MDP3Quantity(vol);
};
auto total_book_volume = [&](const InstrumentSnapshot* snap) {
return MDP3Quantity(book_volume(snap, api::Side::SIDE_BUY).value() +
book_volume(snap, api::Side::SIDE_SELL).value());
};
bts::spartan::table_display::print_table(
out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(const InstrumentSnapshot*)>>> {
{" Security ", 16, [](const auto* s) { return std::string(s->symbol()); }},
{" ID ", 10, [](const auto* s) { return FMT(s->security_id()); }},
{"Asset", 5, [](const auto* s) { return std::string(s->definition().asset()); }},
{"Group", 5, [](const auto* s) { return std::string(s->definition().security_group()); }},
{"Mat", 3, [](const auto* s) { return s->definition().match_algorithm(); }},
{" Status ", 16, [&](const auto* s) { return FMT(s->trading_status()); }},
{" El. Vol ", 10, [](const auto* s) { return FMT(s->electronic_volume()); }},
{" Cl. Vol ", 10, [](const auto* s) { return FMT(s->cleared_volume()); }},
{" Bid Vol ", 10, [&](const auto* s) { return FMT(book_volume(s, api::Side::SIDE_BUY)); }},
{" Ask Vol ", 10, [&](const auto* s) { return FMT(book_volume(s, api::Side::SIDE_SELL)); }},
{" Tot Vol ", 10, [&](const auto* s) { return FMT(total_book_volume(s)); }},
{" Total Orders ", 14, [](const auto* s) { return FMT(s->orders().size()); }},
{"Order Changes", 14, [](const auto* s) { return FMT(s->order_changes().size()); }},
{" Total Fills ", 14, [](const auto* s) { return FMT(s->fills().size()); }},
{" Qty ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_BUY)[0].quantity); }},
{" Bid ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_BUY)[0].price); }},
{" Ask ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_SELL)[0].price); }},
{" Qty ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_SELL)[0].quantity); }},
{"DImp", 4, [](const auto* s) { return std::string((s->definition().implied_matching() ? "Yes" : "No")); }},
{"SImp", 4, [](const auto* s) { return std::string((s->implied_matching_status() ? "Yes" : "No")); }},
{" I Qty ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_BUY)[0].quantity); }},
{" I Bid ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_BUY)[0].price); }},
{" I Ask ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_SELL)[0].price); }},
{" I Qty ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_SELL)[0].quantity); }}
});
out << std::endl;
}
template<class TIter>
inline void print_bbo(std::ostream& out, TIter first, TIter last) {
bts::spartan::table_display::print_table(out, first, last,
std::vector<std::tuple<std::string, int, std::function<std::string(std::tuple<std::string, bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice, MDP3Quantity>)>>> {
{ " Security ", 12, [](auto r) { return std::get<0>(r); } },
{ " Qty ", 10, [](auto r) { return FMT(std::get<2>(r)); } },
{ " Bid ", 10, [](auto r) { return FMT(std::get<1>(r)); } },
{ " Ask ", 10, [](auto r) { return FMT(std::get<3>(r)); } },
{ " Qty ", 10, [](auto r) { return FMT(std::get<4>(r)); } }
});
out << std::endl;
}
template<class TIter>
inline void print_many_bbo(std::ostream& out, TIter first, TIter last) {
std::vector<std::tuple<std::string, bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice, MDP3Quantity>> rows;
while (first != last) {
const InstrumentSnapshot* s = *first;
const auto& bids = s->book(api::Side::SIDE_BUY);
const auto& asks = s->book(api::Side::SIDE_SELL);
rows.push_back(std::make_tuple(std::string(s->symbol()), bids[0].price, bids[0].quantity, asks[0].price, asks[0].quantity));
first++;
}
print_bbo(out, rows.begin(), rows.end());
}
template<class TIter>
inline void print_many_bbo_implied(std::ostream& out, TIter first, TIter last) {
std::vector<std::tuple<std::string, bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice, MDP3Quantity>> rows;
while (first != last) {
const InstrumentSnapshot* s = *first;
const auto& bids = s->implied_book(api::Side::SIDE_BUY);
const auto& asks = s->implied_book(api::Side::SIDE_SELL);
rows.push_back(std::make_tuple(std::string(s->symbol()), bids[0].price, bids[0].quantity, asks[0].price, asks[0].quantity));
first++;
}
print_bbo(out, rows.begin(), rows.end());
}
template<class TIter>
inline void print_many_bbo(TIter first, TIter last, bool direct, bool implied, std::ostream& out = std::cout) {
if (direct && !implied) {
print_many_bbo(out, first, last);
} else if (implied && !direct) {
print_many_bbo_implied(out, first, last);
} else if (direct && implied) {
std::vector<std::tuple<
std::string, bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice, MDP3Quantity,
bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice, MDP3Quantity>>
rows;
while (first != last) {
const InstrumentSnapshot* s = *first;
const auto& bids = s->book(api::Side::SIDE_BUY);
const auto& asks = s->book(api::Side::SIDE_SELL);
const auto& ibids = s->implied_book(api::Side::SIDE_BUY);
const auto& iasks = s->implied_book(api::Side::SIDE_SELL);
rows.push_back(std::make_tuple(
std::string(s->symbol()),
bids[0].price, bids[0].quantity, asks[0].price, asks[0].quantity,
ibids[0].price, ibids[0].quantity, iasks[0].price, iasks[0].quantity));
first++;
}
bts::spartan::table_display::print_table(
out, rows.begin(), rows.end(),
std::vector<std::tuple<
std::string, int,
std::function<std::string(std::tuple<
std::string, bts::md::IntPrice, MDP3Quantity,
bts::md::IntPrice, MDP3Quantity, bts::md::IntPrice,
MDP3Quantity, bts::md::IntPrice, MDP3Quantity>)>>>{
{" Security ", 12, [](auto r) { return std::get<0>(r); }},
{"I Qty", 6, [](auto r) { return FMT(std::get<6>(r)); }},
{" I Bid ", 8, [](auto r) { return FMT(std::get<5>(r)); }},
{" Qty ", 6, [](auto r) { return FMT(std::get<2>(r)); }},
{" Bid ", 8, [](auto r) { return FMT(std::get<1>(r)); }},
{" Ask ", 8, [](auto r) { return FMT(std::get<3>(r)); }},
{" Qty ", 6, [](auto r) { return FMT(std::get<4>(r)); }},
{" I Ask ", 8, [](auto r) { return FMT(std::get<7>(r)); }},
{"I Qty", 6, [](auto r) { return FMT(std::get<8>(r)); }}});
out << std::endl;
}
}
} // namespace bts::spartan::mdp3::diagnostics
@@ -0,0 +1,747 @@
#include <catch/catch.hpp>
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include <google/protobuf/timestamp.pb.h>
#include <google/protobuf/util/time_util.h>
#include <optional>
#include <random>
#include "../MatchingCommon.h"
using namespace google::protobuf::util;
namespace bts::spartan::mdp3::tests {
class RandomGenerator{
public:
std::random_device rd;
double random_double(double min, double max){
//Standard mersenne_twister_engine seeded with rd()
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis(min, max);
return dis(gen);
}
int random_int (int min, int max){
return static_cast<int>(random_double(min,max));
}
api::Side random_side(){
return random_int(1,100)%2==0 ? api::Side::SIDE_BUY : api::Side::SIDE_SELL;
}
};
MDP3Order create_order(uint64_t key, int32_t sec, api::Side side,
uint64_t priority, double price, uint32_t qty,
std::optional<IcebergQuantity> ib = { }) {
auto rv = MDP3Order{MDP3OrderKey(key),
MDP3SecurityID(sec),
side,
MDP3OrderPriority(priority),
md::IntPrice_from_double<7>(price),
MDP3Quantity(qty),
ib};
return rv;
}
MDP3Order create_empty_order(uint64_t key) {
return create_order(key,0,api::Side::SIDE_BUY,0,0.0,0);
}
TEST_CASE("fifo_matching_engine_single_level", "[matcher]") {
auto sec_key = 10;
double price = 100.0;
api::Side side = api::Side::SIDE_BUY;
Trace trace;
std::map<MDP3OrderKey, MatcherOrder> master_ord_map;
FifoPriceLevelMatcher price_level(master_ord_map,
MDP3SecurityID(0),
md::IntPrice_from_double<7>(price),
side,
trace);
auto create_book = [&](int n_orders) -> void {
for (int i = 1; i <= n_orders; i++) {
auto ord_key = static_cast<uint64_t>(i);
auto o = create_order(ord_key, sec_key, side, ord_key, price,
static_cast<uint32_t>(i));
price_level.on_new_same_side_order(o);
}
};
SECTION("add") {
// add 100 orders
create_book(100);
REQUIRE(price_level.n_orders() == 100);
REQUIRE(price_level.quantity().value() == 5050);
}
SECTION("cancel") {
create_book(100);
REQUIRE(price_level.n_orders() == 100);
REQUIRE(price_level.quantity().value() == 5050);
// remove last 10 orders, 91-100
for (int i = 91; i <= 100; i++) {
auto ord_key = static_cast<uint64_t>(i);
price_level.on_order_remove(MDP3OrderKey(ord_key));
}
REQUIRE(price_level.n_orders() == 90);
REQUIRE(price_level.quantity().value() == 4095);
}
SECTION("sweep") {
create_book(100);
REQUIRE(price_level.n_orders() == 100);
REQUIRE(price_level.quantity().value() == 5050);
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(5100),false);
REQUIRE(price_level.n_orders() == 0);
REQUIRE(price_level.quantity().value() == 0);
}
SECTION("full_fill") {
std::vector<MatcherOrder> out_ords;
create_book(100);
// take out the first 20 completely
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(210),false);
REQUIRE(price_level.n_orders() == 80);
REQUIRE(price_level.quantity().value() == 4840);
price_level.copy_values_to(out_ords);
REQUIRE(out_ords.at(79).key.value() == 100);
}
SECTION("partial_fill") {
create_book(100);
// take out the firdt 20 completely
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(193),false);
REQUIRE(price_level.n_orders() == 81);
REQUIRE(price_level.quantity().value() == 4857);
}
SECTION("qty_decrease") {
std::vector<MatcherOrder> out_ords;
create_book(100);
price_level.copy_values_to(out_ords);
auto &ord_59 = out_ords.at(59);
REQUIRE(ord_59.quantity.value() == 60);
auto ord_59_mod = create_order(ord_59.key.value(), 60,
api::Side::SIDE_BUY, 0, price, 20);
price_level.on_order_update(ord_59_mod);
// stays at the same level
price_level.copy_values_to(out_ords);
REQUIRE(out_ords.at(59).quantity.value() == 20);
REQUIRE(out_ords.at(59).key.value() == 60);
REQUIRE(price_level.n_orders() == 100);
REQUIRE(price_level.quantity().value() == 5010);
}
SECTION("qty_increase") {
std::vector<MatcherOrder> out_ords;
create_book(100);
price_level.copy_values_to(out_ords);
auto &ord_59 = out_ords.at(59);
REQUIRE(ord_59.quantity.value() == 60);
auto ord_59_mod = create_order(ord_59.key.value(), 60,
api::Side::SIDE_BUY, 0, price, 61);
price_level.on_order_update(ord_59_mod);
// goes to the end
price_level.copy_values_to(out_ords);
REQUIRE(out_ords.at(99).quantity.value() == 61);
REQUIRE(price_level.n_orders() == 100);
REQUIRE(price_level.quantity().value() == 5051);
}
SECTION("iceberg_simple") {
auto k = 0;
auto price = 100.0;
auto o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, 100.0, 1);
price_level.on_new_same_side_order(o);
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 2);
price_level.on_new_same_side_order(o);
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 3);
price_level.on_new_same_side_order(o);
// iceberg, show qty=2, total_qty=100
o = create_order(
k++, sec_key, api::Side::SIDE_BUY, 0, price, 2,
IcebergQuantity{MDP3Quantity(2), MDP3Quantity(100)});
price_level.on_new_same_side_order(o);
// non ib, to get the visible qty to 15
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 7);
price_level.on_new_same_side_order(o);
REQUIRE(price_level.quantity().value() == 15);
REQUIRE(price_level.total_quantity().value() == 113);
// cross with a 10 lot, this should result in
// {id=4,qty=5},{id=3,qty=2,total_qty=98}
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(10),false);
auto [n_ords, vis_qty] = price_level.n_orders_qty(true);
REQUIRE(n_ords == 2);
REQUIRE(vis_qty == 7);
auto total_qty = price_level.total_quantity();
REQUIRE(total_qty.value() == 103);
// another 5 lot should only leave the IB {id=3,qty=2,total_qty=98}
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(5),false);
REQUIRE(price_level.n_orders() == 1);
REQUIRE(price_level.quantity().value() == 2);
REQUIRE(price_level.total_quantity().value() == 98);
// cross the remaining 98 IB
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(49),false);
REQUIRE(price_level.n_orders() == 1);
REQUIRE(price_level.quantity().value() == 1);
REQUIRE(price_level.total_quantity().value() == 49);
// 1 lots
for (auto i = 0; i < 49; i++)
price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(1),false);
REQUIRE(price_level.n_orders() == 0);
REQUIRE(price_level.quantity().value() == 0);
REQUIRE(price_level.total_quantity().value() == 0);
}
}
TEST_CASE("fifo_matching_engine_book", "[matcher]") {
auto verify_book_level = [](const BookLevel &bl, double price, uint32_t qty,
uint32_t n_orders) {
REQUIRE(bl.price == md::IntPrice_from_double<7>(price));
REQUIRE(bl.n_orders == n_orders);
REQUIRE(bl.quantity.value() == qty);
REQUIRE(bl.n_orders == n_orders);
};
auto sec_key = 10;
FifoSecurityMatcher sec_me(MDP3SecurityID(100));
auto order_key = 0;
auto priority = 0;
auto add_to_book = [&](uint32_t n_orders, api::Side side,
double price,
bool affect_book = true) -> void {
for (uint32_t i = 1; i <= n_orders; i++) {
auto o =
create_order(order_key++, sec_key, side, priority++, price, i);
sec_me.on_order_change(
SmallOrderChange(OrderChange{MDP3Order{},o,OrderChange::added},affect_book));
}
};
SECTION("add") {
// bids
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
// verify bids
{
Book bid_bk = sec_me.book(api::Side::SIDE_BUY);
double exp_price = 15.09;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = 100 - i * 10;
auto exp_qty =
(static_cast<double>(exp_n_orders) / 2) * (1 + exp_n_orders);
verify_book_level(bid_bk.at(i), exp_price, exp_qty, exp_n_orders);
exp_price -= 0.01;
}
}
// verify asks
{
Book ask_bk = sec_me.book(api::Side::SIDE_SELL);
double exp_price = 15.10;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = (i + 1) * 10;
auto exp_qty =
(static_cast<double>(exp_n_orders) / 2) * (1 + exp_n_orders);
verify_book_level(ask_bk.at(i), exp_price, exp_qty, exp_n_orders);
exp_price += 0.01;
}
}
}
SECTION("add_sim_no_effect") {
// bids
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
auto verify_bids = [&](){
Book bid_bk = sec_me.book(api::Side::SIDE_BUY);
double exp_price = 15.09;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = 100 - i * 10;
auto exp_qty =
(static_cast<double>(exp_n_orders) / 2) * (1 + exp_n_orders);
verify_book_level(bid_bk.at(i), exp_price, exp_qty, exp_n_orders);
exp_price -= 0.01;
}
};
auto verify_asks = [&](){
Book ask_bk = sec_me.book(api::Side::SIDE_SELL);
double exp_price = 15.10;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = (i + 1) * 10;
auto exp_qty =
(static_cast<double>(exp_n_orders) / 2) * (1 + exp_n_orders);
verify_book_level(ask_bk.at(i), exp_price, exp_qty, exp_n_orders);
exp_price += 0.01;
}
};
verify_bids();
verify_asks();
// bids: 15.00-15.09
// asks: 15.10-15.19
// add a passive order, should have no effect on the markets
add_to_book(100, api::Side::SIDE_BUY, 15.09,false);
verify_bids();
verify_asks();
// add an aggressive order
add_to_book(100, api::Side::SIDE_BUY, 15.19,false);
verify_bids();
verify_asks();
}
SECTION("remove") {
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
// remove one order at a time from the second level
// 90 orders starting with id 360
auto b = 360;
auto init_n_orders = 90;
auto init_qty = 4095;
auto remvd = 0;
for (auto i = 0; i < 90; i++) {
auto id = MDP3OrderKey(i + b);
sec_me.on_order_change(OrderChange{
MDP3Order{},
create_empty_order(id.value()),
OrderChange::removed
});
auto qty = i + 1;
remvd += qty;
auto now_qty = init_qty - remvd;
auto now_n_orders = init_n_orders - (i + 1);
Book bk = sec_me.book(api::Side::SIDE_BUY);
if (now_qty == 0)
// whole level got removed
// now level 1 should be 15.07
verify_book_level(bk.at(1), 15.07, 3240, 80);
else
verify_book_level(bk.at(1), 15.08, now_qty, now_n_orders);
}
Book bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bk.at(1), 15.07, 3240, 80);
}
SECTION("modify") {
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
// modify qty of all orders to 1
auto b = 360;
for (auto i = 0; i < 90; i++) {
auto id = i + b;
auto ord =
create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.08, 1);
sec_me.on_order_change(OrderChange{
create_empty_order(ord.key.value()),
ord,
OrderChange::updated
});
}
Book bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bk.at(1), 15.08, 90, 90);
// now change all these prices to move to a diff level
for (auto i = 0; i < 90; i++) {
auto id = i + b;
auto ord =
create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.09, 1);
sec_me.on_order_change(OrderChange{
create_empty_order(ord.key.value()),
ord,
OrderChange::updated
});
}
// 15.08 is gone
bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bk.at(1), 15.07, 3240, 80);
verify_book_level(bk.at(0), 15.09, 5140, 190);
}
SECTION("crosses") {
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
// send a new sell at 15.09 to take the first 10 orders
auto ord =
create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.09, 55);
sec_me.on_order_change(OrderChange{MDP3Order{},ord,OrderChange::added});
auto bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bk.at(0), 15.09, 4995, 90);
verify_book_level(bk.at(1), 15.08, 4095, 90);
verify_book_level(bk.at(2), 15.07, 3240, 80);
// now completely sweep 2 complete levels and make the rest a working
// order
ord = create_order(10001, sec_key, api::Side::SIDE_SELL, 10001, 15.08,
10000);
sec_me.on_order_change(OrderChange{MDP3Order{},ord,OrderChange::added});
bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bk.at(0), 15.07, 3240, 80);
// failing (11/16)
auto bk_ask = sec_me.book(api::Side::SIDE_SELL);
verify_book_level(bk_ask.at(0), 15.08, 910, 1);
}
SECTION("IB_cross") {
// create 10 level book and cross the first 3 levels completely.
// verify the resting IB is correct
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
auto bid_bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bid_bk.at(0), 15.09, 5050, 100);
// the bid book is
// 15.09 5050 100
// 15.08 4095 90
// 15.07 3240 80
// 15.06 2485 70
// send a new sell at 15.07 to take the first 3 levels
auto cross_ib =
create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.07, 13,
IcebergQuantity{MDP3Quantity(13), MDP3Quantity(12500)});
sec_me.on_order_change(OrderChange{MDP3Order{},cross_ib,OrderChange::added});
bid_bk = sec_me.book(api::Side::SIDE_BUY);
verify_book_level(bid_bk.at(0), 15.06, 2485, 70);
auto ask_bk = sec_me.book(api::Side::SIDE_SELL);
verify_book_level(ask_bk.at(0), 15.07, 13, 1);
}
}
TEST_CASE("perf", "[matcher]") {
return;
RandomGenerator rnd_gen;
const size_t n_levels = 500;
const double tick_size = 0.01;
const double top_bid = 100.0;
const MDP3SecurityID sec_id = MDP3SecurityID(1);
auto fill_ord = [&](MDP3Order& ord,
MDP3OrderKey key){
ord.key = key;
ord.security = sec_id;
ord.side = rnd_gen.random_side();
ord.priority = MDP3OrderPriority(key.value());
int p_lev = rnd_gen.random_int(0,n_levels-1);
bts_assert(p_lev >= 0);
double price;
if(ord.side == api::Side::SIDE_BUY)
price = top_bid - (p_lev*tick_size);
else
price = (top_bid+tick_size) + (p_lev*tick_size);
ord.price = md::IntPrice_from_double<7>(price);
ord.quantity = MDP3Quantity(rnd_gen.random_int(5,20));
};
auto add_n_orders = [&](std::vector<OrderChange>& out_v,
size_t n_orders,
size_t start_k){
out_v.reserve(n_orders);
for(size_t i=0;i<n_orders;i++){
OrderChange ord_change;
ord_change.change = OrderChange::ChangeType::added;
fill_ord(ord_change.order, MDP3OrderKey(start_k++));
out_v.push_back(ord_change);
}
};
size_t n_orders = 10000, inc = 10000, n_runs = 10;
//size_t n_orders = 1000000, inc = 10000, n_runs = 3;
SECTION("add orders") {
for(size_t i=0;i<n_runs;i++){
FifoSecurityMatcher sec_m(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
MDP3SecurityID(100),
0,
false
});
std::vector<OrderChange> aux;
// takes 22 secs to add 100,000 orders straght up
add_n_orders(aux, n_orders, 0);
auto t0 = TimeUtil::GetCurrentTime();
for (const auto& oc : aux)
sec_m.on_order_change(oc);
auto t1 = TimeUtil::GetCurrentTime();
auto nsecs = TimeUtil::TimestampToNanoseconds(t1) -
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Adding " << aux.size() << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl;
size_t comp_n_orders =
sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
}
SECTION("cxl_orders") {
for(size_t i=0;i<n_runs;i++){
FifoSecurityMatcher sec_m(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
MDP3SecurityID(100),
0,
false
});
std::vector<OrderChange> aux;
// takes 22 secs to add 100,000 orders straght up
add_n_orders(aux, n_orders, 0);
for (const auto& oc : aux)
sec_m.on_order_change(oc);
auto t0 = TimeUtil::GetCurrentTime();
for (auto& oc : aux){
oc.change = OrderChange::ChangeType::removed;
sec_m.on_order_change(oc);
}
auto t1 = TimeUtil::GetCurrentTime();
auto nsecs = TimeUtil::TimestampToNanoseconds(t1) -
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Removing " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl;
n_orders += inc;
size_t comp_n_orders =
sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
REQUIRE(comp_n_orders == 0);
}
}
SECTION("update_dec_orders") {
for(size_t i=0;i<n_runs;i++){
FifoSecurityMatcher sec_m(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
MDP3SecurityID(100),
0,
false
});
std::vector<OrderChange> aux;
// takes 22 secs to add 100,000 orders straght up
add_n_orders(aux, n_orders, 0);
for (const auto& oc : aux)
sec_m.on_order_change(oc);
auto t0 = TimeUtil::GetCurrentTime();
for (auto& oc : aux){
oc.order.quantity = MDP3Quantity(oc.order.quantity.value()-1);
oc.change = OrderChange::ChangeType::updated;
sec_m.on_order_change(oc);
}
auto t1 = TimeUtil::GetCurrentTime();
auto nsecs = TimeUtil::TimestampToNanoseconds(t1) -
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Updaing (qty dec) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl;
size_t comp_n_orders =
sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
}
SECTION("update_inc_orders") {
for(size_t i=0;i<n_runs;i++){
FifoSecurityMatcher sec_m(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
MDP3SecurityID(100),
0,
false
});
std::vector<OrderChange> aux;
// takes 22 secs to add 100,000 orders straght up
add_n_orders(aux, n_orders, 0);
for (const auto& oc : aux)
sec_m.on_order_change(oc);
auto t0 = TimeUtil::GetCurrentTime();
for (auto& oc : aux){
oc.order.quantity = MDP3Quantity(oc.order.quantity.value()+1);
oc.change = OrderChange::ChangeType::updated;
sec_m.on_order_change(oc);
}
auto t1 = TimeUtil::GetCurrentTime();
auto nsecs = TimeUtil::TimestampToNanoseconds(t1) -
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Updaing (qty inc) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl;
size_t comp_n_orders =
sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
}
SECTION("order_price_change_no_cross") {
for(size_t i=0;i<n_runs;i++){
FifoSecurityMatcher sec_m(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
MDP3SecurityID(100),
0,
false
});
std::vector<OrderChange> aux;
// takes 22 secs to add 100,000 orders straght up
add_n_orders(aux, n_orders, 0);
for (const auto& oc : aux)
sec_m.on_order_change(oc);
auto t0 = TimeUtil::GetCurrentTime();
for (auto& oc : aux){
oc.change = OrderChange::ChangeType::updated;
double inc = tick_size;
if(oc.order.side == api::Side::SIDE_BUY)
inc = -1*inc;
oc.order.price =
md::IntPrice_from_double<7>(oc.order.price.price() + inc),
sec_m.on_order_change(oc);
}
auto t1 = TimeUtil::GetCurrentTime();
auto nsecs = TimeUtil::TimestampToNanoseconds(t1) -
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Updating (price change no cross) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl;
size_t comp_n_orders =
sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
}
SECTION("new_order_cross") {
for(size_t i=0;i<n_runs;i++){
FifoSecurityMatcher sec_m(
FifoSecurityMatcher::FifoSecurityMatcherArgs{
MDP3SecurityID(100),
0,
false
});
std::vector<OrderChange> aux;
// takes 22 secs to add 100,000 orders straght ups
add_n_orders(aux, n_orders, 0);
for (const auto& oc : aux)
sec_m.on_order_change(oc);
// send one lot buy orders to completely cross
// the offer market
size_t buy_orders_to_send = sec_m.total_qty(api::Side::SIDE_SELL).value();
OrderChange ochange;
ochange.change = OrderChange::ChangeType::added;
ochange.order = MDP3Order{
MDP3OrderKey(0),MDP3SecurityID(100),
api::Side::SIDE_BUY,MDP3OrderPriority(0),
md::IntPrice_from_double<7>(10000),MDP3Quantity(1),
std::nullopt
};
std::vector<MatchEffect> exec_rep;
auto t0 = TimeUtil::GetCurrentTime();
for(size_t i=0; i<buy_orders_to_send; i++){
ochange.order.key = MDP3OrderKey(n_orders+i+1);
sec_m.on_order_change(ochange,&exec_rep);
bts_assert(exec_rep.size() == (i+1)*2);
auto l_it = exec_rep.end()-1;
MatchEffect& eff = *l_it;
auto& upd = std::get<OrderUpdate>(eff);
REQUIRE(upd.last_fills.value()==1);
}
auto t1 = TimeUtil::GetCurrentTime();
auto nsecs = TimeUtil::TimestampToNanoseconds(t1) -
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Crossing (with new orders) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl;
// whole sell side should be gone
REQUIRE(sec_m.total_n_orders(api::Side::SIDE_SELL) == 0);
n_orders += inc;
}
}
}
} // namespace bts::spartan::mdp3::tests
@@ -0,0 +1,59 @@
#pragma once
#include <iostream>
#include <string>
namespace bts::spartan::mdp3 {
enum EventFlags {
None = 0,
IcebergOrders = 1,
DirectFills = 1 << 1,
ImpliedFills = 1 << 2,
MultipleSecurityAggressors = 1 << 3
};
const char* event_flags_name(const EventFlags& f) {
switch (f) {
case EventFlags::None: return "None";
case EventFlags::DirectFills: return "DirectFills";
case EventFlags::ImpliedFills: return "ImpliedFills";
case EventFlags::MultipleSecurityAggressors: return "MultipleSecurityAggressors";
default: return "Unknown";
}
}
inline EventFlags operator|(const EventFlags& lhs, const EventFlags& rhs) {
return static_cast<EventFlags>(static_cast<int>(lhs) | static_cast<int>(rhs));
}
inline EventFlags& operator|=(EventFlags& lhs, const EventFlags& rhs) {
lhs = static_cast<EventFlags>(static_cast<int>(lhs) | static_cast<int>(rhs));
return lhs;
}
inline EventFlags operator&(const EventFlags& lhs, const EventFlags& rhs) {
return static_cast<EventFlags>(static_cast<int>(lhs) & static_cast<int>(rhs));
}
inline EventFlags& operator&=(EventFlags& lhs, const EventFlags& rhs) {
lhs = static_cast<EventFlags>(static_cast<int>(lhs) & static_cast<int>(rhs));
return lhs;
}
inline EventFlags operator^(const EventFlags& lhs, const EventFlags& rhs) {
return static_cast<EventFlags>(static_cast<int>(lhs) ^ static_cast<int>(rhs));
}
inline EventFlags& operator^=(EventFlags& lhs, const EventFlags& rhs) {
lhs = static_cast<EventFlags>(static_cast<int>(lhs) ^ static_cast<int>(rhs));
return lhs;
}
inline std::ostream& operator<<(std::ostream& out, const EventFlags& flags) {
int fcount = 0;
auto impl = [&](EventFlags f) {
if ((flags & f) == f) {
if (fcount++ > 0)
out << " | ";
out << event_flags_name(f);
}
};
impl(EventFlags::DirectFills);
impl(EventFlags::ImpliedFills);
impl(EventFlags::MultipleSecurityAggressors);
return out;
}
}
@@ -0,0 +1,81 @@
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h"
#include "lib/logging/logging.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <functional>
#include <iostream>
#include <range/v3/numeric.hpp>
#include <range/v3/view.hpp>
#include <vector>
#define _LOG(level, msg) LOG(OrderMan, Info, "{} {}.", _log_header(), msg)
#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
namespace bts::spartan::mdp3::iceberg_file_cache {
// increment this anytime the deduced icebergs change
// so old .iceberg files will be regenerated
const ushort VERSION = 3;
inline std::string _log_header() { return std::string("IcebergFileCache"); }
inline std::string _iceberg_file_path(const std::string& pcap) {
auto dir = spartan::file::dir_from_path(pcap);
auto name = bts::spartan::file::file_name_from_path(pcap);
return dir + R"(/)" + name + ".icebergs";
}
std::unordered_map<MDP3OrderKey, IcebergInput> find_icebergs(const std::string& pcap, bool force_regen) {
std::string path = _iceberg_file_path(pcap);
// goes without saying: read_op and write_op must
// agree with one another, wrt the sequence of fields
std::unordered_map<MDP3OrderKey, IcebergInput> dest;
auto read_op = [&](std::istream& in) -> std::istream& {
auto ok = file::read_from_bin<uint64_t>(in);
auto total_qty = file::read_from_bin<int>(in);
std::vector<std::tuple<size_t, MDP3Quantity>> show_quantities;
for (auto num_states = file::read_from_bin<ushort>(in); num_states > 0; num_states--) {
size_t event_id = file::read_from_bin<size_t>(in);
auto show_qty = file::read_from_bin<int>(in);
show_quantities.emplace_back(event_id, MDP3Quantity(show_qty));
}
std::vector<std::tuple<size_t, MDP3Quantity>> fill_quantities;
for (auto num_fill_events = file::read_from_bin<ushort>(in); num_fill_events > 0; num_fill_events--) {
size_t event_id = file::read_from_bin<size_t>(in);
auto fill_qty = file::read_from_bin<int>(in);
fill_quantities.emplace_back(event_id, fill_qty);
}
dest.emplace(
MDP3OrderKey(ok),
IcebergInput(MDP3Quantity(total_qty), show_quantities, fill_quantities));
return in;
};
if (force_regen || !bts::spartan::file::read_from_file(path, read_op, VERSION)) {
std::cout << "sweeping for icebergs..." << std::endl;
dest = IcebergFinder::find_all(pcap);
auto write_op =
[](std::ostream& out,
const std::pair<MDP3OrderKey, IcebergInput>& p) -> std::ostream& {
file::write_to_bin(out, p.first.value());
file::write_to_bin(out, p.second.total_qty().value());
file::write_to_bin(out, static_cast<ushort>(p.second.show_quantities().size()));
for (auto [event_id, show_qty] : p.second.show_quantities()) {
file::write_to_bin(out, event_id);
file::write_to_bin(out, show_qty.value());
}
file::write_to_bin(out, static_cast<ushort>(p.second.fill_quantities().size()));
for (auto [event_id, fill_qty] : p.second.fill_quantities()) {
file::write_to_bin(out, event_id);
file::write_to_bin(out, fill_qty.value());
}
return out;
};
bts::spartan::file::write_to_file(dest.begin(), dest.end(), path, write_op, VERSION);
} else {
INFO("read ", dest.size(), " icebergs from '", path, "'");
}
return dest;
}
} // namespace bts::spartan::mdp3::iceberg_file_cache
@@ -0,0 +1,14 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
#include <string>
namespace bts::spartan::mdp3::iceberg_file_cache {
// First, looks in the working dir for a file named <pcap>.icebergs.
// If the file does exists, the icebergs are simply read from the file.
// If the file doesn't exist, the pcap is swept for icebergs,
// and the results are written to the file <pcap>.icebergs,
// for faster retrieval on subsequent runs with the specified pcap.
std::unordered_map<MDP3OrderKey, IcebergInput>
find_icebergs(const std::string& pcap, bool force_regen = false);
} // namespace bts::spartan::mdp3::iceberg_file_cache
@@ -0,0 +1,476 @@
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/event_flags.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h"
#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/event_ordinal.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/security_comp.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer.h"
#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h"
#include "server/sparkd/cmds/spartan/util/table_display.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h"
#include <boost/container/flat_map.hpp>
#include <functional>
#include <iostream>
// Error logs are automatically logged to the console
// (w/ default logging configuration), but they lag behind
// immediate console output (i.e., calls to std::cout).
// If _ECHO_ON is defined, the _ERROR_LEVEL macro is set to Info
// to avoid duplicate, and (perhaps more importantly) unsynchronized output.
// #define _ECHO_ON
#ifdef _ECHO_ON
#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__);
#define _ERROR_LEVEL Info
#else
#define _ECHO(...)
#define _ERROR_LEVEL Error
#endif
#define _LOG(level, msg) \
_ECHO(_log_header(), " ", msg, "\n") \
LOG(OrderMan, level, "{} {}", _log_header(), msg)
#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__))
#define ERROR(...) _LOG(_ERROR_LEVEL, bts::orderman::Formatter::fmt(__VA_ARGS__))
// SUMMARIZE macro wrappers
#define _SUMMARIZE SUMMARIZE(_log_header())
#define _SUMMARIZE_TRANSACTION(...) SUMMARIZE_TRANSACTION(_log_header(), FMT(__VA_ARGS__))
#define _SUMMARIZE_TRANSACTION_NAMED(name, ...) SUMMARIZE_TRANSACTION_NAMED(name, _log_header(), FMT(__VA_ARGS__))
namespace bts::spartan::mdp3 {
using namespace bts::spartan::mdp3::iceberg_helper;
// aliases
using TradingStatus = MatcherValidationObserver::TradingStatus;
template<typename K, typename V>
using flat_map = boost::container::flat_map<K, V>;
// consts
const size_t NO_EVENT = 0, MIN_OBSERVED_EVENT = 100;
enum class Verbosity { Minimal = 0, Normal = 1, Verbose = 2 };
///////////////////////
// stateless helpers
///////////////////////
inline std::string _log_header() { return "MatcherValidationObserver"; }
inline bool _is_interesting_sec(const InstrumentSnapshot& s) {
return s.definition().asset() == "ES";
//return strcmp(s.definition().asset(), "SI") == 0;
}
void _sync_orders(
size_t event_id,
const InstrumentSnapshot& s,
Orders& orders,
const std::unordered_map<MDP3OrderKey, IcebergInput>& icebergs) {
//
for (auto ch : s.order_changes()) {
switch (ch.change) {
case OrderChange::added:
case OrderChange::updated:
case OrderChange::updated_price:
add_iceberg_annotation(ch.order, ch.change, icebergs, event_id, true);
orders[ch.order.key] = ch.order;
break;
case OrderChange::removed: orders.erase(ch.order.key); break;
}
}
}
bool _has_multiple_aggressor_secs(const MDP3Event& ev) {
bool aggr_sec_found = false;
for (auto* s : ev.affected_securities) {
for (auto f : s->fills()) {
switch (f.aggressor) {
case MDP3Fill::Aggressor::THIS_BUY:
case MDP3Fill::Aggressor::THIS_SELL:
if (aggr_sec_found)
return true;
aggr_sec_found = true;
break;
default: continue;
}
break;
}
}
return false;
}
inline std::ostream& operator<<(std::ostream& ostr, const TradingStatus& ts) {
switch (ts) {
case TradingStatus::None: ostr << "None"; break;
case TradingStatus::Preopen: ostr << "Preopen"; break;
case TradingStatus::Uninitialized: ostr << "Uninitialized"; break;
case TradingStatus::Initialized: ostr << "Initialized"; break;
default: ostr << "TradingStatus_" << static_cast<int>(ts); break;
}
return ostr;
}
template<class TIter>
inline void print_if_not_empty(TIter first, TIter last, std::string title, std::ostream& out = std::cout) {
bool printed_title = false;
while(first != last) {
if (!printed_title) {
out << title << std::endl;
printed_title = true;
}
out << *first << std::endl;
first++;
}
}
inline bool fill_equals(const MDP3Fill& a, const MDP3Fill& b) {
return a.aggressor == b.aggressor && a.quantity == b.quantity && orderman::price_comp::is_equal(a.price, b.price);
}
inline bool match_fills(const FillDepot& fd, std::vector<const InstrumentSnapshot*> snaps, std::ostream& out = std::cout) {
std::unordered_map<MDP3SecurityID, std::vector<MDP3Fill>> fill_map;
fd.copy_to(fill_map);
bool errors = false;
for (const auto* s : snaps) {
auto sec_fills_it = fill_map.find(s->security_id());
if (sec_fills_it == fill_map.end()) {
print_if_not_empty(s->fills().begin(), s->fills().end(), FMT(s->symbol(), " unmatched fills"), out);
continue;
}
auto& act_fills = sec_fills_it->second;
auto exp_it = s->fills().begin();
std::vector<MDP3Fill> unmatched_exp;
while (exp_it != s->fills().end()) {
bool matched = false;
for (auto act_it = act_fills.begin(); act_it != act_fills.end(); act_it++) {
if (fill_equals(*act_it, *exp_it)) {
act_fills.erase(act_it);
matched = true;
break;
}
}
if (!matched) {
unmatched_exp.push_back(*exp_it);
}
exp_it++;
}
errors |= (act_fills.size() || unmatched_exp.size());
print_if_not_empty(act_fills.begin(), act_fills.end(), FMT(s->symbol(), " unexpected fills"), out);
print_if_not_empty(unmatched_exp.begin(), unmatched_exp.begin(), FMT(s->symbol(), " unmatched fills"), out);
}
return !errors;
}
///////////////////////
// private classes
///////////////////////
struct _SecurityData {
_SecurityData() : status(TradingStatus::None), current_orders(), previous_orders() {}
TradingStatus status;
Orders current_orders, previous_orders;
size_t last_observed_event = NO_EVENT;
const InstrumentSnapshot* snapshot = nullptr;
std::string symbol() const {
return snapshot == nullptr ? "N/A" : snapshot->symbol();
}
MDP3SecurityID security_id() const {
return snapshot == nullptr ? MDP3SecurityID(0) : snapshot->security_id();
}
};
struct MatcherValidationObserver::PImpl {
PImpl() {}
PImpl(std::unordered_map<MDP3OrderKey, IcebergInput> icebergs, flat_map<size_t, size_t> interesting_event_ranges)
: icebergs(icebergs), interesting_event_ranges(interesting_event_ranges) {}
flat_map<MDP3SecurityID, _SecurityData> security_cache;
std::unordered_map<MDP3OrderKey, IcebergInput> icebergs;
flat_map<size_t, size_t> interesting_event_ranges;
flat_map<MDP3OrderKey, MDP3Order> interesting_orders;
// number of 'interesting events' observed so far.
uint32_t interesting_event_count = 0;
// actual event_id (from the exchange) of most recently observed event.
size_t event_id = NO_EVENT;
FifoMatcherContext exch_matcher;
Verbosity verbosity = Verbosity::Minimal;
EventFlags ignore_flags = EventFlags::MultipleSecurityAggressors;
// event_id is used to transition the security's trading status from
// Preopen to Uninitialized
_SecurityData& get_security_data(const InstrumentSnapshot& s, size_t event_id) {
auto it = security_cache.find(s.security_id());
if (it != security_cache.end()) {
auto& data = it->second;
if (event_id != data.last_observed_event) {
data.last_observed_event = event_id;
switch (data.status) {
case TradingStatus::Preopen:
data.status = TradingStatus::Uninitialized;
break;
case TradingStatus::None:
if (s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade)
data.status = TradingStatus::Preopen;
break;
default: break;
}
}
data.snapshot = &s;
return data;
}
// this is the first time this security has been observed; initialize its data.
auto& data = security_cache.emplace(s.security_id(), _SecurityData()).first->second;
data.snapshot = &s;
data.last_observed_event = event_id;
data.previous_orders = s.orders();
for (auto& o : data.previous_orders)
add_iceberg_annotation(o.second, OrderChange::ChangeType::added, icebergs, event_id);
// the first event at the open isn't processed bc
// we don't have a clear understanding of how preopen matching works.
data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade
? TradingStatus::Preopen
: TradingStatus::None;
data.snapshot = &s;
return data;
}
EventFlags get_event_flags(const MDP3Event& ev) {
EventFlags flags = EventFlags::None;
if (_has_multiple_aggressor_secs(ev))
flags |= EventFlags::MultipleSecurityAggressors;
for (const auto* s : ev.affected_securities) {
for (auto& f : s->fills()) {
if (icebergs.find(f.key) != icebergs.end())
flags |= EventFlags::IcebergOrders;
switch (f.aggressor) {
case MDP3Fill::Aggressor::THIS_BUY:
case MDP3Fill::Aggressor::THIS_SELL:
flags |= EventFlags::DirectFills;
break;
case MDP3Fill::Aggressor::IMPLIED:
flags |= EventFlags::ImpliedFills;
break;
default: break;
}
}
}
return flags;
}
};
///////////////////////
// constructors
///////////////////////
MatcherValidationObserver::MatcherValidationObserver(
std::unordered_map<MDP3OrderKey, IcebergInput> icebergs)
: MatcherValidationObserver(icebergs, flat_map<size_t, size_t>{ }) {}
MatcherValidationObserver::MatcherValidationObserver(
std::unordered_map<MDP3OrderKey, IcebergInput> icebergs,
flat_map<size_t, size_t> interesting_event_ranges)
: _me(new MatcherValidationObserver::PImpl(icebergs, interesting_event_ranges)) { }
MatcherValidationObserver::~MatcherValidationObserver() {}
///////////////////////
// misc private methods
///////////////////////
TradingStatus MatcherValidationObserver::_trading_status(const InstrumentSnapshot& s, size_t event_id) {
return _me->get_security_data(s, event_id).status;
}
bool MatcherValidationObserver::_is_ready_to_trade(const InstrumentSnapshot& s, size_t event_id) {
switch (_trading_status(s, event_id)) {
case TradingStatus::Uninitialized:
case TradingStatus::Initialized: return true;
default: return false;
}
}
void MatcherValidationObserver::_sync_current_orders(MDP3Event const& ev) {
for (auto s : ev.affected_securities) {
auto& new_orders = _me->get_security_data(*s, ev.event_id).current_orders;
_sync_orders(ev.event_id, *s, new_orders, _me->icebergs);
}
}
void MatcherValidationObserver::_sync_previous_orders(const MDP3Event& ev) {
for (auto s : ev.affected_securities) {
auto& old_orders = _me->get_security_data(*s, ev.event_id).previous_orders;
_sync_orders(ev.event_id, *s, old_orders, _me->icebergs);
}
}
void MatcherValidationObserver::_reset_matcher() {
// remove orders, and change [actively trading] securities
// trading status back to Preopen so they will be added back
// in the next update (the state must be re-initialized
// before updates can be consumed incrementally again).
if (_me->verbosity > Verbosity::Normal)
INFO("clearing orders to reset the matcher's state");
_me->exch_matcher.clear();
for (auto& kv : _me->security_cache) {
if (kv.second.status != TradingStatus::None)
kv.second.status = TradingStatus::Preopen;
}
}
///////////////////////
// IChannelObserver
///////////////////////
void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap) {
_SUMMARIZE;
for (auto s : snap) {
auto& sec_data = _me->get_security_data(*s, NO_EVENT);
sec_data.current_orders.clear();
_sync_orders(NO_EVENT, *s, sec_data.current_orders, _me->icebergs);
sec_data.previous_orders = sec_data.current_orders;
}
}
void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) {
bts_assert(ev.event_id > _me->event_id);
_me->event_id = ev.event_id;
switch (get_event_ordinal(_me->interesting_event_ranges, ev.event_id, MIN_OBSERVED_EVENT)) {
case bts::spartan::EventOrdinal::Waiting:
// just need to synchronize the order caches.
// the matcher is only validated during 'interesting events.'
_sync_current_orders(ev);
_sync_previous_orders(ev);
return;
case bts::spartan::EventOrdinal::Interesting:
// _sync_previous_orders should be called after
// the matcher has digested the order changes.
break;
case bts::spartan::EventOrdinal::Done:
// nothing left to do (might as well kill the process).
bts_assert_msg(false, "no further events of interest.");
return;
}
auto flags = _me->get_event_flags(ev);
if ((flags & _me->ignore_flags) != EventFlags::None) {
if (_me->verbosity > Verbosity::Minimal)
INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags));
_sync_current_orders(ev);
_sync_previous_orders(ev);
_reset_matcher();
return;
}
++_me->interesting_event_count;
if (_me->verbosity > Verbosity::Minimal)
INFO("Update No ", _me->interesting_event_count, "(event id ", ev.event_id, ")");
// Note: the exchange processes the order updates per security,
// in a particular order, sorted hierarchically by symbol, then expiration.
// 'sorted_secs' will contain the proper order of securities.
std::vector<const InstrumentSnapshot*> sorted_secs;
for (auto* s : ev.affected_securities) {
if (_is_interesting_sec(*s))
sorted_secs.push_back(s);
}
if (sorted_secs.empty())
return;
std::sort(sorted_secs.begin(), sorted_secs.end(), security_comp::SymbolComp());
// securities must be added to the matcher before the order
// changes are processed to ensure the correct implied markets are used.
// Fills are also indexed in this pass, so they can be added to the iceberg qty
std::unordered_map<MDP3OrderKey, MDP3Quantity> fill_map;
for (const auto* s : sorted_secs) {
accumulate_fills(s->fills().begin(), s->fills().end(), fill_map);
_me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs());
_me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching());
}
// initialize matcher with previous state (only applies
// to securities whose orders the matcher hasn't seen yet).
for (const auto* s : sorted_secs) {
if (_trading_status(*s, ev.event_id) != TradingStatus::Uninitialized)
continue;
auto it = _me->security_cache.find(s->security_id());
bts_assert(it != _me->security_cache.end());
auto prev_ords = it->second.previous_orders |
ranges::views::transform([](auto& p) { return p.second; });
_me->exch_matcher.reset(s->security_id(), prev_ords.begin(), prev_ords.end());
_me->security_cache[s->security_id()].status = TradingStatus::Initialized;
}
// process changes with matcher
FillDepot fills;
for (const auto* s : sorted_secs) {
switch (_trading_status(*s, ev.event_id)) {
case TradingStatus::Initialized: break;
default: continue;
}
auto deduced = s->deduced_order_changes();
if (deduced.size()) {
std::vector<SmallOrderChange> deduced_sm;
for (auto& ch : deduced) {
add_iceberg_annotation(ch.order, ch.change, _me->icebergs, ev.event_id);
deduced_sm.emplace_back(ch);
}
_me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills);
}
}
// Verify the matcher's orders are identitical to the exchange's.
_sync_current_orders(ev);
bool orders_match = true;
for (const auto* s : sorted_secs) {
if (!_is_ready_to_trade(*s, ev.event_id))
continue;
auto& sec_data = _me->security_cache[s->security_id()];
auto& curr_orders = sec_data.current_orders;
auto error = _me->exch_matcher.has_same_orders(s->security_id(), curr_orders);
if (error) {
ERROR(
"order discrepancies detected for ", s->symbol(), " on event ", ev.event_id,
"\nError: ", error.value(), "\nOld Orders:\n",
_me->security_cache[s->security_id()].previous_orders);
orders_match = false;
}
}
bool fills_match = match_fills(fills, sorted_secs);
if (!orders_match || !fills_match) {
// todo: add the iceberg fills back, to get to the previous state.
// describe current state for diagnostic purposes
std::stringstream msg;
msg << "Spartan Top of Books:" << std::endl;
_me->exch_matcher.print_all_bbos(msg);
msg << std::endl;
// spartan fills
msg << "Spartan Total Fill Qty: " << fills.total_fill_qty() << std::endl;
auto fill_cmp = [](const auto& lhs, const auto& rhs) {
return lhs.trade_entry_id == rhs.trade_entry_id
? static_cast<int>(lhs.aggressor) > static_cast<int>(rhs.aggressor)
: lhs.trade_entry_id < rhs.trade_entry_id;
};
std::vector<MDP3Fill> spartan_fills;
fills.copy_to(spartan_fills);
std::sort(spartan_fills.begin(), spartan_fills.end(), fill_cmp);
msg << "Spartan Fills (ordered by Trade ID):" << std::endl;
diagnostics::print_fills(spartan_fills.begin(), spartan_fills.end(), msg);
// exch top of books
msg << "Exchange Top of Books:" << std::endl;
diagnostics::print_many_bbo(
sorted_secs.begin(), sorted_secs.end(), true, true, msg);
int exch_fill_qty = 0;
for (auto* s : sorted_secs) {
for (auto f : s->fills())
exch_fill_qty += f.quantity.value();
}
// exch fills
msg << "Exchange Total Fill Qty: " << exch_fill_qty << std::endl;
std::vector<MDP3Fill> all_exch_fills;
for (auto* s : sorted_secs) {
for (auto f : s->fills())
all_exch_fills.push_back(f);
}
std::sort(all_exch_fills.begin(), all_exch_fills.end(), fill_cmp);
msg << "Exchange Fills (ordered by Trade ID):" << std::endl;
diagnostics::print_fills(all_exch_fills.begin(), all_exch_fills.end(), msg);
INFO(msg.str());
// reset the matcher to avoid cascading/compounding errors in subsequent events.
_reset_matcher();
}
_sync_previous_orders(ev);
}
void MatcherValidationObserver::on_real_time_channel_loss() { bts_assert(false); }
} // namespace bts::spartan::mdp3
@@ -0,0 +1,59 @@
#pragma once
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
#include <boost/container/flat_map.hpp>
#include <memory>
#include <unordered_map>
namespace bts::spartan::mdp3 {
struct MatcherValidationObserver : IChannelObserver {
MatcherValidationObserver(std::unordered_map<MDP3OrderKey, IcebergInput> icebergs);
// interesting_event_ranges:
// - keys indicate the first event in an interesting range
// - values indicate the number of events in the range (i.e., the range's length).
MatcherValidationObserver(
std::unordered_map<MDP3OrderKey, IcebergInput> icebergs,
boost::container::flat_map<size_t, size_t> interesting_event_ranges);
~MatcherValidationObserver();
void on_recovery_complete(InitialSnapshot const& snap) override;
void on_real_time_event(MDP3Event const& ev) override;
void on_real_time_channel_loss() override;
// the first time a security's actual trading status
// (i.e., mdp3::SecurityTradingStatus from the exchange)
// is observed to be 'ReadyToTrade,' its local TradingStatus
// (this enum) is switched from 'None' to Preopen.' The security's
// changes in that event are ignored, but on the subsequent event
// its TradingStatus will be flipped from 'Preopen' to 'Uninitialized'
// (in the beginning of the callback). That will be the first event
// in which order changes will be processed for the security.
// As such, the matcher will initialize the state for the security
// by adding every resting order it had prior to the event.
// After that, the TradingStatus will be flipped to 'Initialized.'
// From then on, updates for the security will be processed incrementally.
enum class TradingStatus {
None,
Preopen,
Uninitialized,
Initialized
};
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
TradingStatus _trading_status(const InstrumentSnapshot& s, size_t event_id);
// returns true if the security's TradingStatus is 'Uninitialized', or 'Initialized'
// (in other words, if it isn't 'None,' or 'Preopen')
bool _is_ready_to_trade(const InstrumentSnapshot& s, size_t event_id);
// std::tuple<bool, std::string> _event_filter(const MDP3Event&);
// state maintenance
void _sync_current_orders(MDP3Event const& ev);
void _sync_previous_orders(MDP3Event const& ev);
void _reset_matcher();
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,87 @@
syntax="proto3";
package bts.spartan.protos;
/*
algo: "Sniper"
n_cores: 4
base_algo_latency_ns: 1000
base_algo_latency_ns: 5000
base_algo_latency_ns: 10000
transaction_latency_ns: 50000
param {key: "" value {item: "1" item: "2"}}
parameter {key: "Qty" value {item: "1" item: "2"}}
*/
// The parameters for an entire simulation session, consisting of
// multiple runs.
message SimSessionOptions {
// The algo we are testing
string algo = 1;
// the channel we are testing
string channel = 2;
// Wheter we log every fill/book update/order. Use with care, this
// results in large quantities of log information.
bool log_all_transactions = 10;
// Wheter we should clear all previous runs before starting. False by default.
bool clear_db = 11;
// The max number of cores to use for the simulation. By default all
// cores will be used.
uint32 n_cores = 4 ;
// the days we are testing this out
repeated string market_date = 5;
// A base latency added to any orders produced by an algo. By
// default 22000.
repeated uint64 base_algo_latency_ns = 6;
// An estimate for the average transaction latency at the
// exchange. We fallback on this value when we cannot use the send
// time of a packet to estimate transaction duration. By default
// 60000
repeated uint64 transaction_latency_ns = 7;
// Wheter we should use the first packet or the last packet send
// times when calculating when an algo would see market data
repeated bool use_first_packet_st = 9;
repeated bool simulate_icebergs = 13;
// The start and stop time for the simulation in the form:
// 23:00 04:000. Blank for whatever is in the pcap.
repeated string sim_start_stop_time = 12;
// Algo specific parameters used for this session. The simulation
// wll run through all combinations of parameters, recording the
// outcome of each.
message Param {
string name = 1;
string value = 2;
}
repeated Param param = 8;
}
// The parameters for a single simulation run.
message SimRunOptions {
string algo = 1;
string channel = 2;
string market_date = 3;
bool log_all_transactions = 9;
uint32 param_set_id = 10;
string sim_start_stop_time = 11;
bool use_first_packet_st = 8;
uint64 base_algo_latency_ns = 4;
uint64 transaction_latency_ns = 6;
bool simulate_icebergs = 12;
map<string,string> param = 7;
}
message DayFiles {
repeated string file = 1;
}
@@ -0,0 +1,267 @@
#include "run_sim.h"
#include "server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.h"
#include "server/sparkd/cmds/spartan/back_testing/algos/SniperModel.h"
#include "server/sparkd/cmds/spartan/back_testing/db/SimDB.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/Sim.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.h"
#include "lib/utils/PBUtils.h"
#include <boost/filesystem.hpp>
#include <fstream>
#include <queue>
namespace bts::spartan::mdp3 {
std::string pcap_location(std::string const &channel, std::string const &date) {
auto base_path = os::bts_data_path();
auto pcap_path = fmt::format("{}pcaps/{}/{}_{}_24h.pcap", base_path, channel,
date, channel);
auto xz_pcap_path = pcap_path + ".xz";
if (boost::filesystem::exists(pcap_path))
return pcap_path;
else if (boost::filesystem::exists(xz_pcap_path))
return xz_pcap_path;
else {
bts_assert_msg(false,
fmt::format("Could not find pcap for channel {} on {}",
channel, date));
return "";
}
}
std::vector<size_t>
calc_param_dimensions(protos::SimSessionOptions const &options) {
std::vector<size_t> dimensions;
dimensions.push_back(options.base_algo_latency_ns().size());
dimensions.push_back(options.transaction_latency_ns().size());
dimensions.push_back(options.use_first_packet_st().size());
dimensions.push_back(options.simulate_icebergs().size());
dimensions.push_back(options.sim_start_stop_time().size());
std::map<std::string, size_t> param_dims;
for (auto const &p : options.param())
param_dims[p.name()]++;
for (auto const &p : param_dims) {
dimensions.push_back(p.second);
}
return dimensions;
}
void generate_param_combinations(std::vector<size_t> const &dimensionality,
std::vector<int> &current_combination,
size_t current_idx,
std::vector<std::vector<int>> *out) {
bts_assert(dimensionality.size() == current_combination.size());
if (current_idx == dimensionality.size())
out->push_back(current_combination);
else {
if (dimensionality[current_idx] == 0)
generate_param_combinations(dimensionality, current_combination,
current_idx + 1, out);
else
for (size_t i = 0; i < dimensionality[current_idx]; i++) {
current_combination[current_idx] = i;
generate_param_combinations(dimensionality, current_combination,
current_idx + 1, out);
}
}
}
std::vector<std::vector<int>>
gen_param_combinations(protos::SimSessionOptions const &options) {
auto dimensions = calc_param_dimensions(options);
std::vector<int> first_combination(dimensions.size(), -1);
std::vector<std::vector<int>> all_combinations;
generate_param_combinations(dimensions, first_combination, 0,
&all_combinations);
return all_combinations;
}
std::vector<protos::SimRunOptions>
gen_runs(protos::SimSessionOptions const &options) {
auto combinations = gen_param_combinations(options);
std::vector<protos::SimRunOptions> runs;
for (auto &date : options.market_date()) {
size_t param_set_id = 1;
for (auto &c : combinations) {
auto &this_run = runs.emplace_back();
this_run.set_param_set_id(param_set_id++);
this_run.set_algo(options.algo());
this_run.set_channel(options.channel());
this_run.set_log_all_transactions(options.log_all_transactions());
this_run.set_market_date(date);
size_t param_idx = 0;
// base algo latency
if (c[param_idx] == -1)
this_run.set_base_algo_latency_ns(22000);
else
this_run.set_base_algo_latency_ns(
options.base_algo_latency_ns(c[param_idx]));
param_idx++;
// transaction latency
if (c[param_idx] == -1)
this_run.set_transaction_latency_ns(60000);
else
this_run.set_transaction_latency_ns(
options.transaction_latency_ns(c[param_idx]));
param_idx++;
// use first packet st
if (c[param_idx] == -1)
this_run.set_use_first_packet_st(false);
else
this_run.set_use_first_packet_st(
options.use_first_packet_st(c[param_idx]));
param_idx++;
// simulate icebergs
if (c[param_idx] == -1)
this_run.set_simulate_icebergs(true);
else
this_run.set_simulate_icebergs(
options.simulate_icebergs(c[param_idx]));
param_idx++;
// start/stop time
if (c[param_idx] == -1)
*(this_run.mutable_sim_start_stop_time()) = "21:45 21:00";
else
*(this_run.mutable_sim_start_stop_time()) =
options.sim_start_stop_time(c[param_idx]);
param_idx++;
std::map<std::string, std::vector<std::string>> algo_params;
for (auto const &p : options.param())
algo_params[p.name()].push_back(p.value());
for (auto const &[name, values] : algo_params) {
auto this_parameter_name = name;
auto this_parameter_value = values[c[param_idx++]];
this_run.mutable_param()->insert(
{this_parameter_name, this_parameter_value});
}
}
}
return runs;
}
class PCapEventSourceCache {
struct SrcRec {
std::mutex lock;
std::optional<PCapEventSource> source;
};
std::mutex lock;
std::unordered_map<std::string, std::unique_ptr<SrcRec>> ev_source_cache;
public:
PCapEventSource &get_source(std::string const &pcap_loc) {
SrcRec *rec = nullptr;
{
std::lock_guard<std::mutex> l(lock);
auto &ptr = ev_source_cache[pcap_loc];
if (!ptr)
ptr.reset(new SrcRec());
rec = ptr.get();
}
{
std::lock_guard<std::mutex> l(rec->lock);
if (!rec->source)
rec->source.emplace(pcap_loc);
return *rec->source;
}
}
};
PCapEventSourceCache ev_source_cache;
void run_sim(SimDB &db, SimDB::SessionID sid, std::atomic<size_t> &remaining,
protos::SimRunOptions const &raw_options) {
std::unique_ptr<IAlgoModel> algo;
if (raw_options.algo() == "Sniper")
algo.reset(new SniperModel());
else if (raw_options.algo() == "PolyObserver")
algo.reset(new PolyObserver());
else
bts_assert(false);
auto options = algo->fill_defaults(raw_options);
auto rid = db.begin_run(sid, options);
LOG(Spartan, Warn,
"Starting preload. run: {} date: {} remaining: {} "
"----------------------------------------------",
rid, raw_options.market_date(), (size_t)remaining);
auto logger = [rid, &db](size_t sim_sec, std::string const &src, auto ts,
auto const &header, auto const &entry_type,
auto const &args) {
db.log(rid, sim_sec, src, ts, header, entry_type, args);
};
auto pcap_loc = pcap_location(options.channel(), options.market_date());
auto &source = ev_source_cache.get_source(pcap_loc);
LOG(Spartan, Warn,
"Starting simulation. run: {} date: {} remaining: {} "
"----------------------------------------------",
rid.value(), raw_options.market_date(), (size_t)remaining);
auto event_gen = source.get_event_generator();
Sim sim(*algo, options, source.instruments(), source.icebergs(),
source.polygons(), event_gen, logger);
auto pnl = sim.run();
db.end_run(rid, pnl.cash, pnl.commission, pnl.position);
remaining--;
LOG(Spartan, Warn,
"Completed simulation. run: {} date: {} pandl: {} remaining: {} "
"----------------------------------------------",
rid.value(), raw_options.market_date(), pnl.cash, (size_t)remaining);
}
void run_sim(std::string const &path_to_session_options) {
spartan::protos::SimSessionOptions options;
std::ifstream proto_in(path_to_session_options);
std::string str((std::istreambuf_iterator<char>(proto_in)),
std::istreambuf_iterator<char>());
bts_assert(utils::pb::from_string(str, &options));
SimDB db;
auto sid = db.create_session(options.algo(), options.clear_db());
std::mutex lock;
std::vector<std::thread> workers(options.n_cores() ? options.n_cores()
: os::num_cores());
auto run_options = gen_runs(options);
std::queue<spartan::protos::SimRunOptions> runs;
for (auto &ru : run_options)
runs.push(ru);
std::atomic<size_t> remaining = runs.size();
for (auto &w : workers) {
w = std::thread{[&remaining, &lock, &runs, &db, sid]() {
while (true) {
spartan::protos::SimRunOptions these_options;
{
std::lock_guard<std::mutex> l(lock);
if (runs.size()) {
these_options = runs.front();
runs.pop();
} else
return;
}
run_sim(db, sid, remaining, these_options);
}
}};
}
for (auto &w : workers)
w.join();
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,14 @@
#pragma once
#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h"
#include "server/sparkd/cmds/spartan/back_testing/db/SimLogger.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
namespace bts::spartan::mdp3 {
class MDP3EventGenerator;
class AlgoModel;
void run_sim(protos::SimRunOptions const&);
void run_sim(std::string const&path_to_run_options);
} // namespace bts::spartan::mdp3
@@ -0,0 +1,56 @@
#pragma once
#include <vector>
#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h"
#include "lib/utils/bts_assert.h"
#include <sstream>
namespace bts::spartan::mdp3{
struct EventHeader;
struct MBPUpdate;
struct OrderUpdate;
}
namespace bts::spartan::mdp3 {
class ISim;
class IAlgoModel {
public:
virtual ~IAlgoModel() {}
virtual protos::SimRunOptions fill_defaults (protos::SimRunOptions const&in) {return in;}
virtual void init(ISim &, protos::SimRunOptions const&) = 0;
virtual void finit() {}
virtual void on_replay_start(EventHeader const &,
std::vector<MBPUpdate> const &) {}
virtual void on_order_update(EventHeader const &, OrderUpdate const &) {}
virtual void on_mbp(EventHeader const &, MBPUpdate const &) {}
virtual void on_time() {}
};
inline std::string get_param(protos::SimRunOptions const&in, std::string const&name) {
auto it = in.param().find(name);
bts_assert(it!= in.param().end());
return it->second;
}
template<class T> T get_param(protos::SimRunOptions const&in,std::string const&name) {
auto it = in.param().find(name);
bts_assert(it!= in.param().end());
T result;
std::stringstream{it->second} >> result;
return result;
}
template<class T> void set_param_if_empty(protos::SimRunOptions *in,std::string const&name, T const&v) {
auto it = in->param().find(name);
if (it == in->param().end()) {
std::stringstream str_v;
str_v << v;
auto mmap = in->mutable_param();
(*mmap)[name] = str_v.str();
}
}
}
@@ -0,0 +1,7 @@
#include "ISim.h"
namespace bts::spartan::mdp3 {
ISim::~ISim() {}
}
@@ -0,0 +1,37 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
namespace bts::spartan::mdp3 {
class ISim {
public:
virtual ~ISim();
// Queries
virtual PolygonMap const &polygons() const = 0;
virtual InstrumentMap const &instruments() const = 0;
virtual InstrumentDefinition const &get_sec(MDP3SecurityID id) const = 0;
virtual MDP3TimeStamp time() const = 0;
// Actions
virtual MDP3OrderKey send(SmallOrderChange const &, size_t delay_ns = 0) = 0;
virtual void start_simulation() = 0;
virtual void stop_simulation() = 0;
virtual void log_pandl() const = 0;
template <class... T> void log(std::string const &ty, T &&... args) const;
private:
virtual void algo_log(std::string const &log_entry_type,
std::vector<std::string> const &log_args) const = 0;
};
template <class... T>
void ISim::log(std::string const &log_entry_type, T &&... args) const {
auto lst = std::vector<std::string>{to_str(std::forward<T>(args))...};
algo_log(log_entry_type, lst);
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,763 @@
#include "Sim.h"
#include "lib/logging/logging.h"
#include "lib/utils/datetime.h"
#include "server/sparkd/cmds/spartan/back_testing/db/SimLogger.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h"
#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <queue>
namespace bts::spartan::mdp3 {
enum class Sim::State {
REPLAY, // Get book state from pcap, do not log.
PRESIM, // Get book state from pcap, log
SIM // Get book state from matcher, log
};
enum class Sim::PendingSimOP { NONE, START, STOP };
struct Sim::Simulation {
MergedMatcher matcher = {};
// CME does not guarantee uniqueness of order keys during a
// session, though of course, they must guarantee uniqueness of
// *alive* order keys. During a simulation its possible that the
// life cycle of a CME order changes from what happened in reality,
// which could lead to key colision if we used the cme key. For
// this reason we must remap the CME keys to ones we know will be
// unique during the entire simulation.
std::unordered_map<MDP3OrderKey, MDP3OrderKey> cme_key_map;
// The matcher does not currently support non-trading security
// states. For this reason we have to use this map to detect
// changes of trading state in securities so that when a security
// starts trading we can seed the matcher with existing orders and
// when it stops trading we can remove its orders
std::unordered_set<MDP3SecurityID> ready_secs;
};
struct Sim::AlgoCBFilter : IAlgoModel {
AlgoCBFilter(Sim &owner, IAlgoModel &real_algo)
: owner(owner), real_algo(real_algo) {}
Sim &owner;
IAlgoModel &real_algo;
void init(ISim &s, protos::SimRunOptions const &options) override {
real_algo.init(s, options);
}
void finit() override { real_algo.finit(); }
void on_replay_start(EventHeader const &h,
std::vector<MBPUpdate> const &b) override {
real_algo.on_replay_start(h, b);
}
void on_order_update(EventHeader const &h, OrderUpdate const &o) override {
if (!owner.is_stopping_sim())
real_algo.on_order_update(h, o);
}
void on_mbp(EventHeader const &h, MBPUpdate const &mbp) override {
if (!owner.is_stopping_sim())
real_algo.on_mbp(h, mbp);
}
void on_time() override {
if (!owner.is_stopping_sim())
real_algo.on_time();
}
};
struct Sim::PImpl {
AlgoCBFilter algo;
protos::SimRunOptions options;
InstrumentMap const &instruments;
IcebergMap const &ib_map;
PolygonMap const &polygons;
PCapEventGenerator &event_generator;
SimLogger logger;
std::unordered_map<MDP3SecurityID, InstrumentSnapshot> snapshots = {};
State state = State::REPLAY;
MDP3TimeStamp transact_time = MDP3TimeStamp{0};
MDP3TimeStamp time = MDP3TimeStamp{0};
AlgoEventID event_id = {0, 0};
std::optional<Simulation> simulation = {};
MDP3OrderKey last_order_key =
MDP3OrderKey{std::numeric_limits<MDP3OrderKey::Value>::max()};
std::vector<OrderUpdate> my_fills = {};
PendingSimOP pending_sim_op = PendingSimOP::NONE;
MDP3TimeStamp next_time_event = MDP3TimeStamp{0};
std::queue<AlgoAction> pending_algo_actions = {};
size_t sim_seq = 0;
size_t next_mapped_key = 1;
TradingWindow trading_window = {};
};
Sim::Sim(IAlgoModel &algo, protos::SimRunOptions const &options,
InstrumentMap const &instruments, IcebergMap const &ice,
PolygonMap const &polys, PCapEventGenerator &gen, SimLogger const &l)
: me(new PImpl{AlgoCBFilter{*this, algo}, options, instruments, ice, polys,
gen, l}) {
me->trading_window = calculate_trading_window(
me->options.market_date(), me->options.sim_start_stop_time());
}
Sim::~Sim() {}
PandLSummary Sim::run() {
me->algo.init(*this, me->options);
// Consume pcap events but do not send them to the algo up until
// trading window opens up
while (not_yet_in_trading_window()) {
if (!me->event_generator.top())
return {};
else {
auto const &ev = *me->event_generator.top();
me->transact_time = ev.transact_time;
me->time = get_send_time(ev);
me->event_id.first = ev.event_id;
for (auto const &snap : ev.affected_securities)
me->snapshots[snap.security_id()].update_with(snap);
me->event_generator.move_next();
}
}
// Now initialize the algo with whatever books we currently have
on_algo_replay_start();
// Extract events from the pcap and forward them to the algo while
// we are still in the trading window
while (me->event_generator.top() && still_in_trading_window()) {
bts_assert(!has_pending_sim_op());
next_event();
if (has_pending_sim_op())
handle_pending_sim_op();
}
// All done, log pandl and exit
log_pandl();
return pnl();
}
void Sim::handle_pending_sim_op() {
switch (me->pending_sim_op) {
case PendingSimOP::NONE:
break;
case PendingSimOP::START:
bts_assert(me->state == State::REPLAY);
me->pending_sim_op = PendingSimOP::NONE;
me->state = State::PRESIM;
me->sim_seq++;
sim_log("presim_start");
break;
case PendingSimOP::STOP:
bts_assert(me->state == State::SIM);
me->pending_sim_op = PendingSimOP::NONE;
_stop_simulation();
break;
}
}
void Sim::next_event() {
bts_assert(me->event_generator.top());
auto &next_pcap = *me->event_generator.top();
switch (me->state) {
case State::REPLAY:
if (time_event_due_before(next_pcap.transact_time))
on_time();
else
on_pcap_event_during_replay();
break;
case State::PRESIM: {
bts_assert(!me->simulation);
auto &pending_actions = me->pending_algo_actions;
if (pending_actions.size() &&
pending_actions.front().transact_time < next_pcap.transact_time) {
// We have a pending algo action with a transact time ealier
// than our next pcap event
auto &next_algo_action = pending_actions.front();
if (time_event_due_before(next_algo_action.transact_time))
on_time();
else {
_start_simulation();
on_algo_action();
}
} else {
// The pcap event is the next in line
if (time_event_due_before(next_pcap.transact_time))
on_time();
else
on_pcap_event_during_replay();
}
break;
}
case State::SIM: {
bts_assert(me->simulation);
auto &pending_actions = me->pending_algo_actions;
if (pending_actions.size() &&
pending_actions.front().transact_time < next_pcap.transact_time) {
// We have a pending algo action with a transact time ealier
// than our next pcap event
auto &next_algo_action = pending_actions.front();
if (time_event_due_before(next_algo_action.transact_time))
on_time();
else
on_algo_action();
} else {
// The pcap event is the next in line
if (time_event_due_before(next_pcap.transact_time))
on_time();
else
on_pcap_event_during_simulation();
}
break;
}
}
// We do not want to start the timer at 0 because we would have to
// process timer events for billions of years before a pcap or algo
// action event was due.
if (me->next_time_event == MDP3TimeStamp{0} && me->time.value())
me->next_time_event = me->time;
}
void Sim::on_pcap_event_during_replay() {
bts_assert(me->state == State::REPLAY || me->state == State::PRESIM);
bts_assert(me->event_generator.top());
bts_assert(!me->simulation);
auto const &ev = *me->event_generator.top();
me->transact_time = ev.transact_time;
me->time =
MDP3TimeStamp{std::max(me->time.value() + 1, get_send_time(ev).value())};
me->event_id.first = ev.event_id;
for (auto const &snap : ev.affected_securities)
me->snapshots[snap.security_id()].update_with(snap);
EventHeader header{{ev.event_id, 0}, ev.transact_time, me->time};
for (auto &snap : ev.affected_securities) {
if (snap.ready_to_trade()) {
MBPUpdate mbp{
snap.security_id(),
snap.book(api::SIDE_BUY),
snap.book(api::SIDE_SELL),
snap.implied_book(api::SIDE_BUY),
snap.implied_book(api::SIDE_SELL),
};
if (me->state == State::PRESIM && me->options.log_all_transactions()) {
auto changes = snap.deduced_order_changes();
for (auto &c : changes) {
sim_log("presim_order", ev.transact_time.value(),
change_name(c.change),
get_sec(c.order.security).symbol(),
c.order.price.price(), c.order.quantity.value(),
api::Side_Name(c.order.side), c.order.key.value(),
c.order.key.value());
}
sim_log("presim_book", get_sec(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.price(),
mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.price(),
mbp.mbp_sell[0].quantity.value());
}
me->algo.on_mbp(header, mbp);
}
}
me->event_generator.move_next();
}
void Sim::on_pcap_event_during_simulation() {
bts_assert(me->state == State::SIM);
bts_assert(me->event_generator.top());
bts_assert(me->simulation);
auto const &ev = *me->event_generator.top();
me->transact_time = ev.transact_time;
me->time =
MDP3TimeStamp{std::max(me->time.value() + 1, get_send_time(ev).value())};
me->event_id.first = ev.event_id;
for (auto &snap : ev.affected_securities)
me->snapshots[snap.security_id()].update_with(snap);
for (auto &snap : ev.affected_securities) {
auto this_sid = snap.security_id();
bool was_ready = me->simulation->ready_secs.count(this_sid);
bool is_ready = snap.ready_to_trade();
// The security was not ready for trading but it is now. We
// need to boostrap the matcher with its current order book.
if (!was_ready && is_ready) {
sim_log("sec_started_trading", snap.symbol());
bootstrap_simulated_order_book(ev.transact_time, this_sid);
}
// The security was trading but is no longer. We need to
// clear it from the matcher.
else if (was_ready && !is_ready) {
sim_log("sec_stoped_trading", snap.symbol());
clear_simulated_order_book(ev.transact_time, this_sid);
}
// The security was and remains ready. We need to incorporate
// the latest deduced orders in the matcher and send out any
// matcher effects.
else if (was_ready && is_ready) {
auto changes = snap.deduced_order_changes();
SmallOrderChangeVec matcher_input;
for (auto &c : changes) {
auto corder = correct_pcap_order(c);
if (me->options.log_all_transactions())
sim_log("pcap_order", ev.transact_time.value(),
change_name(corder.change),
get_sec(corder.order.security).symbol(),
corder.order.price.price(),
corder.order.quantity.value(),
api::Side_Name(corder.order.side),
corder.order.key.value(), corder.original_key.value());
matcher_input.push_back(corder);
}
match_and_send_effects(ev.transact_time, matcher_input);
}
}
me->event_generator.move_next();
}
void Sim::on_algo_action() {
bts_assert(me->state == State::SIM);
bts_assert(me->simulation);
bts_assert(me->pending_algo_actions.size());
auto &algo_action = me->pending_algo_actions.front();
auto const &change = algo_action.change;
me->transact_time = algo_action.transact_time;
me->time.value() += me->options.transaction_latency_ns();
me->event_id.second++;
sim_log("algo_order", algo_action.transact_time.value(),
change_name(change.change), get_sec(change.order.security).symbol(),
change.order.price.price(), change.order.quantity.value(),
api::Side_Name(change.order.side), change.order.key.value(), 0);
match_and_send_effects(algo_action.transact_time, {algo_action.change});
me->pending_algo_actions.pop();
}
void Sim::_start_simulation() {
bts_assert(me->state == State::PRESIM);
bts_assert(!me->simulation);
me->state = State::SIM;
auto &simulation = me->simulation.emplace();
auto &matcher = simulation.matcher;
sim_log("simulation_start");
for (auto &[sec_id, snap] : me->snapshots) {
matcher.add_def(snap.definition());
if (snap.ready_to_trade()) {
simulation.ready_secs.insert(sec_id);
std::vector<MDP3Order> resting_orders;
for (const auto &p : snap.orders())
resting_orders.emplace_back(
correct_pcap_order({p.second, p.second, OrderChange::added},
true)
.order);
std::vector<MergedMatcher::MatchEffect> effects;
matcher.reset(sec_id, resting_orders, &effects);
for (auto &ef : effects)
std::visit(
overloaded{[&](MBPUpdate &mbp) {
sim_log("sim_book_init",
get_sec(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.price(),
mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.price(),
mbp.mbp_sell[0].quantity.value());
},
[&](OrderUpdate const &) { bts_assert(false); }},
ef);
}
}
validate_matcher_books(api::SIDE_BUY);
validate_matcher_books(api::SIDE_SELL);
}
void Sim::_stop_simulation() {
bts_assert(me->state == State::SIM);
me->simulation.reset();
me->pending_algo_actions = {};
me->state = State::REPLAY;
me->event_id.second = 0;
on_algo_replay_start();
}
void Sim::on_algo_replay_start() {
std::vector<MBPUpdate> collected_books;
for (auto &[id, snap] : me->snapshots)
if (snap.ready_to_trade()) {
collected_books.emplace_back(
MBPUpdate{id, snap.book(api::SIDE_BUY), snap.book(api::SIDE_SELL),
snap.implied_book(api::SIDE_BUY),
snap.implied_book(api::SIDE_SELL)});
}
me->algo.on_replay_start(
EventHeader{me->event_id, me->transact_time, me->time}, collected_books);
}
void Sim::match_and_send_effects(MDP3TimeStamp transact_time,
SmallOrderChangeVec const &matcher_input) {
bts_assert(me->state == State::SIM);
bts_assert(me->simulation);
auto &simulation = *me->simulation;
std::vector<MergedMatcher::MatchEffect> effects;
simulation.matcher.match(MergedMatcher::SIMULATOR_INPUT, matcher_input,
&effects);
std::sort(effects.begin(), effects.end(),
[](auto const &ef1, auto const &ef2) {
auto rankf1 = std::holds_alternative<MBPUpdate>(ef1) ? 0 : 1;
auto rankf2 = std::holds_alternative<MBPUpdate>(ef2) ? 0 : 1;
return rankf1 < rankf2;
});
EventHeader header{me->event_id, transact_time, me->time};
for (auto &ef : effects)
std::visit(
overloaded{
[&](MBPUpdate const &mbp) {
if (me->options.log_all_transactions())
sim_log("sim_book", get_sec(mbp.security_id).symbol(),
mbp.mbp_buy[0].price.price(),
mbp.mbp_buy[0].quantity.value(),
mbp.mbp_sell[0].price.price(),
mbp.mbp_sell[0].quantity.value());
me->algo.on_mbp(header, mbp);
},
[&](OrderUpdate const &order_update) {
if (order_update.key.value() >= me->last_order_key.value()) {
sim_log("algo_fill", order_update.status_str(),
api::Side_Name(order_update.side),
get_sec(order_update.security_id).symbol(),
order_update.last_price.price(),
order_update.last_fills.value(),
order_update.key.value(), 0);
me->my_fills.push_back(order_update);
me->algo.on_order_update(header, order_update);
} else {
if (me->options.log_all_transactions())
sim_log("fill", order_update.status_str(),
api::Side_Name(order_update.side),
get_sec(order_update.security_id).symbol(),
order_update.last_price.price(),
order_update.last_fills.value(),
order_update.key.value(),
me->simulation->cme_key_map[order_update.key]
.value());
}
}},
ef);
}
void Sim::bootstrap_simulated_order_book(MDP3TimeStamp transact_time,
MDP3SecurityID sid) {
bts_assert(me->state == State::SIM);
bts_assert(me->simulation);
auto &simulation = *me->simulation;
simulation.ready_secs.insert(sid);
auto &snap = me->snapshots[sid];
SmallOrderChangeVec matcher_input;
for (auto const &p : snap.orders())
matcher_input.push_back(correct_pcap_order(
OrderChange{p.second, p.second, OrderChange::added}, true));
match_and_send_effects(transact_time, matcher_input);
}
void Sim::clear_simulated_order_book(MDP3TimeStamp transact_time,
MDP3SecurityID sid) {
bts_assert(me->state == State::SIM);
bts_assert(me->simulation);
auto &simulation = *me->simulation;
simulation.ready_secs.erase(sid);
auto &snap = me->snapshots[sid];
SmallOrderChangeVec matcher_input;
for (auto const &p : snap.orders())
matcher_input.push_back(correct_pcap_order(
OrderChange{p.second, p.second, OrderChange::removed}));
match_and_send_effects(transact_time, matcher_input);
}
SmallOrderChange Sim::correct_pcap_order(OrderChange const &o,
bool set_exact_ib_state) {
bts_assert(me->state == State::SIM);
bts_assert(me->simulation);
auto &key_map = me->simulation->cme_key_map;
bts_assert(o.order.key.value());
MDP3OrderKey &our_key = key_map[o.order.key];
SmallOrderChange result(o);
result.original_key = o.order.key;
switch (o.change) {
case OrderChange::added: {
our_key = MDP3OrderKey{me->next_mapped_key++};
result.order.key = our_key;
break;
}
case OrderChange::removed: {
bts_assert(our_key != MDP3OrderKey{});
result.order.key = our_key;
our_key = MDP3OrderKey{};
break;
}
case OrderChange::updated:
case OrderChange::updated_price: {
bts_assert(our_key != MDP3OrderKey{});
result.order.key = our_key;
}
}
if (me->options.simulate_icebergs()) {
if (o.change != OrderChange::ChangeType::removed) {
// IcebergMap is keyed by exchange keys
auto ib_it = me->ib_map.find(result.original_key);
if (ib_it != me->ib_map.end())
iceberg_helper::add_iceberg_annotation(
result.order, result.change, ib_it->second, me->event_id.first,
set_exact_ib_state);
}
}
return result;
}
PolygonMap const &Sim::polygons() const { return me->polygons; }
InstrumentMap const &Sim::instruments() const { return me->instruments; }
InstrumentDefinition const &Sim::get_sec(MDP3SecurityID id) const {
auto it = me->instruments.find(id);
bts_assert(it != me->instruments.end());
return *it->second;
}
MDP3TimeStamp Sim::time() const { return me->time; }
double Sim::price_at_market(MDP3SecurityID id, MDP3Quantity qty,
api::Side s) const {
auto it = me->snapshots.find(id);
bts_assert(it != me->snapshots.end());
auto &snap = it->second;
auto &book = snap.book(s);
double cummulative_flow = 0;
auto remaining_qty = qty.value();
for (auto &l : book) {
if (remaining_qty == 0)
return cummulative_flow;
else {
auto from_this_level = std::min(remaining_qty, l.quantity.value());
cummulative_flow +=
(from_this_level * get_sec(id).to_cash(l.price.price()));
remaining_qty -= from_this_level;
}
}
if (remaining_qty)
return os::nan();
else
return cummulative_flow;
}
PandLSummary Sim::pnl() const {
auto mkt_pricer = [this](MDP3SecurityID id, MDP3Quantity qty, api::Side s) {
return price_at_market(id, qty, s);
};
PandL pl{me->instruments, mkt_pricer};
auto outright_pos = pl.outright_position(me->my_fills);
auto cash = pl.net_cash(me->my_fills);
auto commission = pl.commission_cost(me->my_fills);
auto pos = pl.position_liquidation_value(outright_pos);
return {cash - commission, pos, commission};
}
void Sim::log_pandl() const {
auto pnl_summary = pnl();
log("pnl", pnl_summary.cash, pnl_summary.commission, pnl_summary.position);
}
MDP3OrderKey Sim::send(SmallOrderChange const &change,
size_t propagation_delay) {
if (me->state == State::REPLAY)
start_simulation();
auto transact_time =
MDP3TimeStamp{me->time.value() + me->options.base_algo_latency_ns() +
propagation_delay};
if (change.change == OrderChange::added) {
auto cpy = change;
cpy.order.key = MDP3OrderKey{me->last_order_key.value()--};
me->pending_algo_actions.push(AlgoAction{transact_time, cpy});
return cpy.order.key;
} else {
bts_assert(change.order.key != MDP3OrderKey{});
me->pending_algo_actions.push(AlgoAction{transact_time, change});
return change.order.key;
}
}
void Sim::algo_log(std::string const &log_entry_type,
std::vector<std::string> const &log_args) const {
_log("algo", log_entry_type, log_args);
}
void Sim::sim_log(std::string const &log_entry_type,
std::vector<std::string> const &log_args) const {
_log("sim", log_entry_type, log_args);
}
void Sim::_log(std::string const &src, std::string const &log_entry_type,
std::vector<std::string> const &log_args) const {
std::ostringstream str;
str << to_time_string(me->time) << " " << src << " " << log_entry_type
<< " ";
for (auto const &a : log_args)
str << a << " ";
LOG(Spartan, Info, "{}", str.str());
me->logger(me->sim_seq, src, me->time,
EventHeader{me->event_id, me->transact_time, me->time},
log_entry_type, log_args);
}
bool Sim::time_event_due_before(MDP3TimeStamp ts) const {
return me->next_time_event.value() &&
me->next_time_event.value() < ts.value();
}
void Sim::on_time() {
me->algo.on_time();
me->next_time_event.value() += 1000 * 1000 * 100;
}
void Sim::start_simulation() {
bts_assert(me->state == State::REPLAY);
bts_assert(me->pending_sim_op == PendingSimOP::NONE ||
me->pending_sim_op == PendingSimOP::START);
me->pending_sim_op = PendingSimOP::START;
}
void Sim::stop_simulation() {
bts_assert(me->state == State::SIM);
bts_assert(me->pending_sim_op == PendingSimOP::NONE);
me->pending_sim_op = PendingSimOP::STOP;
}
void Sim::validate_matcher_books(api::Side side) {
bts_assert(me->state == State::SIM);
bts_assert(me->simulation);
auto &simulation = *me->simulation;
for (auto &[sec_id, snap] : me->snapshots) {
if (snap.ready_to_trade()) {
if (snap.book(side) != simulation.matcher.book(sec_id, side)) {
LOG(Spartan, Error,
"Mismatch between matcher and pcap books on event: {}. "
"Side: "
"{}",
me->event_id.first, api::Side_Name(side));
LOG(Spartan, Error, "PCap Book:");
LOG(Spartan, Error, "{}", snap.book(side));
LOG(Spartan, Error, "Matcher Book:");
LOG(Spartan, Error, "{}", simulation.matcher.book(sec_id, side));
bts_assert(false);
}
}
}
}
bool Sim::still_in_trading_window() const {
if (me->state == State::REPLAY) {
return me->time.value() <= me->trading_window.stop.value();
} else
return true;
}
bool Sim::not_yet_in_trading_window() const {
bts_assert(me->state == State::REPLAY);
return me->time.value() < me->trading_window.start.value();
}
bool Sim::has_pending_sim_op() const {
return me->pending_sim_op != PendingSimOP::NONE;
}
bool Sim::is_stopping_sim() const {
return me->pending_sim_op == PendingSimOP::STOP;
}
MDP3TimeStamp Sim::get_send_time(PCapEvent const &ev) const {
if (me->options.use_first_packet_st())
return ev.first_packet_send_time;
else
return ev.last_packet_send_time;
}
Sim::TradingWindow
Sim::calculate_trading_window(std::string const &mkt_date,
std::string const &start_stop) const {
if (start_stop.empty()) {
return {MDP3TimeStamp{0},
MDP3TimeStamp{std::numeric_limits<MDP3TimeStamp::Value>::max()}};
} else {
std::vector<std::string> split_vec;
boost::algorithm::split(split_vec, start_stop,
boost::algorithm::is_any_of(" "));
bts_assert(split_vec.size() == 2);
auto start = split_vec[0];
auto stop = split_vec[1];
auto day_start = datetime::parse_duration("21:45");
auto adjust_day = [&](std::string const &time_str) {
auto dt = datetime::parse_simple_date_time_string(mkt_date);
auto t = datetime::parse_duration(time_str);
if (t < day_start)
dt += boost::gregorian::date_duration(1);
return datetime::PTime{dt.date(), t};
};
auto start_time = adjust_day(start);
auto stop_time = adjust_day(stop);
bts_assert(stop_time > start_time);
return {to_mdp3time(start_time), to_mdp3time(stop_time)};
}
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,103 @@
#pragma once
#include "server/sparkd/cmds/spartan/back_testing/db/SimLogger.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pandl/PandLSummary.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
namespace bts::spartan::mdp3 {
struct EventHeader;
class PCapEvent;
struct MBPUpdate;
struct OrderUpdate;
} // namespace bts::spartan::mdp3
namespace bts::spartan::protos {
class SimRunOptions;
}
namespace bts::spartan::mdp3 {
class IAlgoModel;
class PCapEventGenerator;
class Sim : ISim {
public:
Sim(IAlgoModel &algo, protos::SimRunOptions const &, InstrumentMap const &im,
IcebergMap const &, PolygonMap const &, PCapEventGenerator &,
SimLogger const &);
~Sim();
PandLSummary run();
private:
// From ISim
PolygonMap const &polygons() const override;
InstrumentMap const &instruments() const override;
InstrumentDefinition const &get_sec(MDP3SecurityID id) const override;
MDP3TimeStamp time() const override;
void log_pandl() const override;
MDP3OrderKey send(SmallOrderChange const &,
size_t propagation_delay) override;
void start_simulation() override;
void stop_simulation() override;
void algo_log(std::string const &log_entry_type,
std::vector<std::string> const &log_args) const override;
// End ISim
PandLSummary pnl() const;
double price_at_market(MDP3SecurityID id, MDP3Quantity qty,
api::Side s) const;
void next_event();
void on_pcap_event_during_replay();
void on_pcap_event_during_simulation();
void on_algo_action();
void on_time();
bool time_event_due_before(MDP3TimeStamp) const;
void bootstrap_simulated_order_book(MDP3TimeStamp transact_time,
MDP3SecurityID);
void clear_simulated_order_book(MDP3TimeStamp transact_time, MDP3SecurityID);
void match_and_send_effects(MDP3TimeStamp transact_time,
SmallOrderChangeVec const &);
SmallOrderChange correct_pcap_order(OrderChange const &pcap_order,
bool set_exact_ib_state = false);
void sim_log(std::string const &log_entry_type,
std::vector<std::string> const &log_args) const;
template <class... T>
void sim_log(std::string const &log_entry_type, T &&... args) const;
void _log(std::string const &src, std::string const &log_entry_type,
std::vector<std::string> const &log_args) const;
void _start_simulation();
void _stop_simulation();
void validate_matcher_books(api::Side);
bool has_pending_sim_op() const;
void on_algo_replay_start();
bool not_yet_in_trading_window() const;
bool still_in_trading_window() const;
bool is_stopping_sim() const;
MDP3TimeStamp get_send_time(PCapEvent const &) const;
void handle_pending_sim_op();
struct TradingWindow {
MDP3TimeStamp start;
MDP3TimeStamp stop;
};
TradingWindow calculate_trading_window(std::string const &mkt_date,
std::string const &start_stop) const;
enum class PendingSimOP;
enum class State;
struct Simulation;
struct PImpl;
struct AlgoCBFilter;
std::unique_ptr<PImpl> me;
};
template <class... T>
void Sim::sim_log(std::string const &log_entry_type, T &&... args) const {
auto lst = std::vector<std::string>{to_str(std::forward<T>(args))...};
_log("sim", log_entry_type, lst);
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,109 @@
#include "PandL.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h"
namespace bts::spartan::mdp3 {
struct PandL::PImpl {
InstrumentMap const &instruments;
Pricer pricer;
};
PandL::PandL(InstrumentMap const &m, Pricer const &p) : me(new PImpl{m, p}) {}
PandL::~PandL() {}
double PandL::net_cash(std::vector<OrderUpdate> const &fills) const {
double net_cash_flow = 0;
for (auto const &f : fills) {
auto cash = price_to_cash(f.security_id,
f.last_fills.value() * f.last_price.price());
auto flow = mkt_to_cash_flow(f.side);
net_cash_flow += cash * flow;
}
return net_cash_flow;
}
double PandL::commission_cost(std::vector<OrderUpdate> const &fills) const {
double cost = 0;
for (auto const &f : fills) {
auto n_legs = get_sec(f.security_id).legs().size();
if (n_legs == 0)
cost += f.last_fills.value() * 1.23;
else
cost += f.last_fills.value() * n_legs * 0.86;
}
return cost;
}
PandL::SymbolNetCash
PandL::symbol_net_cash(std::vector<OrderUpdate> const &fills) const {
std::unordered_map<std::string, std::vector<OrderUpdate>> fills_per_symbol;
for (auto const &f : fills)
fills_per_symbol[get_sec(f.security_id).asset()].push_back(f);
SymbolNetCash result;
for (auto const &[symbol, symbol_fills] : fills_per_symbol) {
result[symbol] = net_cash(symbol_fills);
}
return result;
}
PandL::OutrightPosition
PandL::outright_position(std::vector<OrderUpdate> const &fills) const {
OutrightPosition result;
auto add_outright_pos = [&](auto sec_id, int64_t change) {
auto &this_pos = result[sec_id];
this_pos += change;
};
for (auto const &f : fills) {
auto &sec = get_sec(f.security_id);
if (sec.legs().size()) {
for (auto &leg : sec.legs()) {
auto leg_contract_change =
mkt_to_contract_flow(f.side) * f.last_fills.value() *
mkt_to_contract_flow(leg.side) * leg.ratio.value();
add_outright_pos(leg.security_id, leg_contract_change);
}
} else
add_outright_pos(f.security_id,
mkt_to_contract_flow(f.side) * f.last_fills.value());
}
return result;
}
double PandL::position_liquidation_value(OutrightPosition const&outright_position) const {
double liquidation_proceeds = 0;
for (auto &[id, this_pos] : outright_position) {
auto mkt_action = api::SIDE_UNDEFINED;
if (this_pos > 0)
mkt_action = api::SIDE_SELL;
else
mkt_action = api::SIDE_BUY;
auto cash_flow = me->pricer(id, MDP3Quantity{(int)this_pos},mkt_action);
liquidation_proceeds += cash_flow;
}
return liquidation_proceeds;
}
InstrumentDefinition const &PandL::get_sec(MDP3SecurityID id) const {
return mdp3::get_sec(id,me->instruments);
}
double PandL::price_to_cash(MDP3SecurityID id, double p) const {
return get_sec(id).to_cash(p);
}
int PandL::mkt_to_cash_flow(api::Side s) const {
if (s == api::SIDE_BUY)
return -1;
else
return 1;
}
int PandL::mkt_to_contract_flow(api::Side s) const {
return -1 * mkt_to_cash_flow(s);
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,42 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h"
#include "lib/os/os.h"
#include <cstdint>
#include <vector>
#include <unordered_map>
namespace bts::spartan::mdp3 {struct OrderUpdate;}
namespace bts::spartan::mdp3 {
class PandL {
public:
using SymbolNetCash =
std::unordered_map<std::string, double>;
using OutrightPosition=
std::unordered_map<MDP3SecurityID, int64_t>;
PandL(InstrumentMap const&, Pricer const& = [](auto,auto,auto) {return os::nan();});
~PandL();
double net_cash(std::vector<OrderUpdate> const &fills) const;
SymbolNetCash symbol_net_cash(std::vector<OrderUpdate> const &fills) const;
OutrightPosition outright_position(std::vector<OrderUpdate> const &fills) const;
double position_liquidation_value(OutrightPosition const&) const;
double commission_cost(std::vector<OrderUpdate> const &fills) const;
private:
struct PImpl;
InstrumentDefinition const &get_sec(MDP3SecurityID id) const ;
double price_to_cash(MDP3SecurityID id, double p) const;
int mkt_to_cash_flow(api::Side s) const;
int mkt_to_contract_flow(api::Side s) const;
std::unique_ptr<PImpl> me;
};
}
@@ -0,0 +1,17 @@
#pragma once
namespace bts::spartan::mdp3 {
struct PandLSummary {
double cash = 0; // net of commission
double position = 0;
double commission = 0;
PandLSummary &operator+=(PandLSummary const &rhs) {
cash += rhs.cash;
position += rhs.position;
commission += rhs.commission;
return *this;
}
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,20 @@
#pragma once
#include <boost/container/small_vector.hpp>
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
namespace bts::spartan::mdp3 {
class PCapEvent {
public:
MDP3TimeStamp transact_time = MDP3TimeStamp{0};
size_t event_id = 0;
MDP3TimeStamp first_packet_send_time = MDP3TimeStamp{0};
MDP3TimeStamp last_packet_send_time = MDP3TimeStamp{0};
uint32_t first_packet_sn = 0;
uint32_t last_packet_sn = 0;
boost::container::small_vector<InstrumentSnapshot,1> affected_securities = {};
};
}
@@ -0,0 +1,106 @@
#include "PCapEventGenerator.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include <queue>
namespace bts::spartan::mdp3 {
struct PCapEventGenerator::PImpl {
PImpl(std::string const &pcap, IChannelObserver &obs)
: streamer(pcap, obs), done(false) {}
PCapStreamer streamer;
bool done;
std::vector<PCapEvent> buffered_events;
size_t idx = 0;
};
PCapEventGenerator::PCapEventGenerator(std::string const &pcap)
: me(new PImpl(pcap, *this)) {
move_next();
}
PCapEventGenerator::PCapEventGenerator(PCapEventGenerator &&src)
: me(std::move(src.me)) {}
PCapEventGenerator::~PCapEventGenerator() {}
PCapEventGenerator &PCapEventGenerator::operator=(PCapEventGenerator &&src) {
me = std::move(src.me);
return *this;
}
PCapEvent const *PCapEventGenerator::top() {
if (me->done)
return nullptr;
else {
bts_assert(me->idx < me->buffered_events.size());
return &me->buffered_events[me->idx];
}
}
void PCapEventGenerator::move_next() {
bts_assert(!me->done);
if (me->buffered_events.size() && me->idx < me->buffered_events.size() - 1)
me->idx++;
else {
me->idx = 0;
me->buffered_events.clear();
for (;;) {
auto ok = me->streamer.next();
if (!ok) {
me->done = true;
return;
} else if (me->buffered_events.size())
return;
}
}
}
void PCapEventGenerator::on_recovery_complete(InitialSnapshot const &snaps) {
auto &ev = me->buffered_events.emplace_back();
for (auto snap_ptr : snaps) {
auto &snap_cpy = ev.affected_securities.emplace_back();
// We want to normalize the snapshots we show as PCapEvents so
// that they always carry order changes that when applied to the
// previous snapshot (if any) produce an up to date order
// book. The PCapStreamer does not guarantee this to
// on_recovery_complete, that is sometimes initial snapshots
// will simply contain an initial order state and no
// updates. The code below this normalizes the snapshots we are
// copying so that they always can be relied on to carry the
// delta between snapshot state changes.
snap_cpy.light_copy(*snap_ptr);
snap_cpy.clear_order_changes();
bts_assert(snap_cpy.fills().empty());
bts_assert(snap_cpy.orders().empty());
for (auto const &o : snap_ptr->orders())
snap_cpy.insert_order(o.second);
bts_assert(snap_cpy.orders().size() == snap_ptr->orders().size());
bts_assert(snap_cpy.orders().size() == snap_cpy.order_changes().size());
}
}
void PCapEventGenerator::on_real_time_event(MDP3Event const &ev_in) {
auto &ev = me->buffered_events.emplace_back();
ev.transact_time = ev_in.transact_time;
ev.event_id = ev_in.event_id;
ev.first_packet_send_time = ev_in.first_packet_send_time;
ev.last_packet_send_time = ev_in.last_packet_send_time;
ev.first_packet_sn = ev.first_packet_sn;
ev.last_packet_sn = ev.last_packet_sn;
for (auto snap_ptr : ev_in.affected_securities) {
auto &snap_cpy = ev.affected_securities.emplace_back();
// light_copy() does not copy the orders, just the order changes
// Observers of this class should see a continuous stream of
// order changes which they can use to recreate the entire order
// book..
snap_cpy.light_copy(*snap_ptr);
}
}
void PCapEventGenerator::on_real_time_channel_loss() { bts_assert(false); }
} // namespace bts::spartan::mdp3
@@ -0,0 +1,31 @@
#pragma once
#include <functional>
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include <memory>
namespace bts::spartan::mdp3 {
class PCapEventGenerator : IChannelObserver {
public:
PCapEventGenerator(std::string const&pcap);
PCapEventGenerator(PCapEventGenerator &&);
PCapEventGenerator &operator=(PCapEventGenerator &&src);
~PCapEventGenerator();
PCapEvent const *top();
void move_next() ;
private:
void on_recovery_complete(InitialSnapshot const &) override;
void on_real_time_event(MDP3Event const &) override;
void on_real_time_channel_loss() override;
struct PImpl;
std::unique_ptr<PImpl> me;
};
}
@@ -0,0 +1,41 @@
#include "PCapEventSource.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonFinder.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h"
namespace bts::spartan::mdp3 {
struct PCapEventSource::PImpl {
std::string pcap_file;
DefinitionCollector dc = {};
PolygonFinder pf = {};
IcebergFinder ice_finder = {};
};
PCapEventSource::PCapEventSource(std::string const &pcap)
: me(new PImpl{pcap}) {
observe_channel(pcap, {&me->dc, &me->pf, &me->ice_finder});
}
PCapEventSource::~PCapEventSource() {}
InstrumentMap const& PCapEventSource::instruments() const {
return me->dc.ptr_result;
}
std::unordered_map<size_t,Polygon> const& PCapEventSource::polygons() const {
return me->pf.result;
}
IcebergMap const&PCapEventSource::icebergs() const {
return me->ice_finder.result;
}
PCapEventGenerator PCapEventSource::get_event_generator() const {
return PCapEventGenerator {me->pcap_file};
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,36 @@
#pragma once
#include <vector>
#include <string>
#include <optional>
#include <memory>
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
namespace bts::spartan::mdp3 {
class InstrumentDefinition;
class MDP3Event;
}
namespace bts::spartan::mdp3 {
class PCapEventSource {
public:
PCapEventSource(std::string const &pcap);
~PCapEventSource();
InstrumentMap const &instruments() const ;
std::unordered_map<size_t,Polygon> const &polygons() const;
IcebergMap const&icebergs() const;
PCapEventGenerator get_event_generator() const;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
}
@@ -0,0 +1,89 @@
#pragma once
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
namespace bts::spartan::mdp3 {
class IAlgoModel;
using AlgoEventID = std::pair<size_t,size_t> ;
struct EventHeader {
AlgoEventID event_id = {0,0};
MDP3TimeStamp transact_time = MDP3TimeStamp {0};
MDP3TimeStamp observable_time = MDP3TimeStamp {0};
};
struct AlgoAction {
MDP3TimeStamp transact_time = MDP3TimeStamp{0};
SmallOrderChange change;
};
struct MBPUpdate {
MDP3SecurityID security_id;
Book mbp_buy;
Book mbp_sell;
ImpliedBook implied_mbp_buy;
ImpliedBook implied_mbp_sell;
Book const &mbp_side(api::Side side) const {
if (side == api::SIDE_BUY)
return mbp_buy;
else
return mbp_sell;
}
const ImpliedBook& implied_mbp_side(api::Side side) const {
if (side == api::SIDE_BUY)
return implied_mbp_buy;
else
return implied_mbp_sell;
}
};
struct OrderUpdate {
MDP3OrderKey key;
enum Status { FILLED, PARTIALLY_FILLED, CANCELED };
Status status;
MDP3SecurityID security_id;
api::Side side;
MDP3Quantity last_fills;
md::IntPrice last_price;
enum FillType { PASSIVE, AGGRESSIVE };
// non null when Status == FILLED or PARTIALLY_FILLED
std::optional<FillType> fill_type;// = std::nullopt;
const char*status_str() const {
switch (status) {
case FILLED: return "filled";
case PARTIALLY_FILLED: return "partial";
case CANCELED: return "canceled";
default:
bts_assert(false);
return "";
}
}
};
using MatchEffect =
std::variant<MBPUpdate,OrderUpdate>;
struct AlgoNotification {
EventHeader header;
std::variant<MBPUpdate,OrderUpdate> event;
};
inline std::ostream& operator <<(std::ostream& ostr, const OrderUpdate::Status& status) {
switch (status) {
case OrderUpdate::Status::FILLED: ostr << "FILLED"; break;
case OrderUpdate::Status::PARTIALLY_FILLED: ostr << "PARTIALLY_FILLED"; break;
case OrderUpdate::Status::CANCELED: return ostr << "CANCELED";
default: ostr << "UNKNOWN_" << static_cast<int>(status); break;
}
return ostr;
}
inline std::ostream& operator <<(std::ostream& ostr, const OrderUpdate& upd) {
ostr << "key: " << upd.key.value() << ", status: " << upd.status
<< ", last fills: " << upd.last_fills.value() << ", last price: " << upd.last_price.price();
return ostr;
}
}
@@ -0,0 +1,312 @@
// #include "lib/orderman/orders/algo/IAlgoOrder.h"
// #include "lib/orderman/orders/algo/IAlgoServices.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/ISimServices.h"
// #include "AlgoServicesBridge.h"
// #include "lib/orderman/actors/order_manager/OrderBuilder.h"
// #include "lib/orderman/orders/algo/ChildTemplate.h"
// #include "lib/orderman/orderman_utils/Book.h"
// #include "BridgeCommon.h"
// #include "BeforeLaunchBridge.h"
// using namespace bts::spartan::mdp3;
// using namespace bts::api;
// using namespace bts::api::orderman;
// namespace bts {
// namespace orderman {
// struct OrderState{
// // spartan outgoing order
// MDP3Order order;
// // spark callback object
// ChildState child_state;
// };
// struct AlgoServicesBridge::PImpl {
// std::vector<PositionUpdate> pos;
// BeforeLaunchBridge& bl_bridge;
// IAlgoOrder& algo_order;
// ISimServices& sim_services;
// std::unordered_map<MDP3SecurityID,Book> book_map;
// std::unordered_map<MDP3OrderKey,OrderState> order_map;
// OrderDefinition order_def;
// uint32_t running_ok = 0;
// PImpl(BeforeLaunchBridge& bl_bridge,
// IAlgoOrder& algo_order,
// ISimServices& sim_services)
// :
// bl_bridge(bl_bridge),
// algo_order(algo_order),
// sim_services(sim_services),
// order_def(bl_bridge.definition())
// {
// }
// };
// void absorb_order_callback(const OrderUpdate& new_update,
// ChildState& curr_state){
// uint32_t tot_fills = new_update.last_fills.value() +
// curr_state.total_fills.value();
// double avg_price = 0;
// if (tot_fills>0){
// avg_price =
// curr_state.avg_price.value()*curr_state.total_fills.value() +
// new_update.last_price.price()*new_update.last_fills.value();
// avg_price = avg_price / tot_fills;
// }
// curr_state.total_fills = Quantity(tot_fills);
// curr_state.avg_price = Price(avg_price);
// curr_state.last_fills = Quantity(new_update.last_fills.value());
// curr_state.last_price = Price(new_update.last_price.price());
// switch (new_update.status)
// {
// case OrderUpdate::Status::FILLED:
// curr_state.status = ChildStatus::COMPLETED;
// break;
// case OrderUpdate::Status::PARTIALLY_FILLED:
// curr_state.status = ChildStatus::WORKING;
// break;
// case OrderUpdate::Status::CANCELED:
// curr_state.status = ChildStatus::CANCELED;
// break;
// }
// }
// AlgoServicesBridge::AlgoServicesBridge(BeforeLaunchBridge& bl_bridge,
// IAlgoOrder& algo_order,
// ISimServices& sim_services)
// :
// me(new PImpl(bl_bridge,algo_order, sim_services)) {
// }
// Price AlgoServicesBridge::price() const{
// return Price(0);
// }
// Quantity AlgoServicesBridge::quantity() const {
// return Quantity(0);
// }
// Quantity AlgoServicesBridge::total_fills() const{
// return Quantity(0);
// }
// bool AlgoServicesBridge::is_shutting_down() const {
// return false;
// }
// const std::vector<PositionUpdate>&
// AlgoServicesBridge::current_positions() const {
// bts_assert_msg(false, "Not implemented");
// return me->pos;
// }
// trace::CorID AlgoServicesBridge::get_correlation_id() const {
// trace::CorID cid;
// return cid;
// }
// void AlgoServicesBridge::subscribe_book(SecurityKey k) {
// auto st_sec_key = map_sec_key(k);
// if(me->book_map.find(st_sec_key) == me->book_map.end())
// me->book_map.insert({st_sec_key,Book()});
// }
// void AlgoServicesBridge::unsubscribe_book(SecurityKey k) {
// auto st_sec_key = map_sec_key(k);
// if(me->book_map.find(st_sec_key) != me->book_map.end())
// me->book_map.erase(st_sec_key);
// }
// void AlgoServicesBridge::subscribe_settlement(SecurityKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::unsubscribe_settlement(SecurityKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::subscribe_child_change(OrderKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::unsubscribe_child_change(OrderKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::subscribe_timer() {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::unsubscribe_timer() {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::subscribe_ticker(SecurityKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::unsubscribe_ticker(SecurityKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::subscribe_status(SecurityKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::unsubscribe_status(SecurityKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::subscribe_position() {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::unsubscribe_position() {
// bts_assert_msg(false, "Not implemented");
// }
// OrderKey AlgoServicesBridge::launch_child(ChildTemplate &,
// Price p, Quantity q,
// bool subscribe) {
// return launch_child(me->order_def,p,q,subscribe);
// }
// OrderKey AlgoServicesBridge::launch_child(
// const api::orderman::OrderDefinition& ord_def,
// Price p, Quantity q,
// bool) {
// auto ord_key = me->sim_services.generate_order_key();
// auto sec_key = map_sec_key(SecurityKey(ord_def.security_key()));
// auto ret = me->order_map.insert(
// {
// ord_key,
// OrderState{
// MDP3Order {
// ord_key,
// sec_key,
// ord_def.side(),
// MDP3OrderPriority{},
// md::IntPrice_from_double<7>(p.value()),
// MDP3Quantity(q.value()),
// std::nullopt
// },
// ChildState{
// p,q,Quantity(0),Price(0),Quantity(0),Price(0),
// ChildStatus::WORKING,Quantity(0)
// }
// }});
// bts_assert(ret.second);
// OrderState& ord_state = ret.first->second;
// SmallOrderChange ord_change(ord_state.order,
// OrderChange::ChangeType::added);
// me->sim_services.send(ord_change);
// return map_ord_key(ord_key);
// }
// // Child order maintenance
// void AlgoServicesBridge::update_child(
// OrderKey k,
// const api::orderman::Update& u){
// auto sp_ord_key = map_ord_key(k);
// auto it = me->order_map.find(sp_ord_key);
// if(it != me->order_map.end()){
// OrderState& ord_state = it->second;
// ord_state.order.quantity = MDP3Quantity(u.quantity());
// ord_state.order.price = md::IntPrice_from_double<7>(u.price());
// ord_state.child_state.price = Price(u.price());
// ord_state.child_state.quantity = Quantity(u.quantity());
// SmallOrderChange ord_change(ord_state.order,
// OrderChange::ChangeType::updated);
// me->sim_services.send(ord_change);
// }
// else
// LOG(OrderMan, Warn, "AlgoServicesBridge::update_child OrderKey:{} "
// "does not exist", k);
// }
// void AlgoServicesBridge::cancel_child(OrderKey k) {
// auto sp_ord_key = map_ord_key(k);
// auto it = me->order_map.find(sp_ord_key);
// if(it != me->order_map.end()){
// OrderState& ord_state = it->second;
// SmallOrderChange ord_change(ord_state.order,
// OrderChange::ChangeType::removed);
// me->sim_services.send(ord_change);
// }
// else
// LOG(OrderMan, Warn, "AlgoServicesBridge::cancel_child OrderKey:{} "
// "does not exist", k);
// }
// void AlgoServicesBridge::detach_child(OrderKey) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::detach_all_children() {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::cancel(const std::string&) {
// for (const auto& p : me->order_map)
// me->sim_services.send(
// SmallOrderChange(p.second.order, OrderChange::ChangeType::removed));
// }
// void AlgoServicesBridge::terminate_error(const std::string& msg) {
// cancel(msg);
// }
// void AlgoServicesBridge::complete() {
// }
// void AlgoServicesBridge::set_ladder_visibility(bool) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::update_custom_state(
// const api::orderman::DynamicOrderState&) {
// bts_assert_msg(false, "Not implemented");
// }
// void AlgoServicesBridge::update_fill_state(Quantity, Price) {
// }
// // IAlgoModel
// void AlgoServicesBridge::init(bts::spartan::mdp3::ISimServices&,
// bts::spartan::protos::SimRunOptions const&){
// }
// void AlgoServicesBridge::on_notification(
// const bts::spartan::mdp3::AlgoNotification& ev){
// if(std::holds_alternative<MBPUpdate>(ev.event)){
// // book update
// const MBPUpdate& new_bk = std::get<MBPUpdate>(ev.event);
// auto it = me->book_map.find(new_bk.security_id);
// if(it != me->book_map.end()){
// Book& bk = it->second;
// translate_book(new_bk,bk);
// me->algo_order.on_book(map_sec_key(new_bk.security_id), bk);
// }
// }
// else{
// const OrderUpdate& u = std::get<OrderUpdate>(ev.event);
// auto it = me->order_map.find(u.key);
// if(it != me->order_map.end()){
// OrderState& ord_state = it->second;
// absorb_order_callback(u,ord_state.child_state);
// me->algo_order.on_child_change(
// map_ord_key(ord_state.order.key),ord_state.child_state);
// }
// else
// LOG(OrderMan, Error, "AlgoServicesBridge::on_notification "
// "OrderKey:{} does not exist", u.key);
// }
// }
// } // namespace orderman
// } // namespace bts
@@ -0,0 +1,103 @@
// #pragma once
// #include "lib/orderman/orders/algo/IAlgoOrder.h"
// #include "lib/orderman/orders/algo/IAlgoServices.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/IAlgoModel.h"
// namespace bts {namespace api {namespace orderman{class CustomOrderDefinition;}}}
// namespace bts {namespace spartan {namespace mdp3{class ISimServices;}}}
// namespace bts::spartan::protos { class SimRunOptions; }
// namespace bts {
// namespace orderman {
// class ChildTemplate;
// class IAlgoOrder;
// class BeforeLaunchBridge;
// class AlgoServicesBridge : public IAlgoServices, public bts::spartan::mdp3::IAlgoModel{
// public:
// AlgoServicesBridge(BeforeLaunchBridge& bl_bridge,
// IAlgoOrder& algo_order,
// bts::spartan::mdp3::ISimServices&);
// ~AlgoServicesBridge(){}
// Price price() const override;
// Quantity quantity() const override;
// Quantity total_fills() const override;
// bool is_shutting_down() const override;
// std::vector<PositionUpdate> const& current_positions() const override;
// trace::CorID get_correlation_id() const override;
// void subscribe_book(SecurityKey) override;
// void unsubscribe_book(SecurityKey) override;
// void subscribe_settlement(SecurityKey) override;
// void unsubscribe_settlement(SecurityKey) override;
// void subscribe_child_change(OrderKey) override;
// void unsubscribe_child_change(OrderKey) override;
// void subscribe_timer() override;
// void unsubscribe_timer() override;
// void subscribe_ticker(SecurityKey) override;
// void unsubscribe_ticker(SecurityKey) override;
// void subscribe_status(SecurityKey) override;
// void unsubscribe_status(SecurityKey) override;
// void subscribe_position() override;
// void unsubscribe_position() override;
// OrderKey launch_child(ChildTemplate &, Price, Quantity,
// bool subscribe = true) override;
// OrderKey launch_child(api::orderman::OrderDefinition const &, Price,
// Quantity, bool subscribe = true) override;
// // Child order maintenance
// void update_child(OrderKey child, api::orderman::Update const &) override;
// void cancel_child(OrderKey child) override;
// void detach_child(OrderKey child) override;
// void detach_all_children() override;
// void cancel(std::string const &msg = "") override;
// void terminate_error(std::string const &msg) override;
// void complete() override;
// void set_ladder_visibility(bool is_visible) override;
// void update_custom_state(api::orderman::DynamicOrderState const&) override;
// void update_fill_state(Quantity total_fills, Price last_price) override;
// // IAlgoModel
// void init(bts::spartan::mdp3::ISimServices&,
// bts::spartan::protos::SimRunOptions const&) override;
// void on_notification(bts::spartan::mdp3::AlgoNotification const&) override;
// private:
// struct PImpl;
// std::unique_ptr<PImpl> me;
// };
// } // namespace orderman
// } // namespace bts
@@ -0,0 +1,120 @@
// #include "BeforeLaunchBridge.h"
// #include "../ISimServices.h"
// #include "BridgeCommon.h"
// #include "lib/orderman/actors/order_manager/OrderBuilder.h"
// #include "lib/orderman/orders/algo/ChildTemplate.h"
// namespace bts {
// namespace orderman {
// using namespace bts::spartan::mdp3;
// using namespace bts::api::orderman;
// typedef std::map<InstrumentDefinition const*, api::Security,
// std::function<bool(InstrumentDefinition const*,
// InstrumentDefinition const*)> > InsMap;
// struct BeforeLaunchBridge::PImpl {
// OrderKey ok;
// MDP3SecurityID sec_id;
// ISimServices& sim_services;
// const api::orderman::OrderDefinition& order_def;
// ChildTemplate* child_temp = nullptr;
// api::Security security;
// TimeStamp ts;
// std::map<std::unique_ptr<ChildTemplate>,OrderDefinition> _child_temp_map;
// InsMap sec_map = InsMap([](const auto& a, const auto& b) {
// return a->security_id().value() < b->security_id().value();
// });
// PImpl(
// OrderKey ok,
// MDP3SecurityID sec_id,
// ISimServices& sim_services,
// const api::orderman::OrderDefinition& order_def)
// :
// ok(ok),
// sec_id(sec_id),
// sim_services(sim_services),
// order_def(order_def) {
// }
// };
// // TODO:
// // order def translation (ex sniper)
// // ChildTemplate
// // timestamp
// BeforeLaunchBridge::BeforeLaunchBridge(
// ISimServices& sim_services,
// const api::orderman::OrderDefinition& order_def)
// :
// me(new PImpl(OrderKey(0),MDP3SecurityID(0),sim_services,order_def)){
// for (const auto& p : me->sim_services.instruments()){
// auto mdp3_sec_ptr = p.second;
// api::Security sec = convert_to_spark_security(
// mdp3_sec_ptr,
// me->sim_services.instruments());
// me->sec_map.insert({mdp3_sec_ptr,sec});
// }
// bool found = false;
// for (const auto& p : me->sec_map)
// if(p.second.id() == order_def.security_key()){
// me->security = p.second;
// found = true;
// }
// bts_assert(found);
// }
// const api::orderman::OrderDefinition& BeforeLaunchBridge::definition() const{
// return me->order_def;
// }
// const api::Security& BeforeLaunchBridge::security() const{
// return me->security;
// }
// OrderKey BeforeLaunchBridge::order_key() const{
// return me->ok;
// }
// OrderKey BeforeLaunchBridge::parent_order_key() const{
// // not supported in spartan sim
// return order_key();
// }
// api::Security const* BeforeLaunchBridge::lookup_security(SecurityKey k) const{
// for (const auto& p : me->sec_map)
// if(p.second.id() == k.value())
// return &p.second;
// return nullptr;
// }
// std::vector<api::Security const*> BeforeLaunchBridge::known_securities() const{
// std::vector<api::Security const*> rv;
// for (const auto& p : me->sec_map)
// rv.push_back(&p.second);
// return rv;
// }
// bool BeforeLaunchBridge::is_attaching() const{
// return false;
// }
// ChildTemplate& BeforeLaunchBridge::make_child_template(
// const api::orderman::OrderDefinition& d,
// size_t){
// auto ret = me->_child_temp_map.insert(std::make_pair(std::make_unique<ChildTemplate>(),d));
// bts_assert(ret.second);
// const std::unique_ptr<ChildTemplate>& rv = ret.first->first;
// return *rv;
// }
// // Get the current time
// TimeStamp BeforeLaunchBridge::get_time() const{
// // TODO
// return me->ts;
// }
// }}
@@ -0,0 +1,49 @@
// #pragma once
// #include "lib/orderman/orders/algo/IAlgoOrder.h"
// #include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h"
// namespace bts {namespace api {namespace orderman{class CustomOrderDefinition;}}}
// namespace bts {namespace spartan {namespace mdp3{class ISimServices;}}}
// namespace bts {
// namespace orderman {
// class ChildTemplate;
// class BeforeLaunchBridge : public IBeforeLaunchAlgoServices{
// public:
// BeforeLaunchBridge(
// bts::spartan::mdp3::ISimServices&,
// const api::orderman::OrderDefinition&);
// virtual ~BeforeLaunchBridge(){}
// api::orderman::OrderDefinition const &definition() const override;
// api::Security const &security() const override;
// OrderKey order_key() const override;
// OrderKey parent_order_key() const override;
// api::Security const *lookup_security(SecurityKey k) const override;
// std::vector<api::Security const*> known_securities() const override;
// bool is_attaching() const override;
// ChildTemplate &
// make_child_template(api::orderman::OrderDefinition const &child_def,
// size_t reserve = 1) override;
// TimeStamp get_time() const override;
// private:
// struct PImpl;
// //std::unique_ptr<PImpl> me;
// PImpl* me;
// };
// } // namespace orderman
// } // namespace bts
@@ -0,0 +1,99 @@
// #pragma once
// #include "lib/orderman/orders/algo/IAlgoOrder.h"
// //#include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h"
// //#include "../ISimServices.h"
// #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
// #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h"
// #include "lib/orderman/orderman_utils/Book.h"
// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h"
// #include "server/sparkd/cmds/spartan/back_testing/simulator/IAlgoModel.h"
// namespace bts{
// namespace orderman{
// inline OrderKey map_ord_key(bts::spartan::mdp3::MDP3OrderKey k){
// // uint64_t -> uint32_t
// return OrderKey(static_cast<uint32_t>(k.value()));
// }
// inline bts::spartan::mdp3::MDP3OrderKey map_ord_key(OrderKey k){
// // uint32_t -> uint64_t
// return bts::spartan::mdp3::MDP3OrderKey(static_cast<uint64_t>(k.value()));
// }
// inline SecurityKey map_sec_key(bts::spartan::mdp3::MDP3SecurityID k){
// // int -> uint32_t
// return SecurityKey(static_cast<uint32_t>(k.value()));
// }
// inline bts::spartan::mdp3::MDP3SecurityID map_sec_key(SecurityKey k){
// // uint32_t -> int
// return bts::spartan::mdp3::MDP3SecurityID(static_cast<int>(k.value()));
// }
// inline void translate_book(const bts::spartan::mdp3::MBPUpdate& u_in,
// bts::orderman::Book& b_out){
// // TODO: implieds
// size_t i=-1;
// for (auto const& bl : u_in.mbp_buy)
// if(++i<5)
// b_out.side(api::Side::SIDE_BUY)[i] =
// md::PriceSize (bl.price,bl.quantity.value());
// i=-1;
// for (auto const& bl : u_in.mbp_sell)
// if(++i<5)
// b_out.side(api::Side::SIDE_SELL)[i] =
// md::PriceSize (bl.price,bl.quantity.value());
// }
// inline void fill_future(const bts::spartan::mdp3::InstrumentDefinition&,
// bts::api::Security& out_sec){
// // TODO, set future expiration dates
// out_sec.mutable_future();
// }
// inline bts::api::Security convert_to_spark_security (
// bts::spartan::mdp3::InstrumentDefinition const* def_ptr,
// const bts::spartan::mdp3::InstrumentMap& sec_map){
// auto fill_common = [](const auto& def, auto out_sec){
// out_sec.set_id(map_sec_key(def.security_id()).value());
// out_sec.set_symbol(def.security_group());
// out_sec.set_exchange(bts::api::VENUE_GLBX);
// out_sec.set_currency(bts::api::CURRENCY_USD);
// out_sec.mutable_contract_spec()->set_low_tick(def.min_price_increment_ammount());
// out_sec.mutable_contract_spec()->set_high_tick(def.min_price_increment_ammount());
// };
// const auto& def = *def_ptr;
// bts::api::Security rv_sec;
// fill_common(def,rv_sec);
// switch(def.security_type()){
// case bts::api::SECURITYTYPE_FUTURE:
// fill_future(def,rv_sec);
// break;
// case bts::api::SECURITYTYPE_SPREAD:{
// auto& spr = *rv_sec.mutable_spread();
// for(const auto& l : def.legs()){
// // getting inst def for the leg
// auto it = sec_map.find(l.security_id);
// bts_assert(it != sec_map.end());
// const auto& leg_insdef = *it->second;
// // creating spread
// bts::api::ContractLeg& spr_leg = *spr.add_legs();
// auto& leg_sec = *spr_leg.mutable_security();
// fill_common(leg_insdef, leg_sec);
// fill_future(leg_insdef,leg_sec);
// spr_leg.set_quantity(l.ratio.value());
// spr_leg.set_side(l.side);
// spr_leg.set_price_multiplier(spr_leg.quantity());
// }
// break;
// }
// default:
// break;
// }
// return rv_sec;
// }
// }}