From ae7b25e120d8dcc22a017fa4d2494c6dbee8e714 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 29 Dec 2020 20:01:48 -0500 Subject: [PATCH] Refactor. --- .../back_testing/algos/PolyObserver.cpp | 172 ++++ .../spartan/back_testing/algos/PolyObserver.h | 33 + .../back_testing/algos/SniperModel.cpp | 569 +++++++++++++ .../spartan/back_testing/algos/SniperModel.h | 47 ++ .../algos/spreader/SimOrderGateway.cpp | 166 ++++ .../algos/spreader/SimOrderGateway.h | 40 + .../algos/spreader/SimSpreader.cpp | 114 +++ .../back_testing/algos/spreader/SimSpreader.h | 33 + .../spreader/tests/sim_spreader_tests.cpp | 359 ++++++++ .../cmds/spartan/back_testing/db/SimDB.cpp | 299 +++++++ .../cmds/spartan/back_testing/db/SimDB.h | 48 ++ .../cmds/spartan/back_testing/db/SimLogger.h | 16 + .../matcher/FifoImpliedSecurityMatcher.cpp | 207 +++++ .../matcher/FifoImpliedSecurityMatcher.h | 48 ++ .../matcher/FifoMatcherContext.cpp | 242 ++++++ .../back_testing/matcher/FifoMatcherContext.h | 94 +++ .../matcher/FifoMergedSecurityMatcher.cpp | 369 +++++++++ .../matcher/FifoMergedSecurityMatcher.h | 54 ++ .../matcher/FifoPriceLevelMatcher.cpp | 319 ++++++++ .../matcher/FifoPriceLevelMatcher.h | 116 +++ .../matcher/FifoSecurityMatcher.cpp | 552 +++++++++++++ .../matcher/FifoSecurityMatcher.h | 159 ++++ .../matcher/ImpliedFifoSecurityMatcher.h | 46 ++ .../spartan/back_testing/matcher/Matcher.cpp | 266 ++++++ .../spartan/back_testing/matcher/Matcher.h | 102 +++ .../back_testing/matcher/MatchingCommon.cpp | 26 + .../back_testing/matcher/MatchingCommon.h | 131 +++ .../back_testing/matcher/SecurityMatcher.h | 23 + .../back_testing/matcher/fill_depot.cpp | 85 ++ .../spartan/back_testing/matcher/fill_depot.h | 132 +++ .../back_testing/matcher/fill_factory.h | 32 + .../back_testing/matcher/iceberg_helper.cpp | 94 +++ .../back_testing/matcher/iceberg_helper.h | 60 ++ .../matcher/matcher_diagnostics.h | 258 ++++++ .../matcher/tests/matcher_tests.cpp | 747 +++++++++++++++++ .../matcher/validation/event_flags.h | 59 ++ .../matcher/validation/iceberg_file_cache.cpp | 81 ++ .../matcher/validation/iceberg_file_cache.h | 14 + .../matcher_validation_observer.cpp | 476 +++++++++++ .../validation/matcher_validation_observer.h | 59 ++ .../back_testing/run/SimSessionOptions.proto | 87 ++ .../cmds/spartan/back_testing/run/run_sim.cpp | 267 ++++++ .../cmds/spartan/back_testing/run/run_sim.h | 14 + .../spartan/back_testing/sim/IAlgoModel.h | 56 ++ .../cmds/spartan/back_testing/sim/ISim.cpp | 7 + .../cmds/spartan/back_testing/sim/ISim.h | 37 + .../cmds/spartan/back_testing/sim/Sim.cpp | 763 ++++++++++++++++++ .../cmds/spartan/back_testing/sim/Sim.h | 103 +++ .../spartan/back_testing/sim/pandl/PandL.cpp | 109 +++ .../spartan/back_testing/sim/pandl/PandL.h | 42 + .../back_testing/sim/pandl/PandLSummary.h | 17 + .../spartan/back_testing/sim/pcap/PCapEvent.h | 20 + .../sim/pcap/PCapEventGenerator.cpp | 106 +++ .../sim/pcap/PCapEventGenerator.h | 31 + .../back_testing/sim/pcap/PCapEventSource.cpp | 41 + .../back_testing/sim/pcap/PCapEventSource.h | 36 + .../spartan/back_testing/sim/sim_common.h | 89 ++ .../sim/spark_bridge/AlgoServicesBridge.cpp | 312 +++++++ .../sim/spark_bridge/AlgoServicesBridge.h | 103 +++ .../sim/spark_bridge/BeforeLaunchBridge.cpp | 120 +++ .../sim/spark_bridge/BeforeLaunchBridge.h | 49 ++ .../sim/spark_bridge/BridgeCommon.h | 99 +++ 62 files changed, 9255 insertions(+) create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/algos/spreader/tests/sim_spreader_tests.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/db/SimDB.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/db/SimDB.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/db/SimLogger.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/validation/event_flags.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.proto create mode 100644 src/server/sparkd/cmds/spartan/back_testing/run/run_sim.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/run/run_sim.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/ISim.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/ISim.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/Sim.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandLSummary.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.h create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BridgeCommon.h diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp new file mode 100644 index 000000000..bbf0d672b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp @@ -0,0 +1,172 @@ +#include "PolyObserver.h" + +#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" + +namespace bts::spartan::mdp3 { + +static auto constexpr Worthless = -1.0; + +struct PolyObserver::Target { + size_t polygon_id; + Polygon const polygon; + std::string const descriptor; + api::Side side; + std::string const product; + double value = Worthless; + MDP3Quantity quantity = MDP3Quantity{0}; + void mark_as_worthless() { value = Worthless; } + bool has_value() const { return value > securities::market_epsilon(); } +}; + +struct PolyObserver::Opportunity { + Target *target; + double value; + MDP3Quantity quantity; + EventHeader start; +}; + +struct PolyObserver::PImpl { + std::vector> targets; + std::optional current_opportunity; + std::unordered_map books; + EventHeader last_event; + ISim *sim = nullptr; +}; + +PolyObserver::PolyObserver() : me(new PImpl()) {} +PolyObserver::~PolyObserver() {} + +void PolyObserver::init(ISim &s, protos::SimRunOptions const&) { + me->sim = &s; + for (auto &p : s.polygons()) { + auto descriptor = polygon_descriptor(p.second, s.instruments()); + + auto buy_t = Target{p.first, p.second, descriptor, api::SIDE_BUY, + polygon_product(p.second, s.instruments())}; + auto sell_t = Target{p.first, flip(p.second), descriptor, api::SIDE_SELL, + polygon_product(p.second, s.instruments())}; + + me->targets.emplace_back(std::make_unique(buy_t)); + me->targets.emplace_back(std::make_unique(sell_t)); + } +} + +void PolyObserver::finit() {} + +void PolyObserver::on_replay_start( + EventHeader const ¤t_event, + std::vector const ¤t_books) { + me->sim->log("replay_start", current_event.event_id.first); + me->books.clear(); + for (auto const &b : current_books) + me->books[b.security_id] = b; +} + +void PolyObserver::eval() { + for (auto &t : me->targets) { + t->value = 0; + int qty = std::numeric_limits::max(); + for (auto &l : t->polygon.parts) { + auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); + auto &mkt = me->books[l.sec_id].mbp_side(mkt_side); + auto this_qty = mkt[0].quantity.value(); + auto this_ratio = this_qty / std::abs(l.ratio); + if (this_ratio < qty) + qty = this_ratio; + if (qty == 0) { + t->mark_as_worthless(); + break; + } else { + auto cash = me->sim->get_sec(l.sec_id).to_cash(mkt[0].price); + t->quantity.value() = qty; + t->value += -l.ratio * cash; + } + } + } +} + +void PolyObserver::rank() { + std::sort( + me->targets.begin(), me->targets.end(), + [](std::unique_ptr const &ptr1, + std::unique_ptr const &ptr2) { + auto val1 = ptr1->value; // * ptr1->quantity.value(); + auto val2 = ptr2->value; // * ptr2->quantity.value(); + if (std::abs(val1 - val2) > securities::market_epsilon()) + return val1 > val2; + else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size()) + return ptr1->polygon.parts.size() < ptr2->polygon.parts.size(); + else + return ptr1->polygon_id < ptr2->polygon_id; + }); +} + +void PolyObserver::try_start_new_opportunity() { + bts_assert(!me->current_opportunity); + auto best = me->targets.begin()->get(); + if (best->has_value()) { + // We have an opportunity lets record it + me->current_opportunity = + Opportunity{best, best->value, best->quantity, me->last_event}; + me->sim->log( + "opportunity_begin", best->polygon_id, best->descriptor, + api::Side_Name(best->side), best->value, best->quantity.value(), 0, 0, + us_diff(me->last_event.observable_time, me->last_event.transact_time), + me->last_event.event_id.first, me->last_event.event_id.first); + } +} + +void PolyObserver::analyze() { + if (me->current_opportunity) { + auto &opportunity = *me->current_opportunity; + auto &target = *opportunity.target; + if (target.has_value()) { + // Add value and still has value, opportunity is still ongoing + me->sim->log("opportunity_continue", target.polygon_id, + target.descriptor, api::Side_Name(target.side), + target.value, target.quantity.value(), + us_diff(me->last_event.observable_time, + opportunity.start.observable_time), + us_diff(me->last_event.transact_time, + opportunity.start.transact_time), + us_diff(opportunity.start.observable_time, + opportunity.start.transact_time), + opportunity.start.event_id.first); + } else { + // Had value but not anymore, opportunity is concluded + me->sim->log("opportunity_end", target.polygon_id, target.descriptor, + api::Side_Name(target.side), opportunity.value, + opportunity.quantity.value(), + us_diff(me->last_event.observable_time, + opportunity.start.observable_time), + us_diff(me->last_event.transact_time, + opportunity.start.transact_time), + us_diff(opportunity.start.observable_time, + opportunity.start.transact_time), + opportunity.start.event_id.first, + me->last_event.event_id.first - 1); + me->current_opportunity.reset(); + try_start_new_opportunity(); + } + } else { + // We did not have an opportunity before. Maybe we do now? + try_start_new_opportunity(); + } +} + +void PolyObserver::on_order_update(EventHeader const &, OrderUpdate const &) {bts_assert(false);} + +void PolyObserver::on_mbp(EventHeader const &h, MBPUpdate const &mbp) { + me->last_event = h; + me->books[mbp.security_id] = mbp; + eval(); + rank(); + analyze(); +} + +void PolyObserver::on_time() {} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.h b/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.h new file mode 100644 index 000000000..69442cca4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.h @@ -0,0 +1,33 @@ +#pragma once + +#include "../sim/IAlgoModel.h" +#include +#include + +namespace bts::spartan::mdp3 { +struct Polygon; + +class PolyObserver : public IAlgoModel { + public: + PolyObserver(); + ~PolyObserver(); + + void init(ISim &, protos::SimRunOptions const&) override; + void finit() override; + void on_replay_start(EventHeader const &, std::vector const &) override; + void on_order_update(EventHeader const &, OrderUpdate const &) override; + void on_mbp(EventHeader const &, MBPUpdate const &) override; + void on_time() override; + + private: + struct Target; + struct Opportunity; + struct PImpl; + void eval(); + void rank(); + void analyze(); + void try_start_new_opportunity(); + std::unique_ptr me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp new file mode 100644 index 000000000..4f023f3bc --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp @@ -0,0 +1,569 @@ +#include "SniperModel.h" + +#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include + +#include +#include + +namespace bts::spartan::mdp3 { + +enum class SniperModel::State { W_OPPORTUNITY, SHOOTING, PAUSED }; + +struct SniperModel::Target { + constexpr static double Worthless = -1.0; + size_t polygon_id; + Polygon const polygon; + boost::container::small_vector leg_sec_info; + std::string const descriptor; + api::Side side; + double value = Worthless; + MDP3Quantity quantity = MDP3Quantity{0}; + void mark_as_worthless() { value = Worthless; } + bool has_value(double v) const { return value > v; } +}; + +struct SniperModel::WorkingOrder { + SmallOrderChange order; + std::vector fills; +}; + +struct SniperModel::Shot { + Target ⌖ + EventHeader observed; + EventHeader concluded; + double value_at_observation_ts; + MDP3Quantity quantity; + bool forced_completion; + std::unordered_map working; + size_t transaction_duration; +}; + +struct SniperModel::DependentTargets { + InstrumentDefinition definition; + MBPUpdate book; + std::vector dependencies; +}; + +struct SniperModel::PImpl { + ISim *sim = nullptr; + protos::SimRunOptions options; + int min_orders_at_top; + int max_qty; + int min_depth; + double min_value; + size_t max_transaction_latency_us; + bool double_dip; + size_t ms_to_work_shot; + size_t min_cumm_qty; + double min_cumm_edge; + size_t pay_up_ticks; + size_t throttle_ns; + double max_shot_loss; + std::set polys; + + EventHeader current_event; + size_t streak_id = 0; + + State state = State::W_OPPORTUNITY; + std::unordered_map books = {}; + std::vector> targets = {}; + std::optional current_shot{}; +}; + +SniperModel::SniperModel() : me(new PImpl()) {} +SniperModel::~SniperModel() {} + +void SniperModel::init(ISim &s, protos::SimRunOptions const &options) { + me->sim = &s; + me->options = options; +} + +void SniperModel::on_replay_start(EventHeader const &h, + std::vector const &books) { + auto parse_poly_filter = [] (std::string const&pf) { + std::vector polys; + boost::split(polys,pf,boost::is_any_of(",")); + std::set result; + for (std::string &s : polys) { + boost::trim(s); + if (s.size()) + result.insert(s); + } + return result; + }; + + auto sim = me->sim; + auto options = me->options; + auto streak_id = me->streak_id; + + me.reset( + new PImpl{sim, options, get_param(me->options, "min_orders_at_top"), + get_param(me->options, "max_qty"), + get_param(me->options, "min_depth"), + get_param(me->options, "min_value"), + get_param(me->options, "max_transaction_latency_us"), + get_param(me->options, "double_dip"), + get_param(me->options, "ms_to_work_shot"), + get_param(me->options, "min_cumm_qty"), + get_param(me->options, "min_cumm_edge"), + get_param(me->options, "pay_up_ticks"), + get_param(me->options, "throttle_ns"), + get_param(me->options, "max_shot_loss"), + parse_poly_filter(get_param(me->options,"polys")), + h, streak_id}); + + for (auto &p : me->sim->polygons()) { + auto desc = polygon_descriptor(p.second, me->sim->instruments()); + + if (me->polys.size() && !me->polys.count(desc)) + continue; + + auto buy_t = std::make_unique( + Target{p.first, + p.second, + {}, + desc, + api::SIDE_BUY}); + auto sell_t = std::make_unique( + Target{p.first, + flip(p.second), + {}, + desc, + api::SIDE_SELL}); + me->targets.emplace_back(std::move(buy_t)); + me->targets.emplace_back(std::move(sell_t)); + } + + for (auto &t : me->targets) { + for (auto &pp : t->polygon.parts) { + auto &dt = me->books[pp.sec_id]; + dt.book.security_id = pp.sec_id; + dt.definition = me->sim->get_sec(pp.sec_id); + t->leg_sec_info.push_back(&dt); + auto &dependencies = dt.dependencies; + dependencies.push_back(t.get()); + } + } + + me->sim->log("replay_start"); + for (auto const &mbp : books) { + auto it = me->books.find(mbp.security_id); + if (it != me->books.end()) { + me->sim->log( + "book_at_replay_start", me->sim->get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value()); + it->second.book = mbp; + } + } +} + +void SniperModel::on_mbp(EventHeader const &h, const MBPUpdate &mbp) { + me->current_event = h; + auto it = me->books.find(mbp.security_id); + if (it == me->books.end()) + return; + auto &book = it->second; + book.book = mbp; + + eval(book.dependencies); + + switch (me->state) { + case State::W_OPPORTUNITY: { + auto best_shot = pick(); + if (best_shot) { + me->streak_id++; + me->sim->start_simulation(); + transition(State::SHOOTING); + fire(best_shot.value()); + } + break; + } + case State::SHOOTING: + break; + case State::PAUSED: + check_if_pause_over(); + break; + } +} + +void SniperModel::on_order_update(EventHeader const &h, + const OrderUpdate &update) { + me->current_event = h; + bts_assert(me->state == State::SHOOTING); + bts_assert(me->current_shot); + auto &in_flight = me->current_shot.value(); + + bts_assert(in_flight.working.count(update.key)); + in_flight.working[update.key].fills.push_back(update); + + auto pending_orders = in_flight.working.size(); + bool clean_fills = true; + for (auto const &p : in_flight.working) { + auto &these_fills = p.second.fills; + for (auto &f : these_fills) { + clean_fills = clean_fills && (f.fill_type == OrderUpdate::AGGRESSIVE); + if (f.status == OrderUpdate::CANCELED || + f.status == OrderUpdate::FILLED) { + pending_orders--; + break; + } + } + } + + if (pending_orders == 0) { + std::vector all_fills; + for (auto const &p : in_flight.working) { + auto &these_fills = p.second.fills; + all_fills.insert(all_fills.end(), these_fills.begin(), + these_fills.end()); + } + PandL pandl(me->sim->instruments()); + auto cash = pandl.net_cash(all_fills); + auto commission = pandl.commission_cost(all_fills); + auto net = cash - commission; + in_flight.concluded = h; + + me->sim->log( + "shot_outcome", in_flight.target.descriptor, + api::Side_Name(in_flight.target.side), + in_flight.observed.event_id.first, in_flight.observed.event_id.second, + in_flight.value_at_observation_ts * in_flight.quantity.value(), net, + commission, in_flight.quantity.value(), + clean_fills && !me->current_shot->forced_completion, + me->current_shot->forced_completion, + in_flight.transaction_duration); + me->sim->log_pandl(); + if (!me->double_dip) + transition(State::PAUSED); + else { + auto best_shot = pick(); + if (best_shot) { + me->sim->log("double_dip", me->streak_id); + fire(best_shot.value()); + } else + transition(State::PAUSED); + } + } +} + +void SniperModel::on_time() { + + switch (me->state) { + case State::W_OPPORTUNITY: { + break; + } + case State::SHOOTING: { + maybe_force_fills(); + break; + } + case State::PAUSED: + check_if_pause_over(); + break; + } +} + +void SniperModel::maybe_force_fills() { + bts_assert(me->state == State::SHOOTING); + bts_assert(me->current_shot); + auto ts = me->sim->time(); + auto shot_observed_ts = me->current_shot->observed.observable_time; + auto shot_duration = ts.value() - shot_observed_ts.value(); + + if (me->current_shot->forced_completion) + return; + + if (me->ms_to_work_shot) { + if (shot_duration > 1000 * 1000 * me->ms_to_work_shot) { + me->sim->log("forcing_fills_timeout", shot_observed_ts.value()); + me->current_shot->forced_completion = true; + } else if (me->current_shot->target.value < + -me->max_shot_loss) { + me->sim->log("forcing_fills_maxloss", shot_observed_ts.value()); + me->current_shot->forced_completion = true; + } + } + + if (!me->current_shot->forced_completion) + return; + + auto &in_flight = me->current_shot.value(); + for (auto &p : in_flight.working) { + auto &order = p.second.order; + auto &these_fills = p.second.fills; + auto total_fills = 0; + for (auto &f : these_fills) { + bts_assert(f.status != OrderUpdate::CANCELED); + total_fills += f.last_fills.value(); + } + if (order.order.quantity.value() > total_fills) { + md::IntPrice worst_price = order.order.price; + if (order.order.side == api::SIDE_BUY) + worst_price += me->sim->get_sec(order.order.security) + .min_price_increment_int_price() * + 100; + else + worst_price -= me->sim->get_sec(order.order.security) + .min_price_increment_int_price() * + 100; + order.change = OrderChange::updated_price; + order.order.price = worst_price; + me->sim->send(order); + } + } +} + +void SniperModel::eval(std::vector const &targets) { + for (auto &t : targets) { + t->value = 0; + int qty = me->max_qty; + size_t p_idx = 0; + for (auto &l : t->polygon.parts) { + auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); + auto &leg_sec_info = t->leg_sec_info[p_idx++]; + auto &mkt = leg_sec_info->book.mbp_side(mkt_side); + auto this_qty = mkt[0].quantity.value(); + auto this_ratio = this_qty / std::abs(l.ratio); + if (this_ratio < qty) + qty = this_ratio; + if (qty == 0 || mkt[0].n_orders < me->min_orders_at_top || + depth(mkt).value() < me->min_depth) { + t->mark_as_worthless(); + break; + } else { + auto cash = leg_sec_info->definition.to_cash(mkt[0].price.price()); + t->quantity.value() = qty; + t->value += -l.ratio * cash; + } + } + } +} + +bool SniperModel::within_safety_margin(Target const &t) const { + Book synth_book; + using LegInfo = std::tuple; + std::vector leg_mkts; + + for (auto &leg : t.polygon.parts) { + auto &leg_mkt = leg_mkts.emplace_back(); + std::get<0>(leg_mkt) = MDP3Quantity{leg.ratio} ; + std::get<1>(leg_mkt) = {}; + auto leg_book_side = securities::other_side(side_from_ratio(leg.ratio)); + std::get<2>(leg_mkt) = me->books[leg.sec_id].book.mbp_side(leg_book_side); + } + + book_utils::compute_synthetic_book(synth_book, leg_mkts, + std::tuple_size::value); + + auto cumm_qty = 0ul; + auto min_price = me->min_cumm_edge; + + // This is not right, exactly, we are assuming all the legs are + // expressed in the same terms as the first. + auto &first_leg_def = t.leg_sec_info[0]->definition; + + auto p_value = t.value ; + // The synth book calculation uses a different convention. if a + // polygon has value the synth book will have a < 0 price. + auto s_value = - (first_leg_def.to_cash(synth_book[0].price)); + // The synth book should match the first level of the polygon value. + bts_assert(std::abs(p_value - s_value) < securities::market_epsilon()); + + for (auto &level : synth_book) { + // Again, we need to flip the sign because the convention of the + // synth book calculation is to return <0 for polygons with + // value. + auto level_cash_price = first_leg_def.to_cash(-level.price); + // We crossed the threshold that allows us to ignore the quantity. + if (level_cash_price >= min_price) + return true; + else if (level_cash_price >= me->min_value) + cumm_qty += level.quantity.value(); + else + break; + } + return cumm_qty >= me->min_cumm_qty; +} + +SniperModel::MaybeShot SniperModel::pick() { + std::sort( + me->targets.begin(), me->targets.end(), + [](std::unique_ptr const &ptr1, + std::unique_ptr const &ptr2) { + auto val1 = ptr1->value * ptr1->quantity.value(); + auto val2 = ptr2->value * ptr2->quantity.value(); + if (std::abs(val1 - val2) > securities::market_epsilon()) + return val1 > val2; + else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size()) + return ptr1->polygon.parts.size() < ptr2->polygon.parts.size(); + else + return ptr1->polygon_id < ptr2->polygon_id; + }); + + auto &best_target = *me->targets[0]; + if (best_target.has_value(me->min_value)) { + + bool has_any_qty = me->max_qty; + auto transaction_duration = + me->current_event.observable_time.value() - + me->current_event.transact_time.value(); + bool transaction_is_fresh = transaction_duration < + 1000 * me->max_transaction_latency_us; + + if (has_any_qty && transaction_is_fresh && + within_safety_margin(best_target)) + return Shot{best_target, + me->current_event, + {}, + best_target.value, + best_target.quantity, + false, + {}, + transaction_duration}; + else + return {}; + } else + return {}; +} + +void SniperModel::fire(Shot const &s) { + auto &shot = me->current_shot.emplace(s); + me->sim->log("shot_begin", s.target.descriptor, + api::Side_Name(s.target.side), s.value_at_observation_ts, + s.quantity.value(), me->streak_id); + + for (auto const &b : me->books) { + auto const &mbp = b.second.book; + me->sim->log( + "book_at_shot_start", me->sim->get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value()); + } + + if (me->ms_to_work_shot == 0) { + shot.forced_completion = true; + for (auto &p : shot.target.polygon.parts) { + auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL; + auto mkt_side = securities::other_side(part_side); + auto &book = me->books[p.sec_id].book.mbp_side(mkt_side); + + md::IntPrice worst_price; + size_t available_qty = 0; + for (int i = book.size() - 1; i >= 0; i--) { + if (book[i].quantity.value()) { + worst_price = book[i].price; + for (int n = 0; n <= i; n++) + available_qty += book[n].quantity.value(); + break; + } + } + if (part_side == api::SIDE_BUY) + worst_price += + me->sim->get_sec(p.sec_id).min_price_increment_int_price() * + 100; + else + worst_price -= + me->sim->get_sec(p.sec_id).min_price_increment_int_price() * + 100; + + auto order_change = SmallOrderChange{}; + order_change.order.security = p.sec_id; + order_change.order.price = worst_price; + order_change.order.quantity = + MDP3Quantity{s.quantity.value() * std::abs(p.ratio)}; + order_change.order.side = part_side; + order_change.change = OrderChange::added; + order_change.order.key = me->sim->send(order_change); + shot.working[order_change.order.key].order = order_change; + order_change.change = OrderChange::removed; + me->sim->send(order_change); + } + } else { + for (auto &p : shot.target.polygon.parts) { + auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL; + auto mkt_side = securities::other_side(part_side); + auto &book = me->books[p.sec_id].book.mbp_side(mkt_side); + + auto limit_price = book[0].price; + auto pay_up_adj = + me->sim->get_sec(p.sec_id).min_price_increment_int_price() * + me->pay_up_ticks; + + if (part_side == api::SIDE_BUY) + limit_price += pay_up_adj; + else + limit_price -= pay_up_adj; + auto order_change = SmallOrderChange{}; + order_change.order.security = p.sec_id; + order_change.order.price = limit_price; + order_change.order.quantity = + MDP3Quantity{s.quantity.value() * std::abs(p.ratio)}; + order_change.order.side = part_side; + order_change.change = OrderChange::added; + order_change.order.key = me->sim->send(order_change); + me->sim->log("sent_order", api::Side_Name(part_side), + me->sim->get_sec(p.sec_id).symbol(), + book[0].price.price(), book[0].price.price(), + book[0].quantity.value()); + shot.working[order_change.order.key].order = order_change; + } + } +} + +void SniperModel::check_if_pause_over() { + bts_assert(me->state == State::PAUSED); + bts_assert(me->current_shot); + bts_assert(me->throttle_ns); + auto ns_since_shot_concluded = + me->sim->time().value() - + me->current_shot->concluded.observable_time.value(); + if (ns_since_shot_concluded > me->throttle_ns) + me->sim->stop_simulation(); + else + me->sim->log("paused"); +} + +const char *SniperModel::state_name(State s) { + switch (s) { + case State::SHOOTING: + return "WAIT_FILLS"; + case State::W_OPPORTUNITY: + return "WAIT_OPPORTUNITY"; + case State::PAUSED: + return "WAIT_OPPORTUNITY_CLEAR"; + } + return nullptr; +} + +void SniperModel::transition(State new_state) { + me->sim->log("transition", state_name(me->state), state_name(new_state)); + me->state = new_state; +} + +protos::SimRunOptions +SniperModel::fill_defaults(protos::SimRunOptions const &in) { + protos::SimRunOptions result = in; + set_param_if_empty(&result, "min_orders_at_top", 1); + set_param_if_empty(&result, "max_qty", 1); + set_param_if_empty(&result, "min_depth", 0); + set_param_if_empty(&result, "min_value", 2.0); + set_param_if_empty(&result, "max_transaction_latency_us", 10000); + set_param_if_empty(&result, "double_dip", false); + set_param_if_empty(&result, "ms_to_work_shot", 0); + set_param_if_empty(&result, "min_cumm_edge", 17.5); + set_param_if_empty(&result, "min_cumm_qty", 10); + set_param_if_empty(&result, "pay_up_ticks", 10); + set_param_if_empty(&result, "throttle_ns", 1 * 1000 * 1000 * 1000); + set_param_if_empty(&result, "max_shot_loss", 200); + set_param_if_empty(&result, "polys", ""); + return result; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.h b/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.h new file mode 100644 index 000000000..fb2baaa91 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include "server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h" + +namespace bts::spartan::mdp3 { +struct Polygon; +} + +namespace bts::spartan::mdp3 { + +class SniperModel : public IAlgoModel { +public: + SniperModel(); + ~SniperModel(); + + void init(ISim &, protos::SimRunOptions const&) override; + protos::SimRunOptions fill_defaults (protos::SimRunOptions const&) override; + void on_replay_start(EventHeader const &, std::vector const &) override; + void on_mbp(EventHeader const &, MBPUpdate const &) override; + void on_order_update(EventHeader const &, OrderUpdate const &) override; + void on_time() override; + +private: + enum class State; + struct PImpl; + struct Target; + struct Shot; + struct WorkingOrder; + struct DependentTargets; + using MaybeShot = std::optional; + + void eval(std::vector const&); + MaybeShot pick(); + void fire(Shot const&); + bool within_safety_margin(Target const&) const; + void check_if_pause_over(); + void maybe_force_fills(); + void transition(State); + const char *state_name(State); + std::unique_ptr me; +}; + +} + diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.cpp new file mode 100644 index 000000000..e3f359957 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.cpp @@ -0,0 +1,166 @@ +// #include "SimOrderGateway.h" +// #include "SimSpreader.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/ISimServices.h" +// #include "lib/api/services/OrderService.pb.h" +// #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h" + + +// using namespace bts::orderman; +// using namespace bts::api::orderman; + +// namespace bts::spartan::mdp3 { + +// struct OrderTuple{ +// const OrderKey key; +// OrderChange ochange; +// uint32_t total_fills = 0; +// double avg_filled_price = 0; +// bool order_send_acked = false; +// OrderTuple (OrderKey key, +// OrderChange ochange) +// : +// key(key), +// ochange(ochange) +// {} +// }; + +// OrderStatus translate_status(const OrderUpdate& ord){ +// switch(ord.status){ +// case(OrderUpdate::Status::CANCELED): +// return OrderStatus::ORDER_STATUS_CANCELED; +// case(OrderUpdate::Status::FILLED): +// return OrderStatus::ORDER_STATUS_COMPLETED; +// case(OrderUpdate::Status::PARTIALLY_FILLED): +// return OrderStatus::ORDER_STATUS_WORKING; +// } +// bts_assert(false); +// return OrderStatus::ORDER_STATUS_WORKING; +// } + +// OrderTuple* find_ord(OrderKey key, +// std::vector& orders){ +// for(auto& o : orders) +// if(o.key == key) +// return &o; +// return nullptr; +// } + +// OrderTuple* find_ord(MDP3OrderKey key, +// std::vector& orders){ +// for(auto& o : orders) +// if(o.ochange.order.key == key) +// return &o; +// return nullptr; +// } + +// struct SimOrderGateway::PImpl { +// ISimServices* sim_services = nullptr; +// SpreaderOrder* spreader = nullptr; +// std::vector orders; +// }; + +// SimOrderGateway::SimOrderGateway() +// : +// me(new PImpl()){ +// } + +// void SimOrderGateway::init(ISimServices& sim_services, +// SpreaderOrder& spreader){ +// me->sim_services = &sim_services; +// me->spreader = &spreader; +// } + +// void SimOrderGateway:: place_order(SecurityKey skey, +// OrderKey okey, +// api::Side side, +// Price p, +// Quantity q){ +// bts_assert(me->sim_services!=nullptr); +// const auto& t = +// me->orders.emplace_back( +// okey, +// OrderChange{ +// MDP3Order(), +// MDP3Order{ +// me->sim_services->generate_order_key(), +// MDP3SecurityID{static_cast(skey.value())}, +// side, +// MDP3OrderPriority{}, +// md::IntPrice_from_double<7>(p.value()), +// MDP3Quantity{static_cast(q.value())}, +// {} +// }, +// OrderChange::added +// }); +// me->sim_services->send(t.ochange); +// } + +// void SimOrderGateway::modify_order(OrderKey okey, +// Price p, +// Quantity q){ +// bts_assert(me->sim_services!=nullptr); +// OrderTuple* t = find_ord(okey,me->orders); +// bts_assert(t != nullptr); +// bts_assert(t->order_send_acked); +// auto& ochange = t->ochange; +// ochange.previous_order = ochange.order; +// ochange.order.price = md::IntPrice_from_double<7>(p.value()); +// ochange.order.quantity = MDP3Quantity{static_cast(q.value())}; +// ochange.change = OrderChange::updated; +// me->sim_services->send(ochange); +// } + +// void SimOrderGateway::cancel_order(OrderKey okey){ +// bts_assert(me->sim_services!=nullptr); +// OrderTuple* t = find_ord(okey,me->orders); +// bts_assert(t != nullptr); +// bts_assert(t->order_send_acked); +// auto& ochange = t->ochange; +// ochange.change = OrderChange::removed; +// me->sim_services->send(ochange); +// } + +// OrderKey SimOrderGateway::generate_key(){ +// auto rv = +// static_cast(bts::Timer::now().time_since_epoch().count()); +// return OrderKey(rv); +// } + +// void SimOrderGateway::poll(){ +// for(auto& t : me->orders){ +// if(!t.order_send_acked){ +// t.order_send_acked=true; +// route_update_to_spreader( +// OrderUpdate{ +// t.ochange.order.key, +// OrderUpdate::Status::PARTIALLY_FILLED, +// t.ochange.order.security, +// t.ochange.order.side, +// MDP3Quantity{}, +// md::IntPrice() +// }); +// } +// } +// } + +// void SimOrderGateway::route_update_to_spreader(const OrderUpdate& u){ +// OrderTuple* t = find_ord(u.key,me->orders); +// bts_assert(t!=nullptr); +// uint32_t total_fills = t->total_fills + +// static_cast(u.last_fills.value()); +// double avg_filled_price = 0; +// if(total_fills > 0){ +// avg_filled_price = (t->total_fills*t->avg_filled_price) + +// (u.last_fills.value() * u.last_price.price()); +// avg_filled_price = avg_filled_price / total_fills; +// } +// t->total_fills = total_fills; +// t->avg_filled_price = avg_filled_price; +// me->spreader->test_only_on_child_order_update( +// t->key, +// translate_status(u), +// Quantity(t->total_fills), +// Price(t->avg_filled_price) +// ); +// } +// } diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.h b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.h new file mode 100644 index 000000000..c75c891fc --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimOrderGateway.h @@ -0,0 +1,40 @@ +// #pragma once + +// #include "lib/orderman/orderman_utils/orderman_common.h" +// #include "lib/domain_model/Security.pb.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h" +// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" + +// #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/IAlternateOrderGateway.h" + +// namespace bts { namespace orderman {class SpreaderOrder;}} + +// namespace bts::spartan::mdp3 { + +// class ISimServices; + +// class SimOrderGateway : public bts::orderman::IAlternateOrderGateway{ +// private: +// struct PImpl; +// PImpl* me; +// public: +// SimOrderGateway(); +// ~SimOrderGateway(){} +// void init(ISimServices&, +// bts::orderman::SpreaderOrder&); +// // IAlternateOrderGateway functions +// // (SpreaderChildOrderState's calls need to end up here) +// void place_order(SecurityKey, +// bts::orderman::OrderKey, +// api::Side, +// Price, +// Quantity) override; +// void modify_order(bts::orderman::OrderKey, Price, Quantity) override; +// void cancel_order(bts::orderman::OrderKey) override; +// bts::orderman::OrderKey generate_key() override; +// // Sends ISimServices's updates to spreader +// void poll(); +// void route_update_to_spreader(const OrderUpdate&); +// }; + +// } diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.cpp new file mode 100644 index 000000000..ed8183b2d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.cpp @@ -0,0 +1,114 @@ +// #include "SimSpreader.h" +// #include "lib/domain_model/Security.pb.h" +// #include "lib/api/services/OrderService.pb.h" +// #include "lib/sparkproto/SparkProtos.pb.h" +// #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h" +// #include "lib/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.h" +// #include "lib/orderman/orderman_utils/Book.h" +// #include "SimOrderGateway.h" + +// using namespace bts::orderman; +// using namespace bts::api::orderman; + +// namespace bts::spartan::mdp3 { + +// void translate_book(const MBPUpdate& u_in, +// bts::orderman::Book& b_out){ +// size_t i=-1; +// for (auto const& bl : u_in.mbp_buy) +// if(++i<5) +// b_out.side(api::Side::SIDE_BUY)[i] = +// md::PriceSize (bl.price,bl.quantity.value()); +// i=-1; +// for (auto const& bl : u_in.mbp_sell) +// if(++i<5) +// b_out.side(api::Side::SIDE_SELL)[i] = +// md::PriceSize (bl.price,bl.quantity.value()); +// } + +// bool compare_keys (SecurityKey spark_k, +// MDP3SecurityID spartan_k){ +// return static_cast(spark_k.value()) == spartan_k.value(); +// } + +// bts::orderman::Book* find_book(MDP3SecurityID id, +// std::vector& books){ +// for (auto& b : books) +// if(compare_keys(b.security_key(),id)) +// return &b; +// return nullptr; +// } + +// struct SimSpreader::PImpl { +// const OrderDefinition order_def; +// const SpreaderPreActivationOrderServices pao_services; +// SpreaderOrderServices so_services; +// ISimServices* sim_services = nullptr; +// SimOrderGateway sim_order_gateway; +// SpreaderOrder spreader; +// Price price; +// Quantity quantity; +// std::vector books; +// PImpl(const OrderDefinition& order_def, +// const bts::api::Security& sec, +// Price price, +// Quantity quantity) +// : +// order_def(order_def), +// pao_services(sec), +// so_services(nullptr), +// sim_order_gateway(), +// spreader(&sim_order_gateway), +// price(price), +// quantity(quantity), +// books(){ +// for (const auto& l : sec.synthetic().legs()) +// books.emplace_back(SecurityKey(l.security().id())); +// } +// }; + +// SimSpreader::SimSpreader(const OrderDefinition& order_def, +// const bts::api::Security& sec, +// Price price, +// Quantity qty) +// : +// me(new PImpl(order_def,sec,price,qty)){ +// } +// const SpreaderOrder& SimSpreader::spreader() const{ +// return me->spreader; +// } + +// // IAlgoModel ///////////////////////////////////////////////////////////////// +// /////////////////////////////////////////////////////////////////////////////// +// void SimSpreader::init(ISimServices& sim_services, protos::SimRunOptions const&){ +// me->sim_services = &sim_services; +// me->sim_order_gateway.init(sim_services,me->spreader); +// // TODO (no test_ blah) +// me->spreader.test_only_on_init_new (me->order_def, +// me->pao_services); +// me->spreader.test_only_on_activate(me->price, +// me->quantity, +// me->so_services); +// } +// void SimSpreader::on_time(MDP3TimeStamp){ +// me->sim_order_gateway.poll(); +// } +// void SimSpreader::on_notification(AlgoNotification const& ev) { +// if(std::holds_alternative(ev.event)){ +// // book update +// const MBPUpdate& bk = std::get(ev.event); +// // copy book +// bts::orderman::Book* leg_bk = find_book(bk.security_id, me->books); +// if(leg_bk != nullptr){ +// auto& leg_bk_r = *leg_bk; +// translate_book(bk, leg_bk_r); +// me->spreader.test_only_on_book_update(leg_bk_r); +// } +// } +// else{ +// const OrderUpdate& u = std::get(ev.event); +// me->sim_order_gateway.route_update_to_spreader(u); +// } +// } + +// } diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.h b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.h new file mode 100644 index 000000000..d8e7b02e2 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.h @@ -0,0 +1,33 @@ +// #pragma once + +// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/IAlgoModel.h" +// #include "SimOrderGateway.h" + +// namespace bts { namespace api { namespace orderman {class OrderDefinition;}}} +// namespace bts { namespace api { class OrderDefinition;}} +// namespace bts { namespace orderman { class SpreaderOrder;}} + +// namespace bts::spartan::mdp3 { + +// class ISimServices; + +// class SimSpreader : public IAlgoModel { +// private: +// struct PImpl; +// //std::unique_ptr me; +// PImpl* me; +// public: +// SimSpreader(const bts::api::orderman::OrderDefinition&, +// const bts::api::Security&, +// Price, Quantity); +// const bts::orderman::SpreaderOrder& spreader() const; +// // IAlgoModel +// void init(ISimServices&, protos::SimRunOptions const&) override; +// void on_time(MDP3TimeStamp) override; +// void on_notification(AlgoNotification const&) override; +// ~SimSpreader(){} +// }; + +// } diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/tests/sim_spreader_tests.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/tests/sim_spreader_tests.cpp new file mode 100644 index 000000000..c0362ead6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/spreader/tests/sim_spreader_tests.cpp @@ -0,0 +1,359 @@ +// #include +// #include "lib/domain_model/Security.pb.h" + +// #include "server/sparkd/cmds/spartan/back_testing/algos/spreader/SimSpreader.h" +// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h" +// #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/ISimServices.h" +// #include "lib/orderman/orders/spreader_orders/tests/tests_common.h" +// #include "server/sparkd/cmds/spartan/back_testing/orchestrator/SimSessionOptions.pb.h" + +// using namespace bts::spartan::mdp3; + +// namespace bts { +// namespace orderman { + +// using namespace api; +// using namespace spark; + +// class TestSimServices : public bts::spartan::mdp3::ISimServices{ +// private: +// InstrumentMap _instruments; +// std::unordered_map _polygons; + +// protected: +// void _log(std::string const &, +// std::initializer_list const &) override {} +// public: +// SmallOrderChange last_change; +// size_t n_updates = 0; +// size_t key; +// public: +// TestSimServices () +// : +// key(0){ +// } +// InstrumentMap const &instruments() const override{ +// bts_assert(false); +// return _instruments; +// } +// void terminate_simulation() override {} +// void log_pandl() override {} +// std::unordered_map const&polygons() const override { +// return _polygons; +// } + +// spartan::mdp3::Book book(MDP3SecurityID,api::Side) const override { +// return {}; +// } +// MDP3OrderKey generate_order_key() override{ +// return MDP3OrderKey(key++); +// } +// MDP3TimeStamp now() const override{ +// return MDP3TimeStamp(0); +// } +// void wait(uint64_t) override{ +// } +// void send(const SmallOrderChange& c, uint64_t) override{ +// last_change = c; +// n_updates++; +// } +// ~TestSimServices(){} +// }; + +// bts::spartan::mdp3::MBPUpdate get_book_update( +// const std::vector>& bids, +// const std::vector>& asks, +// uint32_t key){ +// bts::spartan::mdp3::MBPUpdate rv; +// size_t i=0; +// for(auto[p,q] : bids) +// rv.mbp_buy[i++] = BookLevel{ +// md::IntPrice_from_double<7>(p), +// bts::spartan::mdp3::MDP3Quantity(q), +// 0 +// }; +// i=0; +// for(auto[p,q] : asks) +// rv.mbp_sell[i++] = BookLevel{ +// md::IntPrice_from_double<7>(p), +// bts::spartan::mdp3::MDP3Quantity(q), +// 0 +// }; +// rv.security_id = MDP3SecurityID(key); +// return rv; +// } + +// TEST_CASE ("sim_spreader", "[spreader]"){ +// bts::orderman::OrderKey order_key(1); +// // security +// ContractLeg leg_1, leg_2; +// fill_future_contract_leg (leg_1, 1, "CL", +// bts::api::Venue::VENUE_GLBX, +// 2020, 11, 15, +// 1.0, 1.0, +// api::Side::SIDE_BUY, +// 1.0, 0.01); +// fill_future_contract_leg (leg_2, 2, "CL", +// bts::api::Venue::VENUE_GLBX, +// 2020, 12, 15, +// -1.0, -1.0, +// api::Side::SIDE_SELL, +// 1.0, 0.01); +// std::vector legs; +// legs.push_back(leg_1); +// legs.push_back(leg_2); +// Security sec; +// fill_synthetic_security (sec, 2000, "simspreader_2legs", legs); +// // spreader params +// LegSpreaderParams lsp_1, lsp_2, lsp_3, lsp_4; +// fill_leg_spreader_params(lsp_1, 1.0, false, 1.0); +// fill_leg_spreader_params(lsp_2, -1.0, true, 1.0); +// // post hedge time out here +// std::vector params; +// params.push_back(lsp_1); +// params.push_back(lsp_2); +// // putting it together in the order def +// bts::api::orderman::OrderDefinition order_def; +// fill_spreader_order_definition(order_def, Side::SIDE_BUY, sec, params); +// bts::spartan::mdp3::SimSpreader sim_spreader(order_def,sec,Price(-10),Quantity(1)); +// //ISimServices* sim_services = new TestSimServices(); +// TestSimServices sim_services; +// spartan::protos::SimRunOptions options; +// sim_spreader.init(*(ISimServices*)&sim_services,options); +// SECTION("requoting"){ +// std::vector> bids = +// { +// {100,5}, +// {101,5}, +// {102,5} +// }, +// asks = +// { +// {103,5}, +// {104,5}, +// {105,5} +// }; +// auto md_udpate_leg_1 = get_book_update(bids,asks,leg_1.security().id()); +// // send leg_1 book update => leg_2 should quote +// sim_spreader.on_notification( +// { +// {}, +// md_udpate_leg_1 +// } +// ); +// sim_spreader.on_time(bts::spartan::mdp3::MDP3TimeStamp()); +// REQUIRE(sim_services.n_updates==1); +// REQUIRE(sim_services.last_change.order.side == api::Side::SIDE_SELL); +// REQUIRE(sim_services.last_change.order.quantity == bts::spartan::mdp3::MDP3Quantity(1)); +// REQUIRE( +// bts::securities::is_same(Price(sim_services.last_change.order.price.price()), +// Price(113))); +// // send multiple book updates +// double p = 103; +// for(int i=0; i<100; i++){ +// asks = +// { +// {p,5}, +// {p+1,5}, +// {p+2,5} +// }; +// auto md_udpate_leg_1 = get_book_update(bids,asks,leg_1.security().id()); +// sim_spreader.on_notification( +// { +// {}, +// md_udpate_leg_1 +// }); +// REQUIRE(sim_services.n_updates==i+1); // +1 because order_place is also counted +// REQUIRE(sim_services.last_change.order.security == +// bts::spartan::mdp3::MDP3SecurityID(2)); +// REQUIRE(sim_services.last_change.order.side == api::Side::SIDE_SELL); +// REQUIRE(sim_services.last_change.order.quantity == bts::spartan::mdp3::MDP3Quantity(1)); +// REQUIRE( +// bts::securities::is_same(Price(sim_services.last_change.order.price.price()), +// Price(p+10))); +// p++; +// } +// //force a quoting leg fill +// auto q_ord = sim_services.last_change.order; +// sim_spreader.on_notification( +// { +// {}, +// bts::spartan::mdp3::OrderUpdate{ +// q_ord.key, +// bts::spartan::mdp3::OrderUpdate::FILLED, +// q_ord.security, +// q_ord.side, +// bts::spartan::mdp3::MDP3Quantity(1), +// md::IntPrice_from_double<7>(202) +// } +// }); +// // the above should have sent a hedge order, verify that +// auto hedge_ord = sim_services.last_change.order; +// REQUIRE(hedge_ord.security == bts::spartan::mdp3::MDP3SecurityID(1)); +// REQUIRE(hedge_ord.side == api::Side::SIDE_BUY); +// REQUIRE(hedge_ord.quantity == bts::spartan::mdp3::MDP3Quantity(1)); +// REQUIRE( +// bts::securities::is_same(Price(hedge_ord.price.price()), +// Price(192))); +// REQUIRE(sim_spreader.spreader().test_only_func_spreader_state() != +// SpreaderState::END); +// sim_spreader.on_time(bts::spartan::mdp3::MDP3TimeStamp()); +// // send the hedge fill and it should go to end state +// sim_spreader.on_notification( +// { +// {}, +// bts::spartan::mdp3::OrderUpdate{ +// hedge_ord.key, +// bts::spartan::mdp3::OrderUpdate::FILLED, +// hedge_ord.security, +// hedge_ord.side, +// bts::spartan::mdp3::MDP3Quantity(1), +// md::IntPrice_from_double<7>(192) +// } +// }); +// REQUIRE(sim_spreader.spreader().test_only_func_spreader_state() == +// SpreaderState::END); +// } +// } + +// class RV +// { +// public: +// int i; +// RV(int i_arg) +// : +// i(i_arg){ +// } +// RV()=delete; +// RV(const RV&&)=delete; +// RV (RV&& other){ +// i=other.i; +// other.i=0; +// } +// RV& operator=(const RV&)=delete; +// RV& operator=(RV &&src){ +// i=src.i; +// src.i=0; +// return *this; +// } +// int foo (double d1, double d2){ +// return static_cast(d1+d2); +// } +// }; +// class TTemp{ +// }; +// class TInterface{ +// virtual void foo() = 0; +// }; +// template +// class TestContainer : public TInterface{ +// private: +// T _t; +// std::array _at; +// public: +// void add (T &t){ +// _at[0] = t; +// } +// T get() { +// return _t; +// } +// void foo () override{ +// } +// }; +// class RVUser{ +// public: +// RV rv; +// RVUser(RV& rv_arg) +// : rv(std::move(rv_arg)) +// { +// } +// }; +// TEST_CASE ("scratch_pad", "[spreader]"){ +// int (RV::*f_ptr)(double, double) = &RV::foo; +// char* f_addr = (char*&)f_ptr; +// TInterface* r = nullptr; +// int n = gen_rand(0.0, 10.0); +// if (n <= 5) +// r = new TestContainer(); +// else +// r = new TestContainer(); +// REQUIRE(r != nullptr); +// RV rv_1(100); +// RVUser rvuser_1(rv_1); +// REQUIRE(rv_1.i==0); +// REQUIRE(rvuser_1.rv.i==100); +// } +// TEST_CASE ("scratch_pad_2", "[spreader]"){ +// time_t rawtime; +// time (&rawtime); +// struct tm* ptm = gmtime (&rawtime); +// LOG(OrderMan, Info, "hrs:{} mins:{} sec:{}", ptm->tm_hour, ptm->tm_min, ptm->tm_sec); +// } +// // vector insert ////////////////////////////////////// +// /////////////////////////////////////////////////////// +// class Elem{ +// public: +// int i; +// Elem() +// : +// i(-1) +// {} +// Elem (int i_arg) +// : +// i(i_arg) +// {} +// Elem(const Elem& other){ +// i = other.i; +// } +// Elem (Elem&& other){ +// i=other.i; +// } +// Elem& operator=(const Elem& other){ +// i=other.i; +// return *this; +// } +// Elem& operator=(Elem &&src){ +// i = src.i; +// return *this; +// } +// bool operator==(const Elem& e) const {return i == e.i;} +// bool operator!=(const Elem& e) const {return i != e.i;} +// bool operator<(const Elem& e) const {return i < e.i;} +// bool operator>(const Elem& e) const {return i > e.i;} +// }; +// TEST_CASE ("scratch_pad_3", "[spreader]"){ +// std::vector v; +// for(int i=1;i<10;i++) +// v.emplace_back(i); +// Elem e(0); +// // move op= is called when inserting or erasing +// v.insert(v.begin()+2,e); +// v.erase(v.begin()+2); +// std::random_shuffle(v.begin(),v.end()); +// // flat set +// boost::container::flat_set fs; +// for(int i=0;i<20;i+=2){ +// Elem e(i); +// fs.insert(e); +// } +// Elem e9(9); +// // move op is called when inserting +// fs.insert(e9); +// for (const auto& e : fs) +// std::cout << e.i << std::endl; +// // vector assignment vs move assignment +// std::vector vs; +// int n_elems = 10; +// for(int i=1;i vt; +// // new elems are cresated in vt from the elem's in vs, +// // with their copy cons +// vt = vs; +// std::vector vt_2; +// vt_2 = std::move(vs); +// } + +// }} diff --git a/src/server/sparkd/cmds/spartan/back_testing/db/SimDB.cpp b/src/server/sparkd/cmds/spartan/back_testing/db/SimDB.cpp new file mode 100644 index 000000000..5a8eb1ce6 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/db/SimDB.cpp @@ -0,0 +1,299 @@ +#include "SimDB.h" + +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_chrono.h" +#include "lib/utils/datetime.h" +#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" + +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { + +struct SimDB::PendingLog { + RunID rid; + size_t sim_seq; + std::string src; + MDP3TimeStamp ts; + EventHeader h; + std::string log_entry_type; + std::vector args; +}; + +struct SimDB::PImpl { + PImpl(std::string const &loc) + : db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE), db_lock() {} + + SQLite::Database db; + std::mutex db_lock; + std::mutex pending_log_lock; + std::vector pending; + std::unordered_map log_entry_order; + std::thread logger_thread; + std::atomic done = false; +}; + +SimDB::SimDB(std::string const &loc) { + auto parent = boost::filesystem::path(loc).parent_path(); + if (!boost::filesystem::is_directory(parent)) + bts_assert(boost::filesystem::create_directories(parent)); + + if (!boost::filesystem::is_regular_file(loc)) { + auto sql_path = os::sparkd_wd() + "/data_files/spartan/sim_db.sql"; + std::ifstream f(sql_path); + std::string str((std::istreambuf_iterator(f)), + (std::istreambuf_iterator())); + SQLite::Database db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + db.exec(str); + } + me = std::make_unique(loc); + std::lock_guard l(me->db_lock); + SQLite::Statement sync_off(me->db, "PRAGMA synchronous = OFF;"); + sync_off.executeStep(); + SQLite::Statement journal_mem(me->db, "PRAGMA journal_mode = MEMORY;"); + journal_mem.executeStep(); + me->logger_thread = std::thread{[this]() { do_work(); }}; +} + +SimDB::~SimDB() { + me->done = true; + me->logger_thread.join(); +} + +SimDB::SessionID SimDB::create_session(std::string const &algo, + bool clear_all) { + std::lock_guard l(me->db_lock); + + if (clear_all) { + SQLite::Statement dl(me->db, "delete from log"); + dl.exec(); + SQLite::Statement dr(me->db, "delete from run"); + dr.exec(); + } + + auto max_session_query = "select max(rowid) from run"; + SQLite::Statement statement(me->db, max_session_query); + size_t session_id = 0; + while (statement.executeStep()) { + session_id = (long long)statement.getColumn(0); + } + return SessionID{session_id, algo}; +} + +SimDB::RunID SimDB::begin_run(SessionID sid, + protos::SimRunOptions const &options) { + std::lock_guard l(me->db_lock); + + auto insert_run = "insert into run(" + "session_id," + "algo," + "begin_run_time," + "end_run_time," + "market_date," + "channel," + "param_set_id," + "sim_start_stop_time," + "base_algo_latency_ns," + "transaction_latency_ns," + "use_first_packet_st," + "simulate_icebergs," + "realized_pandl," + "commission," + "position_pandl," + "param1," + "param2," + "param3," + "param4," + "param5," + "param6," + "param7," + "param8," + "param9," + "param10," + "param11," + "param12," + "param13," + "param14," + "param15" + ") values (" + ":session_id," + ":algo," + ":begin_run_time," + ":end_run_time," + ":market_date," + ":channel," + ":param_set_id," + ":sim_start_stop_time," + ":base_algo_latency_ns," + ":transaction_latency_ns," + ":use_first_packet_st," + ":simulate_icebergs," + ":realized_pandl," + ":commission," + ":position_pandl," + ":param1," + ":param2," + ":param3," + ":param4," + ":param5," + ":param6," + ":param7," + ":param8," + ":param9," + ":param10," + ":param11," + ":param12," + ":param13," + ":param14," + ":param15" + ")"; + + SQLite::Statement statement(me->db, insert_run); + statement.bind(":session_id", (long long)sid.id); + statement.bind(":algo", sid.algo); + statement.bind(":begin_run_time", datetime::time_string(datetime::now())); + statement.bind(":end_run_time", ""); + statement.bind(":market_date", options.market_date()); + statement.bind(":channel", options.channel()); + statement.bind(":param_set_id", options.param_set_id()); + statement.bind(":sim_start_stop_time", options.sim_start_stop_time()); + statement.bind(":base_algo_latency_ns", + (long long)options.base_algo_latency_ns()); + statement.bind(":transaction_latency_ns", + (long long)options.transaction_latency_ns()); + statement.bind(":use_first_packet_st", + (long long)options.use_first_packet_st()); + statement.bind(":simulate_icebergs", + (long long)options.simulate_icebergs()); + statement.bind(":realized_pandl", 0); + statement.bind(":commission", 0); + statement.bind(":position_pandl", 0); + + std::map sorted_params(options.param().begin(), + options.param().end()); + + size_t p_idx = 1; + for (auto const &[name, val] : sorted_params) + statement.bind(fmt::format(":param{}", p_idx++), + fmt::format("{}: {}", name, val)); + + for (auto i = p_idx; i <= 15; i++) + statement.bind(fmt::format(":param{}", i), ""); + + statement.exec(); + auto run_id = RunID{last_insert_rowid()}; + me->log_entry_order[run_id] = 1; + return run_id; +} + +void SimDB::end_run(RunID run_id, double realized_pandl, double commission, + double position_pandl) { + std::lock_guard l(me->db_lock); + + auto update_run = "update run " + "set " + "end_run_time=:end_run_time, " + "realized_pandl=:realized_pandl," + "commission=:commission," + "position_pandl=:position_pandl " + "where " + "run_id=:run_id"; + + SQLite::Statement statement(me->db, update_run); + statement.bind(":end_run_time", datetime::time_string(datetime::now())); + statement.bind(":run_id", run_id.value()); + statement.bind(":realized_pandl", realized_pandl); + statement.bind(":commission", commission); + statement.bind(":position_pandl", position_pandl); + statement.exec(); +} + +void SimDB::log(RunID rid, size_t sim_seq, std::string const &src, + MDP3TimeStamp now, EventHeader const &h, + std::string const &entry_type, + std::vector const &log_args) { + std::lock_guard l(me->pending_log_lock); + me->pending.emplace_back( + PendingLog{rid, sim_seq, src, now, h, entry_type, log_args}); +} + +void SimDB::_log(std::vector const &log) { + std::lock_guard l(me->db_lock); + me->db.exec("begin"); + for (auto &l : log) { + auto rid = l.rid; + auto now = l.ts; + auto &entry_type = l.log_entry_type; + auto &log_args = l.args; + auto &log_entry_order = me->log_entry_order[rid]; + std::ostringstream insert_log_entry; + insert_log_entry << "insert into log(" + "run_id," + "log_entry_order," + "sim_seq," + "src," + "sim_time," + "h_sim_time," + "entry_type," + "ev_eid1," + "ev_eid2," + "ev_t_time," + "ev_o_time"; + + for (size_t i = 0; i < log_args.size(); i++) + insert_log_entry << ",val" << i + 1; + insert_log_entry << ") values " + "(:run_id,:log_entry_order,:sim_seq,:src,:sim_time,:h_sim_" + "time,:entry_type," + ":ev_eid1,:ev_eid2,:ev_t_time,:ev_o_time"; + for (size_t i = 0; i < log_args.size(); i++) + insert_log_entry << ",:val" << i + 1; + insert_log_entry << ")"; + + SQLite::Statement statement(me->db, insert_log_entry.str()); + statement.bind(":run_id", rid.value()); + statement.bind(":log_entry_order", (long long)log_entry_order++); + statement.bind(":sim_seq", (long long)l.sim_seq); + statement.bind(":src", l.src); + statement.bind(":sim_time", (long long)now.value()); + statement.bind(":h_sim_time", to_time_string(now)); + statement.bind(":entry_type", entry_type); + statement.bind(":ev_eid1", (long long)l.h.event_id.first); + statement.bind(":ev_eid2", (long long)l.h.event_id.second); + statement.bind(":ev_t_time", (long long)l.h.transact_time.value()); + statement.bind(":ev_o_time", (long long)l.h.observable_time.value()); + + size_t i = 1; + for (auto &arg : log_args) { + statement.bind(fmt::format(":val{}", i++), arg); + } + statement.exec(); + } + me->db.exec("commit"); +} + +void SimDB::do_work() { + while (!me->done) { + std::vector to_store; + { + std::lock_guard l(me->pending_log_lock); + to_store = std::move(me->pending); + } + if (to_store.size()) + _log(to_store); + sleep_for(100ms); + } + _log(me->pending); +} + +int SimDB::last_insert_rowid() { + SQLite::Statement statement(me->db, "select last_insert_rowid()"); + statement.executeStep(); + return statement.getColumn("last_insert_rowid()"); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/db/SimDB.h b/src/server/sparkd/cmds/spartan/back_testing/db/SimDB.h new file mode 100644 index 000000000..cd0c06d22 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/db/SimDB.h @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include "lib/persistence/persistence_utils.h" +#include "lib/utils/ValueWrapper.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" + +namespace bts::spartan::protos { +class SimRunOptions; +} + +namespace bts::spartan::mdp3 { + +class SimDB { + public: + SimDB(std::string const &location = persistence::storage_location("sim_db")); + ~SimDB(); + + struct SessionID { + size_t id; + std::string algo; + }; + SessionID create_session(std::string const &algo, bool clear_all = false); + + using RunID = utils::ValueWrapper; + RunID begin_run(SessionID, protos::SimRunOptions const &); + void end_run(RunID,double realized_pandl, double commission,double position_pandl); + + void log(RunID, + size_t sim_seq, + std::string const&src, + MDP3TimeStamp, + EventHeader const&h, + std::string const &log_entry_type, + std::vector const &log_args); + + private: + struct PendingLog; + struct PImpl; + void do_work(); + void _log(std::vector const& ); + int last_insert_rowid(); + std::unique_ptr me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/db/SimLogger.h b/src/server/sparkd/cmds/spartan/back_testing/db/SimLogger.h new file mode 100644 index 000000000..102e1c68a --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/db/SimLogger.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" + +namespace bts::spartan::mdp3 { + +using SimLogger = + std::function const &log_args)>; +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp new file mode 100644 index 000000000..0b614b736 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp @@ -0,0 +1,207 @@ +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h" +#include "lib/domain_model/Security.h" +#include "lib/md/IntPrice.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "lib/orderman/orderman_utils/PriceComp.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include +#include + +// Error logs are automatically logged to the console +// (w/ default logging configuration), but they lag behind +// immediate console output (i.e., calls to std::cout). +// If _ECHO_ON is defined, the _ERROR_LEVEL macro is set to Info +// to avoid duplicate, and (perhaps more importantly) unsynchronized output. +//#define _ECHO_ON +#ifdef _ECHO_ON +#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__) +#define _ERROR_LEVEL Info +#else +#define _ECHO(...) +#define _ERROR_LEVEL Error +#endif +#define _LOG(level, msg) \ + _ECHO(_log_header(), " ", msg, "\n"); \ + LOG(OrderMan, level, "{} {}", _log_header(), msg) +#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define ERROR(...) _LOG(_ERROR_LEVEL, bts::orderman::Formatter::fmt(__VA_ARGS__)) + +// SUMMARIZE macro wrappers +#define _SUMMARIZE SUMMARIZE(_log_header()) +#define _SUMMARIZE_TRANSACTION(...) SUMMARIZE_TRANSACTION(_log_header(), FMT(__VA_ARGS__)) +#define _SUMMARIZE_TRANSACTION_NAMED(name, ...) SUMMARIZE_TRANSACTION_NAMED(name, _log_header(), FMT(__VA_ARGS__)) + +namespace bts::spartan::mdp3 { +using _BookLevel = std::tuple; + +struct FifoImpliedSecurityMatcher::PImpl { + PImpl(MDP3SecurityID security_id, std::string symbol) + : security_id(security_id), implied_mkt(), symbol(symbol), leg_symbols() {} + + MDP3SecurityID security_id; + std::map> implied_mkt; + // symbol/leg_symbols serve diagnostic purposes + // (easier to read logs w/ actual symbols than sec ids) + std::string symbol; + std::map leg_symbols; +}; + +/////////////////////// +// constructors +/////////////////////// +FifoImpliedSecurityMatcher::FifoImpliedSecurityMatcher(MDP3SecurityID security_id) + : FifoImpliedSecurityMatcher(security_id, std::string()) {} + +FifoImpliedSecurityMatcher::FifoImpliedSecurityMatcher( + MDP3SecurityID security_id, std::string symbol) + : _me(new FifoImpliedSecurityMatcher::PImpl(security_id, symbol)) {} + +FifoImpliedSecurityMatcher::~FifoImpliedSecurityMatcher() {} + +/////////////////////// +// diagnostic +/////////////////////// +MDP3SecurityID FifoImpliedSecurityMatcher::security_id() const { + return _me->security_id; +} +std::string FifoImpliedSecurityMatcher::symbol() const { return _me->symbol; } + +std::string FifoImpliedSecurityMatcher::_log_header() const { + return FMT("FifoImpliedSecurityMatcher (", security_id(), ", ", symbol(), ")"); +} + +void FifoImpliedSecurityMatcher::print_legs(std::ostream& out) const { + bool append_bbo = true; + for (const auto& kv : _me->implied_mkt) { + auto leg_size = std::get<1>(kv.second); + std::string leg_sym; + auto it = _me->leg_symbols.find(kv.first); + if (it != _me->leg_symbols.end()) { + leg_sym = it->second; + } + out << std::setw(12) << kv.first << " " << std::setw(12) << leg_sym << " " << leg_size; + if (append_bbo) { + append_bbo = false; + auto bid = top_of_book(api::Side::SIDE_BUY); + auto ask = top_of_book(api::Side::SIDE_SELL); + out << std::setw(8) << FMT("(", bid.quantity, ") ") << std::setw(6) << bid.price << std::setw(6) << ask.price << + FMT("(", ask.quantity, ")"); + } + out << std::endl; + } +} + +/////////////////////// +// book related +/////////////////////// +ImpliedBook FifoImpliedSecurityMatcher::book(api::Side side) const { + // this isn't the best way to do this (copying), + // but the ImpliedBook only has 2 levels in practice, + // so it should be fine. + SmallBook tuple_ib; + ImpliedBook ib; + _book(side, tuple_ib, ib.size()); + for (uint32_t i = 0; i < tuple_ib.size() && i < ib.size(); i++) { + ib[i].price = tuple_ib[i].price; + ib[i].quantity = tuple_ib[i].quantity; + } + return ib; +} + +SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(api::Side side) const { + SmallBook b; + _book(side, b, 1); + if (b.empty()) + return {md::IntPrice(), MDP3Quantity(0)}; + return b[0]; +} + +void FifoImpliedSecurityMatcher::_book(api::Side side, SmallBook& dest, uint32_t depth) const { + std::vector> im_vec; + for (const auto& p : _me->implied_mkt) { + SmallBook bk; + auto [sec_matcher, leg_qty] = p.second; + bts_assert(leg_qty.value() != 0); + auto leg_side = leg_qty.value() > 0 ? securities::other_side(side) : side; + for (auto bl : sec_matcher->book(leg_side)) + bk.emplace_back(bl.price, bl.quantity); + im_vec.emplace_back(leg_qty, md::INT_NAN, bk); + } + return book_utils::compute_synthetic_book(dest, im_vec, depth); +} + +void FifoImpliedSecurityMatcher::add_leg(MDP3SecurityID sec_id, FifoSecurityMatcher* m, MDP3Quantity unit_qty, std::string leg_sym) { + _me->implied_mkt.insert({sec_id, {m, unit_qty}}); + _me->leg_symbols.insert_or_assign(sec_id, leg_sym); +} + +MDP3Quantity FifoImpliedSecurityMatcher::fill(MDP3OrderKey key, api::Side side, MDP3Quantity qty, std::vector* effects, FillDepot& fills) { + SmallBook b; + auto oside = securities::other_side(side); + _book(oside, b, MDP3_MAX_IMPLIED_BOOK_DEPTH); + auto orig_qty = qty; + // can't fill more than the volume in the book + qty = std::min(book_volume(b), qty); + md::IntPrice impl_fill_price = top_of_book(oside).price; + bool has_added_aggr_fills = false; + int fill_qty = 0; + bool local_fills = effects == nullptr; + for (auto kv : _me->implied_mkt) { + auto leg_size = std::get<1>(kv.second); + // todo: support other leg sizes (keeping it simple for now) + bts_assert(leg_size.value() == 1 || leg_size.value() == -1); + api::Side leg_side = leg_size.value() > 0 ? side : securities::other_side(side); + FifoSecurityMatcher& m = *std::get<0>(kv.second); + if (local_fills) + effects = new std::vector(); + uint32_t leg_fills_pos = effects->size(); + auto raw_leg_fill_qty = qty * abs(leg_size); + // INFO("expecting to fill ", raw_leg_fill_qty); + auto raw_leg_rem_qty = m.fill(key, leg_side, md::IntPrice(), raw_leg_fill_qty, effects, false); + // INFO("remaining (unfilled) qty: ", raw_leg_rem_qty); + bts_assert(raw_leg_rem_qty.value() == 0); + for (auto it = effects->begin() + leg_fills_pos; it != effects->end(); it++) { + auto& passive = *it; + bts_assert(std::holds_alternative(passive)); + fills.add_fill(kv.first, key, _to_passive_fill(std::get(passive), true)); + } + // the implied matchers should not produce partial leg fills (modulus leg_size should be 0) + bts_assert(raw_leg_fill_qty.value() == 0 || raw_leg_fill_qty.value() % abs(leg_size).value() == 0); + uint32_t leg_fill_qty = (raw_leg_fill_qty / abs(leg_size)).value(); + bts_assert(leg_fill_qty == static_cast(qty.value())); + if (!has_added_aggr_fills) { + // This only needs to be done once. + // Any leg's passive fills can serve as a reference for + // the fill qty wrt the implied market: impl_fill_qty = leg_fill_qty / abs(leg_size), + // as the ratio raw_leg_fill_qty:abs(leg_size) + // should be the same for all legs. + // It might not be a bad idea to assert this is the case: + // that all legs produce the exact same aggressor fills. + for (auto it = effects->begin() + leg_fills_pos; it != effects->end(); it++) { + auto& f = *it; + bts_assert(std::holds_alternative(f)); + MDP3Fill aggr; + aggr.key = key; + aggr.aggressor = _to_aggressor(side); + aggr.price = impl_fill_price; + aggr.quantity = std::get(f).last_fills / abs(leg_size); + fills.add_fill(security_id(), key, aggr); + fill_qty += aggr.quantity.value(); + } + has_added_aggr_fills = true; + } + if (local_fills) { + delete effects; + effects = nullptr; + } + } + bts_assert(fill_qty <= orig_qty.value()); + return MDP3Quantity(orig_qty.value() - fill_qty); +} +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h new file mode 100644 index 000000000..b255fc680 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h @@ -0,0 +1,48 @@ +#pragma once +#include "lib/md/IntPrice.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { +struct FifoImpliedSecurityMatcher { + FifoImpliedSecurityMatcher(MDP3SecurityID security_id); + // Symbol/sec id serve diagnostic purposes, but aren't absolutely neccessary. + // Logs made by this class begin with a header that includes the symbol/sec id. + FifoImpliedSecurityMatcher(MDP3SecurityID security_id, std::string symbol); + ~FifoImpliedSecurityMatcher(); + + ImpliedBook book(api::Side) const; + MDP3Quantity fill(MDP3OrderKey, api::Side, MDP3Quantity, std::vector*, FillDepot&); + // Note: this class is NOT responsible for deleting any FifoSecurityMatcher pointers. + // The FifoMatcherContext creates one FifoMergedSecurityMatcher per each security, + // and each FifoMergedSecurityMatcher creates/owns a FifoSecurityMatcher pointer + // (and is responsible for deleting it). + // The FifoSecurityMatcher pointer passed to this method should be owned by a + // FifoMergedSecurityMatcher. + // Note the absense of a 'side' parameter: unit_qty should be positive for Buys, + // negative for Sells. + void add_leg(MDP3SecurityID sec_id, FifoSecurityMatcher* m, MDP3Quantity unit_qty, std::string leg_sym); + MDP3SecurityID security_id() const; + std::string symbol() const; + SmallBookLevel top_of_book(api::Side side) const; + void print_legs(std::ostream& out = std::cout) const; + + private: + struct PImpl; + std::unique_ptr _me; + // hide copy ctr + FifoImpliedSecurityMatcher(const FifoImpliedSecurityMatcher&); + + std::string _log_header() const; + void _book(api::Side side, SmallBook& dest, uint32_t depth) const; + uint32_t _fill_order(const MDP3Order& order); +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp new file mode 100644 index 000000000..8d72c6777 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp @@ -0,0 +1,242 @@ +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/implied/implied_market.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/security_comp.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" +#include +#include +#include +#include +#include + +using namespace bts::spartan::diagnostics; + +template +using flat_map = boost::container::flat_map; + +namespace bts::spartan::mdp3 { +/////////////////////// +// stateless helpers +/////////////////////// +//static inline std::string _log_header() { return "FifoMatcherContext"; } + +static inline MDP3Quantity _norm_qty(api::Side side, MDP3Quantity qty) { + switch (side) { + case api::Side::SIDE_BUY: return qty; + case api::Side::SIDE_SELL: return -qty; + default: + bts_assert_msg(false, FMT("unexpected side '", side, "'")); + return qty; + } +} + +struct FifoMatcherContext::PImpl { + const bool ignore_ops_on_deleted_orders; + // _implied_candidates_dirty should be set to true anytime a security is added/removed. + // This signals that implied market candidates must be re-evaluated + // before any susbequent order-matching (re-evaluation is performed lazily). + bool implied_candidates_dirty; + ImpliedMarket contract_matrix; + flat_map matchers; + flat_map symbols; + + PImpl() : ignore_ops_on_deleted_orders(false) {} + + PImpl(const InitArgs& args) + : ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders) {} + + ~PImpl() { + for (auto& p : matchers) + delete p.second; + matchers.clear(); + } +}; + +/////////////////////// +// constructors +/////////////////////// +FifoMatcherContext::FifoMatcherContext() : FifoMatcherContext({false}) {} + +FifoMatcherContext::FifoMatcherContext(const InitArgs& args) : _me(new PImpl(args)) {} + +FifoMatcherContext::~FifoMatcherContext() { } + +/////////////////////// +// security accessors +/////////////////////// +bool FifoMatcherContext::add_security(MDP3SecurityID sec_id, const std::string& symbol, const std::vector& legs) { + auto curr = _me->matchers.find(sec_id); + if (curr == _me->matchers.end()) { + _me->symbols.insert(std::make_pair(sec_id, symbol)); + auto& ic = _me->contract_matrix.add_contract(sec_id); + // Add the legs explicitly (ImpliedMarketBuilder doesn't know about + // 'InstrumentDefinition,' or its leg structure 'LegDef'). + if (legs.empty()) { + // must be an outright (to normalize outrights and spreads, + // the former is treated like a 'single leg spread') + ic.add_leg(sec_id, MDP3Quantity(1)); + } else { + for (auto l : legs) + ic.add_leg(l.security_id, _norm_qty(l.side, l.ratio)); + } + _me->matchers.insert(std::make_pair(sec_id, new FifoMergedSecurityMatcher({sec_id, _me->ignore_ops_on_deleted_orders}))); + } else { + // Not sure why, but [John]I've seen _me->symbols contain empty strings. + // Overwrite the existing value to ensure a valid symbol. + _me->symbols[sec_id] = symbol; + return false; + } + _me->implied_candidates_dirty = true; + return true; +} + +bool FifoMatcherContext::remove_security(MDP3SecurityID sec_id) { + auto curr = _me->symbols.find(sec_id); + if (curr == _me->symbols.end()) + return false; + bts_assert(_me->symbols[sec_id] == curr->second); + _me->contract_matrix.remove_contract(sec_id); + _me->symbols.erase(sec_id); + _me->implied_candidates_dirty = true; + return true; +} + +bool FifoMatcherContext::implied_matching_status(MDP3SecurityID sec_id) const { + auto* m = _get_matcheri(sec_id); + return m != nullptr && m->implied_matching_status(); +} + +void FifoMatcherContext::implied_matching_status(MDP3SecurityID sec_id, bool value) { + auto* m = _get_or_add_matcher(sec_id); + if (m->implied_matching_status() != value) { + m->implied_matching_status(value); + _me->implied_candidates_dirty = true; + } +} + +/////////////////////// +// book +/////////////////////// +Book FifoMatcherContext::book(MDP3SecurityID sec_id, api::Side side) { + return _get_matcher(sec_id)->book(side); +} + +ImpliedBook FifoMatcherContext::implied_book(MDP3SecurityID sec_id, api::Side side) { + return _get_matcher(sec_id)->implied_book(side); +} + +/////////////////////// +// matcher accessors +/////////////////////// +FifoMergedSecurityMatcher* FifoMatcherContext::_get_matcher(MDP3SecurityID sec_id) { + auto m = _me->matchers.find(sec_id); + return m == _me->matchers.end() ? nullptr : m->second; +} + +const FifoMergedSecurityMatcher* FifoMatcherContext::_get_matcheri(MDP3SecurityID sec_id) const { + auto m = _me->matchers.find(sec_id); + return m == _me->matchers.end() ? nullptr : m->second; +} + +FifoMergedSecurityMatcher* FifoMatcherContext::_get_or_add_matcher(MDP3SecurityID sec_id) { + auto m = _me->matchers.find(sec_id); + if (m == _me->matchers.end()) { + auto* new_m = new FifoMergedSecurityMatcher({sec_id, _me->ignore_ops_on_deleted_orders}); + _me->matchers.insert(std::make_pair(sec_id, new_m)); + return new_m; + } + return m->second; +} + +std::tuple +FifoMatcherContext::find_order(MDP3SecurityID sec_id, MDP3OrderKey ok) { + auto* m = _get_matcher(sec_id); + if (m != nullptr) + return m->matcher()->find_order(ok); + return { nullptr, nullptr }; +} + +/////////////////////// +// order callbacks +/////////////////////// +void FifoMatcherContext::on_order_change( + MDP3SecurityID sec_id, + const std::vector& changes, + std::vector* effects, + FillDepot& fills) { + // + // SUMMARIZE(_log_header()); + _ensure_sync_implied_candidates(); + auto* matcher = _get_or_add_matcher(sec_id); + bts_assert(matcher != nullptr); + for (auto ch : changes) + matcher->on_order_change(ch, effects, fills); +} + +void FifoMatcherContext::on_order_change( + const SmallOrderChange& change, std::vector* effects, FillDepot& fills) { + // + on_order_change(change.order.security, {change}, effects, fills); +} + +/////////////////////// +// diagnostic +/////////////////////// +std::optional FifoMatcherContext::has_same_orders(MDP3SecurityID sec, const Orders& orders) { + return _get_matcher(sec)->matcher()->has_same_orders(orders); +} + +void FifoMatcherContext::clear() { + for (auto kv : _me->matchers) + kv.second->matcher()->clear(); +} + +using BBORow = std::tuple; + +void FifoMatcherContext::print_all_bbos(std::ostream& out) { + std::vector rows; + for (auto kv : _me->matchers) { + auto bid = kv.second->matcher()->book_level(api::Side::SIDE_BUY, 0); + auto ask = kv.second->matcher()->book_level(api::Side::SIDE_SELL, 0); + rows.emplace_back( + std::make_tuple(_me->symbols[kv.first], std::get<0>(bid), std::get<1>(bid), + std::get<0>(ask), std::get<1>(ask))); + } + std::sort(rows.begin(), rows.end(), [](const BBORow& lhs, const BBORow& rhs) { + return security_comp::SymbolComp()({std::get<0>(lhs)}, {std::get<0>(rhs)}); + }); + bts::spartan::table_display::print_table(out, rows.begin(), rows.end(), + std::vector>> { + { " Security ", 12, [](auto r) { return std::get<0>(r); } }, + { " Qty ", 10, [](auto r) { return FMT(std::get<2>(r)); } }, + { " Bid ", 10, [](auto r) { return FMT(std::get<1>(r)); } }, + { " Ask ", 10, [](auto r) { return FMT(std::get<3>(r)); } }, + { " Qty ", 10, [](auto r) { return FMT(std::get<4>(r)); } } + }); +} + +void FifoMatcherContext::_ensure_sync_implied_candidates() { + if (!_me->implied_candidates_dirty) + return; + for (auto p : _me->matchers) { + std::vector candidates = _me->contract_matrix.implied_mkts(p.first); + for (auto can : candidates) { + auto* imatcher = p.second->add_implied_matcher(p.first); + for (auto& leg : can) { + std::string sym; + auto it = _me->symbols.find(leg.first); + if (it != _me->symbols.end()) + sym = it->second; + imatcher->add_leg( + leg.first, _get_or_add_matcher(leg.first)->matcher(), + std::get<1>(leg.second), sym); + } + } + } + _me->implied_candidates_dirty = false; +} +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h new file mode 100644 index 000000000..f903d35e9 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h @@ -0,0 +1,94 @@ +#pragma once +#include "lib/orderman/orderman_utils/PriceComp.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { +struct FifoMatcherContext { + struct InitArgs { + bool ignore_ops_on_deleted_orders; + }; + + FifoMatcherContext(); + FifoMatcherContext(const InitArgs& args); + ~FifoMatcherContext(); + + bool add_security(MDP3SecurityID sec_id, const std::string& symbol, const std::vector& legs); + bool remove_security(MDP3SecurityID sec_id); + void implied_matching_status(MDP3SecurityID sec_id, bool value); + bool implied_matching_status(MDP3SecurityID sec_id) const; + void clear(); + Book book(MDP3SecurityID sec_id, api::Side side); + ImpliedBook implied_book(MDP3SecurityID sec_id, api::Side side); + + // the Fills param records individual fills (potentially more than one per order) + void on_order_change(MDP3SecurityID sec_id, const std::vector& changes, std::vector* effects, FillDepot& fills); + void on_order_change(const SmallOrderChange& change, std::vector* effects, FillDepot& fills); + void print_all_bbos(std::ostream& out); + std::optional has_same_orders(MDP3SecurityID, const Orders&); + std::tuple find_order(MDP3SecurityID, MDP3OrderKey); + + private: + struct PImpl; + std::shared_ptr _me; + //FifoMatcherContext(const FifoMatcherContext& other); + + FifoMergedSecurityMatcher* _get_matcher(MDP3SecurityID sec_id); + const FifoMergedSecurityMatcher* _get_matcheri(MDP3SecurityID sec_id) const; + FifoMergedSecurityMatcher* _get_or_add_matcher(MDP3SecurityID sec_id); + void _ensure_sync_implied_candidates(); + + public: + FifoMergedSecurityMatcher* get_matcher(MDP3SecurityID sec_id) { + return _get_matcher(sec_id); + } + // This method doesn't check for crosses, as it assumes all of the orders + // are resting. The expectation is that this method will be called + // to initialize/reset the books (i.e., in response to the first callback, + // or after reset events). + template + inline void reset(MDP3SecurityID sec_id, TIter first, TIter last, std::vector* effects = nullptr) { + _ensure_sync_implied_candidates(); + auto* matcher = _get_or_add_matcher(sec_id); + // The 'order' in which orders are added to the matcher matters. + // The matcher uses an internal order priority counter + // so it can integrate custom orders (i.e., generated by our algos) + // into its books during simulations. + // The scope of an order's priority does not extend beyond its price level. + struct Comp { + bool operator()(const MDP3Order& a, const MDP3Order& b) { + if (a.side == b.side) { + return bts::orderman::price_comp::is_better(a.side, a.price, b.price) || + (a.price == b.price && a.priority < b.priority); + } + return a.side == api::Side::SIDE_BUY; + } + }; + std::set sorted_orders; + while (first != last) { + sorted_orders.emplace(*first); + first++; + } + matcher->matcher()->reset(sorted_orders.begin(), sorted_orders.end()); + if (effects != nullptr) { + effects->emplace_back(MBPUpdate()); + MBPUpdate& mbp = std::get(effects->back()); + mbp.security_id = sec_id; + mbp.mbp_buy = matcher->book(api::Side::SIDE_BUY); + mbp.mbp_sell = matcher->book(api::Side::SIDE_SELL); + mbp.implied_mbp_buy = matcher->implied_book(api::Side::SIDE_BUY); + mbp.implied_mbp_sell = matcher->implied_book(api::Side::SIDE_SELL); + } + } +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp new file mode 100644 index 000000000..680faded4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp @@ -0,0 +1,369 @@ +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h" +#include "lib/md/IntPrice.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "lib/orderman/orderman_utils/PriceComp.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" +#include +#include +#include + +#define _LOG(level, msg) LOG(OrderMan, level, "{} {}", _log_header(), msg) +#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define WARN(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define ERROR(...) _LOG(Error, bts::orderman::Formatter::fmt(__VA_ARGS__)) + +using namespace ranges; +using namespace bts::spartan::mdp3::book_utils; + +namespace bts::spartan::mdp3 { +struct FifoMergedSecurityMatcher::PImpl { + PImpl(const FifoMergedSecurityMatcher::InitArgs& args) + : security_id(args.security_id), + ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders), + implied_matching_status(true), + matcher(new FifoSecurityMatcher({args.security_id, 0, args.ignore_ops_on_deleted_orders, nullptr})), + implied_matchers() {} + + ~PImpl() { + for (uint32_t i = 0; i < implied_matchers.size(); i++) + delete implied_matchers[i]; + implied_matchers.clear(); + } + + const MDP3SecurityID security_id; + const bool ignore_ops_on_deleted_orders; + bool implied_matching_status; + // the Security matcher is kept in a shared_ptr + // so it can be passed around to one or more ImpliedMatchers. + // Note: if a signficant performance penalty is attributed + // to the use of shared_ptrs here, it should be fine if this + // class deletes them in its destructor. + // It seems inappropriate for an ImpliedMatcher to outlive + // the SecurityMatchers that make up its legs anyways. + std::shared_ptr matcher; + std::vector implied_matchers; +}; + +/////////////////////// +// constructors +/////////////////////// +FifoMergedSecurityMatcher::FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher::InitArgs& args) +: _me(new FifoMergedSecurityMatcher::PImpl(args)) {} + +FifoMergedSecurityMatcher::~FifoMergedSecurityMatcher() { } +/////////////////////// +// diagnostic related +/////////////////////// +std::string FifoMergedSecurityMatcher::_log_header() const { + return FMT("[", security_id(), "]", "FifoMergedSecurityMatcher"); +} + +MDP3SecurityID FifoMergedSecurityMatcher::security_id() const { return _me->security_id; } +/////////////////////// +// matcher accessors +/////////////////////// +FifoSecurityMatcher* FifoMergedSecurityMatcher::matcher() { return _me->matcher.get(); } +const FifoSecurityMatcher* FifoMergedSecurityMatcher::matcheri() const { + return _me->matcher.get(); +} +const std::vector& FifoMergedSecurityMatcher::implied_matchers() { + return _me->implied_matchers; +} +FifoImpliedSecurityMatcher* FifoMergedSecurityMatcher::add_implied_matcher(MDP3SecurityID sec) { + auto* m = new FifoImpliedSecurityMatcher(sec); + _me->implied_matchers.push_back(m); + return m; +} +FifoImpliedSecurityMatcher* +FifoMergedSecurityMatcher::_best_implied_matcher(api::Side side) { + if (!_me->implied_matching_status) + return nullptr; + bts::orderman::SidePriceComp cmp(side); + FifoImpliedSecurityMatcher* rv = nullptr; + md::IntPrice best_price; + for (auto& im : _me->implied_matchers) { + auto itop = im->top_of_book(side); + if (itop.quantity.value() > 0 && cmp(itop.price, best_price)) { + rv = im; + best_price = itop.price; + } + } + return rv; +} +bool FifoMergedSecurityMatcher::implied_matching_status() const { + return _me->implied_matching_status; +} +void FifoMergedSecurityMatcher::implied_matching_status(bool value) { + _me->implied_matching_status = value; +} + +void FifoMergedSecurityMatcher::on_order_change(const SmallOrderChange& ch, std::vector* effects, FillDepot& fills) { + if (!implied_matching_status()) { + // there is no "merged matching" when implied matching is off, + // just pass the order changes the direct matcher. + matcher()->on_order_change(ch, effects); + return; + } + bts_assert_msg(ch.affect_book, "affect-free matching is not supported in markets with implied matching."); + switch (ch.change) { + case OrderChange::ChangeType::updated_price: + case OrderChange::ChangeType::updated: + { + auto prev_ord_tup = matcher()->find_order(ch.order.key); + auto prev_ord = std::get<1>(prev_ord_tup); + if (prev_ord == nullptr) { + bts_assert_msg(_me->ignore_ops_on_deleted_orders, "unable to process order-update event: failed to find order."); + break; + } + if (orderman::price_comp::is_better(ch.order.side, ch.order.price, prev_ord->price)) { + // price improved, try to fill the order + auto new_ch = ch; + _fill(new_ch.order, effects, fills, false); + if (new_ch.order.quantity.value() == 0) { + matcher()->on_order_completed(ch.order.key); + } else { + matcher()->on_order_change(new_ch, effects); + } + } else { + // As far as we know, the only type of update that could trigger fills is a + // price improvement. It is worth noting, however, that there is the + // potential for this assumption to be invalid. In a very strange market, we + // may observe down the road: an implied market with a leg for this security's + // same side market, that also has a leg size greater than 1. + matcher()->on_order_change(ch); + } + break; + } + case OrderChange::ChangeType::added: { + auto new_ch = ch; + _fill(new_ch.order, effects, fills, false); + if (get_total_qty(new_ch.order).value() > 0) + matcher()->on_order_change(new_ch); + break; + } + case OrderChange::ChangeType::removed: { + matcher()->on_order_change(ch); + break; + } + } +} +///////////////////////// +// books +///////////////////////// +Book FifoMergedSecurityMatcher::book(api::Side side) { + return matcher()->book(side); +} +ImpliedBook FifoMergedSecurityMatcher::implied_book(api::Side side) { + if (_me->implied_matchers.empty()) + return ImpliedBook(); + // In theory, computing the quantity of any price level beyond the top of the book + // should be done in a hypothetical scenario, where an order with the specified price + // and infinite qty is partially filled. + // + // This accounts for the possibility that implied markets may have securities in + // common, and therefore filling an order in one implied market may impact another + // implied market. + // + // However, in practice, the implied market candidates the exchange uses are independent. + // + // TODO: this should be a linked list, as + // the only functionality needed is forward iteration, + // and erase (the latter for optimization). + std::vector> ibooks; + auto oside = bts::securities::other_side(side); + bts::orderman::SidePriceComp cmp(oside); + for (auto it = _me->implied_matchers.begin(); it != _me->implied_matchers.end(); it++) { + auto res = ibooks.emplace_back((*it)->book(oside), 0); + auto& ib = std::get<0>(res); + if (ib[0].price.is_nan()) + ibooks.pop_back(); + } + auto is_exhausted = [&](const std::tuple& ib_tup) { + const auto& ib = std::get<0>(ib_tup); + auto i = std::get<1>(ib_tup); + return static_cast(i) >= ib.size() || ib[i].price.is_nan(); + }; + auto rem_exhausted = [&]() { + ibooks.erase(std::remove_if(ibooks.begin(), ibooks.end(), is_exhausted)); + }; + ImpliedBook dest; + for (auto& curr : dest) { + // find best price + curr.price = std::get<0>(*max_element(ibooks, cmp, + [](const auto& ibt) { return std::get<0>(ibt)[std::get<1>(ibt)].price.price(); }) + )[0].price; + if (curr.price.is_nan()) + break; + curr.quantity = MDP3Quantity(0); + // combine quantities of implied books with the same top price, as there could be multiple. + for (auto& ib_tup : ibooks) { + auto& ib = std::get<0>(ib_tup); + auto& index = std::get<1>(ib_tup); + if (ib[index].price == curr.price) + curr.quantity = MDP3Quantity(curr.quantity.value() + ib[index++].quantity.value()); + } + rem_exhausted(); + } + return dest; +} + +void FifoMergedSecurityMatcher::merged_book( + api::Side side, std::vector& dest) { + // + std::vector> ibooks; + auto oside = bts::securities::other_side(side); + bts::orderman::SidePriceComp cmp(oside); + for (auto it = _me->implied_matchers.begin(); it != _me->implied_matchers.end(); it++) { + auto res = ibooks.emplace_back((*it)->book(oside), 0); + auto& ib = std::get<0>(res); + if (ib[0].price.is_nan()) + ibooks.pop_back(); + } + auto dbook = matcher()->book(side); + int dindex = 0; + auto is_exhausted = [&](const std::tuple& ib_tup) { + const auto& ib = std::get<0>(ib_tup); + auto i = std::get<1>(ib_tup); + return static_cast(i) >= ib.size() || ib[i].price.is_nan(); + }; + while (true) { + // erase empties + ibooks.erase(std::remove_if(ibooks.begin(), ibooks.end(), is_exhausted), ibooks.end()); + // dir top price/qty + bts::md::IntPrice dprice( + static_cast(dindex) < dbook.size() ? dbook[dindex].price : bts::md::IntPrice()); + MDP3Quantity dqty(dprice.is_nan() ? 0 : dbook[dindex].quantity.value()); + // init best price/qty + bts::md::IntPrice best_price(dprice); + MDP3Quantity curr_qty(0); + // find best impl price + for (auto& ib_tup : ibooks) { + auto& ib = std::get<0>(ib_tup); + auto index = std::get<1>(ib_tup); + if (cmp(ib[index].price, best_price)) + best_price = ib[index].price; + } + // combine quantities of books with the same top price + for (auto& ib_tup : ibooks) { + auto& ib = std::get<0>(ib_tup); + auto& index = std::get<1>(ib_tup); + auto& bl = ib[index]; + if (bl.price == best_price) { + curr_qty = MDP3Quantity(curr_qty.value() + bl.quantity.value()); + index++; + } + } + if (dprice == best_price) { + curr_qty = MDP3Quantity(curr_qty.value() + dqty.value()); + dindex++; + } + if (curr_qty.value() == 0) + break; + dest.push_back({best_price, curr_qty}); + } +} + +void FifoMergedSecurityMatcher::_fill(MDP3Order& order, std::vector* effects, FillDepot& fills, bool reset_show_qty) { + if (order.iceberg_quantity.has_value()) { + // the order's quantity cannot exceed the show qty (but it can be smaller). + bts_assert(order.quantity <= order.iceberg_quantity.value().show); + } + auto rem = _fill(order.key, order.side, order.price, get_total_qty(order), effects, fills); + reset_show_qty |= rem == get_total_qty(order); + set_total_qty(order, rem, reset_show_qty); +} + +inline SmallBookLevel tuple_to_small_book_level(std::tuple t) { + return SmallBookLevel(std::get<0>(t), std::get<1>(t)); +} + +MDP3Quantity FifoMergedSecurityMatcher::_fill( + MDP3OrderKey key, + api::Side side, + bts::md::IntPrice price, + MDP3Quantity quantity, + std::vector* effects, + FillDepot& fills) { + // + auto oside = bts::securities::other_side(side); + bts::orderman::SidePriceComp oside_comp(oside); + while (quantity.value() > 0) { + // Fill one price level at a time. + // While one market may have the best 1st price level, + // another may have the best 2nd price level. + auto* imatcher = _best_implied_matcher(oside); + auto itop = imatcher == nullptr ? + SmallBookLevel() : + imatcher->top_of_book(oside); + auto dtop = tuple_to_small_book_level(matcher()->book_level(oside, 0)); + bool is_dcross = !dtop.price.is_nan() && !oside_comp(price, dtop.price); + bool is_icross = !itop.price.is_nan() && !oside_comp(price, itop.price); + if (!is_dcross && !is_icross) + break; + // the order crossed at least one of the markets + auto ord_desc = [&]() { return fmt_order(key, side, price, quantity); }; + if (!is_dcross || (is_icross && oside_comp(itop.price, dtop.price))) { + // fill with the implied market + DIAG("filling ", ord_desc(), " in implied market ", fmt_price_level(oside, itop)); + auto target_fill_qty = std::min(quantity, itop.quantity); + auto rem = imatcher->fill(key, side, target_fill_qty, effects, fills); + auto fill_qty = target_fill_qty - rem; + quantity -= fill_qty; + DIAG("filled ", fill_qty, ". Order now: ", ord_desc()); + } else { + // fill with the direct market + DIAG("filling ", ord_desc(), " in direct market ", fmt_price_level(oside, dtop)); + bool local_effects = effects == nullptr; + if (local_effects) + effects = new std::vector(); + uint32_t fills_pos = effects->size(); + auto target_fill_qty = std::min(quantity, dtop.quantity); + auto rem = matcher()->fill(key, side, dtop.price, target_fill_qty, effects, true); + auto fill_qty = target_fill_qty - rem; + quantity -= fill_qty; + DIAG("filled ", fill_qty, ". Order now: ", ord_desc()); + MDP3Fill aggr_fill; + bts_assert(aggr_fill.price.is_nan()); + aggr_fill.key = key; + aggr_fill.aggressor = _to_aggressor(side); + for (auto it = effects->begin() + fills_pos; it != effects->end(); it++) { + auto& df = *it; + bts_assert(std::holds_alternative(df)); + auto& upd = std::get(df); + if (upd.key == key) { + if (aggr_fill.price.is_nan()) { + aggr_fill.price = upd.last_price; + aggr_fill.quantity = upd.last_fills; + } else if (aggr_fill.price == upd.last_price) { + aggr_fill.quantity = MDP3Quantity(aggr_fill.quantity.value() + upd.last_fills.value()); + } else { + fills.add_fill(security_id(), key, aggr_fill); + aggr_fill.quantity = MDP3Quantity(0); + aggr_fill.price = bts::md::IntPrice(); + bts_assert(aggr_fill.price.is_nan()); + } + } else { + fills.add_fill(security_id(), key, _to_passive_fill(upd, false)); + } + } + if (local_effects) { + delete effects; + effects = nullptr; + } + if (!aggr_fill.price.is_nan()) { + fills.add_fill(security_id(), key, aggr_fill); + } + } + } + bts_assert(quantity.value() >= 0); + return quantity; +} +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h new file mode 100644 index 000000000..0d24f9e24 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h @@ -0,0 +1,54 @@ +#pragma once +#include "lib/md/IntPrice.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include +#include +#include + +namespace bts::spartan::mdp3 { +struct FifoMergedSecurityMatcher { + struct InitArgs { + MDP3SecurityID security_id; + bool ignore_ops_on_deleted_orders; + }; + + FifoMergedSecurityMatcher(const InitArgs& args); + ~FifoMergedSecurityMatcher(); + + MDP3SecurityID security_id() const; + void implied_matching_status(bool value); + bool implied_matching_status() const; + void on_order_change(const SmallOrderChange&, std::vector*, FillDepot&); + FifoImpliedSecurityMatcher* add_implied_matcher(MDP3SecurityID); + FifoSecurityMatcher* matcher(); + const FifoSecurityMatcher* matcheri() const; + const std::vector& implied_matchers(); + // books + Book book(api::Side); + ImpliedBook implied_book(api::Side); + void merged_book(api::Side, std::vector&); + + private: + struct PImpl; + std::unique_ptr _me; + // hide copy constr + FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher&); + + FifoImpliedSecurityMatcher* _best_implied_matcher(api::Side); + // the order's params (quantity, and iceberg_quantity if applicable) + // will be appropriately adjusted according to the number of fills. + void _fill(MDP3Order&, std::vector*, FillDepot&, bool reset_show_qty); + MDP3Quantity _fill( + MDP3OrderKey, + api::Side, + bts::md::IntPrice, + MDP3Quantity, + std::vector*, + FillDepot&); + std::string _log_header() const; +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp new file mode 100644 index 000000000..cd06d48e7 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp @@ -0,0 +1,319 @@ +#include "FifoPriceLevelMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" + +namespace bts::spartan::mdp3 { + +// private helpers //////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +// orders can cease to be part of this level if there is +// a null slot (ex when the order is physcially moved to another +// price level or at the end of the queue) or the order is cxl'ed +// with the is_active marker +inline bool in_this_level(MatcherOrder const* o_ptr){ + return o_ptr!=nullptr && o_ptr->is_active; +} + +api::Side _opp_side(api::Side side){ + return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL : + api::Side::SIDE_BUY; +} +// O(n) linear search +MatcherOrder *FifoPriceLevelMatcher::_find(MDP3OrderKey key) { + for (MatcherOrder* op : _orders) + if (in_this_level(op) && op->key == key) + return op; + return nullptr; +} + +void FifoPriceLevelMatcher::_logically_delete(MatcherOrder &o) { + _log_order(o,"FifoPriceLevelMatcher::_logically_delete"); + o.is_active = false; +} +void FifoPriceLevelMatcher::_add_order(const MDP3Order &ord) { + MatcherOrder new_ord (ord.key,ord.side,ord.price,ord.quantity, + _priority++,ord.iceberg_quantity); + auto[p,res] = _master_order_map->try_emplace(new_ord.key,new_ord); + MatcherOrder* new_ord_ptr = &p->second; + _orders.push_back(new_ord_ptr); + if(!res) + // it exists, so this order is moved from another price level + // to this price level + *new_ord_ptr = new_ord; +} +// O(n) iteration through all orders to detect loadable ib's +void FifoPriceLevelMatcher::_reload_icebergs() { + _aux_usage.clear(); + for (auto it=_orders.begin();it!=_orders.end();it++){ + MatcherOrder* o_ptr = *it; + if(in_this_level(o_ptr) && o_ptr->ready_to_load_ib()){ + _aux_usage.push_back(o_ptr); + *it = nullptr; + } + } + // add it back + for (auto o : _aux_usage) { + o->reload_ib(); + o->priority = _priority++; + _orders.push_back(o); + } +} +std::tuple +FifoPriceLevelMatcher::_one_pass_cross_order(MDP3OrderKey cross_ord_key, + MDP3Quantity cross_ord_qty, + bool add_agg_side_to_match_effect, + std::vector* res) { + // go through all the orders and see which ones get filled + // (orders are sorted by priority) + for (MatcherOrder* op : _orders) { + if (in_this_level(op)) { + auto& o = *op; + auto [o_rem, cross_rem, filled] = o.cross(cross_ord_qty.value()); + cross_ord_qty = MDP3Quantity(cross_rem); + if(res != nullptr){ + OrderUpdate passive_upd = + { + o.key, + o.total_qty().value() > 0 ? + OrderUpdate::Status::PARTIALLY_FILLED : + OrderUpdate::Status::FILLED, + _sec, + _side, + MDP3Quantity(filled), + _price, + OrderUpdate::FillType::PASSIVE + }; + MatcherUtils::validate_outgoing_update(passive_upd); + res->push_back(passive_upd); + if(add_agg_side_to_match_effect){ + OrderUpdate agg_upd = + { + cross_ord_key, + cross_ord_qty.value() > 0 ? + OrderUpdate::Status::PARTIALLY_FILLED : + OrderUpdate::Status::FILLED, + _sec, + _opp_side(_side), + MDP3Quantity(filled), + _price, + OrderUpdate::FillType::AGGRESSIVE + }; + MatcherUtils::validate_outgoing_update(agg_upd); + res->push_back(agg_upd); + } + } + if(_trace.log(o.key) || _trace.log(cross_ord_key)) + LOG(OrderMan, Diag, + "FifoPriceLevelMatcher::_one_pass_cross_order " + "{} (new_qty:{}) crossed by {} (new_qty:{})", + o.key, o_rem, cross_ord_key, cross_rem); + + if (cross_rem > 0) + bts_assert(o_rem == 0); // visible qty should be completely crossed + else + break; // crossed order is completely filled + } + } + if (cross_ord_qty.value() > 0) + bts_assert(quantity().value() == 0); + // reload the filled ib's and put them at the back + _reload_icebergs(); + return std::make_tuple(total_quantity(), cross_ord_qty); +} +void FifoPriceLevelMatcher::_log_order(const MatcherOrder& ord, + const std::string& msg){ + if(_trace.log(ord.key)) + LOG(OrderMan, Diag, + "{} key:{} side:{} qty:{} ib_qty:{} price:{}", + msg, ord.key, static_cast(side()), ord.quantity, + ord.ib.has_value() ? ord.ib.value().total : MDP3Quantity(0), + price()); +} +void FifoPriceLevelMatcher::_log_order(const MDP3Order& ord, + const std::string& msg){ + if(_trace.log(ord.key)) + LOG(OrderMan, Diag, + "{} key:{} side:{} qty:{} ib_qty:{} price:{}", + msg, ord.key, static_cast(ord.side), ord.quantity, + ord.iceberg_quantity.has_value() ? FMT(ord.iceberg_quantity.value()) + : FMT(MDP3Quantity(0)), + ord.price.price()); +} +// constructors /////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +FifoPriceLevelMatcher::FifoPriceLevelMatcher( + std::map& master_order_map, + MDP3SecurityID sec, + md::IntPrice price, + api::Side side, + const Trace& trace) + : _master_order_map(&master_order_map), + _sec(sec), _price(price), _side(side), _orders(), _aux_usage(), + _priority(0), _trace(trace) {} +FifoPriceLevelMatcher::FifoPriceLevelMatcher(const FifoPriceLevelMatcher &o) { + *this = o; +} +FifoPriceLevelMatcher::FifoPriceLevelMatcher(FifoPriceLevelMatcher &&o) { + *this = std::move(o); +} +// operator overloads ///////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +FifoPriceLevelMatcher & +FifoPriceLevelMatcher::operator=(const FifoPriceLevelMatcher &o) { + _master_order_map = o._master_order_map; + _sec = o._sec; + _price = o._price; + _side = o._side; + _orders = o._orders; + _aux_usage = o._aux_usage; + _priority = o._priority; + return *this; +} +FifoPriceLevelMatcher & +FifoPriceLevelMatcher::operator=(FifoPriceLevelMatcher &&o) { + _master_order_map = std::move(o._master_order_map); + _sec = std::move(o._sec); + _price = std::move(o._price); + _side = std::move(o._side); + _orders = std::move(o._orders); + _aux_usage = std::move(o._aux_usage); + _priority = std::move(o._priority); + return *this; +} +bool FifoPriceLevelMatcher::operator<(const FifoPriceLevelMatcher &o) const { + bts_assert(_side == o._side); + // bid book is sorted high to low (since highest is the top of the book) + return _side == api::Side::SIDE_SELL ? _price < o._price : o._price < _price; +} +// simple accessors /////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +std::tuple FifoPriceLevelMatcher::n_orders_qty( + bool visible_only) const{ + auto [c, q] = std::make_tuple(0, 0); + for (MatcherOrder* o : _orders) + if(in_this_level(o)){ + c++; + q += visible_only ? o->working_qty().value() : o->total_qty().value(); + } + return std::make_tuple(c, q); +} + +bool FifoPriceLevelMatcher::populated() const { + for (auto* o : _orders) + if (in_this_level(o) && o->working_qty().value() > 0) + return true; + return false; +} + +void FifoPriceLevelMatcher::copy_values_to(std::vector& out_v) const{ + out_v.clear(); + for(MatcherOrder* o : _orders) + if(in_this_level(o)) + out_v.push_back(*o); +} +// order interactions ///////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +void FifoPriceLevelMatcher::on_new_same_side_order(const MDP3Order &new_ord) { + bts_assert(new_ord.price == _price); + bts_assert(new_ord.side == _side); + _add_order(new_ord); +} + +std::tuple FifoPriceLevelMatcher::on_new_opp_order( + MDP3OrderKey opp_ord_key, + MDP3Quantity cross_qty, + bool add_agg_side_to_match_effect, + std::vector* res) { + auto rv = _one_pass_cross_order(opp_ord_key, cross_qty, + add_agg_side_to_match_effect, res); + // continue until either the crossing order is consumed + // or the price level is consumed + while (std::get<0>(rv).value() > 0 && std::get<1>(rv).value() > 0) + rv = _one_pass_cross_order(opp_ord_key, std::get<1>(rv), + add_agg_side_to_match_effect, res); + return rv; +} + +void FifoPriceLevelMatcher::on_order_update( + const MDP3Order &upd_ord, + MatcherOrder* ord_ptr_hint) { + // ord_ptr_hint is a hint, if its populated use, else find + bts_assert(upd_ord.side == _side); + bts_assert(upd_ord.price == _price); + if(ord_ptr_hint == nullptr){ + auto map_it = _master_order_map->find(upd_ord.key); + bts_assert(map_it != _master_order_map->end()); + ord_ptr_hint = &map_it->second; + } + MatcherOrder& ord = *ord_ptr_hint; + if (upd_ord.quantity.value() > ord.quantity.value()) { + // remove it from the current location O(N) and add it to + // the end. unlike qty decreases this is a lot more expensive + remove_from_price_level(ord.key); + _orders.push_back(&ord); + // udpate value + ord.quantity = upd_ord.quantity; + } else { + ord.quantity = upd_ord.quantity; + if (ord.ib.has_value() && upd_ord.iceberg_quantity.has_value() && ord.ib.value() != upd_ord.iceberg_quantity.value()) + ord.ib = upd_ord.iceberg_quantity; + } +} +void FifoPriceLevelMatcher::on_order_cancel(MatcherOrder& order, + std::vector* res){ + _logically_delete(order); + OrderUpdate o = { + order.key, + OrderUpdate::Status::CANCELED, + _sec, + _side, + MDP3Quantity(0), + _price, + std::nullopt + }; + if(res != nullptr) + res->push_back(o); +} +void FifoPriceLevelMatcher::on_order_remove(MDP3OrderKey k, + std::vector* res) { + MatcherOrder *ord_p = _find(k); + bts_assert(ord_p != nullptr); + on_order_cancel(*ord_p,res); +} +void FifoPriceLevelMatcher::remove_from_price_level(MDP3OrderKey k) { + // puts a nullptr in place of where this order was + // (perioric cleanup will physically delete the slot) + auto it = std::find_if(_orders.begin(), _orders.end(), + [&](MatcherOrder* o) { + return in_this_level(o) && o->key == k; + }); + bts_assert(it != _orders.end()); + bts_assert(*it != nullptr); + *it = nullptr; +} + +void FifoPriceLevelMatcher::cleanup() { + _orders.erase(std::remove_if(_orders.begin(), _orders.end(), + [](const MatcherOrder* o) { + return !in_this_level(o); + }), + _orders.end()); +} +std::string FifoPriceLevelMatcher::print_mbo(size_t n_orders) const{ + std::ostringstream buff; + auto[n_ords,qty] = n_orders_qty(); + buff << (side()==api::Side::SIDE_BUY ? "B" : "S") << "," << price(); + buff << "," << qty << "," << n_ords << ":"; + size_t i=0; + for (MatcherOrder* o : _orders){ + if(in_this_level(o)){ + if(++i>n_orders) + break; + buff << o->key << "," << o->quantity << ":"; + } + } + return buff.str(); +} +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h new file mode 100644 index 000000000..356fa3668 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h @@ -0,0 +1,116 @@ +#pragma once + +// did not want to fwd decl everything specially +// the templated classes (will fix later) +#include "MatchingCommon.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" + +namespace bts::spartan::mdp3 { + +// Builds the order state for all orders correponding to one +// price level for one security +class FifoPriceLevelMatcher { + private: + // this is the order_map that the sec level matcher owns + // for all the orders for the security. orders are created + // here but added to the map mantained by the + // sec level matcher + std::map* _master_order_map; + MDP3SecurityID _sec; + md::IntPrice _price; + api::Side _side; + std::vector _orders; + std::vector _aux_usage; + uint32_t _priority; + Trace _trace; +// helpers //////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + private: + MatcherOrder *_find(MDP3OrderKey key); + void _logically_delete(MatcherOrder &o); + void _add_order(const MDP3Order &ord); + // crosses the cross_order with existing orders. returns + // the remaining qty in the level and the cross order + void _reload_icebergs(); + // crosses the order in the args with the orders at this level. + // returns the tuple of rem qty at this level and the rem qty + // of the cross order + std::tuple + _one_pass_cross_order(MDP3OrderKey cross_ord_key, + MDP3Quantity cross_ord_qty, + bool add_agg_side_to_match_effect, + std::vector* res); + void _log_order(const MatcherOrder& ord, + const std::string& msg); + void _log_order(const MDP3Order& ord, const std::string& msg); + public: + // constructors //////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////// + FifoPriceLevelMatcher(std::map&, + MDP3SecurityID sec, md::IntPrice price, + api::Side side, const Trace&); + FifoPriceLevelMatcher(const FifoPriceLevelMatcher &); + FifoPriceLevelMatcher(FifoPriceLevelMatcher &&); + // operator overloads ////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////// + FifoPriceLevelMatcher &operator=(const FifoPriceLevelMatcher &o); + FifoPriceLevelMatcher &operator=(FifoPriceLevelMatcher &&o); + bool operator==(const FifoPriceLevelMatcher &o) const { + bts_assert(_side == o._side); + return _price == o._price; + } + bool operator!=(const FifoPriceLevelMatcher &o) const { + bts_assert(_side == o._side); + return _price != o._price; + } + bool operator<(const FifoPriceLevelMatcher &o) const; + // simple accessors //////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////// + std::tuple n_orders_qty(bool visible_only = true) const; + MDP3Quantity quantity() const { + return MDP3Quantity(std::get<1>(n_orders_qty())); + } + MDP3Quantity total_quantity() const { + return MDP3Quantity(std::get<1>(n_orders_qty(false))); + } + uint32_t n_orders() const { return std::get<0>(n_orders_qty()); } + bool populated() const; + md::IntPrice price() const { return _price; } + double price_double() const { return _price.price(); } + api::Side side() const { return _side; } + MatcherOrder *find_order(MDP3OrderKey key) { return _find(key); } + BookLevel book_level() const { + return BookLevel{_price, quantity(), static_cast(n_orders())}; + } + // used for testing + void copy_values_to(std::vector& out_v) const; + // mutators (silly reason) ///////////////////////////////////////////////// + void set_price_side(md::IntPrice p, api::Side s) { + _price = p; + _side = s; + } + // order interactions ////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////// + // order with the same side and price as this price level + void on_new_same_side_order(const MDP3Order &new_ord); + // order update such that only the qty changed (other changes + // are handled at a higher level) + // new order such that it crosses this price level + std::tuple on_new_opp_order( + MDP3OrderKey opp_ord_key, + MDP3Quantity opp_ord_qty, + bool add_agg_side_to_match_effect, + std::vector* = nullptr); + void on_order_update(const MDP3Order& upd_ord, + MatcherOrder* ord_hint = nullptr); + void on_order_cancel(MatcherOrder& order, + std::vector* = nullptr); + void on_order_remove(MDP3OrderKey k, + std::vector* = nullptr); + void remove_from_price_level(MDP3OrderKey k); + void cleanup(); + std::string print_mbo(size_t n_orders) const; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp new file mode 100644 index 000000000..a76339474 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp @@ -0,0 +1,552 @@ +#include "FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include + +namespace bts::spartan::mdp3 { + +// helpers //////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +inline bool _is_cross(const MDP3Order &ord, + const FifoPriceLevelMatcher &price_level) { + return ord.side != price_level.side() && + ((ord.side == api::Side::SIDE_BUY && + ord.price >= price_level.price()) || + (ord.side == api::Side::SIDE_SELL && + ord.price <= price_level.price())); +} + +inline MDP3Quantity total_quantity(const MDP3Order &ord) { + return ord.iceberg_quantity.has_value() ? ord.iceberg_quantity.value().total + : ord.quantity; +} + +inline api::Side _opp_side(api::Side side) { + return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL + : api::Side::SIDE_BUY; +} + +// O(lg(N)) on the price levels +FifoPriceLevelMatcher * +FifoSecurityMatcher::_find_price_level(api::Side side, md::IntPrice price) { + auto &bk = _book(side); + _search_tok.set_price_side(price, side); + auto it = bk.find(_search_tok); + return it != bk.end() ? &*it : nullptr; +} + +std::tuple +FifoSecurityMatcher::_find_order(MDP3OrderKey key) { + auto map_it = _order_map.find(key); + if(map_it != _order_map.end()){ + auto& ord = map_it->second; + if(ord.is_active){ + FifoPriceLevelMatcher* p_level = _find_price_level(ord.side,ord.price); + return std::make_tuple(p_level,&ord); + } + } + return std::tuple( + nullptr, nullptr); +} + +FifoPriceLevelMatcher* FifoSecurityMatcher::_top(api::Side side){ + for(auto& plevel : _book(side)) + if(plevel.populated()) + return &plevel; + return nullptr; +} + +// this is expensive +void FifoSecurityMatcher::_cleanup() { + // remove orders from the price levels + for (auto& p : _bids.book()) + p.cleanup(); + for (auto& p : _asks.book()) + p.cleanup(); + // remove empty levels + auto rem_if = [](const auto &p_lev) { return !p_lev.populated(); }; + _bids.book().erase(std::remove_if(_bids.book().begin(), _bids.book().end(), rem_if), + _bids.book().end()); + _asks.book().erase(std::remove_if(_asks.book().begin(), _asks.book().end(), rem_if), + _asks.book().end()); + // remove inactive orders from map + std::experimental::erase_if(_order_map, [](const auto& p) { + auto& ord = std::get<1>(p); + return !ord.is_active; + }); +} + +FifoPriceLevelMatcher* FifoSecurityMatcher::_get_or_add_price_level(api::Side side, md::IntPrice price) { + auto* price_level = _find_price_level(side, price); + if (price_level == nullptr) { + auto& same_side_book = _book(side); + // this is the only place where a new price level is created + auto p = same_side_book.emplace(_order_map, _sec, price, side, _trace); + bts_assert(std::get<1>(p)); + price_level = &*std::get<0>(p); + } + return price_level; +} + +void FifoSecurityMatcher::_add_resting_order(const MDP3Order& ord) { + _get_or_add_price_level(ord.side, ord.price)->on_new_same_side_order(ord); +} + +void FifoSecurityMatcher::_on_new_order(const MDP3Order &ord, + std::vector* res) { + _log_trace_order(ord,"FifoSecurityMatcher::_on_new_order begin"); + auto& opp_book = _book(_opp_side(ord.side)); + uint32_t new_ord_rem = total_quantity(ord).value(); + for (FifoPriceLevelMatcher &pl : opp_book) { + if (_is_cross(ord, pl)){ + auto t = pl.on_new_opp_order(ord.key, MDP3Quantity(new_ord_rem), + true, res); + new_ord_rem = std::get<1>(t).value(); + } + else + break; + if (new_ord_rem == 0) + break; + bts_assert(pl.quantity().value() == 0); + // stops crossing when the price level is not a cross or there + // no qty left + } + if (new_ord_rem != 0) { + // either the price did not cross into the opp market + // or it cleared out the entire opposite side + // in either case, it becomes a resting order + MDP3Order ord_corrected = ord; + if (ord_corrected.iceberg_quantity.has_value()) { + auto ibq = ord.iceberg_quantity.value(); + auto filled_qty = ibq.total - MDP3Quantity(new_ord_rem); + // important: don't subtract filled_qty from the total + // of the IcebergQuantity (ibq) param passed to 'get_reload_qty.' + ord_corrected.quantity = iceberg_helper::get_reload_qty(ord.quantity, filled_qty, ibq); + ord_corrected.iceberg_quantity.value().total.value() = new_ord_rem; + } else + ord_corrected.quantity.value() = new_ord_rem; + _add_resting_order(ord_corrected); + } + else{ + // since _on_new_order is also the same code path + // thats called when an order changes price levels + auto it = _order_map.find(ord.key); + if(it != _order_map.end()){ + MatcherOrder& ex_ord = it->second; + ex_ord.quantity = MDP3Quantity(0); + ex_ord.is_active = false; + } + } +} + +void FifoSecurityMatcher::_on_order_update(const MDP3Order &ord, + bool price_only, + std::vector* res) { + _log_trace_order(ord,"FifoSecurityMatcher::_on_order_update begin"); + auto[price_level,curr_ord] = _find_order(ord.key); + if(price_level == nullptr || curr_ord == nullptr){ + if(_ignore_ops_on_deleted_orders) + return; + } + bts_assert(price_level != nullptr); + bts_assert(curr_ord != nullptr); + auto new_ord = ord; + // ib total qty is considered immutable, and show qty + // changes are intepreted through regular quantity chagnes. + // Once created, the iceberg_quantity field carries no new + // information about the order, so it is expected to be null. + bts_assert(!new_ord.iceberg_quantity.has_value()); + // side can not be changed (or can it?) + bts_assert(price_level->side() == new_ord.side); + if (curr_ord->ib.has_value()) { + // check if show qty changed + if (new_ord.quantity.value() > 0) { + // For iceberg orders, quantity and show quantity + // changes are one in the same. + // Note that quantity updates are bounded by the iceberg's + // total qty, which is considered immutable. + if (new_ord.quantity > curr_ord->ib.value().total) + new_ord.quantity = curr_ord->ib.value().total; + curr_ord->ib.value().show = new_ord.quantity; + } else { + new_ord.quantity = curr_ord->quantity; + } + new_ord.iceberg_quantity = curr_ord->ib; + } + if (price_only) { + new_ord.quantity = curr_ord->quantity; + } + // check for price change + if (price_level->price() != new_ord.price) { + // remove from current level and add to another + price_level->remove_from_price_level(curr_ord->key); + _on_new_order(new_ord, res); + } else { + price_level->on_order_update(new_ord, curr_ord); + } + _log_trace_order(ord,"FifoSecurityMatcher::on_order_update end"); +} + +void FifoSecurityMatcher::_on_order_remove(MDP3OrderKey key, + std::vector* res) { + if(_trace.log(key)) + LOG(OrderMan, Diag, + "FifoSecurityMatcher::_on_order_remove " + "eid:{} key:{}", + _event_id, key); + + auto[price_level,curr_ord] = _find_order(key); + if(price_level == nullptr || curr_ord == nullptr) + if(_ignore_ops_on_deleted_orders) + return; + bts_assert(price_level != nullptr); + bts_assert(curr_ord != nullptr); + price_level->on_order_cancel(*curr_ord,res); + // we can short circuit this by setting order's is_active + // marker to false here, but its more consistent to let + // the price level handlers take care of this + //(which does make it O(lg(M*N)) instead of lg(N), for the + // extra price level lookup) +} +void FifoSecurityMatcher::_on_order_not_affecting_books( + const SmallOrderChange& o, + std::vector* res){ + auto& ord = o.order; + switch (o.change) { + case OrderChange::added: + case OrderChange::updated: + case OrderChange::updated_price: + { + // since this does not affect the market all we need to do + // is deal with the cross + auto& opp_book = _book(_opp_side(ord.side)); + uint32_t new_ord_rem = total_quantity(ord).value(); + for (const FifoPriceLevelMatcher &pl : opp_book) { + if(!pl.populated()) + continue; + if (_is_cross(ord, pl)){ + uint32_t level_qty = + static_cast(pl.total_quantity().value()); + auto[new_level_qty_ret,new_ord_rem_ret] = + MatcherOrder::compute_cross(level_qty,new_ord_rem); + auto fill_size = level_qty - new_level_qty_ret; + new_ord_rem = new_ord_rem_ret; + if(res != nullptr) + res->push_back(OrderUpdate{ + ord.key, + new_ord_rem > 0 ? + OrderUpdate::Status::PARTIALLY_FILLED : + OrderUpdate::Status::FILLED, + ord.security, + ord.side, + MDP3Quantity(fill_size), + pl.price(), + OrderUpdate::FillType::AGGRESSIVE + }); + } + else + break; + if (new_ord_rem == 0) + break; + } + break; + } + case OrderChange::removed: + if(res != nullptr) + res->push_back(OrderUpdate{ + ord.key, + OrderUpdate::Status::CANCELED, + ord.security, + ord.side, + MDP3Quantity(0), + md::IntPrice(), + std::nullopt + }); + // nothing to do since this order was never made a part of the book + break; + } +} +// constructors /////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +FifoSecurityMatcher::FifoSecurityMatcher(MDP3SecurityID security, + size_t event_id) + : + FifoSecurityMatcher( + FifoSecurityMatcher::FifoSecurityMatcherArgs{ + security,event_id,true,nullptr + }) + {} + +FifoSecurityMatcher::FifoSecurityMatcher(MDP3SecurityID security, + size_t event_id, + const Orders& orders) + : + FifoSecurityMatcher( + FifoSecurityMatcher::FifoSecurityMatcherArgs{ + security,event_id,true,&orders + }) + {} + +FifoSecurityMatcher::FifoSecurityMatcher( + FifoSecurityMatcher::FifoSecurityMatcherArgs args) + : + _order_map(), + _bids(), + _asks(), + _search_tok(_order_map, MDP3SecurityID{}, + md::IntPrice(), api::Side::SIDE_BUY, Trace()), + _sec(args.security_id), + _event_id(args.event_id), + _trace(), + _ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders), + _clean_up_op_threshold(2500), + _running_op_count(0) { + if(args.orders != nullptr){ + const auto& orders = *args.orders; + std::vector sorted_orders; + sorted_orders.reserve(orders.size()); + for (const auto &kv : orders) + sorted_orders.emplace_back(kv.second); + + std::sort(sorted_orders.begin(), sorted_orders.end(), + [](const auto &o1, const auto &o2) { + return o1.priority < o2.priority; + }); + LOG(OrderMan, Diag, "FifoSecurityMatcher::cons begin eid:{} n_orders:{}", + _event_id, orders.size()); + for (auto const &o : sorted_orders) + _on_new_order(o); + LOG(OrderMan, Diag, "FifoSecurityMatcher::cons end eid:{}", _event_id); + _log_mbo(10,100); + } + _running_op_count = 0; +} +FifoSecurityMatcher::~FifoSecurityMatcher(){ + _log_trace_order_state("FifoSecurityMatcher::dest"); +} +// SecurityMatcher implementation ///////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +const Book& FifoSecurityMatcher::BookKeeper::mbp() { + if (_is_dirty) { + int i = 0; + for (const auto& p_level : _book) { + if (p_level.populated()) { + if (i < MDP3_MAX_BOOK_DEPTH) + _mbp[i++] = p_level.book_level(); + else + break; + } + } + // Explicitly set the remaining entries to nan/0 + // (they could have stale values since the book is recycled). + while (i < MDP3_MAX_BOOK_DEPTH) { + _mbp[i].price = md::IntPrice(); + _mbp[i++].quantity.value() = 0; + } + _is_dirty = false; + } + return _mbp; +} +Book FifoSecurityMatcher::book(api::Side side) { + return (side == api::Side::SIDE_BUY ? _bids : _asks).mbp(); +} +size_t FifoSecurityMatcher::total_n_orders(api::Side side) const{ + auto rv=0; + for (const auto& p_level : _book(side)) + rv+=p_level.n_orders(); + return rv; +} +MDP3Quantity FifoSecurityMatcher::total_qty(api::Side side) const{ + auto rv=0; + for (const auto& p_level : _book(side)) + rv+=p_level.total_quantity().value(); + return MDP3Quantity(rv); +} +std::tuple FifoSecurityMatcher::book_level ( + api::Side side, size_t idx) const { + size_t i = 0; + for (const auto& p_level : _book(side)) + if (p_level.populated() && i++ == idx) + return std::make_tuple(p_level.price(), p_level.total_quantity()); + return std::make_tuple(md::INT_NAN, MDP3Quantity(0)); +} +void FifoSecurityMatcher::on_order_change(SmallOrderChange const &o, + std::vector* res) { + if(!o.affect_book){ + _on_order_not_affecting_books(o,res); + return; + } + switch (o.change) { + case OrderChange::added: + _on_new_order(o.order, res); + break; + case OrderChange::updated: + _on_order_update(o.order, false,res); + break; + case OrderChange::updated_price: + _on_order_update(o.order, true,res); + break; + case OrderChange::removed: + _on_order_remove(o.order.key, res); + break; + } + _invalidate_mbp(); + if(_running_op_count++ > _clean_up_op_threshold){ + _running_op_count = 0; + _cleanup(); + } +} + +void FifoSecurityMatcher::on_order_completed(MDP3OrderKey key) { + // todo: actual implementation + // (this is a hack that treats completed orders like cancels) + auto [pl, ord] = _find_order(key); + bts_assert(pl != nullptr); + bts_assert_msg(ord != nullptr, "did not find order that was supposedely 'completed.'"); + MDP3Order order; + order.key = key; + order.side = api::SIDE_UNDEFINED; + SmallOrderChange ch (order,OrderChange::ChangeType::removed); + on_order_change(ch); +} + +MDP3Quantity FifoSecurityMatcher::fill(MDP3OrderKey aggr_key, + api::Side side, + md::IntPrice agg_price, + MDP3Quantity qty, + std::vector* res, + bool add_aggr_fill){ + FifoPriceLevelMatcher* top = _top(_opp_side(side)); + if(top != nullptr){ + bts_assert(top->price()==agg_price || agg_price.is_nan()); + auto t = top->on_new_opp_order(aggr_key,qty,add_aggr_fill,res); + _invalidate_mbp(); + return std::get<1>(t); + } + return MDP3Quantity(0); +} + +std::optional +FifoSecurityMatcher::has_same_orders(const Orders &ord_map) { + if(_trace.is_ignored_event(_event_id)) + return std::nullopt; + for (const auto &kv : ord_map) { + const auto &exp_ord = kv.second; + auto [plevel, ord] = _find_order(exp_ord.key); + auto match = plevel != nullptr && ord != nullptr && + plevel->price() == exp_ord.price && + plevel->side() == exp_ord.side && + ord->quantity == exp_ord.quantity && + ((!exp_ord.iceberg_quantity.has_value() || + exp_ord.iceberg_quantity.value().total.value() == 0) || + ord->ib == exp_ord.iceberg_quantity); + if (!match) { + std::ostringstream buff; + buff << "eid:" << _event_id << " n_ords:" << ord_map.size() + << std::endl; + buff << "Book:" << print_book(10) << std::endl; + if (ord == nullptr){ + buff << "Order with Key:" << exp_ord.key.value() << " not found" + << std::endl; + buff << "Expected price:" << exp_ord.price << std::endl; + buff << "Expected qty:" << exp_ord.quantity << std::endl; + buff << "Expected side:" << exp_ord.side << std::endl; + buff << "Expected IB state:" << exp_ord.iceberg_quantity << std::endl; + } + else { + buff << "Order with Key:" << exp_ord.key.value() + << " does not match" << std::endl; + buff << "Expected price:" << exp_ord.price + << " matcher:" << plevel->price() << std::endl; + buff << "Expected qty:" << exp_ord.quantity + << " matcher:" << ord->quantity << std::endl; + buff << "Expected side:" << exp_ord.side + << " matcher:" << plevel->side() << std::endl; + buff << "Expected IB state:" << exp_ord.iceberg_quantity + << " matcher:" << ord->ib << std::endl; + } + return buff.str(); + } + } + LOG(OrderMan, Diag, "FifoSecurityMatcher::has_same_orders passed"); + return std::nullopt; +} + +void FifoSecurityMatcher::clear() { + _bids.book().clear(); + _asks.book().clear(); + _order_map.clear(); + _invalidate_mbp(); + _running_op_count = 0; +} +// logging and printing /////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +std::string FifoSecurityMatcher::print_book(size_t n_levels) { + // auto rem_if = [](const auto& p_lev) { return !p_lev.populated(); }; + std::ostringstream buff; + auto add_to_buff = [&](const auto &pl) { + std::string side = pl.side() == api::Side::SIDE_BUY ? "B" : "S"; + buff << side << ":" << pl.price_double() << ":" << pl.quantity().value() + << ":" << pl.n_orders() << std::endl; + }; + // asks are stored lowest to highest + size_t i=0; + for (const auto &pl : _asks.book()){ + if(++i>n_levels) + break; + add_to_buff(pl); + } + // bids are scored highest to lowest + i=0; + for (const auto &pl : _bids.book()){ + if(++i>n_levels) + break; + add_to_buff(pl); + } + return buff.str(); +} +void FifoSecurityMatcher::_log_trace_order_state(const std::string& msg){ + for (const auto& kv : _trace.traced_orders){ + MDP3OrderKey k = kv.first; + auto[pl,o] = _find_order(k); + if(pl != nullptr && o != nullptr) + LOG(OrderMan, Diag, + "Order state msg:{} eid:{} key:{} qty:{} ib:{} price:{}", + msg, _event_id, o->key, o->quantity, o->ib, pl->price()); + } +} +void FifoSecurityMatcher::_log_trace_order(const MDP3Order& ord, + const std::string& msg){ + if(_trace.log(ord.key,_event_id)){ + LOG(OrderMan, Diag, + "{} eid:{} key:{} side:{} qty:{} ib_qty:{} price:{} book:{}", + msg, _event_id, ord.key, static_cast(ord.side), ord.quantity, + ord.iceberg_quantity, ord.price.price(), print_book()); + _log_trace_order_state(msg); + } +} +void FifoSecurityMatcher::_log_mbo(size_t n_levels, size_t n_orders){ + if(_trace.log(_event_id)){ + std::ostringstream buff; + buff << "eid:" << _event_id << std::endl; + size_t i=0; + for (const auto &pl : _asks.book()){ + if(++i>n_levels) + break; + buff << pl.print_mbo(n_orders) << std::endl; + } + i=0; + for (const auto &pl : _bids.book()){ + if(++i>n_levels) + break; + buff << pl.print_mbo(n_orders) << std::endl; + } + auto msg = buff.str(); + LOG(OrderMan, Diag, "{}", msg); + } +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h new file mode 100644 index 000000000..c96ed3a3f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h @@ -0,0 +1,159 @@ +#pragma once + +// did not want to fwd decl everything specially +// the templated classes (will fix later) +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "FifoPriceLevelMatcher.h" +#include +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "SecurityMatcher.h" +#include + +namespace bts::spartan::mdp3 { + +class FifoSecurityMatcher : public SecurityMatcher { +public: + struct FifoSecurityMatcherArgs{ + MDP3SecurityID security_id; + const size_t event_id = 0; + const bool ignore_ops_on_deleted_orders = true; + Orders const* orders = nullptr; + }; + private: + struct BookKeeper { + boost::container::flat_set& book() { return _book; } + const boost::container::flat_set& booki() const { return _book; } + const Book& mbp(); + void invalidate() { _is_dirty = true; } + + private: + Book _mbp; + bool _is_dirty = true; + boost::container::flat_set _book; + }; + // for orders we need a structure to provide fast access + // and where adding and removing elements does not invalidate + // iterators (or cached pointers) + // https://en.cppreference.com/w/cpp/container#Iterator_invalidation + // while map (trees) are slower than unordered_map (pure hashtables), + // pointers get invalidated when they are rehashed (since we dont + // have a key collision, would it be rehashed?) + std::map _order_map; + // ask prices are sorted lowest to highest and and bid prices + // are sorted highest to lowest (see BookKeeper::_book). + BookKeeper _bids, _asks; + FifoPriceLevelMatcher _search_tok; + const MDP3SecurityID _sec; + const size_t _event_id; + const Trace _trace; + const bool _ignore_ops_on_deleted_orders; + const size_t _clean_up_op_threshold; + size_t _running_op_count; + + private: + // helpers ///////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////// + // complexity: O(lg(n)) + FifoPriceLevelMatcher *_find_price_level(api::Side, md::IntPrice price); + boost::container::flat_set &_book(api::Side side) { + return (side == api::Side::SIDE_BUY ? _bids : _asks).book(); + } + const boost::container::flat_set & + _book(api::Side side) const { + return (side == api::Side::SIDE_BUY ? _bids : _asks).booki(); + } + void _invalidate_mbp() { + _bids.invalidate(); + _asks.invalidate(); + } + // the complexity is O(lgN+lgM == lg(M*N)), since first it first searchess + // for the order, from the order it gets the price, side and searches + // for price level in the book (flat_set) + std::tuple + _find_order(MDP3OrderKey key); + FifoPriceLevelMatcher* _top(api::Side side); + void _cleanup(); + // order interaction + void _on_new_order(const MDP3Order&, std::vector* = nullptr); + void _on_order_update(const MDP3Order&, bool price_only = false,std::vector* = nullptr); + void _on_order_remove(MDP3OrderKey, + std::vector* = nullptr); + void _on_order_not_affecting_books(const SmallOrderChange& o, + std::vector* res); + std::string _log_header() const; + public: + // cons/dest /////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////// + FifoSecurityMatcher(MDP3SecurityID security, size_t event_id = 0); + FifoSecurityMatcher(MDP3SecurityID security, + size_t event_id, + Orders const &); + FifoSecurityMatcher(FifoSecurityMatcher::FifoSecurityMatcherArgs); + ~FifoSecurityMatcher(); + // SecurityMatcher implementation ////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////// + Book book(api::Side) override; + // The qty, and add_aggr_fill params address a couple of issues wrt 'merged matching:' + // Particular quantities of passive fills need to be generated when + // an aggressor order is filled against an implied market, while the aggressor + // fills must be generated independently (there are several passive legs, + // but only one aggressor security). The 'add_aggr_fill' param would be false in + // this case, and the 'qty' param would be limited by either a thinner market in + // another leg, by the volume in this market's top price level, or by the aggressor + // order's quantity itself. + // + // In the case of direct market fills, both aggressor and passive fills + // should be generated (of course, so 'add_aggr_fill' would be true). + // However, the fill qty may also be constricted to a certain number of + // price levels. This is because the market with the best price doesn't + // necessarily have the 2nd best price as well. + MDP3Quantity fill(MDP3OrderKey aggr_key, api::Side side, md::IntPrice price, + MDP3Quantity qty, + std::vector* res, bool add_aggr_fill); + std::tuple book_level( + api::Side side, size_t idx) const override; + void on_order_change(SmallOrderChange const &, + std::vector* = nullptr) override ; + void on_order_completed(MDP3OrderKey key); + // For testing + std::optional has_same_orders(Orders const &); + size_t total_n_orders(api::Side) const; + MDP3Quantity total_qty(api::Side) const; + // Removes existing orders. + void clear(); + // Removes existing orders, then populates the book with the + // specified sequence of MDP3Orders. + template + void reset(Iter first, Iter last) { + clear(); + while (first != last) { + _add_resting_order(*first); + first++; + } + } + + std::tuple find_order(MDP3OrderKey key) { return _find_order(key); } +// debugging logging ////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +public: + std::string print_book(size_t n_levels=3); +private: + void _log_trace_order_state(const std::string& msg); + void _log_trace_order(const MDP3Order& ord, const std::string& msg); + void _log_mbo(size_t n_levels,size_t n_orders); + // Inserts the specified order at the appropriate price level without + // attempting to fill it. While this won't matter in most cases + // (resting orders don't cross the opposite market anyways) 'on_new_order' + // imposes a side effect on iceberg orders which '_add_resting_order' does not: + // + // after attempting to fill the new iceberg order, + // '_on_new_order' resets the order's [visible]quantity + // to the minimum of its show, and total quantity. + // + // This is obviously problematic if the aim is to inject an exact state + // into the matcher, as the order's visible quantity is often less than + // the show quantity (e.g., after partial fills, before the next reload). + void _add_resting_order(const MDP3Order&); + FifoPriceLevelMatcher* _get_or_add_price_level(api::Side side, md::IntPrice); +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h new file mode 100644 index 000000000..bf25956c3 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h @@ -0,0 +1,46 @@ +#pragma once +#include "lib/md/IntPrice.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { +struct ImpliedFifoSecurityMatcher { + ImpliedFifoSecurityMatcher(MDP3SecurityID security_id) + : _security_id(security_id), _implied_mkt() {} + + ImpliedBook book(api::Side side) const; + MDP3Quantity + fill(MDP3OrderKey key, api::Side side, MDP3Quantity qty, FillDepot& fills); + // Note: this class is NOT responsible for deleting any FifoSecurityMatcher pointers. + // The FifoMatcherContext creates one FifoMergedSecurityMatcher per each security, + // and each FifoMergedSecurityMatcher creates/owns a FifoSecurityMatcher pointer + // (and is responsible for deleting it). + // The FifoSecurityMatcher pointer passed to this method should be owned by a + // FifoMergedSecurityMatcher. + // Note the absense of a 'side' parameter: unit_qty should be positive for Buys, + // negative for Sells. + void add_leg(MDP3SecurityID sec_id, FifoSecurityMatcher* m, MDP3Quantity unit_qty) { + _implied_mkt.insert({sec_id, {m, unit_qty}}); + } + + MDP3SecurityID security_id() const { return _security_id; } + std::tuple top_of_book(api::Side side) const; + + private: + MDP3SecurityID _security_id; + std::map> _implied_mkt; + // hide copy ctr + ImpliedFifoSecurityMatcher(const ImpliedFifoSecurityMatcher&); + + void _book(api::Side side, std::vector>& dest) const; + uint32_t _fill_order(const MDP3Order& order); +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp new file mode 100644 index 000000000..844d7c5be --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp @@ -0,0 +1,266 @@ +#include "Matcher.h" +#include "MatchingCommon.h" +#include +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/book_utils.h" +#include +#include +#include + +using namespace google::protobuf::util; + +namespace bts::spartan::mdp3{ + +MergedMatcher::MergedMatcher() + : _matcher_map(), + _trace(), _match_count(0), _match_count_this_per(0), + _per_start_time(TimeUtil::GetCurrentTime()) { } + +MergedMatcher::MergedMatcher(MergedMatcher&& src) { + _matcher_map = std::move(src._matcher_map); +} + +void MergedMatcher::add_def(const InstrumentDefinition& def) { + std::string ass(def.asset()); + _asset_map.emplace(def.security_id(), ass); + auto m = _matcher_map.find(ass); + if (m == _matcher_map.end()) + m = _matcher_map.emplace(ass, FifoMatcherContext({true})).first; + m->second.add_security(def.security_id(), std::string(def.symbol()), def.legs()); + m->second.implied_matching_status(def.security_id(), def.implied_matching()); +} + +std::string MergedMatcher::_get_asset(MDP3SecurityID sec) const { + auto it = _asset_map.find(sec); + return it == _asset_map.end() ? std::string() : it->second; +} + +template +using MBP = std::array; + +template +inline bool mbp_equals(const MBP& a, const MBP& b) { + for (uint32_t i = 0; i < a.size(); i++) { + if (a[i].price != b[i].price || a[i].quantity != b[i].quantity) + return false; + } + return true; +} + +// using structs rather than std::tuples to avoid +// accidentally mixing up before/after, or bids/asks. +struct MBPData { + MBPData() {} + MBPData(FifoMergedSecurityMatcher& mat) + : bids(mat.book(api::Side::SIDE_BUY)), asks(mat.book(api::Side::SIDE_SELL)), + ibids(mat.implied_book(api::Side::SIDE_BUY)), iasks(mat.implied_book(api::Side::SIDE_SELL)) {} + Book bids, asks; + ImpliedBook ibids, iasks; +}; + +inline bool operator==(const MBPData& lhs, const MBPData& rhs) { + auto lev_eq = [](const BookLevel& x, const BookLevel& y) { + return x.price == y.price && x.quantity == y.quantity; + }; + return std::equal(lhs.bids.begin(), lhs.bids.end(), rhs.bids.begin(), lev_eq) && + std::equal(lhs.asks.begin(), lhs.asks.end(), rhs.asks.begin(), lev_eq) && + std::equal(lhs.ibids.begin(), lhs.ibids.end(), rhs.ibids.begin(), lev_eq) && + std::equal(lhs.iasks.begin(), lhs.iasks.end(), rhs.iasks.begin(), lev_eq); +} + +struct MBPSnapshot { + MBPSnapshot() : _matcher(nullptr), _initial_mbp() {} + + MBPSnapshot(FifoMergedSecurityMatcher* matcher) + : _matcher(matcher), _initial_mbp(*_matcher) {} + + std::optional get_new_mbp() { + bts_assert(_matcher != nullptr); + auto curr = MBPData(*_matcher); + if (_initial_mbp == curr) + return {}; + return curr; + } + private: + FifoMergedSecurityMatcher* _matcher; + MBPData _initial_mbp; +}; + +FifoMatcherContext& MergedMatcher::_asset_matcher(MDP3SecurityID sec, bool add_if_missing) { + auto ass_it = _asset_map.find(sec); + if (add_if_missing && ass_it == _asset_map.end()) + ass_it = _asset_map.emplace(sec, std::string()).first; + bts_assert(ass_it != _asset_map.end()); + auto mat_it = _matcher_map.find(ass_it->second); + if (add_if_missing && mat_it == _matcher_map.end()) + mat_it = _matcher_map.emplace(ass_it->second, FifoMatcherContext({true})).first; + bts_assert(mat_it != _matcher_map.end()); + return mat_it->second; +} + +void MergedMatcher::match(ChangeSrc, + SmallOrderChangeVec const&changes, + std::vector* res) { + _match_count_this_per++; + _match_count++; + // This could be better optimized, but the idea is to + // 1) take a snapshot of every security's book/implied book, + // 2) process the order changes, + // 3) compare the current books with the snapshots to detect mbp changes. + // The map's hierarchcy is asset->security->books. + std::map> mbps; + for (const auto& c : changes) { + bts_assert(c.order.key.value() != 0); + auto ass = _get_asset(c.order.security); + auto it = mbps.find(ass); + if (it == mbps.end()) + it = mbps.emplace(ass, std::map()).first; + auto mbp_tup_it = it->second.find(c.order.security); + if (mbp_tup_it == it->second.end()) { + auto* mat = _matcher_map[ass].get_matcher(c.order.security); + bts_assert(mat != nullptr); + it->second.emplace(c.order.security, MBPSnapshot(mat)); + } + } + + for (const auto& c : changes){ + _trace.log_incoming_change(c); + MatcherUtils::validate_incoming_order_change(c); + switch (c.change) { + case OrderChange::ChangeType::added: + bts_assert(_used_order_keys.emplace(c.order.key).second); + break; + case OrderChange::ChangeType::updated: + case OrderChange::ChangeType::updated_price: + case OrderChange::ChangeType::removed: + bts_assert(_used_order_keys.count(c.order.key)); + break; + } + // this holds individual fills, meaning one aggr fill per price, + // and potentially multiple fills per order. + // todo: make the fills param optional, as + // it will often be more info than we care abt. + FillDepot fills; + _asset_matcher(c.order.security).on_order_change(c, res, fills); + } + // log/validate order updates + for(const auto& u : *res) + _trace.log_outgoing_update(std::get(u)); + for(const auto& u : *res) + MatcherUtils::validate_outgoing_update(std::get(u)); + // add mbp updates + for (auto& ass_kv : mbps) { + for (auto& sec_kv : ass_kv.second) { + auto new_mbp = sec_kv.second.get_new_mbp(); + if (new_mbp.has_value()) { + res->emplace_back(MBPUpdate{ + sec_kv.first, + new_mbp.value().bids, + new_mbp.value().asks, + new_mbp.value().ibids, + new_mbp.value().iasks + }); + } + } + } + auto now = TimeUtil::GetCurrentTime(); + auto usecs = TimeUtil::TimestampToMicroseconds(now) - + TimeUtil::TimestampToMicroseconds(_per_start_time); + if(usecs >= 1000000){ + LOG(OrderMan, Diag, "usecs:{} events processed:{} tot events processed:{}", + usecs, _match_count_this_per, _match_count); + _match_count_this_per = 0; + _per_start_time = now; + } +} + +Book MergedMatcher::book(MDP3SecurityID id, api::Side s) { + return _asset_matcher(id).book(id, s); +} + +ImpliedBook MergedMatcher::implied_book(MDP3SecurityID id, api::Side s) { + return _asset_matcher(id).implied_book(id, s); +} + +void MergedMatcher::clear() { + for (auto& mp : _matcher_map) + mp.second.clear(); +} + +///////////////// +// DirectMatcher +//////////////// +DirectMatcher::DirectMatcher() +: +_matcher_map(), +_trace(), +_match_count(0), +_match_count_this_per(0), +_per_start_time(TimeUtil::GetCurrentTime()){ +} + +DirectMatcher::DirectMatcher(DirectMatcher&& src) { + _matcher_map = std::move(src._matcher_map); +} + +DirectMatcher::DirectMatcher(const std::vector& inst_defs) { + for (const auto& sec_id : inst_defs) + _matcher_map.insert(std::pair( + sec_id, FifoSecurityMatcher(sec_id))); +} + +void DirectMatcher::match(ChangeSrc, + SmallOrderChangeVec const&changes, + std::vector* res) { + _match_count_this_per++; + _match_count++; + std::vector> dirty; + dirty.reserve(_matcher_map.size()); + + for (const auto& c : changes) { + auto it = _matcher_map.find(c.order.security); + bts_assert (it != _matcher_map.end()); + auto& sec_matcher = it->second; + _trace.log_incoming_change(c); + MatcherUtils::validate_incoming_order_change(c); + sec_matcher.on_order_change(c, res); + dirty.emplace_back(c.order.security, &it->second); + } + for(const auto& u : *res) + _trace.log_outgoing_update(std::get(u)); + for(const auto& u : *res) + MatcherUtils::validate_outgoing_update(std::get(u)); + // all order activity is not reflecting in the + // books + for (auto cm : dirty) + res->emplace_back( + MBPUpdate + { + cm.first, + cm.second->book(api::Side::SIDE_BUY), + cm.second->book(api::Side::SIDE_SELL), + ImpliedBook(), + ImpliedBook() + }); + auto now = TimeUtil::GetCurrentTime(); + auto usecs = TimeUtil::TimestampToMicroseconds(now) - + TimeUtil::TimestampToMicroseconds(_per_start_time); + if(usecs >= 1000000){ + LOG(OrderMan, Diag, "usecs:{} events processed:{} tot events processed:{}", + usecs, _match_count_this_per, _match_count); + _match_count_this_per = 0; + _per_start_time = now; + } +} + +Book DirectMatcher::book(MDP3SecurityID id, api::Side s) { + auto it = _matcher_map.find(id); + bts_assert(it != _matcher_map.end()); + return it->second.book(s); +} +void DirectMatcher::clear() { + for (auto& p : _matcher_map) + p.second.clear(); +} +} + diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h new file mode 100644 index 000000000..67cf60a2a --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h @@ -0,0 +1,102 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "FifoSecurityMatcher.h" +#include "FifoMatcherContext.h" +#include "MatchingCommon.h" +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { + +class InstrumentDefinition; + +class MergedMatcher { +private: + std::unordered_set _used_order_keys; + std::map _asset_map; + std::map _matcher_map; + Trace _trace; + size_t _match_count; + size_t _match_count_this_per; + google::protobuf::Timestamp _per_start_time; + std::string _get_asset(MDP3SecurityID sec) const; + FifoMatcherContext& _asset_matcher(MDP3SecurityID, bool add_if_missing = true); + +public: + MergedMatcher (); + MergedMatcher (MergedMatcher&&); + MergedMatcher &operator=(MergedMatcher&&) {return *this;} + + using MatchEffect = + std::variant; + + enum ChangeSrc { + ALGO, SIMULATOR_INPUT + }; + + void add_def(const InstrumentDefinition& def); + void match(ChangeSrc, + SmallOrderChangeVec const&, + std::vector *); + + Book book(MDP3SecurityID, api::Side); + ImpliedBook implied_book(MDP3SecurityID, api::Side); + + template + void init(TIter first, TIter last) { + while (first != last) { + add_def(*first); + first++; + } + } + + void reset(MDP3SecurityID sec, std::vector orders, std::vector* effects = nullptr) { + for (const auto& o : orders) + _used_order_keys.emplace(o.key); + _asset_matcher(sec).reset(sec, orders.begin(), orders.end(), effects); + } + void clear(); +}; + +// no implied matching (implied integration perf isn't great, +// need to investigate before DirectMatcher can be retired). +class DirectMatcher { +private: + std::map _matcher_map; + Trace _trace; + size_t _match_count; + size_t _match_count_this_per; + google::protobuf::Timestamp _per_start_time; + +public: + DirectMatcher (); + DirectMatcher (std::vector const&); + DirectMatcher (DirectMatcher&&); + DirectMatcher &operator=(DirectMatcher&&) {return *this;} + + using MatchEffect = + std::variant; + + enum ChangeSrc { + ALGO, SIMULATOR_INPUT + }; + + void match(ChangeSrc, + SmallOrderChangeVec const&, + std::vector *); + + Book book(MDP3SecurityID, api::Side); + ImpliedBook implied_book(MDP3SecurityID, api::Side); + + template + void reset(MDP3SecurityID sec, Iter first, Iter last) { + _matcher_map.try_emplace(sec).first->second.reset(first, last); + } + void clear(); +}; + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.cpp new file mode 100644 index 000000000..5cfa5c671 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.cpp @@ -0,0 +1,26 @@ +#include "MatchingCommon.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" + +namespace bts::spartan::mdp3 { +std::tuple MatcherOrder::cross(uint32_t cross_qty) { + auto [r, c] = compute_cross(quantity.value(), cross_qty); + bts_assert(quantity.value() - r == cross_qty - c); + auto filled = cross_qty - c; + quantity = MDP3Quantity(r); + if (ib.has_value()) { + ib.value().total -= MDP3Quantity(filled); + is_active = ib.value().total.value() > 0; + } else + is_active = quantity.value() > 0; + return std::make_tuple(r, c, filled); +} + +void MatcherOrder::reload_ib() { + if (ready_to_load_ib()) { + is_active = ib.value().total.value() > 0; + if (quantity.value() == 0) + quantity = std::min(ib.value().show, ib.value().total); + is_active = quantity.value() > 0; + } +} +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h new file mode 100644 index 000000000..4346f86ea --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h @@ -0,0 +1,131 @@ +#pragma once + +// did not want to fwd decl everything specially +// the templated classes (will fix later) +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include + +namespace bts::spartan::mdp3 { + +struct Trace{ +public: + const std::map traced_orders = + { + {MDP3OrderKey(765164629193),true} + }; + const std::map traced_events = {}; + const std::map ignored_events = {}; + bool log(MDP3OrderKey key, size_t eid=0) const{ + return traced_orders.find(key) != traced_orders.end() || + traced_events.find(eid) != traced_events.end(); + } + bool log(size_t eid) const{ + return traced_events.find(eid) != traced_events.end(); + } + bool is_ignored_event(size_t eid) const{ + return ignored_events.find(eid) != ignored_events.end(); + } + std::string _side(api::Side s){ + return s == api::Side::SIDE_BUY ? "BUY" : "SELL"; + } + void log_incoming_change(const SmallOrderChange& c){ + if(traced_orders.find(c.order.key) != traced_orders.end()) + LOG(OrderMan, Diag, + "Matcher incoming change:{} ok:{} side:{} price:{} qty:{} ib:{}", + c.change, c.order.key, _side(c.order.side), c.order.price, + c.order.quantity, + c.order.iceberg_quantity); + } + void log_outgoing_update(const OrderUpdate& u){ + if(traced_orders.find(u.key) != traced_orders.end()) + LOG(OrderMan, Diag, + "Matcher outgoing update:{} ok:{} side:{} last_fp:{} last_fq:{}", + u.status, u.key, _side(u.side), u.last_price, u.last_fills); + } +}; + +class MatcherUtils{ +public: + static void validate_incoming_order_change(const SmallOrderChange& c){ + // In sim input, when an iceberg order is updated + // such that only its price changed, the quantity field + // is set to 0 so the matcher knows not to change the show qty, + // At the cme, it appears that the show qty can only be + // changed by updating the 'visible' qty (i.e., the order's qty + // in the mbo). + // todo: use the OrderChange::ChangeType::updated_price enum + // to indicate this kind of update, then integrate that logic + // into the assert statement below. Another possible remedy + // would be to indicate show qty changes through the show qty + // field (in which case the matcher would need to reflect the + // show qty change in the visible qty, instead of the other way around). + // + // if(c.change == OrderChange::ChangeType::added || + // c.change == OrderChange::ChangeType::updated) + if (c.change == OrderChange::ChangeType::added) + bts_assert(c.order.quantity.value() > 0); + } + static void validate_outgoing_update(const OrderUpdate& u){ + if (u.status == OrderUpdate::Status::PARTIALLY_FILLED || + u.status == OrderUpdate::Status::FILLED) + bts_assert(u.last_fills.value() > 0); + if (u.status == OrderUpdate::Status::CANCELED) + bts_assert(u.last_fills.value() == 0); + } +}; + +struct MatcherOrder { + static std::tuple compute_cross(uint32_t resting_qty, + uint32_t cross_qty) { + if (cross_qty >= resting_qty) { + cross_qty -= resting_qty; + resting_qty = 0; + } else { + bts_assert(resting_qty > cross_qty); + resting_qty -= cross_qty; + cross_qty = 0; + } + return std::make_tuple(resting_qty, cross_qty); + } + MDP3OrderKey key; + api::Side side; + md::IntPrice price; + MDP3Quantity quantity; + uint32_t priority = 0; + std::optional ib; + bool is_active = true; + MatcherOrder(MDP3OrderKey key, + api::Side side, + md::IntPrice price, + MDP3Quantity qty, // working qty out there + uint32_t priority, + std::optional ib = {}) + : + key(key), + side(side), + price(price), + quantity(qty), + priority(priority), + ib(ib) {} + // returns the remainder of this order, remainder of opp order + // (ie the new qty NOT the filled qty, for that subtract from + // original) + std::tuple cross(uint32_t cross_qty); + MDP3Quantity total_qty() const { + return ib.has_value() ? ib.value().total : quantity; + } + MDP3Quantity working_qty() const { return quantity; } + bool ready_to_load_ib() const { + return ib.has_value() && quantity.value() == 0 && ib.value().total.value() > 0; + } + void reload_ib(); +}; + +inline std::ostream& operator <<(std::ostream& out, const MatcherOrder& ord) { + out << "[" << ord.key << "] " << ord.side << " " << ord.quantity << " @ " << ord.price + << " (ice " << ord.ib << " active " << (ord.is_active ? "Y" : "N") << ")"; + return out; +} + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h new file mode 100644 index 000000000..cfcc9e79b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h @@ -0,0 +1,23 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" + +namespace bts::spartan::mdp3 { + +class SecurityMatcher { +public: + virtual ~SecurityMatcher() {} + // accessors + // unf the book() method cannot be const right now. + // The FifoSecurityMatcher recomputes books lazily + // for performance reasons. Its internal state will change + // when this method is called if its book is dirty, + // and has to be recomputed. + virtual Book book(api::Side) = 0; + virtual std::tuple book_level( + api::Side, size_t level = 0) const = 0; + // mutators + virtual void on_order_change(SmallOrderChange const &, + std::vector* = nullptr) = 0; +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.cpp new file mode 100644 index 000000000..5573c262d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.cpp @@ -0,0 +1,85 @@ +#include "fill_depot.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include +#include +#include +#include +#include +#include + +using namespace ranges; +namespace bts::spartan::mdp3 { +bool FillDepot::add_fill(MDP3SecurityID sec, MDP3OrderKey aggressor, MDP3Fill f) { + auto sfit = _cache.find(sec); + if (sfit == _cache.end()) { + auto p = _cache.emplace( + sec, std::unordered_map>()); + sfit = p.first; + } + auto ofit = sfit->second.find(f.key); + if (ofit == sfit->second.end()) { + auto p = sfit->second.emplace(f.key, std::unordered_map()); + ofit = p.first; + } + auto pfit = ofit->second.find(f.price.price()); + if (pfit == ofit->second.end()) { + pfit = ofit->second.emplace(f.price.price(), FillNode()).first; + } + if (pfit->second.merge(aggressor, f)) { + _fill_count++; + return true; + } + return false; +} + +MDP3Quantity FillNode::total_fill_qty() const { + auto dfills = accumulate( + direct_passive | views::transform([](auto& f) { return f.quantity.value(); }), 0); + auto ifills = accumulate( + implied_passive | views::transform([](auto& f) { return f.quantity.value(); }), 0); + return MDP3Quantity( + buy_aggressor.quantity.value() + sell_aggressor.quantity.value() + dfills + ifills); +} + +MDP3Quantity FillDepot::total_fill_qty() const { + return MDP3Quantity( + accumulate(_cache | views::transform([](auto& kv) { + return accumulate(kv.second | views::transform([](auto& of) { + return accumulate(of.second | views::transform([](auto& pf) { return pf.second.total_fill_qty().value(); }), 0); + }), 0); + }), 0)); +} + +void FillDepot::copy_to(std::vector& dest) const { + for (auto& sec_fills : _cache) { + for(auto& ord_fills : sec_fills.second) { + for (auto& price_fills : ord_fills.second) + price_fills.second.copy_to(dest); + } + } +} + +void FillDepot::copy_to(std::unordered_map>& dest) const { + for (auto& sec_fills : _cache) { + auto& fills = dest.emplace(sec_fills.first, std::vector()).first->second; + for(auto& ord_fills : sec_fills.second) { + for (auto& price_fills : ord_fills.second) + price_fills.second.copy_to(fills); + } + } +} + +bool FillNode::match(const MDP3Fill& f) const { + switch (f.aggressor) { + case MDP3Fill::Aggressor::THIS_BUY: return _price_size_eq(buy_aggressor, f); + case MDP3Fill::Aggressor::THIS_SELL: return _price_size_eq(sell_aggressor, f); + case MDP3Fill::Aggressor::OTHER: + return any_of(direct_passive, [&](auto& of) { return _price_size_eq(of, f); }); + case MDP3Fill::Aggressor::IMPLIED: + return any_of(implied_passive, [&](auto& of) { return _price_size_eq(of, f); }); + } + return false; +} +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h new file mode 100644 index 000000000..f0e707d5e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h @@ -0,0 +1,132 @@ +#pragma once +#include "lib/md/IntPrice.h" +#include "lib/utils/bts_assert.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { +struct FillNode { + FillNode() : buy_aggressor(), sell_aggressor(), direct_passive(), implied_passive() {} + + private: + static inline bool _price_size_eq(const MDP3Fill& lhs, const MDP3Fill& rhs) { + return lhs.quantity == rhs.quantity; + } + + static inline bool _merge_fill(MDP3Fill& target, const MDP3Fill& fill) { + bool is_new = target.price.is_nan(); + if (is_new) { + target.key = fill.key; + target.price = fill.price; + target.aggressor = fill.aggressor; + } else { + bts_assert(target.key == fill.key); + bts_assert(target.price == fill.price); + bts_assert(target.aggressor == fill.aggressor); + } + target.quantity = MDP3Quantity(fill.quantity.value() + target.quantity.value()); + return is_new; + } + + static inline bool _merge_fill(std::vector& target, const MDP3Fill& fill, bool combine = false) { + if (combine && target.size()) { + _merge_fill(target[target.size() -1], fill); + return false; + } + target.push_back(fill); + return true; + } + + public: + uint32_t fill_count() const { return _fill_count; } + void set_curr_aggressor(MDP3OrderKey k) { + curr_aggressor = k; + } + bool merge(MDP3OrderKey aggressor, MDP3Fill f) { + bool rv = false; + switch (f.aggressor) { + case MDP3Fill::Aggressor::THIS_BUY: rv = _merge_fill(buy_aggressor, f); break; + case MDP3Fill::Aggressor::THIS_SELL: rv = _merge_fill(sell_aggressor, f); break; + case MDP3Fill::Aggressor::OTHER: rv = _merge_fill(direct_passive, f, aggressor == curr_aggressor); break; + case MDP3Fill::Aggressor::IMPLIED: rv = _merge_fill(implied_passive, f, aggressor == curr_aggressor); break; + default: bts_assert(false); return false; + } + if (rv) + _fill_count++; + curr_aggressor = aggressor; + return rv; + } + + bool match(const MDP3Fill& f) const; + + void copy_to(std::vector& dest) const { + if (buy_aggressor.quantity.value() > 0) + dest.push_back(buy_aggressor); + if (sell_aggressor.quantity.value() > 0) + dest.push_back(sell_aggressor); + for (auto f : direct_passive) + dest.push_back(f); + for (auto f : implied_passive) + dest.push_back(f); + } + MDP3Quantity total_fill_qty() const; + +private: + MDP3Fill buy_aggressor; + MDP3Fill sell_aggressor; + MDP3OrderKey curr_aggressor; + std::vector direct_passive, implied_passive; + uint32_t _fill_count = 0; +}; + +struct FillDepot { + bool add_fill(MDP3SecurityID sec, MDP3OrderKey aggressor, MDP3Fill f); + uint32_t fill_count() const { return _fill_count; } + MDP3Quantity total_fill_qty() const; + MDP3Quantity aggressor_fill_qty(MDP3SecurityID sec, MDP3OrderKey ord, int max_fill_items = 9999999) { + bts_assert(max_fill_items >= 0); + if (max_fill_items == 0) + return MDP3Quantity(0); + auto it = _cache.find(sec); + if (it == _cache.end()) { + return MDP3Quantity(0); + } + auto ord_it = it->second.find(ord); + if (ord_it == it->second.end()) { + return MDP3Quantity(0); + } + int res = 0; + for (auto pfit = ord_it->second.begin(); pfit != ord_it->second.end(); pfit++) { + std::vector ord_fills; + pfit->second.copy_to(ord_fills); + for (auto& f : ord_fills) { + switch (f.aggressor) { + case MDP3Fill::Aggressor::THIS_BUY: + case MDP3Fill::Aggressor::THIS_SELL: + res += f.quantity.value(); + if (--max_fill_items == 0) + return MDP3Quantity(res); + break; + default: break; + } + } + } + return MDP3Quantity(res); + } + void copy_to(std::vector& dest) const; + void copy_to(std::unordered_map>& dest) const; + + private: + uint32_t _fill_count = 0; + std::unordered_map>> _cache; +}; + +std::ostream& operator<<(std::ostream& out, const FillDepot& fd); +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h new file mode 100644 index 000000000..1db4f266e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h @@ -0,0 +1,32 @@ +#pragma once +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" + +namespace bts::spartan::mdp3 { +inline MDP3Fill::Aggressor _to_aggressor(api::Side s) { + switch (s) { + case api::Side::SIDE_BUY: return MDP3Fill::Aggressor::THIS_BUY; + case api::Side::SIDE_SELL: return MDP3Fill::Aggressor::THIS_SELL; + default: return MDP3Fill::Aggressor::OTHER; + } +} + +inline MDP3Fill _to_passive_fill(const OrderUpdate& upd, bool implied_aggressor) { + MDP3Fill passive; + passive.key = upd.key; + passive.price = upd.last_price; + passive.quantity = upd.last_fills; + passive.aggressor = + implied_aggressor ? MDP3Fill::Aggressor::IMPLIED : MDP3Fill::Aggressor::OTHER; + return passive; +} + +inline MDP3Fill +_to_aggressor_fill(MDP3OrderKey k, api::Side side, const OrderUpdate& upd) { + MDP3Fill aggr; + aggr.key = k; + aggr.price = upd.last_price; + aggr.quantity = upd.last_fills; + aggr.aggressor = _to_aggressor(side); + return aggr; +} +} // namespace bts::spartan::mdp3 \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.cpp new file mode 100644 index 000000000..2dfe9614c --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.cpp @@ -0,0 +1,94 @@ +#include "iceberg_helper.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "lib/utils/map_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" + +namespace bts::spartan::mdp3::iceberg_helper { + +#define NOT_IMPLEMENTED(...) \ + bts_assert_msg(false, orderman::Formatter::fmt(__VA_ARGS__, " not implemented.")); + +void add_iceberg_annotation( + MDP3Order& o, + OrderChange::ChangeType change_type, + const std::unordered_map& icebergs, + size_t event_id, + bool set_exact_state) { + const auto* ib = find_value(o.key, icebergs); + if (ib != nullptr) + add_iceberg_annotation(o, change_type, *ib, event_id, set_exact_state); +} + +void add_iceberg_annotation( + MDP3Order& o, + OrderChange::ChangeType change_type, + const IcebergInput& ib, + size_t event_id, + bool set_exact_state) { + switch (change_type) { + case OrderChange::ChangeType::added: { + o.iceberg_quantity = IcebergQuantity(); + auto show = std::get<1>(ib.show_qty_at(event_id)); + o.iceberg_quantity.value().total = ib.total_qty(); + o.iceberg_quantity.value().show = show; + bts_assert(o.iceberg_quantity.value().show.value() > 0); + //bts_assert(o.iceberg_quantity.value().total >= o.iceberg_quantity.value().show); + if (set_exact_state) { + // the matcher-validation-observer maintains a snapshot + // of the orderbooks (including iceberg total qty) + // to validate the matcher's internal books. + auto filled = ib.count_fills_by(event_id); + bts_assert(filled < ib.total_qty()); + o.iceberg_quantity.value().total = ib.total_qty() - filled; + } + break; + } + case OrderChange::ChangeType::updated: { + auto [ib_event_id, show] = ib.show_qty_at(event_id); + if (set_exact_state) { + // the matcher-validation-observer maintains a snapshot + // of the orderbooks (including iceberg total qty) + // to validate the matcher's internal books. + auto filled = ib.count_fills_by(event_id); + bts_assert(filled < ib.total_qty()); + o.iceberg_quantity = IcebergQuantity(); + o.iceberg_quantity.value().total = ib.total_qty() - filled; + o.iceberg_quantity.value().show = show; + } else { + // When orders are fed to the matcher, + // the ib total is only included when the order is added. + // On subsequent updates, the iceberg_quantity is set to + // null to indicate that it should be ignored. + o.iceberg_quantity = std::nullopt; + if (ib_event_id == event_id) + o.quantity = show; + else { + // Iceberg show quantities are synchronized with regular quantity changes, + // so if this was a quantity-changing event, the iceberg map would have + // a show qty entry for it. Since ib_event_id != event_id, + // this must be a price change event. + // + // This order's current state in the matcher is not neccessarily the + // same as its state during this event on the actual trading day. + // The simulation may have caused it to diverge. Set the + // quantity to 0 so the matcher knows not to change it. + o.quantity.value() = 0; + } + } + break; + } + case OrderChange::ChangeType::removed: break; + default: NOT_IMPLEMENTED("OrderChange::ChangeType::", change_type); break; + } +} + +MDP3Quantity get_reload_qty( + MDP3Quantity vis_qty, MDP3Quantity fill_qty, const IcebergQuantity& ib_before_fill) { + vis_qty = std::min(vis_qty, std::min(ib_before_fill.show, ib_before_fill.total)) - + (fill_qty % ib_before_fill.show); + if (vis_qty.value() <= 0) + vis_qty += ib_before_fill.show; + return std::min(vis_qty, ib_before_fill.total - fill_qty); +} +} // namespace bts::spartan::mdp3::iceberg_helper diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h new file mode 100644 index 000000000..a36601456 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h @@ -0,0 +1,60 @@ +#pragma once +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" +#include + +namespace bts::spartan::mdp3::iceberg_helper { +// Checks the iceberg-input map for an entry for the specified order. +// If one is found, the order's quantity, and iceberg_quantity members +// are assigned the appropriate values, for the given event +// (see below for details). +// +// Note: when seeding the matcher with a particular exchange state, +// set_exact_state should be true, otherwise probably false +// (see above for details). +void add_iceberg_annotation( + MDP3Order&, + OrderChange::ChangeType, + const std::unordered_map&, + size_t event_id, + bool set_exact_state = false); + +// Populates the iceberg_quantity, and [visible]quantity fields of the +// specified order, with appropriate values for the given event id. +// +// Regardless of 'set_exact_state', if the specified event is the first +// event of the order's life (i.e., when it was added): +// - order.iceberg_quantity.value().total = initial_total; +// - order.iceberg_quantity.value().show = initial_show; +// - order.quantity = initial_show; +// +// If 'set_exact_state' is FALSE, and the specified event coincides with +// a [visible]quantity update (note that visible quantity updates, and +// show quantity updates are treated as one in the same): +// - order.iceberg_quantity = std::nullopt; +// - order.quantity = new_visible_and_show; +// +// If 'set_exact_state' is FALSE, and the specified event doesn't coincide +// with either of the cases described above: +// - order.iceberg_quantity = std::nullopt; +// - order.quantity = 0; +// - this signals to the matcher that it only needs to check +// for a price change. +// +// If 'set_exact_state' is TRUE: +// - order.iceberg_quantity.total = orig_total - num_fills_at_or_before_specified_event; +// - order.iceberg_quantity.show = (most recent show quantity +// at or before the specified event); +// - order.quantity is unchanged, unless the event is the first of +// the order's life, in which case it is synchronized with the show quantity. +// - if the specified event is before the order was actually added, +// the show, and visible quantity will be 0, but the total will be the original total. +void add_iceberg_annotation( + MDP3Order&, + OrderChange::ChangeType, + const IcebergInput&, + size_t event_id, + bool set_exact_state); + +MDP3Quantity get_reload_qty( + MDP3Quantity vis_qty, MDP3Quantity fill_qty, const IcebergQuantity& ib_before_fill); +} // namespace bts::spartan::mdp3::iceberg_helper diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h new file mode 100644 index 000000000..e7aaa5251 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h @@ -0,0 +1,258 @@ +#pragma once +#include "lib/md/IntPrice.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" +#include +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3::diagnostics { + +inline std::ostream& +operator<<(std::ostream& out, const ::mdp3::SecurityTradingStatus::Value& value) { + switch (value) { + case ::mdp3::SecurityTradingStatus::TradingHalt: out << "TradingHalt"; break; + case ::mdp3::SecurityTradingStatus::Close: out << "Close"; break; + case ::mdp3::SecurityTradingStatus::NewPriceIndication: out << "NewPriceIndication"; break; + case ::mdp3::SecurityTradingStatus::ReadyToTrade: out << "ReadyToTrade"; break; + case ::mdp3::SecurityTradingStatus::NotAvailableForTrading: out << "NotTrading"; break; + case ::mdp3::SecurityTradingStatus::UnknownorInvalid: out << "UnknownorInvalid"; break; + case ::mdp3::SecurityTradingStatus::PreOpen: out << "PreOpen"; break; + case ::mdp3::SecurityTradingStatus::PreCross: out << "PreCross"; break; + case ::mdp3::SecurityTradingStatus::Cross: out << "Cross"; break; + case ::mdp3::SecurityTradingStatus::PostClose: out << "PostClose"; break; + case ::mdp3::SecurityTradingStatus::NoChange: out << "NoChange"; break; + case ::mdp3::SecurityTradingStatus::NULL_VALUE: out << "NULL_VALUE"; break; + default: out << "SecurityTradingStatus_" << static_cast(value); break; + } + return out; +} + +template +inline void print_fills(TIter first, TIter last, std::ostream& out = std::cout) { + bts::spartan::table_display::print_table( + out, first, last, + std::vector>>{ + {" Order ", 12, [](MDP3Fill f) { return FMT(f.key); }}, + {" Price ", 12, [](MDP3Fill f) { return FMT(f.price); }}, + {" Qty ", 8, [](MDP3Fill f) { return FMT(f.quantity); }}, + {" S Qty ", 8, [](MDP3Fill f) { return FMT(f.summary_quantity); }}, + {" Agg ", 12, [](MDP3Fill f) { return FMT(f.aggressor); }}, + {" Trade ", 12, [](MDP3Fill f) { return FMT(f.trade_entry_id); }}}); + out << std::endl; +} + +template +inline void print_detailed_fills(std::ostream& out, TIter first, TIter last) { + bts::spartan::table_display::print_table( + out, first, last, + std::vector)>>>{ + {" Security ", 12, [](std::tuple f) { return std::get<0>(f); }}, + {" Order ", 12, [](std::tuple f) { return FMT(std::get<2>(f).key); }}, + {" Side ", 6, [](std::tuple f) { return FMT(std::get<1>(f).side); }}, + {"O Qty", 6, [](std::tuple f) { return FMT(std::get<1>(f).quantity); }}, + {"O Price", 8, [](std::tuple f) { return FMT(std::get<1>(f).price); }}, + {"F Price", 8, [](std::tuple f) { return FMT(std::get<2>(f).price); }}, + {"F Qty", 6, [](std::tuple f) { return FMT(std::get<2>(f).quantity); }}, + {"S Qty", 6, [](std::tuple f) { return FMT(std::get<2>(f).summary_quantity); }}, + {" Agg ", 9, [](std::tuple f) { return FMT(std::get<2>(f).aggressor); }}, + {" Trade ", 10, [](std::tuple f) { return FMT(std::get<2>(f).trade_entry_id); }}}); + out << std::endl; +} + +template +inline void print_orders(std::ostream& out, TIter first, TIter last) { + bts::spartan::table_display::print_table( + out, first, last, + std::vector>>{ + {" Order ", 12, [](MDP3Order o) { return FMT(o.key); }}, + {" Side ", 8, [](MDP3Order o) { return FMT(o.side); }}, + {" Price ", 12, [](MDP3Order o) { return FMT(o.price); }}, + {" Qty ", 8, [](MDP3Order o) { return FMT(o.quantity); }}, + {" Priority ", 12, [](MDP3Order o) { return FMT(o.priority); }}}); + out << std::endl; +} + +template +inline void print_detailed_orders(TIter first, TIter last, std::ostream& out = std::cout) { + bts::spartan::table_display::print_table( + out, first, last, + std::vector)>>>{ + {" Sec ID ", 12, [](auto o) { return FMT(std::get<0>(o)); }}, + {" Symbol ", 12, [](auto o) { return std::get<1>(o); }}, + {" Order ", 12, [](auto o) { return FMT(std::get<2>(o).key); }}, + {" Side ", 8, [](auto o) { return FMT(std::get<2>(o).side); }}, + {" Price ", 12, [](auto o) { return FMT(std::get<2>(o).price); }}, + {" Qty ", 8, [](auto o) { return FMT(std::get<2>(o).quantity); }}, + {" Priority ", 12, [](auto o) { return FMT(std::get<2>(o).priority); }}}); + out << std::endl; +} + +template +inline void print_order_changes(TIter first, TIter last, std::ostream& out = std::cout) { + bts::spartan::table_display::print_table( + out, first, last, + std::vector>> { + { " Order ", 12, [](auto ch) { return FMT(ch.order.key); } }, + { " Ch. ", 8, [](auto ch) { return FMT(ch.change); } }, + {" Side ", 8, [](auto ch){ return FMT(ch.order.side); }}, + {" O Price ", 12, [](auto ch){ return FMT(ch.previous_order.price.price()); }}, + {" N Price ", 12, [](auto ch){ return FMT(ch.order.price.price()); }}, + {" O Qty ", 8, [](auto ch){return FMT(ch.previous_order.quantity); }}, + {" N Qty ", 8, [](auto ch){ return FMT(ch.order.quantity); }}, + {" O Ice ", 8, [](auto ch){ return FMT(ch.previous_order.iceberg_quantity); }}, + {" N Ice ", 8, [](auto ch){ return FMT(ch.order.iceberg_quantity); }}}); + out << std::endl; +} + +template +inline void print_detailed_order_changes(TIter first, TIter last, std::ostream& out = std::cout) { + bts::spartan::table_display::print_table( + out, first, last, + std::vector)>>> { + {" Sec ID ", 12, [](auto ch) { return FMT(std::get<0>(ch)); }}, + {" Symbol ", 12, [](auto ch) { return std::get<1>(ch); }}, + { " Order ", 12, [](auto ch) { return FMT(std::get<2>(ch).order.key); } }, + { " Ch. ", 8, [](auto ch) { return FMT(std::get<2>(ch).change); } }, + {" Side ", 8, [](auto ch){ return FMT(std::get<2>(ch).order.side); }}, + {" O Price ", 12, [](auto ch){ return FMT(std::get<2>(ch).previous_order.price.price()); }}, + {" N Price ", 12, [](auto ch){ return FMT(std::get<2>(ch).order.price.price()); }}, + {" O Qty ", 8, [](auto ch){return FMT(std::get<2>(ch).previous_order.quantity); }}, + {" N Qty ", 8, [](auto ch){ return FMT(std::get<2>(ch).order.quantity); }}}); + out << std::endl; +} + +template +inline void print_instrument_snapshots(TIter first, TIter last, std::ostream& out = std::cout) { + auto book_volume = [](const InstrumentSnapshot* snap, api::Side side) { + const auto& b = snap->book(side); + int vol = ranges::accumulate(b | ranges::views::transform([](const BookLevel& l) { return l.quantity.value(); }), 0); + return MDP3Quantity(vol); + }; + auto total_book_volume = [&](const InstrumentSnapshot* snap) { + return MDP3Quantity(book_volume(snap, api::Side::SIDE_BUY).value() + + book_volume(snap, api::Side::SIDE_SELL).value()); + }; + bts::spartan::table_display::print_table( + out, first, last, + std::vector>> { + {" Security ", 16, [](const auto* s) { return std::string(s->symbol()); }}, + {" ID ", 10, [](const auto* s) { return FMT(s->security_id()); }}, + {"Asset", 5, [](const auto* s) { return std::string(s->definition().asset()); }}, + {"Group", 5, [](const auto* s) { return std::string(s->definition().security_group()); }}, + {"Mat", 3, [](const auto* s) { return s->definition().match_algorithm(); }}, + {" Status ", 16, [&](const auto* s) { return FMT(s->trading_status()); }}, + {" El. Vol ", 10, [](const auto* s) { return FMT(s->electronic_volume()); }}, + {" Cl. Vol ", 10, [](const auto* s) { return FMT(s->cleared_volume()); }}, + {" Bid Vol ", 10, [&](const auto* s) { return FMT(book_volume(s, api::Side::SIDE_BUY)); }}, + {" Ask Vol ", 10, [&](const auto* s) { return FMT(book_volume(s, api::Side::SIDE_SELL)); }}, + {" Tot Vol ", 10, [&](const auto* s) { return FMT(total_book_volume(s)); }}, + {" Total Orders ", 14, [](const auto* s) { return FMT(s->orders().size()); }}, + {"Order Changes", 14, [](const auto* s) { return FMT(s->order_changes().size()); }}, + {" Total Fills ", 14, [](const auto* s) { return FMT(s->fills().size()); }}, + {" Qty ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_BUY)[0].quantity); }}, + {" Bid ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_BUY)[0].price); }}, + {" Ask ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_SELL)[0].price); }}, + {" Qty ", 8, [](const auto* s) { return FMT(s->book(api::Side::SIDE_SELL)[0].quantity); }}, + {"DImp", 4, [](const auto* s) { return std::string((s->definition().implied_matching() ? "Yes" : "No")); }}, + {"SImp", 4, [](const auto* s) { return std::string((s->implied_matching_status() ? "Yes" : "No")); }}, + {" I Qty ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_BUY)[0].quantity); }}, + {" I Bid ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_BUY)[0].price); }}, + {" I Ask ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_SELL)[0].price); }}, + {" I Qty ", 8, [](const auto* s) { return FMT(s->implied_book(api::Side::SIDE_SELL)[0].quantity); }} + }); + out << std::endl; +} + +template +inline void print_bbo(std::ostream& out, TIter first, TIter last) { + bts::spartan::table_display::print_table(out, first, last, + std::vector)>>> { + { " Security ", 12, [](auto r) { return std::get<0>(r); } }, + { " Qty ", 10, [](auto r) { return FMT(std::get<2>(r)); } }, + { " Bid ", 10, [](auto r) { return FMT(std::get<1>(r)); } }, + { " Ask ", 10, [](auto r) { return FMT(std::get<3>(r)); } }, + { " Qty ", 10, [](auto r) { return FMT(std::get<4>(r)); } } + }); + out << std::endl; +} + +template +inline void print_many_bbo(std::ostream& out, TIter first, TIter last) { + std::vector> rows; + while (first != last) { + const InstrumentSnapshot* s = *first; + const auto& bids = s->book(api::Side::SIDE_BUY); + const auto& asks = s->book(api::Side::SIDE_SELL); + rows.push_back(std::make_tuple(std::string(s->symbol()), bids[0].price, bids[0].quantity, asks[0].price, asks[0].quantity)); + first++; + } + print_bbo(out, rows.begin(), rows.end()); +} + +template +inline void print_many_bbo_implied(std::ostream& out, TIter first, TIter last) { + std::vector> rows; + while (first != last) { + const InstrumentSnapshot* s = *first; + const auto& bids = s->implied_book(api::Side::SIDE_BUY); + const auto& asks = s->implied_book(api::Side::SIDE_SELL); + rows.push_back(std::make_tuple(std::string(s->symbol()), bids[0].price, bids[0].quantity, asks[0].price, asks[0].quantity)); + first++; + } + print_bbo(out, rows.begin(), rows.end()); +} + +template +inline void print_many_bbo(TIter first, TIter last, bool direct, bool implied, std::ostream& out = std::cout) { + if (direct && !implied) { + print_many_bbo(out, first, last); + } else if (implied && !direct) { + print_many_bbo_implied(out, first, last); + } else if (direct && implied) { + std::vector> + rows; + while (first != last) { + const InstrumentSnapshot* s = *first; + const auto& bids = s->book(api::Side::SIDE_BUY); + const auto& asks = s->book(api::Side::SIDE_SELL); + const auto& ibids = s->implied_book(api::Side::SIDE_BUY); + const auto& iasks = s->implied_book(api::Side::SIDE_SELL); + rows.push_back(std::make_tuple( + std::string(s->symbol()), + bids[0].price, bids[0].quantity, asks[0].price, asks[0].quantity, + ibids[0].price, ibids[0].quantity, iasks[0].price, iasks[0].quantity)); + first++; + } + bts::spartan::table_display::print_table( + out, rows.begin(), rows.end(), + std::vector)>>>{ + {" Security ", 12, [](auto r) { return std::get<0>(r); }}, + {"I Qty", 6, [](auto r) { return FMT(std::get<6>(r)); }}, + {" I Bid ", 8, [](auto r) { return FMT(std::get<5>(r)); }}, + {" Qty ", 6, [](auto r) { return FMT(std::get<2>(r)); }}, + {" Bid ", 8, [](auto r) { return FMT(std::get<1>(r)); }}, + {" Ask ", 8, [](auto r) { return FMT(std::get<3>(r)); }}, + {" Qty ", 6, [](auto r) { return FMT(std::get<4>(r)); }}, + {" I Ask ", 8, [](auto r) { return FMT(std::get<7>(r)); }}, + {"I Qty", 6, [](auto r) { return FMT(std::get<8>(r)); }}}); + out << std::endl; + } +} +} // namespace bts::spartan::mdp3::diagnostics \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp new file mode 100644 index 000000000..64cd2c804 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp @@ -0,0 +1,747 @@ +#include +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" +#include +#include +#include +#include +#include "../MatchingCommon.h" + +using namespace google::protobuf::util; + +namespace bts::spartan::mdp3::tests { + +class RandomGenerator{ +public: + std::random_device rd; + double random_double(double min, double max){ + //Standard mersenne_twister_engine seeded with rd() + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis(min, max); + return dis(gen); + } + + int random_int (int min, int max){ + return static_cast(random_double(min,max)); + } + + api::Side random_side(){ + return random_int(1,100)%2==0 ? api::Side::SIDE_BUY : api::Side::SIDE_SELL; + } +}; + +MDP3Order create_order(uint64_t key, int32_t sec, api::Side side, + uint64_t priority, double price, uint32_t qty, + std::optional ib = { }) { + auto rv = MDP3Order{MDP3OrderKey(key), + MDP3SecurityID(sec), + side, + MDP3OrderPriority(priority), + md::IntPrice_from_double<7>(price), + MDP3Quantity(qty), + ib}; + return rv; +} + +MDP3Order create_empty_order(uint64_t key) { + return create_order(key,0,api::Side::SIDE_BUY,0,0.0,0); +} + +TEST_CASE("fifo_matching_engine_single_level", "[matcher]") { + auto sec_key = 10; + double price = 100.0; + api::Side side = api::Side::SIDE_BUY; + Trace trace; + std::map master_ord_map; + FifoPriceLevelMatcher price_level(master_ord_map, + MDP3SecurityID(0), + md::IntPrice_from_double<7>(price), + side, + trace); + auto create_book = [&](int n_orders) -> void { + for (int i = 1; i <= n_orders; i++) { + auto ord_key = static_cast(i); + auto o = create_order(ord_key, sec_key, side, ord_key, price, + static_cast(i)); + price_level.on_new_same_side_order(o); + } + }; + SECTION("add") { + // add 100 orders + create_book(100); + REQUIRE(price_level.n_orders() == 100); + REQUIRE(price_level.quantity().value() == 5050); + } + SECTION("cancel") { + create_book(100); + REQUIRE(price_level.n_orders() == 100); + REQUIRE(price_level.quantity().value() == 5050); + // remove last 10 orders, 91-100 + for (int i = 91; i <= 100; i++) { + auto ord_key = static_cast(i); + price_level.on_order_remove(MDP3OrderKey(ord_key)); + } + REQUIRE(price_level.n_orders() == 90); + REQUIRE(price_level.quantity().value() == 4095); + } + SECTION("sweep") { + create_book(100); + REQUIRE(price_level.n_orders() == 100); + REQUIRE(price_level.quantity().value() == 5050); + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(5100),false); + REQUIRE(price_level.n_orders() == 0); + REQUIRE(price_level.quantity().value() == 0); + } + SECTION("full_fill") { + std::vector out_ords; + create_book(100); + // take out the first 20 completely + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(210),false); + REQUIRE(price_level.n_orders() == 80); + REQUIRE(price_level.quantity().value() == 4840); + price_level.copy_values_to(out_ords); + REQUIRE(out_ords.at(79).key.value() == 100); + } + SECTION("partial_fill") { + create_book(100); + // take out the firdt 20 completely + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(193),false); + REQUIRE(price_level.n_orders() == 81); + REQUIRE(price_level.quantity().value() == 4857); + } + SECTION("qty_decrease") { + std::vector out_ords; + create_book(100); + price_level.copy_values_to(out_ords); + auto &ord_59 = out_ords.at(59); + REQUIRE(ord_59.quantity.value() == 60); + auto ord_59_mod = create_order(ord_59.key.value(), 60, + api::Side::SIDE_BUY, 0, price, 20); + price_level.on_order_update(ord_59_mod); + // stays at the same level + price_level.copy_values_to(out_ords); + REQUIRE(out_ords.at(59).quantity.value() == 20); + REQUIRE(out_ords.at(59).key.value() == 60); + REQUIRE(price_level.n_orders() == 100); + REQUIRE(price_level.quantity().value() == 5010); + } + SECTION("qty_increase") { + std::vector out_ords; + create_book(100); + price_level.copy_values_to(out_ords); + auto &ord_59 = out_ords.at(59); + REQUIRE(ord_59.quantity.value() == 60); + auto ord_59_mod = create_order(ord_59.key.value(), 60, + api::Side::SIDE_BUY, 0, price, 61); + price_level.on_order_update(ord_59_mod); + // goes to the end + price_level.copy_values_to(out_ords); + REQUIRE(out_ords.at(99).quantity.value() == 61); + REQUIRE(price_level.n_orders() == 100); + REQUIRE(price_level.quantity().value() == 5051); + } + SECTION("iceberg_simple") { + auto k = 0; + auto price = 100.0; + auto o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, 100.0, 1); + price_level.on_new_same_side_order(o); + o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 2); + price_level.on_new_same_side_order(o); + o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 3); + price_level.on_new_same_side_order(o); + // iceberg, show qty=2, total_qty=100 + o = create_order( + k++, sec_key, api::Side::SIDE_BUY, 0, price, 2, + IcebergQuantity{MDP3Quantity(2), MDP3Quantity(100)}); + price_level.on_new_same_side_order(o); + // non ib, to get the visible qty to 15 + o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 7); + price_level.on_new_same_side_order(o); + REQUIRE(price_level.quantity().value() == 15); + REQUIRE(price_level.total_quantity().value() == 113); + // cross with a 10 lot, this should result in + // {id=4,qty=5},{id=3,qty=2,total_qty=98} + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(10),false); + auto [n_ords, vis_qty] = price_level.n_orders_qty(true); + REQUIRE(n_ords == 2); + REQUIRE(vis_qty == 7); + auto total_qty = price_level.total_quantity(); + REQUIRE(total_qty.value() == 103); + // another 5 lot should only leave the IB {id=3,qty=2,total_qty=98} + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(5),false); + REQUIRE(price_level.n_orders() == 1); + REQUIRE(price_level.quantity().value() == 2); + REQUIRE(price_level.total_quantity().value() == 98); + // cross the remaining 98 IB + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(49),false); + REQUIRE(price_level.n_orders() == 1); + REQUIRE(price_level.quantity().value() == 1); + REQUIRE(price_level.total_quantity().value() == 49); + // 1 lots + for (auto i = 0; i < 49; i++) + price_level.on_new_opp_order(MDP3OrderKey(100001), MDP3Quantity(1),false); + REQUIRE(price_level.n_orders() == 0); + REQUIRE(price_level.quantity().value() == 0); + REQUIRE(price_level.total_quantity().value() == 0); + } +} + +TEST_CASE("fifo_matching_engine_book", "[matcher]") { + auto verify_book_level = [](const BookLevel &bl, double price, uint32_t qty, + uint32_t n_orders) { + REQUIRE(bl.price == md::IntPrice_from_double<7>(price)); + REQUIRE(bl.n_orders == n_orders); + REQUIRE(bl.quantity.value() == qty); + REQUIRE(bl.n_orders == n_orders); + }; + auto sec_key = 10; + FifoSecurityMatcher sec_me(MDP3SecurityID(100)); + auto order_key = 0; + auto priority = 0; + auto add_to_book = [&](uint32_t n_orders, api::Side side, + double price, + bool affect_book = true) -> void { + for (uint32_t i = 1; i <= n_orders; i++) { + auto o = + create_order(order_key++, sec_key, side, priority++, price, i); + sec_me.on_order_change( + SmallOrderChange(OrderChange{MDP3Order{},o,OrderChange::added},affect_book)); + } + }; + SECTION("add") { + // bids + auto n_levels = 10; + // bids + for (struct { + int i = 0; + double p = 15; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + // asks + for (struct { + int i = 0; + double p = 15.10; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + // verify bids + { + Book bid_bk = sec_me.book(api::Side::SIDE_BUY); + double exp_price = 15.09; + for (int i = 0; i < n_levels; i++) { + auto exp_n_orders = 100 - i * 10; + auto exp_qty = + (static_cast(exp_n_orders) / 2) * (1 + exp_n_orders); + verify_book_level(bid_bk.at(i), exp_price, exp_qty, exp_n_orders); + exp_price -= 0.01; + } + } + // verify asks + { + Book ask_bk = sec_me.book(api::Side::SIDE_SELL); + double exp_price = 15.10; + for (int i = 0; i < n_levels; i++) { + auto exp_n_orders = (i + 1) * 10; + auto exp_qty = + (static_cast(exp_n_orders) / 2) * (1 + exp_n_orders); + verify_book_level(ask_bk.at(i), exp_price, exp_qty, exp_n_orders); + exp_price += 0.01; + } + } + } + SECTION("add_sim_no_effect") { + // bids + auto n_levels = 10; + // bids + for (struct { + int i = 0; + double p = 15; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + // asks + for (struct { + int i = 0; + double p = 15.10; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + auto verify_bids = [&](){ + Book bid_bk = sec_me.book(api::Side::SIDE_BUY); + double exp_price = 15.09; + for (int i = 0; i < n_levels; i++) { + auto exp_n_orders = 100 - i * 10; + auto exp_qty = + (static_cast(exp_n_orders) / 2) * (1 + exp_n_orders); + verify_book_level(bid_bk.at(i), exp_price, exp_qty, exp_n_orders); + exp_price -= 0.01; + } + }; + auto verify_asks = [&](){ + Book ask_bk = sec_me.book(api::Side::SIDE_SELL); + double exp_price = 15.10; + for (int i = 0; i < n_levels; i++) { + auto exp_n_orders = (i + 1) * 10; + auto exp_qty = + (static_cast(exp_n_orders) / 2) * (1 + exp_n_orders); + verify_book_level(ask_bk.at(i), exp_price, exp_qty, exp_n_orders); + exp_price += 0.01; + } + }; + verify_bids(); + verify_asks(); + // bids: 15.00-15.09 + // asks: 15.10-15.19 + // add a passive order, should have no effect on the markets + add_to_book(100, api::Side::SIDE_BUY, 15.09,false); + verify_bids(); + verify_asks(); + // add an aggressive order + add_to_book(100, api::Side::SIDE_BUY, 15.19,false); + verify_bids(); + verify_asks(); + } + SECTION("remove") { + auto n_levels = 10; + // bids + for (struct { + int i = 0; + double p = 15; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + // asks + for (struct { + int i = 0; + double p = 15.10; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + // remove one order at a time from the second level + // 90 orders starting with id 360 + auto b = 360; + auto init_n_orders = 90; + auto init_qty = 4095; + auto remvd = 0; + for (auto i = 0; i < 90; i++) { + auto id = MDP3OrderKey(i + b); + sec_me.on_order_change(OrderChange{ + MDP3Order{}, + create_empty_order(id.value()), + OrderChange::removed + }); + auto qty = i + 1; + remvd += qty; + auto now_qty = init_qty - remvd; + auto now_n_orders = init_n_orders - (i + 1); + Book bk = sec_me.book(api::Side::SIDE_BUY); + if (now_qty == 0) + // whole level got removed + // now level 1 should be 15.07 + verify_book_level(bk.at(1), 15.07, 3240, 80); + else + verify_book_level(bk.at(1), 15.08, now_qty, now_n_orders); + } + Book bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bk.at(1), 15.07, 3240, 80); + } + SECTION("modify") { + auto n_levels = 10; + // bids + for (struct { + int i = 0; + double p = 15; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + // asks + for (struct { + int i = 0; + double p = 15.10; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + // modify qty of all orders to 1 + auto b = 360; + for (auto i = 0; i < 90; i++) { + auto id = i + b; + auto ord = + create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.08, 1); + sec_me.on_order_change(OrderChange{ + create_empty_order(ord.key.value()), + ord, + OrderChange::updated + }); + } + Book bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bk.at(1), 15.08, 90, 90); + // now change all these prices to move to a diff level + for (auto i = 0; i < 90; i++) { + auto id = i + b; + auto ord = + create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.09, 1); + sec_me.on_order_change(OrderChange{ + create_empty_order(ord.key.value()), + ord, + OrderChange::updated + }); + } + // 15.08 is gone + bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bk.at(1), 15.07, 3240, 80); + verify_book_level(bk.at(0), 15.09, 5140, 190); + } + SECTION("crosses") { + auto n_levels = 10; + // bids + for (struct { + int i = 0; + double p = 15; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + // asks + for (struct { + int i = 0; + double p = 15.10; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + // send a new sell at 15.09 to take the first 10 orders + auto ord = + create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.09, 55); + sec_me.on_order_change(OrderChange{MDP3Order{},ord,OrderChange::added}); + auto bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bk.at(0), 15.09, 4995, 90); + verify_book_level(bk.at(1), 15.08, 4095, 90); + verify_book_level(bk.at(2), 15.07, 3240, 80); + // now completely sweep 2 complete levels and make the rest a working + // order + ord = create_order(10001, sec_key, api::Side::SIDE_SELL, 10001, 15.08, + 10000); + sec_me.on_order_change(OrderChange{MDP3Order{},ord,OrderChange::added}); + bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bk.at(0), 15.07, 3240, 80); + // failing (11/16) + auto bk_ask = sec_me.book(api::Side::SIDE_SELL); + verify_book_level(bk_ask.at(0), 15.08, 910, 1); + } + SECTION("IB_cross") { + // create 10 level book and cross the first 3 levels completely. + // verify the resting IB is correct + auto n_levels = 10; + // bids + for (struct { + int i = 0; + double p = 15; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + // asks + for (struct { + int i = 0; + double p = 15.10; + void inc() { + i++; + p += 0.01; + } + } lp; + lp.i < n_levels; lp.inc()) + add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + auto bid_bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bid_bk.at(0), 15.09, 5050, 100); + // the bid book is + // 15.09 5050 100 + // 15.08 4095 90 + // 15.07 3240 80 + // 15.06 2485 70 + // send a new sell at 15.07 to take the first 3 levels + auto cross_ib = + create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.07, 13, + IcebergQuantity{MDP3Quantity(13), MDP3Quantity(12500)}); + sec_me.on_order_change(OrderChange{MDP3Order{},cross_ib,OrderChange::added}); + bid_bk = sec_me.book(api::Side::SIDE_BUY); + verify_book_level(bid_bk.at(0), 15.06, 2485, 70); + auto ask_bk = sec_me.book(api::Side::SIDE_SELL); + verify_book_level(ask_bk.at(0), 15.07, 13, 1); + } +} + +TEST_CASE("perf", "[matcher]") { + return; + RandomGenerator rnd_gen; + const size_t n_levels = 500; + const double tick_size = 0.01; + const double top_bid = 100.0; + const MDP3SecurityID sec_id = MDP3SecurityID(1); + + auto fill_ord = [&](MDP3Order& ord, + MDP3OrderKey key){ + ord.key = key; + ord.security = sec_id; + ord.side = rnd_gen.random_side(); + ord.priority = MDP3OrderPriority(key.value()); + int p_lev = rnd_gen.random_int(0,n_levels-1); + bts_assert(p_lev >= 0); + double price; + if(ord.side == api::Side::SIDE_BUY) + price = top_bid - (p_lev*tick_size); + else + price = (top_bid+tick_size) + (p_lev*tick_size); + ord.price = md::IntPrice_from_double<7>(price); + ord.quantity = MDP3Quantity(rnd_gen.random_int(5,20)); + }; + auto add_n_orders = [&](std::vector& out_v, + size_t n_orders, + size_t start_k){ + out_v.reserve(n_orders); + for(size_t i=0;i aux; + // takes 22 secs to add 100,000 orders straght up + add_n_orders(aux, n_orders, 0); + auto t0 = TimeUtil::GetCurrentTime(); + for (const auto& oc : aux) + sec_m.on_order_change(oc); + auto t1 = TimeUtil::GetCurrentTime(); + auto nsecs = TimeUtil::TimestampToNanoseconds(t1) - + TimeUtil::TimestampToNanoseconds(t0); + std::cout << "Adding " << aux.size() << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl; + size_t comp_n_orders = + sec_m.total_n_orders(api::Side::SIDE_BUY) + + sec_m.total_n_orders(api::Side::SIDE_SELL); + REQUIRE(comp_n_orders == n_orders); + n_orders += inc; + } + } + SECTION("cxl_orders") { + for(size_t i=0;i aux; + // takes 22 secs to add 100,000 orders straght up + add_n_orders(aux, n_orders, 0); + for (const auto& oc : aux) + sec_m.on_order_change(oc); + auto t0 = TimeUtil::GetCurrentTime(); + for (auto& oc : aux){ + oc.change = OrderChange::ChangeType::removed; + sec_m.on_order_change(oc); + } + auto t1 = TimeUtil::GetCurrentTime(); + auto nsecs = TimeUtil::TimestampToNanoseconds(t1) - + TimeUtil::TimestampToNanoseconds(t0); + std::cout << "Removing " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl; + n_orders += inc; + size_t comp_n_orders = + sec_m.total_n_orders(api::Side::SIDE_BUY) + + sec_m.total_n_orders(api::Side::SIDE_SELL); + REQUIRE(comp_n_orders == 0); + } + } + SECTION("update_dec_orders") { + for(size_t i=0;i aux; + // takes 22 secs to add 100,000 orders straght up + add_n_orders(aux, n_orders, 0); + for (const auto& oc : aux) + sec_m.on_order_change(oc); + auto t0 = TimeUtil::GetCurrentTime(); + for (auto& oc : aux){ + oc.order.quantity = MDP3Quantity(oc.order.quantity.value()-1); + oc.change = OrderChange::ChangeType::updated; + sec_m.on_order_change(oc); + } + auto t1 = TimeUtil::GetCurrentTime(); + auto nsecs = TimeUtil::TimestampToNanoseconds(t1) - + TimeUtil::TimestampToNanoseconds(t0); + std::cout << "Updaing (qty dec) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl; + size_t comp_n_orders = + sec_m.total_n_orders(api::Side::SIDE_BUY) + + sec_m.total_n_orders(api::Side::SIDE_SELL); + REQUIRE(comp_n_orders == n_orders); + n_orders += inc; + } + } + SECTION("update_inc_orders") { + for(size_t i=0;i aux; + // takes 22 secs to add 100,000 orders straght up + add_n_orders(aux, n_orders, 0); + for (const auto& oc : aux) + sec_m.on_order_change(oc); + auto t0 = TimeUtil::GetCurrentTime(); + for (auto& oc : aux){ + oc.order.quantity = MDP3Quantity(oc.order.quantity.value()+1); + oc.change = OrderChange::ChangeType::updated; + sec_m.on_order_change(oc); + } + auto t1 = TimeUtil::GetCurrentTime(); + auto nsecs = TimeUtil::TimestampToNanoseconds(t1) - + TimeUtil::TimestampToNanoseconds(t0); + std::cout << "Updaing (qty inc) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl; + size_t comp_n_orders = + sec_m.total_n_orders(api::Side::SIDE_BUY) + + sec_m.total_n_orders(api::Side::SIDE_SELL); + REQUIRE(comp_n_orders == n_orders); + n_orders += inc; + } + } + SECTION("order_price_change_no_cross") { + for(size_t i=0;i aux; + // takes 22 secs to add 100,000 orders straght up + add_n_orders(aux, n_orders, 0); + for (const auto& oc : aux) + sec_m.on_order_change(oc); + auto t0 = TimeUtil::GetCurrentTime(); + for (auto& oc : aux){ + oc.change = OrderChange::ChangeType::updated; + double inc = tick_size; + if(oc.order.side == api::Side::SIDE_BUY) + inc = -1*inc; + oc.order.price = + md::IntPrice_from_double<7>(oc.order.price.price() + inc), + sec_m.on_order_change(oc); + } + auto t1 = TimeUtil::GetCurrentTime(); + auto nsecs = TimeUtil::TimestampToNanoseconds(t1) - + TimeUtil::TimestampToNanoseconds(t0); + std::cout << "Updating (price change no cross) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl; + size_t comp_n_orders = + sec_m.total_n_orders(api::Side::SIDE_BUY) + + sec_m.total_n_orders(api::Side::SIDE_SELL); + REQUIRE(comp_n_orders == n_orders); + n_orders += inc; + } + } + SECTION("new_order_cross") { + for(size_t i=0;i aux; + // takes 22 secs to add 100,000 orders straght ups + add_n_orders(aux, n_orders, 0); + for (const auto& oc : aux) + sec_m.on_order_change(oc); + // send one lot buy orders to completely cross + // the offer market + size_t buy_orders_to_send = sec_m.total_qty(api::Side::SIDE_SELL).value(); + OrderChange ochange; + ochange.change = OrderChange::ChangeType::added; + ochange.order = MDP3Order{ + MDP3OrderKey(0),MDP3SecurityID(100), + api::Side::SIDE_BUY,MDP3OrderPriority(0), + md::IntPrice_from_double<7>(10000),MDP3Quantity(1), + std::nullopt + }; + std::vector exec_rep; + auto t0 = TimeUtil::GetCurrentTime(); + for(size_t i=0; i(eff); + REQUIRE(upd.last_fills.value()==1); + } + auto t1 = TimeUtil::GetCurrentTime(); + auto nsecs = TimeUtil::TimestampToNanoseconds(t1) - + TimeUtil::TimestampToNanoseconds(t0); + std::cout << "Crossing (with new orders) " << n_orders << " took " << 1.0*nsecs/1000000 << " msecs" << std::endl; + // whole sell side should be gone + REQUIRE(sec_m.total_n_orders(api::Side::SIDE_SELL) == 0); + n_orders += inc; + } + } +} + +} // namespace bts::spartan::mdp3::tests diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/event_flags.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/event_flags.h new file mode 100644 index 000000000..2678ecc3b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/event_flags.h @@ -0,0 +1,59 @@ +#pragma once +#include +#include + +namespace bts::spartan::mdp3 { +enum EventFlags { + None = 0, + IcebergOrders = 1, + DirectFills = 1 << 1, + ImpliedFills = 1 << 2, + MultipleSecurityAggressors = 1 << 3 +}; + +const char* event_flags_name(const EventFlags& f) { + switch (f) { + case EventFlags::None: return "None"; + case EventFlags::DirectFills: return "DirectFills"; + case EventFlags::ImpliedFills: return "ImpliedFills"; + case EventFlags::MultipleSecurityAggressors: return "MultipleSecurityAggressors"; + default: return "Unknown"; + } +} + +inline EventFlags operator|(const EventFlags& lhs, const EventFlags& rhs) { + return static_cast(static_cast(lhs) | static_cast(rhs)); +} +inline EventFlags& operator|=(EventFlags& lhs, const EventFlags& rhs) { + lhs = static_cast(static_cast(lhs) | static_cast(rhs)); + return lhs; +} +inline EventFlags operator&(const EventFlags& lhs, const EventFlags& rhs) { + return static_cast(static_cast(lhs) & static_cast(rhs)); +} +inline EventFlags& operator&=(EventFlags& lhs, const EventFlags& rhs) { + lhs = static_cast(static_cast(lhs) & static_cast(rhs)); + return lhs; +} +inline EventFlags operator^(const EventFlags& lhs, const EventFlags& rhs) { + return static_cast(static_cast(lhs) ^ static_cast(rhs)); +} +inline EventFlags& operator^=(EventFlags& lhs, const EventFlags& rhs) { + lhs = static_cast(static_cast(lhs) ^ static_cast(rhs)); + return lhs; +} +inline std::ostream& operator<<(std::ostream& out, const EventFlags& flags) { + int fcount = 0; + auto impl = [&](EventFlags f) { + if ((flags & f) == f) { + if (fcount++ > 0) + out << " | "; + out << event_flags_name(f); + } + }; + impl(EventFlags::DirectFills); + impl(EventFlags::ImpliedFills); + impl(EventFlags::MultipleSecurityAggressors); + return out; +} +} \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.cpp new file mode 100644 index 000000000..a1baa66d3 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.cpp @@ -0,0 +1,81 @@ +#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h" +#include "lib/logging/logging.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include +#include +#include +#include +#include + +#define _LOG(level, msg) LOG(OrderMan, Info, "{} {}.", _log_header(), msg) +#define DIAG(...) _LOG(Diag, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) + +namespace bts::spartan::mdp3::iceberg_file_cache { +// increment this anytime the deduced icebergs change +// so old .iceberg files will be regenerated +const ushort VERSION = 3; + +inline std::string _log_header() { return std::string("IcebergFileCache"); } + +inline std::string _iceberg_file_path(const std::string& pcap) { + auto dir = spartan::file::dir_from_path(pcap); + auto name = bts::spartan::file::file_name_from_path(pcap); + return dir + R"(/)" + name + ".icebergs"; +} + +std::unordered_map find_icebergs(const std::string& pcap, bool force_regen) { + std::string path = _iceberg_file_path(pcap); + // goes without saying: read_op and write_op must + // agree with one another, wrt the sequence of fields + std::unordered_map dest; + auto read_op = [&](std::istream& in) -> std::istream& { + auto ok = file::read_from_bin(in); + auto total_qty = file::read_from_bin(in); + std::vector> show_quantities; + for (auto num_states = file::read_from_bin(in); num_states > 0; num_states--) { + size_t event_id = file::read_from_bin(in); + auto show_qty = file::read_from_bin(in); + show_quantities.emplace_back(event_id, MDP3Quantity(show_qty)); + } + std::vector> fill_quantities; + for (auto num_fill_events = file::read_from_bin(in); num_fill_events > 0; num_fill_events--) { + size_t event_id = file::read_from_bin(in); + auto fill_qty = file::read_from_bin(in); + fill_quantities.emplace_back(event_id, fill_qty); + } + dest.emplace( + MDP3OrderKey(ok), + IcebergInput(MDP3Quantity(total_qty), show_quantities, fill_quantities)); + return in; + }; + if (force_regen || !bts::spartan::file::read_from_file(path, read_op, VERSION)) { + std::cout << "sweeping for icebergs..." << std::endl; + dest = IcebergFinder::find_all(pcap); + auto write_op = + [](std::ostream& out, + const std::pair& p) -> std::ostream& { + file::write_to_bin(out, p.first.value()); + file::write_to_bin(out, p.second.total_qty().value()); + file::write_to_bin(out, static_cast(p.second.show_quantities().size())); + for (auto [event_id, show_qty] : p.second.show_quantities()) { + file::write_to_bin(out, event_id); + file::write_to_bin(out, show_qty.value()); + } + file::write_to_bin(out, static_cast(p.second.fill_quantities().size())); + for (auto [event_id, fill_qty] : p.second.fill_quantities()) { + file::write_to_bin(out, event_id); + file::write_to_bin(out, fill_qty.value()); + } + return out; + }; + bts::spartan::file::write_to_file(dest.begin(), dest.end(), path, write_op, VERSION); + } else { + INFO("read ", dest.size(), " icebergs from '", path, "'"); + } + return dest; +} +} // namespace bts::spartan::mdp3::iceberg_file_cache diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h new file mode 100644 index 000000000..6098988b7 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h @@ -0,0 +1,14 @@ +#pragma once +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" +#include + +namespace bts::spartan::mdp3::iceberg_file_cache { +// First, looks in the working dir for a file named .icebergs. +// If the file does exists, the icebergs are simply read from the file. +// If the file doesn't exist, the pcap is swept for icebergs, +// and the results are written to the file .icebergs, +// for faster retrieval on subsequent runs with the specified pcap. +std::unordered_map +find_icebergs(const std::string& pcap, bool force_regen = false); +} // namespace bts::spartan::mdp3::iceberg_file_cache \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.cpp new file mode 100644 index 000000000..815d76868 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.cpp @@ -0,0 +1,476 @@ +#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "lib/orderman/orderman_utils/PriceComp.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/event_flags.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_quantity_operators.h" +#include "server/sparkd/cmds/spartan/md/mdp3/gen/SecurityTradingStatus.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/event_ordinal.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/order_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/security_comp.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer.h" +#include "server/sparkd/cmds/spartan/util/action_summarizer_macros.h" +#include "server/sparkd/cmds/spartan/util/table_display.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h" +#include +#include +#include + +// Error logs are automatically logged to the console +// (w/ default logging configuration), but they lag behind +// immediate console output (i.e., calls to std::cout). +// If _ECHO_ON is defined, the _ERROR_LEVEL macro is set to Info +// to avoid duplicate, and (perhaps more importantly) unsynchronized output. +// #define _ECHO_ON +#ifdef _ECHO_ON +#define _ECHO(...) std::cout << bts::orderman::Formatter::fmt(__VA_ARGS__); +#define _ERROR_LEVEL Info +#else +#define _ECHO(...) +#define _ERROR_LEVEL Error +#endif +#define _LOG(level, msg) \ + _ECHO(_log_header(), " ", msg, "\n") \ + LOG(OrderMan, level, "{} {}", _log_header(), msg) +#define INFO(...) _LOG(Info, bts::orderman::Formatter::fmt(__VA_ARGS__)) +#define ERROR(...) _LOG(_ERROR_LEVEL, bts::orderman::Formatter::fmt(__VA_ARGS__)) + +// SUMMARIZE macro wrappers +#define _SUMMARIZE SUMMARIZE(_log_header()) +#define _SUMMARIZE_TRANSACTION(...) SUMMARIZE_TRANSACTION(_log_header(), FMT(__VA_ARGS__)) +#define _SUMMARIZE_TRANSACTION_NAMED(name, ...) SUMMARIZE_TRANSACTION_NAMED(name, _log_header(), FMT(__VA_ARGS__)) + +namespace bts::spartan::mdp3 { +using namespace bts::spartan::mdp3::iceberg_helper; +// aliases +using TradingStatus = MatcherValidationObserver::TradingStatus; +template +using flat_map = boost::container::flat_map; +// consts +const size_t NO_EVENT = 0, MIN_OBSERVED_EVENT = 100; + +enum class Verbosity { Minimal = 0, Normal = 1, Verbose = 2 }; + +/////////////////////// +// stateless helpers +/////////////////////// +inline std::string _log_header() { return "MatcherValidationObserver"; } + +inline bool _is_interesting_sec(const InstrumentSnapshot& s) { + return s.definition().asset() == "ES"; + //return strcmp(s.definition().asset(), "SI") == 0; +} + +void _sync_orders( + size_t event_id, + const InstrumentSnapshot& s, + Orders& orders, + const std::unordered_map& icebergs) { + // + for (auto ch : s.order_changes()) { + switch (ch.change) { + case OrderChange::added: + case OrderChange::updated: + case OrderChange::updated_price: + add_iceberg_annotation(ch.order, ch.change, icebergs, event_id, true); + orders[ch.order.key] = ch.order; + break; + case OrderChange::removed: orders.erase(ch.order.key); break; + } + } +} + +bool _has_multiple_aggressor_secs(const MDP3Event& ev) { + bool aggr_sec_found = false; + for (auto* s : ev.affected_securities) { + for (auto f : s->fills()) { + switch (f.aggressor) { + case MDP3Fill::Aggressor::THIS_BUY: + case MDP3Fill::Aggressor::THIS_SELL: + if (aggr_sec_found) + return true; + aggr_sec_found = true; + break; + default: continue; + } + break; + } + } + return false; +} + +inline std::ostream& operator<<(std::ostream& ostr, const TradingStatus& ts) { + switch (ts) { + case TradingStatus::None: ostr << "None"; break; + case TradingStatus::Preopen: ostr << "Preopen"; break; + case TradingStatus::Uninitialized: ostr << "Uninitialized"; break; + case TradingStatus::Initialized: ostr << "Initialized"; break; + default: ostr << "TradingStatus_" << static_cast(ts); break; + } + return ostr; +} + +template +inline void print_if_not_empty(TIter first, TIter last, std::string title, std::ostream& out = std::cout) { + bool printed_title = false; + while(first != last) { + if (!printed_title) { + out << title << std::endl; + printed_title = true; + } + out << *first << std::endl; + first++; + } +} + +inline bool fill_equals(const MDP3Fill& a, const MDP3Fill& b) { + return a.aggressor == b.aggressor && a.quantity == b.quantity && orderman::price_comp::is_equal(a.price, b.price); +} + +inline bool match_fills(const FillDepot& fd, std::vector snaps, std::ostream& out = std::cout) { + std::unordered_map> fill_map; + fd.copy_to(fill_map); + bool errors = false; + for (const auto* s : snaps) { + auto sec_fills_it = fill_map.find(s->security_id()); + if (sec_fills_it == fill_map.end()) { + print_if_not_empty(s->fills().begin(), s->fills().end(), FMT(s->symbol(), " unmatched fills"), out); + continue; + } + auto& act_fills = sec_fills_it->second; + auto exp_it = s->fills().begin(); + std::vector unmatched_exp; + while (exp_it != s->fills().end()) { + bool matched = false; + for (auto act_it = act_fills.begin(); act_it != act_fills.end(); act_it++) { + if (fill_equals(*act_it, *exp_it)) { + act_fills.erase(act_it); + matched = true; + break; + } + } + if (!matched) { + unmatched_exp.push_back(*exp_it); + } + exp_it++; + } + errors |= (act_fills.size() || unmatched_exp.size()); + print_if_not_empty(act_fills.begin(), act_fills.end(), FMT(s->symbol(), " unexpected fills"), out); + print_if_not_empty(unmatched_exp.begin(), unmatched_exp.begin(), FMT(s->symbol(), " unmatched fills"), out); + } + return !errors; +} +/////////////////////// +// private classes +/////////////////////// +struct _SecurityData { + _SecurityData() : status(TradingStatus::None), current_orders(), previous_orders() {} + TradingStatus status; + Orders current_orders, previous_orders; + size_t last_observed_event = NO_EVENT; + const InstrumentSnapshot* snapshot = nullptr; + std::string symbol() const { + return snapshot == nullptr ? "N/A" : snapshot->symbol(); + } + MDP3SecurityID security_id() const { + return snapshot == nullptr ? MDP3SecurityID(0) : snapshot->security_id(); + } +}; + +struct MatcherValidationObserver::PImpl { + PImpl() {} + PImpl(std::unordered_map icebergs, flat_map interesting_event_ranges) + : icebergs(icebergs), interesting_event_ranges(interesting_event_ranges) {} + + flat_map security_cache; + std::unordered_map icebergs; + flat_map interesting_event_ranges; + flat_map interesting_orders; + + // number of 'interesting events' observed so far. + uint32_t interesting_event_count = 0; + // actual event_id (from the exchange) of most recently observed event. + size_t event_id = NO_EVENT; + FifoMatcherContext exch_matcher; + Verbosity verbosity = Verbosity::Minimal; + EventFlags ignore_flags = EventFlags::MultipleSecurityAggressors; + + // event_id is used to transition the security's trading status from + // Preopen to Uninitialized + _SecurityData& get_security_data(const InstrumentSnapshot& s, size_t event_id) { + auto it = security_cache.find(s.security_id()); + if (it != security_cache.end()) { + auto& data = it->second; + if (event_id != data.last_observed_event) { + data.last_observed_event = event_id; + switch (data.status) { + case TradingStatus::Preopen: + data.status = TradingStatus::Uninitialized; + break; + case TradingStatus::None: + if (s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade) + data.status = TradingStatus::Preopen; + break; + default: break; + } + } + data.snapshot = &s; + return data; + } + // this is the first time this security has been observed; initialize its data. + auto& data = security_cache.emplace(s.security_id(), _SecurityData()).first->second; + data.snapshot = &s; + data.last_observed_event = event_id; + data.previous_orders = s.orders(); + for (auto& o : data.previous_orders) + add_iceberg_annotation(o.second, OrderChange::ChangeType::added, icebergs, event_id); + // the first event at the open isn't processed bc + // we don't have a clear understanding of how preopen matching works. + data.status = s.trading_status() == ::mdp3::SecurityTradingStatus::ReadyToTrade + ? TradingStatus::Preopen + : TradingStatus::None; + data.snapshot = &s; + return data; + } + + EventFlags get_event_flags(const MDP3Event& ev) { + EventFlags flags = EventFlags::None; + if (_has_multiple_aggressor_secs(ev)) + flags |= EventFlags::MultipleSecurityAggressors; + for (const auto* s : ev.affected_securities) { + for (auto& f : s->fills()) { + if (icebergs.find(f.key) != icebergs.end()) + flags |= EventFlags::IcebergOrders; + switch (f.aggressor) { + case MDP3Fill::Aggressor::THIS_BUY: + case MDP3Fill::Aggressor::THIS_SELL: + flags |= EventFlags::DirectFills; + break; + case MDP3Fill::Aggressor::IMPLIED: + flags |= EventFlags::ImpliedFills; + break; + default: break; + } + } + } + return flags; + } +}; +/////////////////////// +// constructors +/////////////////////// +MatcherValidationObserver::MatcherValidationObserver( + std::unordered_map icebergs) + : MatcherValidationObserver(icebergs, flat_map{ }) {} + +MatcherValidationObserver::MatcherValidationObserver( + std::unordered_map icebergs, + flat_map interesting_event_ranges) + : _me(new MatcherValidationObserver::PImpl(icebergs, interesting_event_ranges)) { } + +MatcherValidationObserver::~MatcherValidationObserver() {} + +/////////////////////// +// misc private methods +/////////////////////// +TradingStatus MatcherValidationObserver::_trading_status(const InstrumentSnapshot& s, size_t event_id) { + return _me->get_security_data(s, event_id).status; +} + +bool MatcherValidationObserver::_is_ready_to_trade(const InstrumentSnapshot& s, size_t event_id) { + switch (_trading_status(s, event_id)) { + case TradingStatus::Uninitialized: + case TradingStatus::Initialized: return true; + default: return false; + } +} + +void MatcherValidationObserver::_sync_current_orders(MDP3Event const& ev) { + for (auto s : ev.affected_securities) { + auto& new_orders = _me->get_security_data(*s, ev.event_id).current_orders; + _sync_orders(ev.event_id, *s, new_orders, _me->icebergs); + } +} + +void MatcherValidationObserver::_sync_previous_orders(const MDP3Event& ev) { + for (auto s : ev.affected_securities) { + auto& old_orders = _me->get_security_data(*s, ev.event_id).previous_orders; + _sync_orders(ev.event_id, *s, old_orders, _me->icebergs); + } +} + +void MatcherValidationObserver::_reset_matcher() { + // remove orders, and change [actively trading] securities + // trading status back to Preopen so they will be added back + // in the next update (the state must be re-initialized + // before updates can be consumed incrementally again). + if (_me->verbosity > Verbosity::Normal) + INFO("clearing orders to reset the matcher's state"); + _me->exch_matcher.clear(); + for (auto& kv : _me->security_cache) { + if (kv.second.status != TradingStatus::None) + kv.second.status = TradingStatus::Preopen; + } +} +/////////////////////// +// IChannelObserver +/////////////////////// +void MatcherValidationObserver::on_recovery_complete(InitialSnapshot const& snap) { + _SUMMARIZE; + for (auto s : snap) { + auto& sec_data = _me->get_security_data(*s, NO_EVENT); + sec_data.current_orders.clear(); + _sync_orders(NO_EVENT, *s, sec_data.current_orders, _me->icebergs); + sec_data.previous_orders = sec_data.current_orders; + } +} + +void MatcherValidationObserver::on_real_time_event(MDP3Event const& ev) { + bts_assert(ev.event_id > _me->event_id); + _me->event_id = ev.event_id; + switch (get_event_ordinal(_me->interesting_event_ranges, ev.event_id, MIN_OBSERVED_EVENT)) { + case bts::spartan::EventOrdinal::Waiting: + // just need to synchronize the order caches. + // the matcher is only validated during 'interesting events.' + _sync_current_orders(ev); + _sync_previous_orders(ev); + return; + case bts::spartan::EventOrdinal::Interesting: + // _sync_previous_orders should be called after + // the matcher has digested the order changes. + break; + case bts::spartan::EventOrdinal::Done: + // nothing left to do (might as well kill the process). + bts_assert_msg(false, "no further events of interest."); + return; + } + auto flags = _me->get_event_flags(ev); + if ((flags & _me->ignore_flags) != EventFlags::None) { + if (_me->verbosity > Verbosity::Minimal) + INFO(FMT("skipping event ", ev.event_id, ", with flags ", flags)); + _sync_current_orders(ev); + _sync_previous_orders(ev); + _reset_matcher(); + return; + } + ++_me->interesting_event_count; + if (_me->verbosity > Verbosity::Minimal) + INFO("Update No ", _me->interesting_event_count, "(event id ", ev.event_id, ")"); + // Note: the exchange processes the order updates per security, + // in a particular order, sorted hierarchically by symbol, then expiration. + // 'sorted_secs' will contain the proper order of securities. + std::vector sorted_secs; + for (auto* s : ev.affected_securities) { + if (_is_interesting_sec(*s)) + sorted_secs.push_back(s); + } + if (sorted_secs.empty()) + return; + std::sort(sorted_secs.begin(), sorted_secs.end(), security_comp::SymbolComp()); + // securities must be added to the matcher before the order + // changes are processed to ensure the correct implied markets are used. + // Fills are also indexed in this pass, so they can be added to the iceberg qty + std::unordered_map fill_map; + for (const auto* s : sorted_secs) { + accumulate_fills(s->fills().begin(), s->fills().end(), fill_map); + _me->exch_matcher.add_security(s->security_id(), s->symbol(), s->definition().legs()); + _me->exch_matcher.implied_matching_status(s->security_id(), s->definition().implied_matching()); + } + // initialize matcher with previous state (only applies + // to securities whose orders the matcher hasn't seen yet). + for (const auto* s : sorted_secs) { + if (_trading_status(*s, ev.event_id) != TradingStatus::Uninitialized) + continue; + auto it = _me->security_cache.find(s->security_id()); + bts_assert(it != _me->security_cache.end()); + auto prev_ords = it->second.previous_orders | + ranges::views::transform([](auto& p) { return p.second; }); + _me->exch_matcher.reset(s->security_id(), prev_ords.begin(), prev_ords.end()); + _me->security_cache[s->security_id()].status = TradingStatus::Initialized; + } + // process changes with matcher + FillDepot fills; + for (const auto* s : sorted_secs) { + switch (_trading_status(*s, ev.event_id)) { + case TradingStatus::Initialized: break; + default: continue; + } + auto deduced = s->deduced_order_changes(); + if (deduced.size()) { + std::vector deduced_sm; + for (auto& ch : deduced) { + add_iceberg_annotation(ch.order, ch.change, _me->icebergs, ev.event_id); + deduced_sm.emplace_back(ch); + } + _me->exch_matcher.on_order_change(s->security_id(), deduced_sm, nullptr, fills); + } + } + // Verify the matcher's orders are identitical to the exchange's. + _sync_current_orders(ev); + bool orders_match = true; + for (const auto* s : sorted_secs) { + if (!_is_ready_to_trade(*s, ev.event_id)) + continue; + auto& sec_data = _me->security_cache[s->security_id()]; + auto& curr_orders = sec_data.current_orders; + auto error = _me->exch_matcher.has_same_orders(s->security_id(), curr_orders); + if (error) { + ERROR( + "order discrepancies detected for ", s->symbol(), " on event ", ev.event_id, + "\nError: ", error.value(), "\nOld Orders:\n", + _me->security_cache[s->security_id()].previous_orders); + orders_match = false; + } + } + bool fills_match = match_fills(fills, sorted_secs); + if (!orders_match || !fills_match) { + // todo: add the iceberg fills back, to get to the previous state. + // describe current state for diagnostic purposes + std::stringstream msg; + msg << "Spartan Top of Books:" << std::endl; + _me->exch_matcher.print_all_bbos(msg); + msg << std::endl; + // spartan fills + msg << "Spartan Total Fill Qty: " << fills.total_fill_qty() << std::endl; + auto fill_cmp = [](const auto& lhs, const auto& rhs) { + return lhs.trade_entry_id == rhs.trade_entry_id + ? static_cast(lhs.aggressor) > static_cast(rhs.aggressor) + : lhs.trade_entry_id < rhs.trade_entry_id; + }; + std::vector spartan_fills; + fills.copy_to(spartan_fills); + std::sort(spartan_fills.begin(), spartan_fills.end(), fill_cmp); + msg << "Spartan Fills (ordered by Trade ID):" << std::endl; + diagnostics::print_fills(spartan_fills.begin(), spartan_fills.end(), msg); + // exch top of books + msg << "Exchange Top of Books:" << std::endl; + diagnostics::print_many_bbo( + sorted_secs.begin(), sorted_secs.end(), true, true, msg); + int exch_fill_qty = 0; + for (auto* s : sorted_secs) { + for (auto f : s->fills()) + exch_fill_qty += f.quantity.value(); + } + // exch fills + msg << "Exchange Total Fill Qty: " << exch_fill_qty << std::endl; + std::vector all_exch_fills; + for (auto* s : sorted_secs) { + for (auto f : s->fills()) + all_exch_fills.push_back(f); + } + std::sort(all_exch_fills.begin(), all_exch_fills.end(), fill_cmp); + msg << "Exchange Fills (ordered by Trade ID):" << std::endl; + diagnostics::print_fills(all_exch_fills.begin(), all_exch_fills.end(), msg); + INFO(msg.str()); + // reset the matcher to avoid cascading/compounding errors in subsequent events. + _reset_matcher(); + } + _sync_previous_orders(ev); +} + +void MatcherValidationObserver::on_real_time_channel_loss() { bts_assert(false); } +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h new file mode 100644 index 000000000..7f2955a92 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h @@ -0,0 +1,59 @@ +#pragma once +#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" +#include +#include +#include + +namespace bts::spartan::mdp3 { +struct MatcherValidationObserver : IChannelObserver { + MatcherValidationObserver(std::unordered_map icebergs); + // interesting_event_ranges: + // - keys indicate the first event in an interesting range + // - values indicate the number of events in the range (i.e., the range's length). + MatcherValidationObserver( + std::unordered_map icebergs, + boost::container::flat_map interesting_event_ranges); + ~MatcherValidationObserver(); + void on_recovery_complete(InitialSnapshot const& snap) override; + void on_real_time_event(MDP3Event const& ev) override; + void on_real_time_channel_loss() override; + // the first time a security's actual trading status + // (i.e., mdp3::SecurityTradingStatus from the exchange) + // is observed to be 'ReadyToTrade,' its local TradingStatus + // (this enum) is switched from 'None' to Preopen.' The security's + // changes in that event are ignored, but on the subsequent event + // its TradingStatus will be flipped from 'Preopen' to 'Uninitialized' + // (in the beginning of the callback). That will be the first event + // in which order changes will be processed for the security. + // As such, the matcher will initialize the state for the security + // by adding every resting order it had prior to the event. + // After that, the TradingStatus will be flipped to 'Initialized.' + // From then on, updates for the security will be processed incrementally. + enum class TradingStatus { + None, + Preopen, + Uninitialized, + Initialized + }; + + private: + struct PImpl; + std::unique_ptr _me; + + TradingStatus _trading_status(const InstrumentSnapshot& s, size_t event_id); + // returns true if the security's TradingStatus is 'Uninitialized', or 'Initialized' + // (in other words, if it isn't 'None,' or 'Preopen') + bool _is_ready_to_trade(const InstrumentSnapshot& s, size_t event_id); + // std::tuple _event_filter(const MDP3Event&); + + // state maintenance + void _sync_current_orders(MDP3Event const& ev); + void _sync_previous_orders(MDP3Event const& ev); + void _reset_matcher(); +}; +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.proto b/src/server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.proto new file mode 100644 index 000000000..0873665cc --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.proto @@ -0,0 +1,87 @@ +syntax="proto3"; +package bts.spartan.protos; + +/* +algo: "Sniper" +n_cores: 4 + +base_algo_latency_ns: 1000 +base_algo_latency_ns: 5000 +base_algo_latency_ns: 10000 +transaction_latency_ns: 50000 + +param {key: "" value {item: "1" item: "2"}} +parameter {key: "Qty" value {item: "1" item: "2"}} +*/ + +// The parameters for an entire simulation session, consisting of +// multiple runs. +message SimSessionOptions { + // The algo we are testing + string algo = 1; + + // the channel we are testing + string channel = 2; + + // Wheter we log every fill/book update/order. Use with care, this + // results in large quantities of log information. + bool log_all_transactions = 10; + + // Wheter we should clear all previous runs before starting. False by default. + bool clear_db = 11; + + // The max number of cores to use for the simulation. By default all + // cores will be used. + uint32 n_cores = 4 ; + + // the days we are testing this out + repeated string market_date = 5; + + // A base latency added to any orders produced by an algo. By + // default 22000. + repeated uint64 base_algo_latency_ns = 6; + + // An estimate for the average transaction latency at the + // exchange. We fallback on this value when we cannot use the send + // time of a packet to estimate transaction duration. By default + // 60000 + repeated uint64 transaction_latency_ns = 7; + + // Wheter we should use the first packet or the last packet send + // times when calculating when an algo would see market data + repeated bool use_first_packet_st = 9; + + repeated bool simulate_icebergs = 13; + + // The start and stop time for the simulation in the form: + // 23:00 04:000. Blank for whatever is in the pcap. + repeated string sim_start_stop_time = 12; + + // Algo specific parameters used for this session. The simulation + // wll run through all combinations of parameters, recording the + // outcome of each. + message Param { + string name = 1; + string value = 2; + } + repeated Param param = 8; +} + +// The parameters for a single simulation run. +message SimRunOptions { + string algo = 1; + string channel = 2; + string market_date = 3; + bool log_all_transactions = 9; + uint32 param_set_id = 10; + string sim_start_stop_time = 11; + bool use_first_packet_st = 8; + uint64 base_algo_latency_ns = 4; + uint64 transaction_latency_ns = 6; + bool simulate_icebergs = 12; + map param = 7; +} + +message DayFiles { + repeated string file = 1; +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/run/run_sim.cpp b/src/server/sparkd/cmds/spartan/back_testing/run/run_sim.cpp new file mode 100644 index 000000000..04dddcc29 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/run/run_sim.cpp @@ -0,0 +1,267 @@ +#include "run_sim.h" + +#include "server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.h" +#include "server/sparkd/cmds/spartan/back_testing/algos/SniperModel.h" +#include "server/sparkd/cmds/spartan/back_testing/db/SimDB.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/Sim.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.h" + +#include "lib/utils/PBUtils.h" +#include +#include +#include + +namespace bts::spartan::mdp3 { + +std::string pcap_location(std::string const &channel, std::string const &date) { + auto base_path = os::bts_data_path(); + auto pcap_path = fmt::format("{}pcaps/{}/{}_{}_24h.pcap", base_path, channel, + date, channel); + auto xz_pcap_path = pcap_path + ".xz"; + if (boost::filesystem::exists(pcap_path)) + return pcap_path; + else if (boost::filesystem::exists(xz_pcap_path)) + return xz_pcap_path; + else { + bts_assert_msg(false, + fmt::format("Could not find pcap for channel {} on {}", + channel, date)); + return ""; + } +} + +std::vector +calc_param_dimensions(protos::SimSessionOptions const &options) { + std::vector dimensions; + dimensions.push_back(options.base_algo_latency_ns().size()); + dimensions.push_back(options.transaction_latency_ns().size()); + dimensions.push_back(options.use_first_packet_st().size()); + dimensions.push_back(options.simulate_icebergs().size()); + dimensions.push_back(options.sim_start_stop_time().size()); + std::map param_dims; + for (auto const &p : options.param()) + param_dims[p.name()]++; + for (auto const &p : param_dims) { + dimensions.push_back(p.second); + } + return dimensions; +} + +void generate_param_combinations(std::vector const &dimensionality, + std::vector ¤t_combination, + size_t current_idx, + std::vector> *out) { + bts_assert(dimensionality.size() == current_combination.size()); + if (current_idx == dimensionality.size()) + out->push_back(current_combination); + else { + if (dimensionality[current_idx] == 0) + generate_param_combinations(dimensionality, current_combination, + current_idx + 1, out); + else + for (size_t i = 0; i < dimensionality[current_idx]; i++) { + current_combination[current_idx] = i; + generate_param_combinations(dimensionality, current_combination, + current_idx + 1, out); + } + } +} + +std::vector> +gen_param_combinations(protos::SimSessionOptions const &options) { + auto dimensions = calc_param_dimensions(options); + std::vector first_combination(dimensions.size(), -1); + std::vector> all_combinations; + generate_param_combinations(dimensions, first_combination, 0, + &all_combinations); + return all_combinations; +} + +std::vector +gen_runs(protos::SimSessionOptions const &options) { + auto combinations = gen_param_combinations(options); + std::vector runs; + for (auto &date : options.market_date()) { + size_t param_set_id = 1; + for (auto &c : combinations) { + + auto &this_run = runs.emplace_back(); + this_run.set_param_set_id(param_set_id++); + this_run.set_algo(options.algo()); + this_run.set_channel(options.channel()); + this_run.set_log_all_transactions(options.log_all_transactions()); + + this_run.set_market_date(date); + + size_t param_idx = 0; + + // base algo latency + if (c[param_idx] == -1) + this_run.set_base_algo_latency_ns(22000); + else + this_run.set_base_algo_latency_ns( + options.base_algo_latency_ns(c[param_idx])); + param_idx++; + + // transaction latency + if (c[param_idx] == -1) + this_run.set_transaction_latency_ns(60000); + else + this_run.set_transaction_latency_ns( + options.transaction_latency_ns(c[param_idx])); + param_idx++; + + // use first packet st + if (c[param_idx] == -1) + this_run.set_use_first_packet_st(false); + else + this_run.set_use_first_packet_st( + options.use_first_packet_st(c[param_idx])); + param_idx++; + + // simulate icebergs + if (c[param_idx] == -1) + this_run.set_simulate_icebergs(true); + else + this_run.set_simulate_icebergs( + options.simulate_icebergs(c[param_idx])); + param_idx++; + + // start/stop time + if (c[param_idx] == -1) + *(this_run.mutable_sim_start_stop_time()) = "21:45 21:00"; + else + *(this_run.mutable_sim_start_stop_time()) = + options.sim_start_stop_time(c[param_idx]); + param_idx++; + + std::map> algo_params; + for (auto const &p : options.param()) + algo_params[p.name()].push_back(p.value()); + + for (auto const &[name, values] : algo_params) { + auto this_parameter_name = name; + auto this_parameter_value = values[c[param_idx++]]; + this_run.mutable_param()->insert( + {this_parameter_name, this_parameter_value}); + } + } + } + return runs; +} + +class PCapEventSourceCache { + struct SrcRec { + std::mutex lock; + std::optional source; + }; + std::mutex lock; + std::unordered_map> ev_source_cache; + + public: + PCapEventSource &get_source(std::string const &pcap_loc) { + + SrcRec *rec = nullptr; + { + std::lock_guard l(lock); + auto &ptr = ev_source_cache[pcap_loc]; + if (!ptr) + ptr.reset(new SrcRec()); + rec = ptr.get(); + } + + { + std::lock_guard l(rec->lock); + if (!rec->source) + rec->source.emplace(pcap_loc); + return *rec->source; + } + } +}; + +PCapEventSourceCache ev_source_cache; + +void run_sim(SimDB &db, SimDB::SessionID sid, std::atomic &remaining, + protos::SimRunOptions const &raw_options) { + + std::unique_ptr algo; + if (raw_options.algo() == "Sniper") + algo.reset(new SniperModel()); + else if (raw_options.algo() == "PolyObserver") + algo.reset(new PolyObserver()); + else + bts_assert(false); + + auto options = algo->fill_defaults(raw_options); + auto rid = db.begin_run(sid, options); + LOG(Spartan, Warn, + "Starting preload. run: {} date: {} remaining: {} " + "----------------------------------------------", + rid, raw_options.market_date(), (size_t)remaining); + auto logger = [rid, &db](size_t sim_sec, std::string const &src, auto ts, + auto const &header, auto const &entry_type, + auto const &args) { + db.log(rid, sim_sec, src, ts, header, entry_type, args); + }; + + auto pcap_loc = pcap_location(options.channel(), options.market_date()); + auto &source = ev_source_cache.get_source(pcap_loc); + LOG(Spartan, Warn, + "Starting simulation. run: {} date: {} remaining: {} " + "----------------------------------------------", + rid.value(), raw_options.market_date(), (size_t)remaining); + auto event_gen = source.get_event_generator(); + + Sim sim(*algo, options, source.instruments(), source.icebergs(), + source.polygons(), event_gen, logger); + auto pnl = sim.run(); + db.end_run(rid, pnl.cash, pnl.commission, pnl.position); + remaining--; + LOG(Spartan, Warn, + "Completed simulation. run: {} date: {} pandl: {} remaining: {} " + "----------------------------------------------", + rid.value(), raw_options.market_date(), pnl.cash, (size_t)remaining); +} + +void run_sim(std::string const &path_to_session_options) { + spartan::protos::SimSessionOptions options; + std::ifstream proto_in(path_to_session_options); + std::string str((std::istreambuf_iterator(proto_in)), + std::istreambuf_iterator()); + bts_assert(utils::pb::from_string(str, &options)); + + SimDB db; + auto sid = db.create_session(options.algo(), options.clear_db()); + + std::mutex lock; + std::vector workers(options.n_cores() ? options.n_cores() + : os::num_cores()); + auto run_options = gen_runs(options); + std::queue runs; + for (auto &ru : run_options) + runs.push(ru); + + std::atomic remaining = runs.size(); + + for (auto &w : workers) { + w = std::thread{[&remaining, &lock, &runs, &db, sid]() { + while (true) { + spartan::protos::SimRunOptions these_options; + { + std::lock_guard l(lock); + if (runs.size()) { + these_options = runs.front(); + runs.pop(); + } else + return; + } + run_sim(db, sid, remaining, these_options); + } + }}; + } + + for (auto &w : workers) + w.join(); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/run/run_sim.h b/src/server/sparkd/cmds/spartan/back_testing/run/run_sim.h new file mode 100644 index 000000000..0dec932a9 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/run/run_sim.h @@ -0,0 +1,14 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/back_testing/db/SimLogger.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" + +namespace bts::spartan::mdp3 { +class MDP3EventGenerator; +class AlgoModel; +void run_sim(protos::SimRunOptions const&); +void run_sim(std::string const&path_to_run_options); + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h b/src/server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h new file mode 100644 index 000000000..b0be454d4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h" +#include "lib/utils/bts_assert.h" +#include + +namespace bts::spartan::mdp3{ +struct EventHeader; +struct MBPUpdate; +struct OrderUpdate; +} + +namespace bts::spartan::mdp3 { + +class ISim; + +class IAlgoModel { + public: + virtual ~IAlgoModel() {} + virtual protos::SimRunOptions fill_defaults (protos::SimRunOptions const&in) {return in;} + virtual void init(ISim &, protos::SimRunOptions const&) = 0; + virtual void finit() {} + virtual void on_replay_start(EventHeader const &, + std::vector const &) {} + virtual void on_order_update(EventHeader const &, OrderUpdate const &) {} + virtual void on_mbp(EventHeader const &, MBPUpdate const &) {} + virtual void on_time() {} +}; + +inline std::string get_param(protos::SimRunOptions const&in, std::string const&name) { + auto it = in.param().find(name); + bts_assert(it!= in.param().end()); + return it->second; +} + +template T get_param(protos::SimRunOptions const&in,std::string const&name) { + auto it = in.param().find(name); + bts_assert(it!= in.param().end()); + + T result; + std::stringstream{it->second} >> result; + return result; +} + +template void set_param_if_empty(protos::SimRunOptions *in,std::string const&name, T const&v) { + auto it = in->param().find(name); + if (it == in->param().end()) { + std::stringstream str_v; + str_v << v; + auto mmap = in->mutable_param(); + (*mmap)[name] = str_v.str(); + } +} + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/ISim.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/ISim.cpp new file mode 100644 index 000000000..de328187d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/ISim.cpp @@ -0,0 +1,7 @@ +#include "ISim.h" + +namespace bts::spartan::mdp3 { + +ISim::~ISim() {} + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/ISim.h b/src/server/sparkd/cmds/spartan/back_testing/sim/ISim.h new file mode 100644 index 000000000..4eaa15e1f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/ISim.h @@ -0,0 +1,37 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" + +namespace bts::spartan::mdp3 { + +class ISim { + public: + virtual ~ISim(); + + // Queries + virtual PolygonMap const &polygons() const = 0; + virtual InstrumentMap const &instruments() const = 0; + virtual InstrumentDefinition const &get_sec(MDP3SecurityID id) const = 0; + virtual MDP3TimeStamp time() const = 0; + + // Actions + virtual MDP3OrderKey send(SmallOrderChange const &, size_t delay_ns = 0) = 0; + virtual void start_simulation() = 0; + virtual void stop_simulation() = 0; + virtual void log_pandl() const = 0; + template void log(std::string const &ty, T &&... args) const; + + private: + virtual void algo_log(std::string const &log_entry_type, + std::vector const &log_args) const = 0; +}; + +template +void ISim::log(std::string const &log_entry_type, T &&... args) const { + auto lst = std::vector{to_str(std::forward(args))...}; + algo_log(log_entry_type, lst); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp new file mode 100644 index 000000000..c09f40957 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp @@ -0,0 +1,763 @@ +#include "Sim.h" + +#include "lib/logging/logging.h" +#include "lib/utils/datetime.h" +#include "server/sparkd/cmds/spartan/back_testing/db/SimLogger.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h" +#include "server/sparkd/cmds/spartan/back_testing/run/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/IAlgoModel.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" + +#include +#include +#include +#include +#include +#include + +namespace bts::spartan::mdp3 { + +enum class Sim::State { + REPLAY, // Get book state from pcap, do not log. + PRESIM, // Get book state from pcap, log + SIM // Get book state from matcher, log +}; + +enum class Sim::PendingSimOP { NONE, START, STOP }; + +struct Sim::Simulation { + MergedMatcher matcher = {}; + + // CME does not guarantee uniqueness of order keys during a + // session, though of course, they must guarantee uniqueness of + // *alive* order keys. During a simulation its possible that the + // life cycle of a CME order changes from what happened in reality, + // which could lead to key colision if we used the cme key. For + // this reason we must remap the CME keys to ones we know will be + // unique during the entire simulation. + std::unordered_map cme_key_map; + + // The matcher does not currently support non-trading security + // states. For this reason we have to use this map to detect + // changes of trading state in securities so that when a security + // starts trading we can seed the matcher with existing orders and + // when it stops trading we can remove its orders + std::unordered_set ready_secs; +}; + +struct Sim::AlgoCBFilter : IAlgoModel { + AlgoCBFilter(Sim &owner, IAlgoModel &real_algo) + : owner(owner), real_algo(real_algo) {} + Sim &owner; + IAlgoModel &real_algo; + + void init(ISim &s, protos::SimRunOptions const &options) override { + real_algo.init(s, options); + } + void finit() override { real_algo.finit(); } + + void on_replay_start(EventHeader const &h, + std::vector const &b) override { + real_algo.on_replay_start(h, b); + } + void on_order_update(EventHeader const &h, OrderUpdate const &o) override { + if (!owner.is_stopping_sim()) + real_algo.on_order_update(h, o); + } + void on_mbp(EventHeader const &h, MBPUpdate const &mbp) override { + if (!owner.is_stopping_sim()) + real_algo.on_mbp(h, mbp); + } + void on_time() override { + if (!owner.is_stopping_sim()) + real_algo.on_time(); + } +}; + +struct Sim::PImpl { + AlgoCBFilter algo; + protos::SimRunOptions options; + InstrumentMap const &instruments; + IcebergMap const &ib_map; + PolygonMap const &polygons; + PCapEventGenerator &event_generator; + SimLogger logger; + std::unordered_map snapshots = {}; + State state = State::REPLAY; + MDP3TimeStamp transact_time = MDP3TimeStamp{0}; + MDP3TimeStamp time = MDP3TimeStamp{0}; + AlgoEventID event_id = {0, 0}; + std::optional simulation = {}; + MDP3OrderKey last_order_key = + MDP3OrderKey{std::numeric_limits::max()}; + std::vector my_fills = {}; + PendingSimOP pending_sim_op = PendingSimOP::NONE; + MDP3TimeStamp next_time_event = MDP3TimeStamp{0}; + std::queue pending_algo_actions = {}; + size_t sim_seq = 0; + size_t next_mapped_key = 1; + TradingWindow trading_window = {}; +}; + +Sim::Sim(IAlgoModel &algo, protos::SimRunOptions const &options, + InstrumentMap const &instruments, IcebergMap const &ice, + PolygonMap const &polys, PCapEventGenerator &gen, SimLogger const &l) + : me(new PImpl{AlgoCBFilter{*this, algo}, options, instruments, ice, polys, + gen, l}) { + me->trading_window = calculate_trading_window( + me->options.market_date(), me->options.sim_start_stop_time()); +} + +Sim::~Sim() {} + +PandLSummary Sim::run() { + me->algo.init(*this, me->options); + + // Consume pcap events but do not send them to the algo up until + // trading window opens up + while (not_yet_in_trading_window()) { + if (!me->event_generator.top()) + return {}; + else { + auto const &ev = *me->event_generator.top(); + me->transact_time = ev.transact_time; + me->time = get_send_time(ev); + me->event_id.first = ev.event_id; + for (auto const &snap : ev.affected_securities) + me->snapshots[snap.security_id()].update_with(snap); + me->event_generator.move_next(); + } + } + + // Now initialize the algo with whatever books we currently have + on_algo_replay_start(); + + // Extract events from the pcap and forward them to the algo while + // we are still in the trading window + while (me->event_generator.top() && still_in_trading_window()) { + bts_assert(!has_pending_sim_op()); + next_event(); + if (has_pending_sim_op()) + handle_pending_sim_op(); + } + + // All done, log pandl and exit + log_pandl(); + return pnl(); +} + +void Sim::handle_pending_sim_op() { + switch (me->pending_sim_op) { + case PendingSimOP::NONE: + break; + case PendingSimOP::START: + bts_assert(me->state == State::REPLAY); + me->pending_sim_op = PendingSimOP::NONE; + me->state = State::PRESIM; + me->sim_seq++; + sim_log("presim_start"); + break; + case PendingSimOP::STOP: + bts_assert(me->state == State::SIM); + me->pending_sim_op = PendingSimOP::NONE; + _stop_simulation(); + break; + } +} + +void Sim::next_event() { + bts_assert(me->event_generator.top()); + auto &next_pcap = *me->event_generator.top(); + + switch (me->state) { + case State::REPLAY: + if (time_event_due_before(next_pcap.transact_time)) + on_time(); + else + on_pcap_event_during_replay(); + break; + case State::PRESIM: { + bts_assert(!me->simulation); + auto &pending_actions = me->pending_algo_actions; + + if (pending_actions.size() && + pending_actions.front().transact_time < next_pcap.transact_time) { + + // We have a pending algo action with a transact time ealier + // than our next pcap event + auto &next_algo_action = pending_actions.front(); + if (time_event_due_before(next_algo_action.transact_time)) + on_time(); + else { + _start_simulation(); + on_algo_action(); + } + } else { + // The pcap event is the next in line + if (time_event_due_before(next_pcap.transact_time)) + on_time(); + else + on_pcap_event_during_replay(); + } + break; + } + case State::SIM: { + bts_assert(me->simulation); + auto &pending_actions = me->pending_algo_actions; + + if (pending_actions.size() && + pending_actions.front().transact_time < next_pcap.transact_time) { + + // We have a pending algo action with a transact time ealier + // than our next pcap event + auto &next_algo_action = pending_actions.front(); + if (time_event_due_before(next_algo_action.transact_time)) + on_time(); + else + on_algo_action(); + } else { + // The pcap event is the next in line + if (time_event_due_before(next_pcap.transact_time)) + on_time(); + else + on_pcap_event_during_simulation(); + } + break; + } + } + + // We do not want to start the timer at 0 because we would have to + // process timer events for billions of years before a pcap or algo + // action event was due. + if (me->next_time_event == MDP3TimeStamp{0} && me->time.value()) + me->next_time_event = me->time; +} + +void Sim::on_pcap_event_during_replay() { + bts_assert(me->state == State::REPLAY || me->state == State::PRESIM); + bts_assert(me->event_generator.top()); + bts_assert(!me->simulation); + + auto const &ev = *me->event_generator.top(); + me->transact_time = ev.transact_time; + me->time = + MDP3TimeStamp{std::max(me->time.value() + 1, get_send_time(ev).value())}; + me->event_id.first = ev.event_id; + + for (auto const &snap : ev.affected_securities) + me->snapshots[snap.security_id()].update_with(snap); + + EventHeader header{{ev.event_id, 0}, ev.transact_time, me->time}; + + for (auto &snap : ev.affected_securities) { + if (snap.ready_to_trade()) { + MBPUpdate mbp{ + snap.security_id(), + snap.book(api::SIDE_BUY), + snap.book(api::SIDE_SELL), + snap.implied_book(api::SIDE_BUY), + snap.implied_book(api::SIDE_SELL), + }; + if (me->state == State::PRESIM && me->options.log_all_transactions()) { + auto changes = snap.deduced_order_changes(); + for (auto &c : changes) { + sim_log("presim_order", ev.transact_time.value(), + change_name(c.change), + get_sec(c.order.security).symbol(), + c.order.price.price(), c.order.quantity.value(), + api::Side_Name(c.order.side), c.order.key.value(), + c.order.key.value()); + } + sim_log("presim_book", get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), + mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), + mbp.mbp_sell[0].quantity.value()); + } + me->algo.on_mbp(header, mbp); + } + } + me->event_generator.move_next(); +} + +void Sim::on_pcap_event_during_simulation() { + bts_assert(me->state == State::SIM); + bts_assert(me->event_generator.top()); + bts_assert(me->simulation); + + auto const &ev = *me->event_generator.top(); + me->transact_time = ev.transact_time; + me->time = + MDP3TimeStamp{std::max(me->time.value() + 1, get_send_time(ev).value())}; + me->event_id.first = ev.event_id; + + for (auto &snap : ev.affected_securities) + me->snapshots[snap.security_id()].update_with(snap); + + for (auto &snap : ev.affected_securities) { + auto this_sid = snap.security_id(); + bool was_ready = me->simulation->ready_secs.count(this_sid); + bool is_ready = snap.ready_to_trade(); + + // The security was not ready for trading but it is now. We + // need to boostrap the matcher with its current order book. + if (!was_ready && is_ready) { + sim_log("sec_started_trading", snap.symbol()); + bootstrap_simulated_order_book(ev.transact_time, this_sid); + } + + // The security was trading but is no longer. We need to + // clear it from the matcher. + else if (was_ready && !is_ready) { + sim_log("sec_stoped_trading", snap.symbol()); + clear_simulated_order_book(ev.transact_time, this_sid); + } + + // The security was and remains ready. We need to incorporate + // the latest deduced orders in the matcher and send out any + // matcher effects. + else if (was_ready && is_ready) { + auto changes = snap.deduced_order_changes(); + SmallOrderChangeVec matcher_input; + for (auto &c : changes) { + auto corder = correct_pcap_order(c); + if (me->options.log_all_transactions()) + sim_log("pcap_order", ev.transact_time.value(), + change_name(corder.change), + get_sec(corder.order.security).symbol(), + corder.order.price.price(), + corder.order.quantity.value(), + api::Side_Name(corder.order.side), + corder.order.key.value(), corder.original_key.value()); + matcher_input.push_back(corder); + } + match_and_send_effects(ev.transact_time, matcher_input); + } + } + + me->event_generator.move_next(); +} + +void Sim::on_algo_action() { + bts_assert(me->state == State::SIM); + bts_assert(me->simulation); + bts_assert(me->pending_algo_actions.size()); + auto &algo_action = me->pending_algo_actions.front(); + auto const &change = algo_action.change; + me->transact_time = algo_action.transact_time; + me->time.value() += me->options.transaction_latency_ns(); + me->event_id.second++; + + sim_log("algo_order", algo_action.transact_time.value(), + change_name(change.change), get_sec(change.order.security).symbol(), + change.order.price.price(), change.order.quantity.value(), + api::Side_Name(change.order.side), change.order.key.value(), 0); + + match_and_send_effects(algo_action.transact_time, {algo_action.change}); + me->pending_algo_actions.pop(); +} + +void Sim::_start_simulation() { + bts_assert(me->state == State::PRESIM); + bts_assert(!me->simulation); + me->state = State::SIM; + auto &simulation = me->simulation.emplace(); + auto &matcher = simulation.matcher; + + sim_log("simulation_start"); + for (auto &[sec_id, snap] : me->snapshots) { + matcher.add_def(snap.definition()); + if (snap.ready_to_trade()) { + simulation.ready_secs.insert(sec_id); + std::vector resting_orders; + for (const auto &p : snap.orders()) + resting_orders.emplace_back( + correct_pcap_order({p.second, p.second, OrderChange::added}, + true) + .order); + std::vector effects; + matcher.reset(sec_id, resting_orders, &effects); + for (auto &ef : effects) + std::visit( + overloaded{[&](MBPUpdate &mbp) { + sim_log("sim_book_init", + get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), + mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), + mbp.mbp_sell[0].quantity.value()); + }, + [&](OrderUpdate const &) { bts_assert(false); }}, + ef); + } + } + + validate_matcher_books(api::SIDE_BUY); + validate_matcher_books(api::SIDE_SELL); +} + +void Sim::_stop_simulation() { + bts_assert(me->state == State::SIM); + me->simulation.reset(); + me->pending_algo_actions = {}; + me->state = State::REPLAY; + me->event_id.second = 0; + on_algo_replay_start(); +} + +void Sim::on_algo_replay_start() { + std::vector collected_books; + for (auto &[id, snap] : me->snapshots) + if (snap.ready_to_trade()) { + collected_books.emplace_back( + MBPUpdate{id, snap.book(api::SIDE_BUY), snap.book(api::SIDE_SELL), + snap.implied_book(api::SIDE_BUY), + snap.implied_book(api::SIDE_SELL)}); + } + me->algo.on_replay_start( + EventHeader{me->event_id, me->transact_time, me->time}, collected_books); +} + +void Sim::match_and_send_effects(MDP3TimeStamp transact_time, + SmallOrderChangeVec const &matcher_input) { + + bts_assert(me->state == State::SIM); + bts_assert(me->simulation); + + auto &simulation = *me->simulation; + + std::vector effects; + simulation.matcher.match(MergedMatcher::SIMULATOR_INPUT, matcher_input, + &effects); + std::sort(effects.begin(), effects.end(), + [](auto const &ef1, auto const &ef2) { + auto rankf1 = std::holds_alternative(ef1) ? 0 : 1; + auto rankf2 = std::holds_alternative(ef2) ? 0 : 1; + return rankf1 < rankf2; + }); + EventHeader header{me->event_id, transact_time, me->time}; + + for (auto &ef : effects) + std::visit( + overloaded{ + [&](MBPUpdate const &mbp) { + if (me->options.log_all_transactions()) + sim_log("sim_book", get_sec(mbp.security_id).symbol(), + mbp.mbp_buy[0].price.price(), + mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), + mbp.mbp_sell[0].quantity.value()); + me->algo.on_mbp(header, mbp); + }, + [&](OrderUpdate const &order_update) { + if (order_update.key.value() >= me->last_order_key.value()) { + sim_log("algo_fill", order_update.status_str(), + api::Side_Name(order_update.side), + get_sec(order_update.security_id).symbol(), + order_update.last_price.price(), + order_update.last_fills.value(), + order_update.key.value(), 0); + me->my_fills.push_back(order_update); + me->algo.on_order_update(header, order_update); + } else { + if (me->options.log_all_transactions()) + sim_log("fill", order_update.status_str(), + api::Side_Name(order_update.side), + get_sec(order_update.security_id).symbol(), + order_update.last_price.price(), + order_update.last_fills.value(), + order_update.key.value(), + me->simulation->cme_key_map[order_update.key] + .value()); + } + }}, + ef); +} + +void Sim::bootstrap_simulated_order_book(MDP3TimeStamp transact_time, + MDP3SecurityID sid) { + bts_assert(me->state == State::SIM); + bts_assert(me->simulation); + auto &simulation = *me->simulation; + + simulation.ready_secs.insert(sid); + auto &snap = me->snapshots[sid]; + SmallOrderChangeVec matcher_input; + for (auto const &p : snap.orders()) + matcher_input.push_back(correct_pcap_order( + OrderChange{p.second, p.second, OrderChange::added}, true)); + match_and_send_effects(transact_time, matcher_input); +} + +void Sim::clear_simulated_order_book(MDP3TimeStamp transact_time, + MDP3SecurityID sid) { + bts_assert(me->state == State::SIM); + bts_assert(me->simulation); + auto &simulation = *me->simulation; + + simulation.ready_secs.erase(sid); + auto &snap = me->snapshots[sid]; + SmallOrderChangeVec matcher_input; + for (auto const &p : snap.orders()) + matcher_input.push_back(correct_pcap_order( + OrderChange{p.second, p.second, OrderChange::removed})); + match_and_send_effects(transact_time, matcher_input); +} + +SmallOrderChange Sim::correct_pcap_order(OrderChange const &o, + bool set_exact_ib_state) { + bts_assert(me->state == State::SIM); + bts_assert(me->simulation); + auto &key_map = me->simulation->cme_key_map; + + bts_assert(o.order.key.value()); + MDP3OrderKey &our_key = key_map[o.order.key]; + + SmallOrderChange result(o); + result.original_key = o.order.key; + + switch (o.change) { + case OrderChange::added: { + our_key = MDP3OrderKey{me->next_mapped_key++}; + result.order.key = our_key; + break; + } + case OrderChange::removed: { + bts_assert(our_key != MDP3OrderKey{}); + result.order.key = our_key; + our_key = MDP3OrderKey{}; + break; + } + case OrderChange::updated: + case OrderChange::updated_price: { + bts_assert(our_key != MDP3OrderKey{}); + result.order.key = our_key; + } + } + if (me->options.simulate_icebergs()) { + if (o.change != OrderChange::ChangeType::removed) { + // IcebergMap is keyed by exchange keys + auto ib_it = me->ib_map.find(result.original_key); + if (ib_it != me->ib_map.end()) + iceberg_helper::add_iceberg_annotation( + result.order, result.change, ib_it->second, me->event_id.first, + set_exact_ib_state); + } + } + return result; +} + +PolygonMap const &Sim::polygons() const { return me->polygons; } + +InstrumentMap const &Sim::instruments() const { return me->instruments; } + +InstrumentDefinition const &Sim::get_sec(MDP3SecurityID id) const { + auto it = me->instruments.find(id); + bts_assert(it != me->instruments.end()); + return *it->second; +} + +MDP3TimeStamp Sim::time() const { return me->time; } + +double Sim::price_at_market(MDP3SecurityID id, MDP3Quantity qty, + api::Side s) const { + + auto it = me->snapshots.find(id); + bts_assert(it != me->snapshots.end()); + auto &snap = it->second; + + auto &book = snap.book(s); + double cummulative_flow = 0; + auto remaining_qty = qty.value(); + for (auto &l : book) { + if (remaining_qty == 0) + return cummulative_flow; + else { + auto from_this_level = std::min(remaining_qty, l.quantity.value()); + cummulative_flow += + (from_this_level * get_sec(id).to_cash(l.price.price())); + remaining_qty -= from_this_level; + } + } + if (remaining_qty) + return os::nan(); + else + return cummulative_flow; +} + +PandLSummary Sim::pnl() const { + auto mkt_pricer = [this](MDP3SecurityID id, MDP3Quantity qty, api::Side s) { + return price_at_market(id, qty, s); + }; + + PandL pl{me->instruments, mkt_pricer}; + auto outright_pos = pl.outright_position(me->my_fills); + + auto cash = pl.net_cash(me->my_fills); + auto commission = pl.commission_cost(me->my_fills); + auto pos = pl.position_liquidation_value(outright_pos); + return {cash - commission, pos, commission}; +} + +void Sim::log_pandl() const { + auto pnl_summary = pnl(); + log("pnl", pnl_summary.cash, pnl_summary.commission, pnl_summary.position); +} + +MDP3OrderKey Sim::send(SmallOrderChange const &change, + size_t propagation_delay) { + + if (me->state == State::REPLAY) + start_simulation(); + + auto transact_time = + MDP3TimeStamp{me->time.value() + me->options.base_algo_latency_ns() + + propagation_delay}; + + if (change.change == OrderChange::added) { + auto cpy = change; + cpy.order.key = MDP3OrderKey{me->last_order_key.value()--}; + me->pending_algo_actions.push(AlgoAction{transact_time, cpy}); + return cpy.order.key; + } else { + bts_assert(change.order.key != MDP3OrderKey{}); + me->pending_algo_actions.push(AlgoAction{transact_time, change}); + return change.order.key; + } +} + +void Sim::algo_log(std::string const &log_entry_type, + std::vector const &log_args) const { + _log("algo", log_entry_type, log_args); +} + +void Sim::sim_log(std::string const &log_entry_type, + std::vector const &log_args) const { + _log("sim", log_entry_type, log_args); +} + +void Sim::_log(std::string const &src, std::string const &log_entry_type, + std::vector const &log_args) const { + std::ostringstream str; + str << to_time_string(me->time) << " " << src << " " << log_entry_type + << " "; + for (auto const &a : log_args) + str << a << " "; + LOG(Spartan, Info, "{}", str.str()); + me->logger(me->sim_seq, src, me->time, + EventHeader{me->event_id, me->transact_time, me->time}, + log_entry_type, log_args); +} + +bool Sim::time_event_due_before(MDP3TimeStamp ts) const { + + return me->next_time_event.value() && + me->next_time_event.value() < ts.value(); +} + +void Sim::on_time() { + me->algo.on_time(); + me->next_time_event.value() += 1000 * 1000 * 100; +} + +void Sim::start_simulation() { + bts_assert(me->state == State::REPLAY); + bts_assert(me->pending_sim_op == PendingSimOP::NONE || + me->pending_sim_op == PendingSimOP::START); + me->pending_sim_op = PendingSimOP::START; +} + +void Sim::stop_simulation() { + bts_assert(me->state == State::SIM); + bts_assert(me->pending_sim_op == PendingSimOP::NONE); + me->pending_sim_op = PendingSimOP::STOP; +} + +void Sim::validate_matcher_books(api::Side side) { + bts_assert(me->state == State::SIM); + bts_assert(me->simulation); + auto &simulation = *me->simulation; + + for (auto &[sec_id, snap] : me->snapshots) { + if (snap.ready_to_trade()) { + if (snap.book(side) != simulation.matcher.book(sec_id, side)) { + + LOG(Spartan, Error, + "Mismatch between matcher and pcap books on event: {}. " + "Side: " + "{}", + me->event_id.first, api::Side_Name(side)); + LOG(Spartan, Error, "PCap Book:"); + LOG(Spartan, Error, "{}", snap.book(side)); + LOG(Spartan, Error, "Matcher Book:"); + LOG(Spartan, Error, "{}", simulation.matcher.book(sec_id, side)); + bts_assert(false); + } + } + } +} + +bool Sim::still_in_trading_window() const { + if (me->state == State::REPLAY) { + return me->time.value() <= me->trading_window.stop.value(); + } else + return true; +} + +bool Sim::not_yet_in_trading_window() const { + bts_assert(me->state == State::REPLAY); + return me->time.value() < me->trading_window.start.value(); +} + +bool Sim::has_pending_sim_op() const { + return me->pending_sim_op != PendingSimOP::NONE; +} + +bool Sim::is_stopping_sim() const { + return me->pending_sim_op == PendingSimOP::STOP; +} + +MDP3TimeStamp Sim::get_send_time(PCapEvent const &ev) const { + if (me->options.use_first_packet_st()) + return ev.first_packet_send_time; + else + return ev.last_packet_send_time; +} + +Sim::TradingWindow +Sim::calculate_trading_window(std::string const &mkt_date, + std::string const &start_stop) const { + if (start_stop.empty()) { + return {MDP3TimeStamp{0}, + MDP3TimeStamp{std::numeric_limits::max()}}; + } else { + std::vector split_vec; + boost::algorithm::split(split_vec, start_stop, + boost::algorithm::is_any_of(" ")); + bts_assert(split_vec.size() == 2); + auto start = split_vec[0]; + auto stop = split_vec[1]; + auto day_start = datetime::parse_duration("21:45"); + auto adjust_day = [&](std::string const &time_str) { + auto dt = datetime::parse_simple_date_time_string(mkt_date); + auto t = datetime::parse_duration(time_str); + if (t < day_start) + dt += boost::gregorian::date_duration(1); + return datetime::PTime{dt.date(), t}; + }; + auto start_time = adjust_day(start); + auto stop_time = adjust_day(stop); + bts_assert(stop_time > start_time); + return {to_mdp3time(start_time), to_mdp3time(stop_time)}; + } +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.h b/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.h new file mode 100644 index 000000000..599d15e46 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.h @@ -0,0 +1,103 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/back_testing/db/SimLogger.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/ISim.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pandl/PandLSummary.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" + +namespace bts::spartan::mdp3 { +struct EventHeader; +class PCapEvent; +struct MBPUpdate; +struct OrderUpdate; +} // namespace bts::spartan::mdp3 +namespace bts::spartan::protos { +class SimRunOptions; +} + +namespace bts::spartan::mdp3 { + +class IAlgoModel; +class PCapEventGenerator; + +class Sim : ISim { + public: + Sim(IAlgoModel &algo, protos::SimRunOptions const &, InstrumentMap const &im, + IcebergMap const &, PolygonMap const &, PCapEventGenerator &, + SimLogger const &); + ~Sim(); + PandLSummary run(); + + private: + // From ISim + PolygonMap const &polygons() const override; + InstrumentMap const &instruments() const override; + InstrumentDefinition const &get_sec(MDP3SecurityID id) const override; + MDP3TimeStamp time() const override; + void log_pandl() const override; + MDP3OrderKey send(SmallOrderChange const &, + size_t propagation_delay) override; + + void start_simulation() override; + void stop_simulation() override; + void algo_log(std::string const &log_entry_type, + std::vector const &log_args) const override; + // End ISim + + PandLSummary pnl() const; + double price_at_market(MDP3SecurityID id, MDP3Quantity qty, + api::Side s) const; + + void next_event(); + void on_pcap_event_during_replay(); + void on_pcap_event_during_simulation(); + void on_algo_action(); + void on_time(); + bool time_event_due_before(MDP3TimeStamp) const; + void bootstrap_simulated_order_book(MDP3TimeStamp transact_time, + MDP3SecurityID); + void clear_simulated_order_book(MDP3TimeStamp transact_time, MDP3SecurityID); + void match_and_send_effects(MDP3TimeStamp transact_time, + SmallOrderChangeVec const &); + SmallOrderChange correct_pcap_order(OrderChange const &pcap_order, + bool set_exact_ib_state = false); + void sim_log(std::string const &log_entry_type, + std::vector const &log_args) const; + + template + void sim_log(std::string const &log_entry_type, T &&... args) const; + + void _log(std::string const &src, std::string const &log_entry_type, + std::vector const &log_args) const; + void _start_simulation(); + void _stop_simulation(); + void validate_matcher_books(api::Side); + bool has_pending_sim_op() const; + void on_algo_replay_start(); + bool not_yet_in_trading_window() const; + bool still_in_trading_window() const; + bool is_stopping_sim() const; + MDP3TimeStamp get_send_time(PCapEvent const &) const; + void handle_pending_sim_op(); + struct TradingWindow { + MDP3TimeStamp start; + MDP3TimeStamp stop; + }; + TradingWindow calculate_trading_window(std::string const &mkt_date, + std::string const &start_stop) const; + enum class PendingSimOP; + enum class State; + struct Simulation; + struct PImpl; + struct AlgoCBFilter; + std::unique_ptr me; +}; + +template +void Sim::sim_log(std::string const &log_entry_type, T &&... args) const { + auto lst = std::vector{to_str(std::forward(args))...}; + _log("sim", log_entry_type, lst); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp new file mode 100644 index 000000000..725c5bc78 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp @@ -0,0 +1,109 @@ +#include "PandL.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/sim_common.h" + +namespace bts::spartan::mdp3 { + +struct PandL::PImpl { + InstrumentMap const &instruments; + Pricer pricer; +}; + +PandL::PandL(InstrumentMap const &m, Pricer const &p) : me(new PImpl{m, p}) {} +PandL::~PandL() {} + +double PandL::net_cash(std::vector const &fills) const { + double net_cash_flow = 0; + for (auto const &f : fills) { + auto cash = price_to_cash(f.security_id, + f.last_fills.value() * f.last_price.price()); + auto flow = mkt_to_cash_flow(f.side); + net_cash_flow += cash * flow; + } + return net_cash_flow; +} + +double PandL::commission_cost(std::vector const &fills) const { + double cost = 0; + for (auto const &f : fills) { + auto n_legs = get_sec(f.security_id).legs().size(); + if (n_legs == 0) + cost += f.last_fills.value() * 1.23; + else + cost += f.last_fills.value() * n_legs * 0.86; + } + return cost; +} + +PandL::SymbolNetCash +PandL::symbol_net_cash(std::vector const &fills) const { + std::unordered_map> fills_per_symbol; + for (auto const &f : fills) + fills_per_symbol[get_sec(f.security_id).asset()].push_back(f); + + SymbolNetCash result; + for (auto const &[symbol, symbol_fills] : fills_per_symbol) { + result[symbol] = net_cash(symbol_fills); + } + return result; +} + +PandL::OutrightPosition +PandL::outright_position(std::vector const &fills) const { + OutrightPosition result; + auto add_outright_pos = [&](auto sec_id, int64_t change) { + auto &this_pos = result[sec_id]; + this_pos += change; + }; + + for (auto const &f : fills) { + auto &sec = get_sec(f.security_id); + + if (sec.legs().size()) { + for (auto &leg : sec.legs()) { + auto leg_contract_change = + mkt_to_contract_flow(f.side) * f.last_fills.value() * + mkt_to_contract_flow(leg.side) * leg.ratio.value(); + add_outright_pos(leg.security_id, leg_contract_change); + } + } else + add_outright_pos(f.security_id, + mkt_to_contract_flow(f.side) * f.last_fills.value()); + } + return result; +} + +double PandL::position_liquidation_value(OutrightPosition const&outright_position) const { + double liquidation_proceeds = 0; + for (auto &[id, this_pos] : outright_position) { + auto mkt_action = api::SIDE_UNDEFINED; + if (this_pos > 0) + mkt_action = api::SIDE_SELL; + else + mkt_action = api::SIDE_BUY; + auto cash_flow = me->pricer(id, MDP3Quantity{(int)this_pos},mkt_action); + liquidation_proceeds += cash_flow; + } + return liquidation_proceeds; +} + +InstrumentDefinition const &PandL::get_sec(MDP3SecurityID id) const { + return mdp3::get_sec(id,me->instruments); +} + +double PandL::price_to_cash(MDP3SecurityID id, double p) const { + return get_sec(id).to_cash(p); +} + +int PandL::mkt_to_cash_flow(api::Side s) const { + if (s == api::SIDE_BUY) + return -1; + else + return 1; +} + +int PandL::mkt_to_contract_flow(api::Side s) const { + return -1 * mkt_to_cash_flow(s); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h new file mode 100644 index 000000000..d291f36f2 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h @@ -0,0 +1,42 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" +#include "lib/os/os.h" + +#include +#include +#include + +namespace bts::spartan::mdp3 {struct OrderUpdate;} + +namespace bts::spartan::mdp3 { + +class PandL { +public: + using SymbolNetCash = + std::unordered_map; + using OutrightPosition= + std::unordered_map; + + PandL(InstrumentMap const&, Pricer const& = [](auto,auto,auto) {return os::nan();}); + ~PandL(); + + double net_cash(std::vector const &fills) const; + SymbolNetCash symbol_net_cash(std::vector const &fills) const; + OutrightPosition outright_position(std::vector const &fills) const; + double position_liquidation_value(OutrightPosition const&) const; + double commission_cost(std::vector const &fills) const; +private: + struct PImpl; + InstrumentDefinition const &get_sec(MDP3SecurityID id) const ; + double price_to_cash(MDP3SecurityID id, double p) const; + int mkt_to_cash_flow(api::Side s) const; + int mkt_to_contract_flow(api::Side s) const; + std::unique_ptr me; +}; + + + + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandLSummary.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandLSummary.h new file mode 100644 index 000000000..d4f1aeb1a --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandLSummary.h @@ -0,0 +1,17 @@ +#pragma once + +namespace bts::spartan::mdp3 { + +struct PandLSummary { + double cash = 0; // net of commission + double position = 0; + double commission = 0; + PandLSummary &operator+=(PandLSummary const &rhs) { + cash += rhs.cash; + position += rhs.position; + commission += rhs.commission; + return *this; + } +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h new file mode 100644 index 000000000..9c96ccb3d --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +class PCapEvent { + public: + MDP3TimeStamp transact_time = MDP3TimeStamp{0}; + size_t event_id = 0; + MDP3TimeStamp first_packet_send_time = MDP3TimeStamp{0}; + MDP3TimeStamp last_packet_send_time = MDP3TimeStamp{0}; + uint32_t first_packet_sn = 0; + uint32_t last_packet_sn = 0; + boost::container::small_vector affected_securities = {}; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.cpp new file mode 100644 index 000000000..02bc70f40 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.cpp @@ -0,0 +1,106 @@ +#include "PCapEventGenerator.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include + +namespace bts::spartan::mdp3 { + +struct PCapEventGenerator::PImpl { + PImpl(std::string const &pcap, IChannelObserver &obs) + : streamer(pcap, obs), done(false) {} + + PCapStreamer streamer; + bool done; + std::vector buffered_events; + size_t idx = 0; +}; + +PCapEventGenerator::PCapEventGenerator(std::string const &pcap) + : me(new PImpl(pcap, *this)) { + move_next(); +} + +PCapEventGenerator::PCapEventGenerator(PCapEventGenerator &&src) + : me(std::move(src.me)) {} + +PCapEventGenerator::~PCapEventGenerator() {} + +PCapEventGenerator &PCapEventGenerator::operator=(PCapEventGenerator &&src) { + me = std::move(src.me); + return *this; +} + +PCapEvent const *PCapEventGenerator::top() { + if (me->done) + return nullptr; + else { + bts_assert(me->idx < me->buffered_events.size()); + return &me->buffered_events[me->idx]; + } +} + +void PCapEventGenerator::move_next() { + bts_assert(!me->done); + if (me->buffered_events.size() && me->idx < me->buffered_events.size() - 1) + me->idx++; + else { + me->idx = 0; + me->buffered_events.clear(); + for (;;) { + auto ok = me->streamer.next(); + if (!ok) { + me->done = true; + return; + } else if (me->buffered_events.size()) + return; + } + } +} + +void PCapEventGenerator::on_recovery_complete(InitialSnapshot const &snaps) { + auto &ev = me->buffered_events.emplace_back(); + for (auto snap_ptr : snaps) { + + auto &snap_cpy = ev.affected_securities.emplace_back(); + + // We want to normalize the snapshots we show as PCapEvents so + // that they always carry order changes that when applied to the + // previous snapshot (if any) produce an up to date order + // book. The PCapStreamer does not guarantee this to + // on_recovery_complete, that is sometimes initial snapshots + // will simply contain an initial order state and no + // updates. The code below this normalizes the snapshots we are + // copying so that they always can be relied on to carry the + // delta between snapshot state changes. + snap_cpy.light_copy(*snap_ptr); + snap_cpy.clear_order_changes(); + bts_assert(snap_cpy.fills().empty()); + bts_assert(snap_cpy.orders().empty()); + for (auto const &o : snap_ptr->orders()) + snap_cpy.insert_order(o.second); + bts_assert(snap_cpy.orders().size() == snap_ptr->orders().size()); + bts_assert(snap_cpy.orders().size() == snap_cpy.order_changes().size()); + } +} + +void PCapEventGenerator::on_real_time_event(MDP3Event const &ev_in) { + auto &ev = me->buffered_events.emplace_back(); + ev.transact_time = ev_in.transact_time; + ev.event_id = ev_in.event_id; + ev.first_packet_send_time = ev_in.first_packet_send_time; + ev.last_packet_send_time = ev_in.last_packet_send_time; + ev.first_packet_sn = ev.first_packet_sn; + ev.last_packet_sn = ev.last_packet_sn; + + for (auto snap_ptr : ev_in.affected_securities) { + auto &snap_cpy = ev.affected_securities.emplace_back(); + // light_copy() does not copy the orders, just the order changes + // Observers of this class should see a continuous stream of + // order changes which they can use to recreate the entire order + // book.. + snap_cpy.light_copy(*snap_ptr); + } +} + +void PCapEventGenerator::on_real_time_channel_loss() { bts_assert(false); } + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h new file mode 100644 index 000000000..5757c8288 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h" +#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" +#include + +namespace bts::spartan::mdp3 { + +class PCapEventGenerator : IChannelObserver { + public: + PCapEventGenerator(std::string const&pcap); + PCapEventGenerator(PCapEventGenerator &&); + PCapEventGenerator &operator=(PCapEventGenerator &&src); + ~PCapEventGenerator(); + + PCapEvent const *top(); + void move_next() ; + +private: + void on_recovery_complete(InitialSnapshot const &) override; + void on_real_time_event(MDP3Event const &) override; + void on_real_time_channel_loss() override; + struct PImpl; + std::unique_ptr me; +}; + + + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.cpp new file mode 100644 index 000000000..b29196053 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.cpp @@ -0,0 +1,41 @@ +#include "PCapEventSource.h" + +#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonFinder.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEvent.h" + +namespace bts::spartan::mdp3 { + +struct PCapEventSource::PImpl { + std::string pcap_file; + DefinitionCollector dc = {}; + PolygonFinder pf = {}; + IcebergFinder ice_finder = {}; +}; + +PCapEventSource::PCapEventSource(std::string const &pcap) + : me(new PImpl{pcap}) { + observe_channel(pcap, {&me->dc, &me->pf, &me->ice_finder}); +} + +PCapEventSource::~PCapEventSource() {} + +InstrumentMap const& PCapEventSource::instruments() const { + return me->dc.ptr_result; +} + +std::unordered_map const& PCapEventSource::polygons() const { + return me->pf.result; +} + +IcebergMap const&PCapEventSource::icebergs() const { + return me->ice_finder.result; +} + + PCapEventGenerator PCapEventSource::get_event_generator() const { + return PCapEventGenerator {me->pcap_file}; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.h new file mode 100644 index 000000000..1e584c800 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventSource.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include + +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.h" +#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" +#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" + +namespace bts::spartan::mdp3 { +class InstrumentDefinition; +class MDP3Event; +} + +namespace bts::spartan::mdp3 { + +class PCapEventSource { +public: + PCapEventSource(std::string const &pcap); + ~PCapEventSource(); + InstrumentMap const &instruments() const ; + std::unordered_map const &polygons() const; + IcebergMap const&icebergs() const; + + PCapEventGenerator get_event_generator() const; +private: + struct PImpl; + std::unique_ptr me; +}; + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h b/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h new file mode 100644 index 000000000..00292cd0b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h @@ -0,0 +1,89 @@ +#pragma once + +#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" + +namespace bts::spartan::mdp3 { + +class IAlgoModel; + +using AlgoEventID = std::pair ; + +struct EventHeader { + AlgoEventID event_id = {0,0}; + MDP3TimeStamp transact_time = MDP3TimeStamp {0}; + MDP3TimeStamp observable_time = MDP3TimeStamp {0}; +}; + +struct AlgoAction { + MDP3TimeStamp transact_time = MDP3TimeStamp{0}; + SmallOrderChange change; +}; + +struct MBPUpdate { + MDP3SecurityID security_id; + Book mbp_buy; + Book mbp_sell; + ImpliedBook implied_mbp_buy; + ImpliedBook implied_mbp_sell; + Book const &mbp_side(api::Side side) const { + if (side == api::SIDE_BUY) + return mbp_buy; + else + return mbp_sell; + } + const ImpliedBook& implied_mbp_side(api::Side side) const { + if (side == api::SIDE_BUY) + return implied_mbp_buy; + else + return implied_mbp_sell; + } +}; + +struct OrderUpdate { + MDP3OrderKey key; + enum Status { FILLED, PARTIALLY_FILLED, CANCELED }; + Status status; + MDP3SecurityID security_id; + api::Side side; + MDP3Quantity last_fills; + md::IntPrice last_price; + enum FillType { PASSIVE, AGGRESSIVE }; + // non null when Status == FILLED or PARTIALLY_FILLED + std::optional fill_type;// = std::nullopt; + const char*status_str() const { + switch (status) { + case FILLED: return "filled"; + case PARTIALLY_FILLED: return "partial"; + case CANCELED: return "canceled"; + default: + bts_assert(false); + return ""; + } + } +}; + +using MatchEffect = + std::variant; + +struct AlgoNotification { + EventHeader header; + std::variant event; +}; + +inline std::ostream& operator <<(std::ostream& ostr, const OrderUpdate::Status& status) { + switch (status) { + case OrderUpdate::Status::FILLED: ostr << "FILLED"; break; + case OrderUpdate::Status::PARTIALLY_FILLED: ostr << "PARTIALLY_FILLED"; break; + case OrderUpdate::Status::CANCELED: return ostr << "CANCELED"; + default: ostr << "UNKNOWN_" << static_cast(status); break; + } + return ostr; +} + +inline std::ostream& operator <<(std::ostream& ostr, const OrderUpdate& upd) { + ostr << "key: " << upd.key.value() << ", status: " << upd.status + << ", last fills: " << upd.last_fills.value() << ", last price: " << upd.last_price.price(); + return ostr; +} + +} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.cpp new file mode 100644 index 000000000..f1ef7ae16 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.cpp @@ -0,0 +1,312 @@ +// #include "lib/orderman/orders/algo/IAlgoOrder.h" +// #include "lib/orderman/orders/algo/IAlgoServices.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/ISimServices.h" +// #include "AlgoServicesBridge.h" +// #include "lib/orderman/actors/order_manager/OrderBuilder.h" +// #include "lib/orderman/orders/algo/ChildTemplate.h" +// #include "lib/orderman/orderman_utils/Book.h" +// #include "BridgeCommon.h" +// #include "BeforeLaunchBridge.h" + + +// using namespace bts::spartan::mdp3; +// using namespace bts::api; +// using namespace bts::api::orderman; + +// namespace bts { +// namespace orderman { + +// struct OrderState{ +// // spartan outgoing order +// MDP3Order order; +// // spark callback object +// ChildState child_state; +// }; + +// struct AlgoServicesBridge::PImpl { +// std::vector pos; +// BeforeLaunchBridge& bl_bridge; +// IAlgoOrder& algo_order; +// ISimServices& sim_services; +// std::unordered_map book_map; +// std::unordered_map order_map; +// OrderDefinition order_def; +// uint32_t running_ok = 0; + +// PImpl(BeforeLaunchBridge& bl_bridge, +// IAlgoOrder& algo_order, +// ISimServices& sim_services) +// : +// bl_bridge(bl_bridge), +// algo_order(algo_order), +// sim_services(sim_services), +// order_def(bl_bridge.definition()) +// { +// } +// }; + +// void absorb_order_callback(const OrderUpdate& new_update, +// ChildState& curr_state){ +// uint32_t tot_fills = new_update.last_fills.value() + +// curr_state.total_fills.value(); +// double avg_price = 0; +// if (tot_fills>0){ +// avg_price = +// curr_state.avg_price.value()*curr_state.total_fills.value() + +// new_update.last_price.price()*new_update.last_fills.value(); +// avg_price = avg_price / tot_fills; +// } +// curr_state.total_fills = Quantity(tot_fills); +// curr_state.avg_price = Price(avg_price); +// curr_state.last_fills = Quantity(new_update.last_fills.value()); +// curr_state.last_price = Price(new_update.last_price.price()); +// switch (new_update.status) +// { +// case OrderUpdate::Status::FILLED: +// curr_state.status = ChildStatus::COMPLETED; +// break; +// case OrderUpdate::Status::PARTIALLY_FILLED: +// curr_state.status = ChildStatus::WORKING; +// break; +// case OrderUpdate::Status::CANCELED: +// curr_state.status = ChildStatus::CANCELED; +// break; +// } +// } + +// AlgoServicesBridge::AlgoServicesBridge(BeforeLaunchBridge& bl_bridge, +// IAlgoOrder& algo_order, +// ISimServices& sim_services) +// : +// me(new PImpl(bl_bridge,algo_order, sim_services)) { +// } + +// Price AlgoServicesBridge::price() const{ +// return Price(0); +// } + +// Quantity AlgoServicesBridge::quantity() const { +// return Quantity(0); +// } + +// Quantity AlgoServicesBridge::total_fills() const{ +// return Quantity(0); +// } + +// bool AlgoServicesBridge::is_shutting_down() const { +// return false; +// } + +// const std::vector& +// AlgoServicesBridge::current_positions() const { +// bts_assert_msg(false, "Not implemented"); +// return me->pos; +// } + +// trace::CorID AlgoServicesBridge::get_correlation_id() const { +// trace::CorID cid; +// return cid; +// } + +// void AlgoServicesBridge::subscribe_book(SecurityKey k) { +// auto st_sec_key = map_sec_key(k); +// if(me->book_map.find(st_sec_key) == me->book_map.end()) +// me->book_map.insert({st_sec_key,Book()}); +// } + +// void AlgoServicesBridge::unsubscribe_book(SecurityKey k) { +// auto st_sec_key = map_sec_key(k); +// if(me->book_map.find(st_sec_key) != me->book_map.end()) +// me->book_map.erase(st_sec_key); +// } + +// void AlgoServicesBridge::subscribe_settlement(SecurityKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::unsubscribe_settlement(SecurityKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::subscribe_child_change(OrderKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::unsubscribe_child_change(OrderKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::subscribe_timer() { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::unsubscribe_timer() { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::subscribe_ticker(SecurityKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::unsubscribe_ticker(SecurityKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::subscribe_status(SecurityKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::unsubscribe_status(SecurityKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::subscribe_position() { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::unsubscribe_position() { +// bts_assert_msg(false, "Not implemented"); +// } + +// OrderKey AlgoServicesBridge::launch_child(ChildTemplate &, +// Price p, Quantity q, +// bool subscribe) { +// return launch_child(me->order_def,p,q,subscribe); +// } + +// OrderKey AlgoServicesBridge::launch_child( +// const api::orderman::OrderDefinition& ord_def, +// Price p, Quantity q, +// bool) { +// auto ord_key = me->sim_services.generate_order_key(); +// auto sec_key = map_sec_key(SecurityKey(ord_def.security_key())); +// auto ret = me->order_map.insert( +// { +// ord_key, +// OrderState{ +// MDP3Order { +// ord_key, +// sec_key, +// ord_def.side(), +// MDP3OrderPriority{}, +// md::IntPrice_from_double<7>(p.value()), +// MDP3Quantity(q.value()), +// std::nullopt +// }, +// ChildState{ +// p,q,Quantity(0),Price(0),Quantity(0),Price(0), +// ChildStatus::WORKING,Quantity(0) +// } +// }}); +// bts_assert(ret.second); +// OrderState& ord_state = ret.first->second; +// SmallOrderChange ord_change(ord_state.order, +// OrderChange::ChangeType::added); +// me->sim_services.send(ord_change); +// return map_ord_key(ord_key); +// } + +// // Child order maintenance +// void AlgoServicesBridge::update_child( +// OrderKey k, +// const api::orderman::Update& u){ +// auto sp_ord_key = map_ord_key(k); +// auto it = me->order_map.find(sp_ord_key); +// if(it != me->order_map.end()){ +// OrderState& ord_state = it->second; +// ord_state.order.quantity = MDP3Quantity(u.quantity()); +// ord_state.order.price = md::IntPrice_from_double<7>(u.price()); +// ord_state.child_state.price = Price(u.price()); +// ord_state.child_state.quantity = Quantity(u.quantity()); +// SmallOrderChange ord_change(ord_state.order, +// OrderChange::ChangeType::updated); +// me->sim_services.send(ord_change); +// } +// else +// LOG(OrderMan, Warn, "AlgoServicesBridge::update_child OrderKey:{} " +// "does not exist", k); +// } + +// void AlgoServicesBridge::cancel_child(OrderKey k) { +// auto sp_ord_key = map_ord_key(k); +// auto it = me->order_map.find(sp_ord_key); +// if(it != me->order_map.end()){ +// OrderState& ord_state = it->second; +// SmallOrderChange ord_change(ord_state.order, +// OrderChange::ChangeType::removed); +// me->sim_services.send(ord_change); +// } +// else +// LOG(OrderMan, Warn, "AlgoServicesBridge::cancel_child OrderKey:{} " +// "does not exist", k); +// } + +// void AlgoServicesBridge::detach_child(OrderKey) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::detach_all_children() { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::cancel(const std::string&) { +// for (const auto& p : me->order_map) +// me->sim_services.send( +// SmallOrderChange(p.second.order, OrderChange::ChangeType::removed)); +// } + +// void AlgoServicesBridge::terminate_error(const std::string& msg) { +// cancel(msg); +// } + +// void AlgoServicesBridge::complete() { +// } + +// void AlgoServicesBridge::set_ladder_visibility(bool) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::update_custom_state( +// const api::orderman::DynamicOrderState&) { +// bts_assert_msg(false, "Not implemented"); +// } + +// void AlgoServicesBridge::update_fill_state(Quantity, Price) { +// } + +// // IAlgoModel + +// void AlgoServicesBridge::init(bts::spartan::mdp3::ISimServices&, +// bts::spartan::protos::SimRunOptions const&){ +// } + +// void AlgoServicesBridge::on_notification( +// const bts::spartan::mdp3::AlgoNotification& ev){ +// if(std::holds_alternative(ev.event)){ +// // book update +// const MBPUpdate& new_bk = std::get(ev.event); +// auto it = me->book_map.find(new_bk.security_id); +// if(it != me->book_map.end()){ +// Book& bk = it->second; +// translate_book(new_bk,bk); +// me->algo_order.on_book(map_sec_key(new_bk.security_id), bk); +// } +// } +// else{ +// const OrderUpdate& u = std::get(ev.event); +// auto it = me->order_map.find(u.key); +// if(it != me->order_map.end()){ +// OrderState& ord_state = it->second; +// absorb_order_callback(u,ord_state.child_state); +// me->algo_order.on_child_change( +// map_ord_key(ord_state.order.key),ord_state.child_state); +// } +// else +// LOG(OrderMan, Error, "AlgoServicesBridge::on_notification " +// "OrderKey:{} does not exist", u.key); +// } + +// } + +// } // namespace orderman +// } // namespace bts diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.h b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.h new file mode 100644 index 000000000..73516e509 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.h @@ -0,0 +1,103 @@ +// #pragma once + +// #include "lib/orderman/orders/algo/IAlgoOrder.h" +// #include "lib/orderman/orders/algo/IAlgoServices.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/IAlgoModel.h" + +// namespace bts {namespace api {namespace orderman{class CustomOrderDefinition;}}} +// namespace bts {namespace spartan {namespace mdp3{class ISimServices;}}} + +// namespace bts::spartan::protos { class SimRunOptions; } + +// namespace bts { +// namespace orderman { + +// class ChildTemplate; +// class IAlgoOrder; +// class BeforeLaunchBridge; + +// class AlgoServicesBridge : public IAlgoServices, public bts::spartan::mdp3::IAlgoModel{ +// public: +// AlgoServicesBridge(BeforeLaunchBridge& bl_bridge, +// IAlgoOrder& algo_order, +// bts::spartan::mdp3::ISimServices&); +// ~AlgoServicesBridge(){} + +// Price price() const override; + +// Quantity quantity() const override; + +// Quantity total_fills() const override; + +// bool is_shutting_down() const override; + +// std::vector const& current_positions() const override; + +// trace::CorID get_correlation_id() const override; + +// void subscribe_book(SecurityKey) override; + +// void unsubscribe_book(SecurityKey) override; + +// void subscribe_settlement(SecurityKey) override; + +// void unsubscribe_settlement(SecurityKey) override; + +// void subscribe_child_change(OrderKey) override; + +// void unsubscribe_child_change(OrderKey) override; + +// void subscribe_timer() override; + +// void unsubscribe_timer() override; + +// void subscribe_ticker(SecurityKey) override; + +// void unsubscribe_ticker(SecurityKey) override; + +// void subscribe_status(SecurityKey) override; + +// void unsubscribe_status(SecurityKey) override; + +// void subscribe_position() override; + +// void unsubscribe_position() override; + +// OrderKey launch_child(ChildTemplate &, Price, Quantity, +// bool subscribe = true) override; + +// OrderKey launch_child(api::orderman::OrderDefinition const &, Price, +// Quantity, bool subscribe = true) override; + +// // Child order maintenance +// void update_child(OrderKey child, api::orderman::Update const &) override; +// void cancel_child(OrderKey child) override; +// void detach_child(OrderKey child) override; +// void detach_all_children() override; + +// void cancel(std::string const &msg = "") override; + +// void terminate_error(std::string const &msg) override; + +// void complete() override; + +// void set_ladder_visibility(bool is_visible) override; + +// void update_custom_state(api::orderman::DynamicOrderState const&) override; + +// void update_fill_state(Quantity total_fills, Price last_price) override; + +// // IAlgoModel + +// void init(bts::spartan::mdp3::ISimServices&, +// bts::spartan::protos::SimRunOptions const&) override; + +// void on_notification(bts::spartan::mdp3::AlgoNotification const&) override; + +// private: +// struct PImpl; +// std::unique_ptr me; +// }; + +// } // namespace orderman +// } // namespace bts diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.cpp new file mode 100644 index 000000000..fa18533e0 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.cpp @@ -0,0 +1,120 @@ +// #include "BeforeLaunchBridge.h" +// #include "../ISimServices.h" +// #include "BridgeCommon.h" +// #include "lib/orderman/actors/order_manager/OrderBuilder.h" +// #include "lib/orderman/orders/algo/ChildTemplate.h" + + +// namespace bts { +// namespace orderman { + +// using namespace bts::spartan::mdp3; +// using namespace bts::api::orderman; + +// typedef std::map > InsMap; + +// struct BeforeLaunchBridge::PImpl { +// OrderKey ok; +// MDP3SecurityID sec_id; +// ISimServices& sim_services; +// const api::orderman::OrderDefinition& order_def; +// ChildTemplate* child_temp = nullptr; +// api::Security security; +// TimeStamp ts; +// std::map,OrderDefinition> _child_temp_map; +// InsMap sec_map = InsMap([](const auto& a, const auto& b) { +// return a->security_id().value() < b->security_id().value(); +// }); + +// PImpl( +// OrderKey ok, +// MDP3SecurityID sec_id, +// ISimServices& sim_services, +// const api::orderman::OrderDefinition& order_def) +// : +// ok(ok), +// sec_id(sec_id), +// sim_services(sim_services), +// order_def(order_def) { +// } +// }; + +// // TODO: +// // order def translation (ex sniper) +// // ChildTemplate +// // timestamp + +// BeforeLaunchBridge::BeforeLaunchBridge( +// ISimServices& sim_services, +// const api::orderman::OrderDefinition& order_def) +// : +// me(new PImpl(OrderKey(0),MDP3SecurityID(0),sim_services,order_def)){ +// for (const auto& p : me->sim_services.instruments()){ +// auto mdp3_sec_ptr = p.second; +// api::Security sec = convert_to_spark_security( +// mdp3_sec_ptr, +// me->sim_services.instruments()); +// me->sec_map.insert({mdp3_sec_ptr,sec}); +// } +// bool found = false; +// for (const auto& p : me->sec_map) +// if(p.second.id() == order_def.security_key()){ +// me->security = p.second; +// found = true; +// } +// bts_assert(found); +// } + +// const api::orderman::OrderDefinition& BeforeLaunchBridge::definition() const{ +// return me->order_def; +// } + +// const api::Security& BeforeLaunchBridge::security() const{ +// return me->security; +// } + +// OrderKey BeforeLaunchBridge::order_key() const{ +// return me->ok; +// } + +// OrderKey BeforeLaunchBridge::parent_order_key() const{ +// // not supported in spartan sim +// return order_key(); +// } + +// api::Security const* BeforeLaunchBridge::lookup_security(SecurityKey k) const{ +// for (const auto& p : me->sec_map) +// if(p.second.id() == k.value()) +// return &p.second; +// return nullptr; +// } + +// std::vector BeforeLaunchBridge::known_securities() const{ +// std::vector rv; +// for (const auto& p : me->sec_map) +// rv.push_back(&p.second); +// return rv; +// } + +// bool BeforeLaunchBridge::is_attaching() const{ +// return false; +// } + +// ChildTemplate& BeforeLaunchBridge::make_child_template( +// const api::orderman::OrderDefinition& d, +// size_t){ +// auto ret = me->_child_temp_map.insert(std::make_pair(std::make_unique(),d)); +// bts_assert(ret.second); +// const std::unique_ptr& rv = ret.first->first; +// return *rv; +// } + +// // Get the current time +// TimeStamp BeforeLaunchBridge::get_time() const{ +// // TODO +// return me->ts; +// } + +// }} diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.h b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.h new file mode 100644 index 000000000..20d8d40e2 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.h @@ -0,0 +1,49 @@ +// #pragma once + +// #include "lib/orderman/orders/algo/IAlgoOrder.h" +// #include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h" + + +// namespace bts {namespace api {namespace orderman{class CustomOrderDefinition;}}} +// namespace bts {namespace spartan {namespace mdp3{class ISimServices;}}} + +// namespace bts { +// namespace orderman { + +// class ChildTemplate; + +// class BeforeLaunchBridge : public IBeforeLaunchAlgoServices{ +// public: +// BeforeLaunchBridge( +// bts::spartan::mdp3::ISimServices&, +// const api::orderman::OrderDefinition&); +// virtual ~BeforeLaunchBridge(){} + +// api::orderman::OrderDefinition const &definition() const override; + +// api::Security const &security() const override; + +// OrderKey order_key() const override; + +// OrderKey parent_order_key() const override; + +// api::Security const *lookup_security(SecurityKey k) const override; + +// std::vector known_securities() const override; + +// bool is_attaching() const override; + +// ChildTemplate & +// make_child_template(api::orderman::OrderDefinition const &child_def, +// size_t reserve = 1) override; + +// TimeStamp get_time() const override; + +// private: +// struct PImpl; +// //std::unique_ptr me; +// PImpl* me; +// }; + +// } // namespace orderman +// } // namespace bts diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BridgeCommon.h b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BridgeCommon.h new file mode 100644 index 000000000..4020da304 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/spark_bridge/BridgeCommon.h @@ -0,0 +1,99 @@ +// #pragma once + +// #include "lib/orderman/orders/algo/IAlgoOrder.h" +// //#include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h" +// //#include "../ISimServices.h" +// #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +// #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentMap.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h" +// #include "lib/orderman/orderman_utils/Book.h" +// #include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/AlgoEvent.h" +// #include "server/sparkd/cmds/spartan/back_testing/simulator/IAlgoModel.h" + +// namespace bts{ +// namespace orderman{ + +// inline OrderKey map_ord_key(bts::spartan::mdp3::MDP3OrderKey k){ +// // uint64_t -> uint32_t +// return OrderKey(static_cast(k.value())); +// } + +// inline bts::spartan::mdp3::MDP3OrderKey map_ord_key(OrderKey k){ +// // uint32_t -> uint64_t +// return bts::spartan::mdp3::MDP3OrderKey(static_cast(k.value())); +// } + +// inline SecurityKey map_sec_key(bts::spartan::mdp3::MDP3SecurityID k){ +// // int -> uint32_t +// return SecurityKey(static_cast(k.value())); +// } + +// inline bts::spartan::mdp3::MDP3SecurityID map_sec_key(SecurityKey k){ +// // uint32_t -> int +// return bts::spartan::mdp3::MDP3SecurityID(static_cast(k.value())); +// } + +// inline void translate_book(const bts::spartan::mdp3::MBPUpdate& u_in, +// bts::orderman::Book& b_out){ +// // TODO: implieds +// size_t i=-1; +// for (auto const& bl : u_in.mbp_buy) +// if(++i<5) +// b_out.side(api::Side::SIDE_BUY)[i] = +// md::PriceSize (bl.price,bl.quantity.value()); +// i=-1; +// for (auto const& bl : u_in.mbp_sell) +// if(++i<5) +// b_out.side(api::Side::SIDE_SELL)[i] = +// md::PriceSize (bl.price,bl.quantity.value()); +// } + +// inline void fill_future(const bts::spartan::mdp3::InstrumentDefinition&, +// bts::api::Security& out_sec){ +// // TODO, set future expiration dates +// out_sec.mutable_future(); +// } + +// inline bts::api::Security convert_to_spark_security ( +// bts::spartan::mdp3::InstrumentDefinition const* def_ptr, +// const bts::spartan::mdp3::InstrumentMap& sec_map){ +// auto fill_common = [](const auto& def, auto out_sec){ +// out_sec.set_id(map_sec_key(def.security_id()).value()); +// out_sec.set_symbol(def.security_group()); +// out_sec.set_exchange(bts::api::VENUE_GLBX); +// out_sec.set_currency(bts::api::CURRENCY_USD); +// out_sec.mutable_contract_spec()->set_low_tick(def.min_price_increment_ammount()); +// out_sec.mutable_contract_spec()->set_high_tick(def.min_price_increment_ammount()); +// }; +// const auto& def = *def_ptr; +// bts::api::Security rv_sec; +// fill_common(def,rv_sec); +// switch(def.security_type()){ +// case bts::api::SECURITYTYPE_FUTURE: +// fill_future(def,rv_sec); +// break; +// case bts::api::SECURITYTYPE_SPREAD:{ +// auto& spr = *rv_sec.mutable_spread(); +// for(const auto& l : def.legs()){ +// // getting inst def for the leg +// auto it = sec_map.find(l.security_id); +// bts_assert(it != sec_map.end()); +// const auto& leg_insdef = *it->second; +// // creating spread +// bts::api::ContractLeg& spr_leg = *spr.add_legs(); +// auto& leg_sec = *spr_leg.mutable_security(); +// fill_common(leg_insdef, leg_sec); +// fill_future(leg_insdef,leg_sec); +// spr_leg.set_quantity(l.ratio.value()); +// spr_leg.set_side(l.side); +// spr_leg.set_price_multiplier(spr_leg.quantity()); +// } +// break; +// } +// default: +// break; +// } +// return rv_sec; +// } +// }}