From f07e018fbd5f7d84f50ef92dcf4c6878b81397a2 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 4 Nov 2020 08:42:43 -0500 Subject: [PATCH] Many changes. --- src/data_files/files/spartan/md.sql | 111 +++------ src/data_files/files/spartan/sim_db.sql | 9 +- src/lib/md/IntPrice.h | 8 +- .../mdp3/back_testing/algos/PolyObserver.cpp | 112 +++++---- .../md/mdp3/back_testing/algos/PolyObserver.h | 3 +- .../md/mdp3/back_testing/algos/Sniper.cpp | 215 +++++++++--------- .../md/mdp3/back_testing/algos/Sniper.h | 3 +- .../spreader/tests/sim_spreader_tests.cpp | 4 + .../matcher/FifoSecurityMatcher.cpp | 2 + .../md/mdp3/back_testing/matcher/Matcher.cpp | 2 - .../mdp3/back_testing/simulator/AlgoEvent.h | 4 +- .../back_testing/simulator/ExchangeSim.cpp | 2 +- .../mdp3/back_testing/simulator/IAlgoModel.h | 10 + .../simulator/agents/AlgoAgent.cpp | 7 +- .../simulator/agents/GatewayAgent.cpp | 119 ++++++---- .../simulator/agents/GatewayAgent.h | 6 +- .../md/mdp3/common/InstrumentDefinition.h | 7 + .../spartan/md/mdp3/utils/polygon_utils.cpp | 4 +- 18 files changed, 329 insertions(+), 299 deletions(-) diff --git a/src/data_files/files/spartan/md.sql b/src/data_files/files/spartan/md.sql index 94f66d495..25cb418fa 100644 --- a/src/data_files/files/spartan/md.sql +++ b/src/data_files/files/spartan/md.sql @@ -167,6 +167,7 @@ CREATE TABLE polygon_values( book_5_eid numeric, book_5_side text ); + CREATE UNIQUE INDEX polygon_value_idx ON polygon_values(vid); drop table if exists polygon_opportunities; create table polygon_opportunities( @@ -184,7 +185,7 @@ create table polygon_opportunities( end_eid numeric, /* last event where this opportunity appeared */ min_n_orders numeric /* smallest number of orders preset for each of the legs of the polygon */ ); - + CREATE UNIQUE INDEX polygon_op_value_idx ON polygon_opportunities(begin_vid,end_vid); /* Usefull Views -------------------------------------------------------------------------------- */ @@ -279,7 +280,8 @@ select events_begin.h_t_time h_time, events_begin.last_st begin_ts, events_end.t_time end_ts, - (events_end.t_time - events_begin.last_st) / 1000 duration, + (events_end.t_time - events_begin.last_st) / 1000 observable_duration, + (events_end.t_time - events_begin.t_time) / 1000 duration, events_begin.t_time t_time, (events_begin.last_st - events_begin.t_time)/ 1000 transaction_duration from @@ -303,89 +305,34 @@ where duration >20 and merged_value > 5; - - -drop view if exists j_tops_v; -drop view if exists sim_md_mismatches; -drop table if exists sim_books ; -CREATE TABLE sim_books( - eid numeric, - sid numeric, -/* regular books */ -/* ---------------------------------------------------- */ -/* price qty n orders */ -/* ---------------------------------------------------- */ - bid_p_0 numeric, bid_q_0 numeric, bid_no_0 numeric, - bid_p_1 numeric, bid_q_1 numeric, bid_no_1 numeric, - bid_p_2 numeric, bid_q_2 numeric, bid_no_2 numeric, - bid_p_3 numeric, bid_q_3 numeric, bid_no_3 numeric, - bid_p_4 numeric, bid_q_4 numeric, bid_no_4 numeric, - bid_p_5 numeric, bid_q_5 numeric, bid_no_5 numeric, - bid_p_6 numeric, bid_q_6 numeric, bid_no_6 numeric, - bid_p_7 numeric, bid_q_7 numeric, bid_no_7 numeric, - bid_p_8 numeric, bid_q_8 numeric, bid_no_8 numeric, - bid_p_9 numeric, bid_q_9 numeric, bid_no_9 numeric, - ask_p_0 numeric, ask_q_0 numeric, ask_no_0 numeric, - ask_p_1 numeric, ask_q_1 numeric, ask_no_1 numeric, - ask_p_2 numeric, ask_q_2 numeric, ask_no_2 numeric, - ask_p_3 numeric, ask_q_3 numeric, ask_no_3 numeric, - ask_p_4 numeric, ask_q_4 numeric, ask_no_4 numeric, - ask_p_5 numeric, ask_q_5 numeric, ask_no_5 numeric, - ask_p_6 numeric, ask_q_6 numeric, ask_no_6 numeric, - ask_p_7 numeric, ask_q_7 numeric, ask_no_7 numeric, - ask_p_8 numeric, ask_q_8 numeric, ask_no_8 numeric, - ask_p_9 numeric, ask_q_9 numeric, ask_no_9 numeric -); -CREATE UNIQUE INDEX sim_book_idx ON sim_books(eid,sid); - -drop view if exists j_tops_v; -create view j_tops_v as +drop view if exists opportunities_detail; +create view opportunities_detail as select - books.eid, - books.sid, - books.bid_p_0 md_bid, sim_books.bid_p_0 sim_bid, - books.bid_q_0 md_bid_qty, sim_books.bid_q_0 sim_bid_qty, - books.ask_p_0 md_ask, sim_books.ask_p_0 sim_ask, - books.ask_q_0 md_ask_qty, sim_books.ask_q_0 sim_ask_qty -from - books, sim_books + opportunities.*, + (polygon_values.s_time-polygon_values.t_time)/1000 detail_transaction_duration, + polygon_values.min_n_orders detail_n_orders, + polygon_values.merged_value detail_value, + polygon_values.merged_qty detail_qty + from + opportunities, + polygon_values where - books.sid = sim_books.sid and - books.eid = sim_books.eid; + polygon_values.vid >= opportunities.begin_vid and + polygon_values.vid <= opportunities.end_vid; -create p_value as +drop view if exists opportunity_summary; +create view opportunity_summary as select - h_t_time, events.eid,t_time,first_st,last_st, - esz0.bid esz0_bid, - esz0.ask esz0_ask, - esu0.bid esu0_bid, - esu0.ask esu0_ask, - esu0_esz0.bid esu0_esz0_bid, - esu0_esz0.ask esu0_esz0_ask, - esu0.bid_qty esu0_bid_qty, - esu0.ask_qty esu0_ask_qty, - esz0.bid_qty esz0_bid_qty, - esz0.ask_qty esz0_ask_qty, - esu0_esz0.bid_qty esu0_esz0_bid_qty, - esu0_esz0.ask_qty esu0_esz0_ask_qty + min(merged_value) merged_value, + min(observable_duration) observable_duration, + min(transaction_duration) transaction_duration, + avg(detail_transaction_duration) avg_detail_transaction_duration, + max(detail_n_orders) max_n_orders, + max(detail_value) max_value, + max(detail_qty) max_detail_qt, + count(*) event_count from - events - left join (select * from top_of_books where symbol='ESU0') esu0 on events.eid=esu0.eid - left join (select * from top_of_books where symbol='ESZ0') esz0 on events.eid=esz0.eid - left join (select * from top_of_books where symbol='ESU0-ESZ0') esu0_esz0 on events.eid=esz0.eid; + opportunities_detail +group by opid; + - - -drop function if exists get_book; -create function get_book(sid_p numeric,eid_p numeric) - returns numeric - language plpgsql -as -$$ -declare - eid_r numeric; -begin - select max(books.eid) into eid_r from books where books.sid=sid_p and books.eid<=eid_p ; - return eid_r; -end; -$$ diff --git a/src/data_files/files/spartan/sim_db.sql b/src/data_files/files/spartan/sim_db.sql index cf90721c6..907edbb19 100644 --- a/src/data_files/files/spartan/sim_db.sql +++ b/src/data_files/files/spartan/sim_db.sql @@ -93,20 +93,23 @@ select position_pandl, param_latency.param_value latency, param_date.param_value date, - param_qty.param_value shot_qty, + param_min_depth.param_value min_depth, + param_min_value.param_value min_value, param_min_orders.param_value min_orders from session, run, (select * from param where param_name='base_algo_latency_ns') param_latency, (select * from param where param_name='market_date') param_date, - (select * from param where param_name='Qty') param_qty, + (select * from param where param_name='MinDepth') param_min_depth, + (select * from param where param_name='MinValue') param_min_value, (select * from param where param_name='MinOrdersAtTop') param_min_orders WHERE session.session_id = run.session_id and run.run_id = param_latency.run_id and run.run_id = param_date.run_id and - run.run_id = param_qty.run_id and + run.run_id = param_min_depth.run_id and + run.run_id = param_min_value.run_id and run.run_id = param_min_orders.run_id order by cast(realized_pandl as real) desc; diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index 4a4dc8a4e..73689cb45 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -81,6 +81,12 @@ class IntPrice { _exponent = temp._exponent; return *this; } + IntPrice& operator -=(const IntPrice& o) { + auto temp = *this - o; + _mantissa = temp._mantissa; + _exponent = temp._exponent; + return *this; + } IntPrice operator * (const IntPrice& other) const { return op(other, [](int64_t m1, int64_t m2) { return m1 * m2; }); } @@ -199,4 +205,4 @@ inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) { return os; } -}} \ No newline at end of file +}} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp index 26692eaf7..729fbd01c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.cpp @@ -33,31 +33,30 @@ struct PolyObserver::Opportunity { MDP3Quantity quantity; }; -struct PolyObserver::DependentTargets { - MBPUpdate book; - std::vector dependencies; -}; - struct PolyObserver::PImpl { ISimServices *sim = nullptr; - std::unordered_map books; + std::unordered_map books; std::vector> targets; - size_t event_id; + AlgoEventID event_id; MDP3TimeStamp transact_time; MDP3TimeStamp observable_time; + size_t max_events; }; PolyObserver::PolyObserver() : me(new PImpl()) {} PolyObserver::~PolyObserver() {} -void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &) { + +void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) { me->sim = &s; me->books.clear(); me->targets.clear(); - me->event_id = 0; + me->event_id = {0, 0}; me->transact_time = MDP3TimeStamp{0}; me->observable_time = MDP3TimeStamp{0}; + me->max_events = + param_or_default(options, "MaxEvents", std::numeric_limits::max()); for (auto &p : s.polygons()) { auto buy_t = @@ -71,28 +70,37 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &) { me->targets.emplace_back(std::make_unique(buy_t)); me->targets.emplace_back(std::make_unique(sell_t)); } - - for (auto &t : me->targets) { - for (auto &pp : t->polygon.parts) { - auto &dependencies = me->books[pp.sec_id].dependencies; - dependencies.push_back(t.get()); - } - } } void PolyObserver::on_notification(AlgoNotification const &n) { - me->transact_time = n.transact_time; - me->observable_time = n.observable_time; - std::visit(*this, n.event); + if (n.event_id.first >= me->max_events) + me->sim->terminate_simulation(); + else { + if (me->event_id.first) { + for (auto i = me->event_id.first+1;i < n.event_id.first;i++) { + me->event_id.first = i; + me->event_id.second = 0; + me->transact_time = n.transact_time; + me->observable_time = n.observable_time; + eval(); + rank(); + log(); + } + } + me->event_id = n.event_id; + me->transact_time = n.transact_time; + me->observable_time = n.observable_time; + std::visit(*this, n.event); + } } -void PolyObserver::eval(std::vector const &targets) { - for (auto &t : targets) { +void PolyObserver::eval() { + for (auto &t : me->targets) { t->value = 0; int qty = 1; for (auto &l : t->polygon.parts) { auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); - auto &mkt = me->books[l.sec_id].book.mbp_side(mkt_side); + auto &mkt = me->books[l.sec_id].mbp_side(mkt_side); auto this_qty = mkt[0].quantity.value(); auto this_ratio = this_qty / std::abs(l.ratio); if (this_ratio < qty) @@ -126,35 +134,45 @@ void PolyObserver::rank() { void PolyObserver::log() { for (auto const &t : me->targets) if (t->has_value()) - me->sim->log("opportunity", me->event_id, t->descriptor, t->value, - t->quantity.value()); + me->sim->log("opportunity", me->event_id.first, me->event_id.second, + t->descriptor, t->value, t->quantity.value()); } -void PolyObserver::log_book_update(MBPUpdate const &update) { - me->sim->log("book", me->sim->get_sec(update.security_id).symbol(), - update.original_event_id, update.mbp_buy[0].price.price(), - update.mbp_buy[0].quantity.value(), - update.mbp_sell[0].price.price(), - update.mbp_sell[0].quantity.value()); -} - -void PolyObserver::operator()(const MBPUpdate &mbp) { - me->sim->log("book", me->sim->get_sec(mbp.security_id).symbol(), - mbp.original_event_id, mbp.mbp_buy[0].price.price(), - mbp.mbp_buy[0].quantity.value(), mbp.mbp_sell[0].price.price(), - mbp.mbp_sell[0].quantity.value()); - - if (mbp.original_event_id > 20000) - me->sim->terminate_simulation(); +std::pair available(Book const&b) { + std::pair result ; + result.first = md::IntPrice{}; + result.second= MDP3Quantity{0}; - me->event_id = mbp.original_event_id; - auto it = me->books.find(mbp.security_id); - if (it == me->books.end()) - return; + for (size_t i = 0; i < b.size();i++) { + if (b[i].quantity.value() == 0) + break; + else { + result.first = b[i].price; + result.second.value() += b[i].quantity.value(); + } + } + return result; +} - auto &book = it->second; - book.book = mbp; - eval(book.dependencies); + +void PolyObserver::operator()(const MBPUpdate &mbp) { + + auto avb = available(mbp.mbp_buy); + auto avs = available(mbp.mbp_sell); + + me->sim->log("book", me->sim->get_sec(mbp.security_id).symbol(), + me->event_id.first, + mbp.mbp_buy[0].price.price(), + mbp.mbp_buy[0].quantity.value(), + mbp.mbp_sell[0].price.price(), + mbp.mbp_sell[0].quantity.value(), + avb.first.price(), + avb.second.value(), + avs.first.price(), + avs.second.value()); + + me->books[mbp.security_id] = mbp; + eval(); rank(); log(); } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h index 64cbc997b..83ca6eab8 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h @@ -23,8 +23,7 @@ private: struct Target; struct Opportunity; struct DependentTargets; - void log_book_update(MBPUpdate const &); - void eval(std::vector const&); + void eval(); void rank(); void log(); std::unique_ptr me; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp index e7f5e39b7..7fb162d2c 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp @@ -13,7 +13,7 @@ namespace bts::spartan::mdp3 { -enum class Sniper::State { WAIT_OPPORTUNITY, WAIT_FILLS }; +enum class Sniper::State { WAIT_OPPORTUNITY, WAIT_FILLS, WAIT_CLEAR }; struct Sniper::Target { constexpr static double Worthless = -1.0; @@ -28,18 +28,17 @@ struct Sniper::Target { void mark_as_worthless() { value = Worthless; } - bool has_value() const { return value > securities::market_epsilon(); } + bool has_value(double v) const { return value > v; } int value_in_ticks() const { return value / tick_size; } }; struct Sniper::Shot { Target ⌖ - size_t observed_event_id; + AlgoEventID observed_event_id; MDP3TimeStamp observed_ts; double value_at_observation_ts; MDP3Quantity quantity; - size_t end_of_opportunity_window; std::unordered_map> fills; }; @@ -55,74 +54,81 @@ struct Sniper::PImpl { int order_qty; int min_depth; bool affect_books; + size_t max_events; + double min_value; + size_t max_transaction_latency; State state = State::WAIT_OPPORTUNITY; std::unordered_map books; - std::vector targets; - std::optional in_flight; - size_t event_id; + std::vector> targets; + std::optional current_shot; + MDP3TimeStamp transact_time; MDP3TimeStamp observable_time; + AlgoEventID current_event; }; Sniper::Sniper() : me(new PImpl()) {} Sniper::~Sniper() {} -int param_or_default(protos::SimRunOptions const &options, - std::string const &name, int default_value) { - auto it = options.parameters().find(name); - if (it != options.parameters().end()) - return std::stoi(it->second); - else - return default_value; -} - void Sniper::init(ISimServices &s, protos::SimRunOptions const &options) { me->sim = &s; me->options = options; me->min_orders_at_top = param_or_default(options, "MinOrdersAtTop", 1); me->order_qty = param_or_default(options, "Qty", 1); - me->min_depth = param_or_default(options, "MinDepth", 100); + me->min_depth = param_or_default(options, "MinDepth", 0); me->affect_books = param_or_default(options, "AffectBooks", true); + me->max_events = + param_or_default(options, "MaxEvents", std::numeric_limits::max()); + me->min_value = param_or_default(options, "MinValue", 2); + me->max_transaction_latency = + param_or_default(options, "MaxTransactionLatency", 100000); me->state = State::WAIT_OPPORTUNITY; me->books.clear(); me->targets.clear(); - me->in_flight.reset(); - me->event_id = 0; + me->current_shot.reset(); + me->current_event = {0, 0}; me->transact_time = MDP3TimeStamp{0}; me->observable_time = MDP3TimeStamp{0}; for (auto &p : s.polygons()) { - auto buy_t = + + auto desc = polygon_descriptor(p.second, me->sim->instruments()); + auto buy_t = std::make_unique( Target{p.second, polygon_descriptor(p.second, me->sim->instruments()), polygon_product(p.second, me->sim->instruments()), abs_contracts(p.second, me->sim->instruments()), - tick_size(p.second, me->sim->instruments())}; - auto sell_t = + tick_size(p.second, me->sim->instruments())}); + auto sell_t = std::make_unique( Target{flip(p.second), polygon_descriptor(flip(p.second), me->sim->instruments()), polygon_product(p.second, me->sim->instruments()), abs_contracts(p.second, me->sim->instruments()), - tick_size(p.second, me->sim->instruments())}; + tick_size(p.second, me->sim->instruments())}); - me->targets.emplace_back(buy_t); - me->targets.emplace_back(sell_t); + me->targets.emplace_back(std::move(buy_t)); + me->targets.emplace_back(std::move(sell_t)); } for (auto &t : me->targets) { - for (auto &pp : t.polygon.parts) { + for (auto &pp : t->polygon.parts) { auto &dependencies = me->books[pp.sec_id].dependencies; - dependencies.push_back(&t); + dependencies.push_back(t.get()); } } } void Sniper::on_notification(AlgoNotification const &n) { - me->transact_time = n.transact_time; - me->observable_time = n.observable_time; - std::visit(*this, n.event); + if (n.event_id.first >= me->max_events) + me->sim->terminate_simulation(); + else { + me->current_event = n.event_id; + me->transact_time = n.transact_time; + me->observable_time = n.observable_time; + std::visit(*this, n.event); + } } void Sniper::eval(std::vector const &targets) { @@ -149,9 +155,10 @@ void Sniper::eval(std::vector const &targets) { } } -Sniper::MaybeShot Sniper::pick(std::vector &targets) { - std::sort(targets.begin(), targets.end(), - [](Target const *ptr1, Target const *ptr2) { +Sniper::MaybeShot Sniper::pick() { + std::sort(me->targets.begin(), me->targets.end(), + [](std::unique_ptr const &ptr1, + std::unique_ptr const &ptr2) { auto val1 = ptr1->value * ptr1->quantity.value(); auto val2 = ptr2->value * ptr2->quantity.value(); if (std::abs(val1 - val2) > securities::market_epsilon()) @@ -161,20 +168,19 @@ Sniper::MaybeShot Sniper::pick(std::vector &targets) { ptr2->polygon.parts.size(); }); - if (targets[0]->has_value()) { - for (auto t : targets) - if (t->has_value()) - me->sim->log("opportunity", me->event_id, t->descriptor, t->product, - t->n_abs_contracts, t->value_in_ticks(), t->value, - t->quantity.value(), t->value * t->quantity.value()); - if (me->order_qty) - return Shot{*targets[0], - me->event_id, - me->sim->now(), - targets[0]->value, - targets[0]->quantity, - 0, - {}}; + if (me->targets[0]->has_value(me->min_value)) { + for (auto const &t : me->targets) + if (t->has_value(me->min_value)) + me->sim->log("opportunity", me->transact_time.value(), + me->observable_time.value(), me->current_event.first, + me->current_event.second, t->descriptor, t->value, + t->quantity.value()); + if (me->order_qty && + (me->observable_time.value() - me->transact_time.value()) < + me->max_transaction_latency) + return Shot{*me->targets[0], me->current_event, + me->sim->now(), me->targets[0]->value, + me->targets[0]->quantity, {}}; else return {}; } else @@ -183,19 +189,29 @@ Sniper::MaybeShot Sniper::pick(std::vector &targets) { void Sniper::fire(Shot const &s) { - auto &shot = me->in_flight.emplace(s); + auto &shot = me->current_shot.emplace(s); for (auto &p : shot.target.polygon.parts) { auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL; auto mkt_side = securities::other_side(part_side); auto &book = me->books[p.sec_id].book.mbp_side(mkt_side); + md::IntPrice worst_price; + size_t available_qty = 0; for (int i = book.size() - 1; i >= 0; i--) { if (book[i].quantity.value()) { worst_price = book[i].price; + for (int n = 0; n <= i; n++) + available_qty += book[n].quantity.value(); break; } } + if (part_side == api::SIDE_BUY) + worst_price += + me->sim->get_sec(p.sec_id).min_price_increment_int_price() * 100; + else + worst_price -= + me->sim->get_sec(p.sec_id).min_price_increment_int_price() * 100; auto order_change = SmallOrderChange{}; order_change.affect_book = me->affect_books; @@ -206,75 +222,63 @@ void Sniper::fire(Shot const &s) { MDP3Quantity{s.quantity.value() * std::abs(p.ratio)}; order_change.order.side = part_side; order_change.change = OrderChange::added; - - me->sim->log("sending_order", api::Side_Name(part_side), + me->sim->log("sending_order", me->current_event.first, + me->current_event.second, api::Side_Name(part_side), me->sim->get_sec(p.sec_id).symbol(), book[0].price.price(), - worst_price.price()); + worst_price.price(), available_qty); me->sim->send(order_change); shot.fills[order_change.order.key] = {}; order_change.change = OrderChange::removed; me->sim->send(order_change); } - me->sim->log("shot", shot.target.descriptor, shot.observed_event_id, - shot.target.product, shot.target.n_abs_contracts, - shot.target.value_in_ticks(), shot.observed_ts.value(), - shot.value_at_observation_ts); -} - -void Sniper::eval_pick_fire(std::vector &targets) { - bts_assert(me->state == State::WAIT_OPPORTUNITY); - eval(targets); - auto best_shot = pick(targets); - if (best_shot) { - fire(best_shot.value()); - transition(State::WAIT_FILLS); - } } void Sniper::operator()(const MBPUpdate &mbp) { - me->sim->log("book", me->sim->get_sec(mbp.security_id).symbol(), - mbp.original_event_id, mbp.mbp_buy[0].price.price(), - mbp.mbp_buy[0].quantity.value(), mbp.mbp_sell[0].price.price(), - mbp.mbp_sell[0].quantity.value()); - - if (mbp.original_event_id > 20000) - me->sim->terminate_simulation(); - - if (mbp.original_event_id) - me->event_id = mbp.original_event_id; auto it = me->books.find(mbp.security_id); if (it == me->books.end()) return; - auto &book = it->second; book.book = mbp; - switch (me->state) { - case State::WAIT_OPPORTUNITY: - eval_pick_fire(book.dependencies); - break; - case State::WAIT_FILLS: - eval({&me->in_flight->target}); - if (!me->in_flight->target.has_value() && !me->in_flight->end_of_opportunity_window) { - me->in_flight->end_of_opportunity_window = me->event_id; - auto t = &me->in_flight->target; - me->sim->log("opportunity_end", me->event_id, t->descriptor, - t->product, t->n_abs_contracts, t->value_in_ticks(), - t->value, t->quantity.value(), - t->value * t->quantity.value()); - } + eval(book.dependencies); - pick(book.dependencies); + switch (me->state) { + case State::WAIT_OPPORTUNITY: { + auto best_shot = pick(); + if (best_shot) { + fire(best_shot.value()); + transition(State::WAIT_FILLS); + } break; } -} // namespace bts::spartan::mdp3 + case State::WAIT_FILLS: + me->sim->log("wait_fills", me->current_event.first, + me->current_event.second); + break; + case State::WAIT_CLEAR: + bts_assert(me->current_shot); + if (!me->current_shot->target.has_value(me->min_value)) { + me->current_shot.reset(); + me->state = State::WAIT_OPPORTUNITY; + auto best_shot = pick(); + if (best_shot) { + fire(best_shot.value()); + transition(State::WAIT_FILLS); + } + } else { + me->sim->log("wait_clear", me->current_event.first, + me->current_event.second); + } + break; + } +} void Sniper::operator()(const OrderUpdate &update) { bts_assert(me->state == State::WAIT_FILLS); - bts_assert(me->in_flight); - auto &in_flight = me->in_flight.value(); + bts_assert(me->current_shot); + auto &in_flight = me->current_shot.value(); bts_assert(in_flight.fills.count(update.key)); in_flight.fills[update.key].push_back(update); @@ -298,22 +302,13 @@ void Sniper::operator()(const OrderUpdate &update) { these_fills.end()); } PandL pandl(me->sim->instruments()); - me->sim->log("shot_outcome", in_flight.observed_event_id, - in_flight.observed_ts.value(), in_flight.target.descriptor, - in_flight.value_at_observation_ts, - in_flight.quantity.value(), me->transact_time.value(), - pandl.net_cash(all_fills), in_flight.target.n_abs_contracts); + me->sim->log("shot_outcome", in_flight.target.descriptor, + in_flight.observed_event_id.first, + in_flight.observed_event_id.second, me->current_event.first, + me->current_event.second, in_flight.value_at_observation_ts, + pandl.net_cash(all_fills)); me->sim->log_pandl(); - transition(State::WAIT_OPPORTUNITY); - auto observed_ts = me->in_flight->observed_ts; - me->in_flight.reset(); - if (observed_ts < me->observable_time) { - // We have received events since we shot, revaluate - std::vector t_ptr; - for (auto &t:me->targets) - t_ptr.push_back(&t); - eval_pick_fire(t_ptr); - } + transition(State::WAIT_CLEAR); } } @@ -323,6 +318,8 @@ const char *Sniper::state_name(State s) { return "WAIT"; case State::WAIT_OPPORTUNITY: return "WATCH"; + case State::WAIT_CLEAR: + return "WAIT_CLEAR"; } return nullptr; } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h index 537db0043..ddd60151a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h @@ -27,9 +27,8 @@ private: using MaybeShot = std::optional; void eval(std::vector const&); - MaybeShot pick(std::vector &); + MaybeShot pick(); void fire(Shot const&); - void eval_pick_fire(std::vector &); void transition(State); const char *state_name(State); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp index 312b5f134..1fb91b090 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp @@ -139,6 +139,7 @@ TEST_CASE ("sim_spreader", "[spreader]"){ // send leg_1 book update => leg_2 should quote sim_spreader.on_notification( { + {}, bts::spartan::mdp3::MDP3TimeStamp{}, bts::spartan::mdp3::MDP3TimeStamp{}, md_udpate_leg_1 @@ -163,6 +164,7 @@ TEST_CASE ("sim_spreader", "[spreader]"){ auto md_udpate_leg_1 = get_book_update(bids,asks,leg_1.security().id()); sim_spreader.on_notification( { + {}, bts::spartan::mdp3::MDP3TimeStamp{}, bts::spartan::mdp3::MDP3TimeStamp{}, md_udpate_leg_1 @@ -181,6 +183,7 @@ TEST_CASE ("sim_spreader", "[spreader]"){ auto q_ord = sim_services.last_change.order; sim_spreader.on_notification( { + {}, bts::spartan::mdp3::MDP3TimeStamp{}, bts::spartan::mdp3::MDP3TimeStamp{}, bts::spartan::mdp3::OrderUpdate{ @@ -206,6 +209,7 @@ TEST_CASE ("sim_spreader", "[spreader]"){ // send the hedge fill and it should go to end state sim_spreader.on_notification( { + {}, bts::spartan::mdp3::MDP3TimeStamp{}, bts::spartan::mdp3::MDP3TimeStamp{}, bts::spartan::mdp3::OrderUpdate{ diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp index ec5c3346f..fbceae06b 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp @@ -182,6 +182,8 @@ void FifoSecurityMatcher::_on_order_not_affecting_books( auto& opp_book = _book(_opp_side(ord.side)); uint32_t new_ord_rem = total_quantity(ord).value(); for (FifoPriceLevelMatcher &pl : opp_book) { + if(!pl.populated()) + continue; if (_is_cross(ord, pl)){ uint32_t level_qty = static_cast(pl.total_quantity().value()); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp index ae1282b9f..8bacb9e9e 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp @@ -177,7 +177,6 @@ void MergedMatcher::match(ChangeSrc, auto new_mbp = sec_kv.second.get_new_mbp(); if (new_mbp.has_value()) { res->emplace_back(MBPUpdate{ - 0, sec_kv.first, new_mbp.value().bids, new_mbp.value().asks, @@ -291,7 +290,6 @@ void DirectMatcher::match(ChangeSrc, res->emplace_back( MBPUpdate { - 0, cm.first, cm.second->matcher.book(api::Side::SIDE_BUY), cm.second->matcher.book(api::Side::SIDE_SELL), diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h index 10fcfd520..868187924 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/AlgoEvent.h @@ -4,8 +4,9 @@ namespace bts::spartan::mdp3 { +using AlgoEventID = std::pair ; + struct MBPUpdate { - size_t original_event_id; MDP3SecurityID security_id; Book mbp_buy; Book mbp_sell; @@ -49,6 +50,7 @@ using MatchEffect = std::variant; struct AlgoNotification { + AlgoEventID event_id; MDP3TimeStamp transact_time; MDP3TimeStamp observable_time; std::variant event; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp index 91c8ab561..5737368e6 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp @@ -80,7 +80,7 @@ void ExchangeSim::prepare_run(std::vector const &algos, // Init gateways, add them to the simulator for (auto &i : me->inputs) { - auto &gw = me->run_state->gateways.emplace_back(i->instruments, options); + auto &gw = me->run_state->gateways.emplace_back(me->logger,i->instruments, options); me->run_state->simulator.add_agent(gw); } diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h index 53121234f..6312509d0 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -1,6 +1,7 @@ #pragma once #include "../../common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" #include "AlgoEvent.h" namespace bts::spartan::protos { @@ -9,6 +10,15 @@ class SimRunOptions; namespace bts::spartan::mdp3 { class ISimServices; +inline int param_or_default(protos::SimRunOptions const &options, + std::string const &name, int default_value) { + auto it = options.parameters().find(name); + if (it != options.parameters().end()) + return std::stoi(it->second); + else + return default_value; +} + class IAlgoModel { public: virtual ~IAlgoModel(); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp index 66228be19..e44617a96 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp @@ -127,6 +127,9 @@ Book AlgoAgent::book(MDP3SecurityID id, api::Side s) const { } void AlgoAgent::send(SmallOrderChange const &change, uint64_t propagation_delay) { + auto total_delay = me->options.base_algo_latency_ns() + propagation_delay; + auto t_time = time() + total_delay; + log("order_action", change_name(change.change), get_sec(change.order.security).symbol(), change.order.price.price(), change.order.quantity.value(), api::Side_Name(change.order.side), @@ -134,8 +137,8 @@ void AlgoAgent::send(SmallOrderChange const &change, uint64_t propagation_delay) me->my_orders.insert(change.order.key); emit_delayed_event( - me->options.base_algo_latency_ns() + propagation_delay, - AlgoAction{MDP3TimeStamp{time()}, change}); + total_delay, + AlgoAction{MDP3TimeStamp{t_time}, change}); } void AlgoAgent::terminate_simulation() { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp index 98e2be51b..6dde308dc 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp @@ -16,7 +16,7 @@ // just uncomment the line below. #define ENABLE_IMPLIED_MATCHING namespace bts::spartan::mdp3 { - + #ifdef ENABLE_IMPLIED_MATCHING using Matcher = MergedMatcher; #else @@ -33,27 +33,33 @@ extract_ids(std::unordered_map const &m) { struct GatewayAgent::PImpl { #ifdef ENABLE_IMPLIED_MATCHING - PImpl(std::unordered_map const &defs, + PImpl(SimLogger const &l, + std::unordered_map const &defs, protos::SimRunOptions const &options) - : my_instruments(defs), options(options), matcher() { - auto def_vals = defs | ranges::views::transform([](const auto& p) { return p.second; }); - matcher.init(def_vals.begin(), def_vals.end()); - } + : my_instruments(defs), options(options), matcher(), + logger(l), current_event{0, 0} { + auto def_vals = defs | ranges::views::transform( + [](const auto &p) { return p.second; }); + matcher.init(def_vals.begin(), def_vals.end()); + } #else PImpl(std::unordered_map const &defs, protos::SimRunOptions const &options) - : my_instruments(defs), options(options), matcher(extract_ids(defs)) { } + : my_instruments(defs), options(options), matcher(extract_ids(defs)) {} #endif std::unordered_map const &my_instruments; protos::SimRunOptions options; Matcher matcher; + SimLogger logger; + AlgoEventID current_event; }; GatewayAgent::GatewayAgent( + SimLogger const &l, std::unordered_map const &defs, protos::SimRunOptions const &options) - : me(new PImpl(defs, options)) { + : me(new PImpl(l, defs, options)) { me->options = options; } @@ -82,6 +88,14 @@ Book GatewayAgent::book(MDP3SecurityID id, api::Side s) { return me->matcher.book(id, s); } +template +void log(SimLogger &l, sim::TimeStamp ts, std::string const &log_entry_type, + T &&... args) { + auto lst = + std::initializer_list{to_str(std::forward(args))...}; + l(MDP3TimeStamp{ts}, log_entry_type, lst); +} + // Handle a recorded order change, that is an order change we got from // a pcap recording of the market data void GatewayAgent::_handle(ExchangeTransactionEvent const &e) { @@ -92,9 +106,11 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) { if (me->my_instruments.count(first_change.order.security)) { - // Advance our clock either to the send time in the recording - // (unless we are already ahead of that time, possibly because - // we got delayed processing algo operations). + me->current_event = AlgoEventID{ev->event_id, 0}; + + // Advance our clock to the send time in the recording (unless + // we are already ahead of that time, possibly because we got + // delayed processing algo operations). auto send_time = ev->last_packet_send_time.value(); if (send_time > time()) forward_clock_by(send_time - time()); @@ -107,49 +123,64 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) { me->matcher.match(Matcher::SIMULATOR_INPUT, ev->order_changes, &effects); for (auto &ef : effects) - std::visit(overloaded{[&](MBPUpdate &mbp) { - mbp.original_event_id = e.ev->event_id; - emit_delayed_event( - me->options.mbp_update_delay_ns(), - AlgoNotification{ev->transact_time, - MDP3TimeStamp{time()}, - mbp}); - }, - [&](OrderUpdate const &order_update) { - emit_delayed_event( - me->options.order_update_delay_ns(), - AlgoNotification{ev->transact_time, - MDP3TimeStamp{time()}, - order_update}); - }}, - ef); + std::visit( + overloaded{ + [&](MBPUpdate &mbp) { + emit_delayed_event( + me->options.mbp_update_delay_ns(), + AlgoNotification{me->current_event, ev->transact_time, + MDP3TimeStamp{time()}, mbp}); + }, + [&](OrderUpdate const &order_update) { + emit_delayed_event( + me->options.order_update_delay_ns(), + AlgoNotification{me->current_event, ev->transact_time, + MDP3TimeStamp{time()}, order_update}); + }}, + ef); } } // Handle an algo order change, that is an order change requested by // one of the algos under test. void GatewayAgent::_handle(AlgoActionEvent const &e) { + me->current_event.second++; auto const &algo_action = e.ev; + auto const &change = algo_action.change; + + + log(me->logger, time(), "gw_algo_order", algo_action.transact_time.value(), + change_name(change.change), get_sec(change.order.security).symbol(), + change.order.price.price(), change.order.quantity.value(), + api::Side_Name(change.order.side), change.order.key.value(), + change.order.security.value(), me->current_event.first, + me->current_event.second); std::vector effects; me->matcher.match(Matcher::ALGO, {algo_action.change}, &effects); for (auto &ef : effects) - std::visit(overloaded{[&](MBPUpdate &mbp) { - mbp.original_event_id = 0; - emit_delayed_event( - me->options.mbp_update_delay_ns(), - AlgoNotification{algo_action.transact_time, - MDP3TimeStamp{time()}, - mbp}); - }, - [&](OrderUpdate const &order_update) { - emit_delayed_event( - me->options.order_update_delay_ns(), - AlgoNotification{algo_action.transact_time, - MDP3TimeStamp{time()}, - order_update}); - }}, - ef); + std::visit( + overloaded{[&](MBPUpdate &mbp) { + emit_delayed_event( + me->options.mbp_update_delay_ns(), + AlgoNotification{me->current_event, + algo_action.transact_time, + MDP3TimeStamp{time()}, mbp}); + }, + [&](OrderUpdate const &order_update) { + emit_delayed_event( + me->options.order_update_delay_ns(), + AlgoNotification{ + me->current_event, algo_action.transact_time, + MDP3TimeStamp{time()}, order_update}); + }}, + ef); } -} // namespace bts::spartan::mdp3 \ No newline at end of file +InstrumentDefinition const &GatewayAgent::get_sec(MDP3SecurityID id) const { + auto it = me->my_instruments.find(id); + bts_assert(it != me->my_instruments.end()); + return it->second; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h index 63c60bb74..12bba6290 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.h @@ -3,6 +3,7 @@ #include "../core/AgentBase.h" #include "../events/SimEvents.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimLogger.h" #include #include @@ -14,7 +15,7 @@ class InstrumentDefinition; class GatewayAgent : public sim::AgentBase { public: - GatewayAgent( + GatewayAgent(SimLogger const &logger, std::unordered_map const &, protos::SimRunOptions const &); GatewayAgent(GatewayAgent &&); @@ -27,8 +28,9 @@ class GatewayAgent : public sim::AgentBase { private: void _handle(ExchangeTransactionEvent const &); void _handle(AlgoActionEvent const &); + InstrumentDefinition const &get_sec(MDP3SecurityID) const; struct PImpl; std::unique_ptr me; }; -} // namespace bts::spartan::mdp3 \ No newline at end of file +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h index d284e5b48..13bc5f193 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h @@ -130,6 +130,13 @@ class InstrumentDefinition { }); } + md::IntPrice min_price_increment_int_price() const { + return get([](auto m) { + return md::IntPrice(m.minPriceIncrement().mantissa(), + m.minPriceIncrement().exponent()); + }); + } + double min_price_increment_ammount() const { auto visitor = overloaded{ [](::mdp3::MDInstrumentDefinitionFuture54 m) { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.cpp index d70335180..ca19781dc 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.cpp @@ -358,7 +358,9 @@ std::string polygon_descriptor(Polygon const &p, str << ")"; str << " "; } - return str.str(); + auto result = str.str(); + result.resize(result.size()-1); + return result; } std::string polygon_product(Polygon const &polygon,