From 524e83c59574635cf6f57bd8a99d5dae28a17c8d Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Tue, 10 Nov 2020 10:49:14 -0500 Subject: [PATCH] Streamlined sim db schema for easier querying. --- src/data_files/files/spartan/sim_db.sql | 265 ++++++------------ .../orchestrator/SimOrchestrator.cpp | 2 +- .../orchestrator/SimSessionOptions.proto | 3 - .../md/mdp3/back_testing/sim_db/SimDB.cpp | 147 ++++++---- .../md/mdp3/back_testing/sim_db/SimDB.h | 17 +- 5 files changed, 177 insertions(+), 257 deletions(-) diff --git a/src/data_files/files/spartan/sim_db.sql b/src/data_files/files/spartan/sim_db.sql index a3693b7e8..4d7c71646 100644 --- a/src/data_files/files/spartan/sim_db.sql +++ b/src/data_files/files/spartan/sim_db.sql @@ -1,59 +1,38 @@ -/* A session is a set of simulation runs for the same algo, covering -multiple combinations of days and parameters. */ - -drop table if exists session; -CREATE TABLE session( - session_id integer primary key, -/* the date when this session was run */ - session_run_date text, -/* the algo under test */ - algo text, -/* an optional descriptor for this session */ - title text -); /* A run is a simulation covering a single algo, in a single day, with a fixed set of parameters. */ - drop table if exists run; create table run( run_id integer primary key, - session_id numeric, -/* Clock time when we started / ended this run */ - begin_time text, - end_time text, + session_id text, + algo text, + begin_run_time text, + end_run_time text, + market_date text, + channel text, + base_algo_latency_ns numeric, + order_update_delay_ns numeric, + mbp_update_delay_ns numeric, + transaction_latency_ns numeric, realized_pandl real, position_pandl real, - foreign key(session_id) references session(session_id) -); - -/* -A run param is the value of a specific parameter on a specific -simulation run. Some parameters are standardized across algos, while -others are algo specific. Some of the standardized parameters include: - -- market_date -> market_date -- channel->* channel -- base_algo_latency_ns -> base_algo_latency_ns -- order_update_delay_ns -> order_update_delay_ns -- mbp_update_delay_ns -> mbp_update_delay_ns -- transaction_latency_ns -> transaction_latency_ns -*/ - -drop table if exists param; -create table param( - param_id integer primary_key, - run_id numeric, - param_name text, - param_value text, - foreign key(run_id) references run(run_id) +/* algo specific parameters */ + param1 text, + param2 text, + param3 text, + param4 text, + param5 text, + param6 text, + param7 text, + param8 text, + param9 text, + param10 text ); /* A log entry is an entry emited during a simulation run. Some of these entries are standardized across algos, while others are algo specific. Some of the standard entries are: - - pnl -> realized pnl, position pnl - symbol_pnl ->* symbol, pos, realized_pnl - outright_pos->* contract, pos @@ -63,8 +42,8 @@ specific. Some of the standard entries are: drop table if exists log; create table log( - log_entry_id integer primary_key, run_id numeric, + log_entry_order numeric, t_time numeric, h_t_time TEXT, entry_type TEXT, @@ -80,24 +59,78 @@ create table log( val10 TEXT, foreign key(run_id) references run(run_id) ); -CREATE INDEX log_idx ON log(run_id,entry_type); +CREATE INDEX log_idx_0 ON log(run_id,entry_type); +CREATE INDEX log_idx_1 ON log(val1); +CREATE INDEX log_idx_2 ON log(val2); +CREATE INDEX log_idx_3 ON log(val3); /* sniper specific views */ +drop view if exists sniper_run; +create view sniper_run as +select + run_id, + session_id , + begin_run_time , + end_run_time , + market_date , + channel , + base_algo_latency_ns, + order_update_delay_ns, + mbp_update_delay_ns, + transaction_latency_ns, + realized_pandl , + position_pandl , + cast(param1 as numeric) MinDepth, + cast(param2 as numeric) MinOrdersAtTop, + cast(param3 as numeric) MinParamue, + cast(param4 as numeric) Qty, + cast(param5 as numeric) MaxTransactionLatency, + cast(param6 as numeric) DoubleDip, + cast(param7 as numeric) MsToWorkShot +from + run +order by run_id ; + + +drop view if exists shot_beginnings; +create view shot_beginnings as +select + run_id, + t_time, + val3 polygon, + cast (val1 as numeric) o_eid, + cast (val2 as numeric) o_eid_2, + cast (val4 as real) target, + cast (val5 as numeric) quantity, + cast (val6 as numeric) streak_id +from + log +where + entry_type='shot_begin'; drop view if exists shot_outcomes; create view shot_outcomes as select - run_id, - t_time, + log.run_id, + log.t_time, val1 polygon, - cast(val2 as numeric) observed_eid, - cast(val4 as numeric) executed_eid, + cast(val2 as numeric) o_eid, + cast(val3 as numeric) o_eid_2, + cast(val4 as numeric) e_eid, + cast(val5 as numeric) e_eid_2, cast(val6 as real) target, - cast(val7 as real) realized + cast(val7 as real) realized, + cast(val8 as numeric) streak_id, + (log.t_time - (lag(log.t_time) over (partition by log.entry_type order by log.t_time)))/1000 t_since_last_shot, + (log.t_time - shot_beginnings.t_time)/1000 cycle_t from - log + log, shot_beginnings where - entry_type='shot_outcome' ; + log.entry_type='shot_outcome' and + cast(val2 as numeric) = shot_beginnings.o_eid and + cast(val3 as numeric) = shot_beginnings.o_eid_2 and + log.run_id = shot_beginnings.run_id; + drop view if exists shot_counts; create view shot_counts as @@ -112,133 +145,3 @@ from group by run_id ; -drop view if exists opportunity_counts; -create view opportunity_counts as -select - run_id, count(*) n_opportunities -from - log -where - entry_type='opportunity' -group by - run_id ; - -drop view if exists sniper_profits; -CREATE VIEW sniper_profits as -select - session.algo algo, - session.session_id, - run.run_id, - realized_pandl, - position_pandl, - n_shots, - good_shots, - lucky_shots, - unlucky_shots, - param_latency.param_value latency, - param_date.param_value date, - param_min_depth.param_value min_depth, - param_min_orders.param_value min_orders, - param_min_value.param_value min_value, - param_qty.param_value qty, - param_max_transaction_latency.param_value max_t_latency, - param_double_dip.param_value double_dip, - param_ms_to_work_shot.param_value ms_to_work -from - session, - run, - shot_counts, - (select * from param where param_name='base_algo_latency_ns') param_latency, - (select * from param where param_name='market_date') param_date, - (select * from param where param_name='MinDepth') param_min_depth, - (select * from param where param_name='MinOrdersAtTop') param_min_orders, - (select * from param where param_name='MinValue') param_min_value, - (select * from param where param_name='Qty') param_qty, - (select * from param where param_name='MaxTransactionLatency') param_max_transaction_latency, - (select * from param where param_name='DoubleDip') param_double_dip, - (select * from param where param_name='MsToWorkShot') param_ms_to_work_shot -WHERE - session.session_id = run.session_id and - run.run_id = shot_counts.run_id and - run.run_id = param_latency.run_id and - run.run_id = param_date.run_id and - run.run_id = param_min_depth.run_id and - run.run_id = param_min_orders.run_id and - run.run_id = param_min_value.run_id and - run.run_id = param_qty.run_id and - run.run_id = param_max_transaction_latency.run_id and - run.run_id = param_double_dip.run_id and - run.run_id = param_ms_to_work_shot.run_id -order by - cast(realized_pandl as real) desc; - - -drop view if exists per_day_ranked_sniper_profits; -create view per_day_ranked_sniper_profits as -select - sniper_profits.*, - row_number() over (partition by date order by realized_pandl desc) rank -from - sniper_profits -where - position_pandl=0 ; - - -drop view if exists best_sniper_run_of_the_day; -create view best_sniper_run_of_the_day as -select - sniper_profits.* -from - sniper_profits, - per_day_ranked_sniper_profits -where - sniper_profits.run_id=per_day_ranked_sniper_profits.run_id and - per_day_ranked_sniper_profits.rank=1 -order by - sniper_profits.date; - -drop view if exists fake_shot_outcomes; -create view fake_shot_outcomes as -select - run_id, - val4 symbol, - cast(val1 as numeric) begin_eid, - cast(val2 as numeric) end_eid, - cast(val5 as real) merged_value, - cast(val6 as numeric) merged_qty, - cast(val3 as numeric) begin_ts, - cast(val7 as numeric) end_ts, - (val7-val3)/1000 duration -from - log -where - entry_type='fake_shot_outcome'; - -drop view if exists top_of_books ; -create view top_of_books as -select - run_id, - val1 symbol, - cast(val2 as numeric) eid, - cast(val3 as real) bid, - cast(val4 as real) bid_qty, - cast(val5 as real) ask, - cast(val6 as real) ask_qty -from - log -where - entry_type = 'book'; - -drop view if exists poly_observer_opportunities; -create view poly_observer_opportunities as -select - val1 symbol, - cast(val2 as real) merged_value, - cast(val3 as numeric) merged_qty, - cast(val4 as numeric) begin_eid, - cast(val5 as numeric) end_eid, - run_id -from - log -where - entry_type='opportunity' ; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp index 7368f6dd6..1ac262ec1 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp @@ -220,7 +220,7 @@ void SimOrchestrator::run() { workers[i % workers.size()].assigned_runs.push_back(&me->runs[i]); auto session_id = - me->db.create_session(me->options.algo(), me->options.session_title()); + me->db.create_session(me->options.algo()); std::atomic run_count = me->runs.size(); for (auto &w : workers) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto index 851ca6bfb..e9ef8555a 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto @@ -36,9 +36,6 @@ message SimSessionOptions { // grouped and used as a unit for each simulation run. repeated string file = 2; - // An optional user friendly title for this session. - string session_title = 3; - // The max number of cores to use for the simulation. By default all // cores will be used. uint32 n_cores = 4 ; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp index d545a4fc4..09e5e8840 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp @@ -27,6 +27,7 @@ struct SimDB::PImpl { SQLite::Database db; std::mutex lock; std::vector pending; + std::unordered_map log_entry_order; }; SimDB::SimDB(std::string const &loc) { @@ -52,26 +53,18 @@ void SimDB::destroy_db(std::string const &loc) { SimDB::~SimDB() {} -SimDB::SessionID SimDB::create_session(std::string const &algo, - std::string const &title) { +SimDB::SessionID SimDB::create_session(std::string const &algo) { std::lock_guard l(me->lock); - auto insert_session = "insert into session (" - "session_run_date," - "algo," - "title" - ") values (" - ":session_run_date," - ":algo," - ":title" - ")"; + auto max_session_query = "select max(rowid) from run"; - SQLite::Statement statement(me->db, insert_session); - statement.bind(":session_run_date", datetime::date_string(datetime::now())); - statement.bind(":algo", algo); - statement.bind(":title", title); - statement.exec(); - return SessionID{last_insert_rowid()}; + SQLite::Statement statement(me->db, max_session_query); + size_t session_id=0; + while (statement.executeStep()) { + session_id = (long long)statement.getColumn(0); + } + + return SessionID{session_id, algo}; } SimDB::RunID SimDB::begin_run(SessionID sid, @@ -80,32 +73,83 @@ SimDB::RunID SimDB::begin_run(SessionID sid, auto insert_run = "insert into run(" "session_id," - "begin_time" + "algo," + "begin_run_time," + "end_run_time," + "market_date," + "channel," + "base_algo_latency_ns," + "order_update_delay_ns," + "mbp_update_delay_ns," + "transaction_latency_ns," + "realized_pandl," + "position_pandl," + "param1," + "param2," + "param3," + "param4," + "param5," + "param6," + "param7," + "param8," + "param9," + "param10" ") values (" ":session_id," - ":begin_time" + ":algo," + ":begin_run_time," + ":end_run_time," + ":market_date," + ":channel," + ":base_algo_latency_ns," + ":order_update_delay_ns," + ":mbp_update_delay_ns," + ":transaction_latency_ns," + ":realized_pandl," + ":position_pandl," + ":param1," + ":param2," + ":param3," + ":param4," + ":param5," + ":param6," + ":param7," + ":param8," + ":param9," + ":param10" ")"; SQLite::Statement statement(me->db, insert_run); - statement.bind(":session_id", sid.value()); - statement.bind(":begin_time", datetime::time_string(datetime::now())); + statement.bind(":session_id", (long long)sid.id); + statement.bind(":algo", sid.algo); + statement.bind(":begin_run_time", datetime::time_string(datetime::now())); + statement.bind(":end_run_time", ""); + statement.bind(":market_date", options.market_date()); + statement.bind(":channel", options.channels(0)); + statement.bind(":base_algo_latency_ns", + (long long)options.base_algo_latency_ns()); + statement.bind(":order_update_delay_ns", + (long long)options.order_update_delay_ns()); + statement.bind(":mbp_update_delay_ns", + (long long)options.mbp_update_delay_ns()); + statement.bind(":transaction_latency_ns", + (long long)options.transaction_latency_ns()); + statement.bind(":realized_pandl", 0); + statement.bind(":position_pandl", 0); + + std::map sorted_params( + options.parameters().begin(), options.parameters().end()); + + size_t p_idx = 1; + for (auto const &p : sorted_params) + statement.bind(fmt::format(":param{}", p_idx++), p.second); + + for (auto i = p_idx; i <= 10; i++) + statement.bind(fmt::format(":param{}", i), ""); + statement.exec(); - - auto run_id = RunID{last_insert_rowid()}; - - // Log parameters - _add_param(run_id, "market_date", options.market_date()); - for (auto const &c : options.channels()) - _add_param(run_id, "channel", c); - _add_param(run_id, "base_algo_latency_ns", options.base_algo_latency_ns()); - _add_param(run_id, "order_update_delay_ns", options.order_update_delay_ns()); - _add_param(run_id, "mbp_update_delay_ns", options.mbp_update_delay_ns()); - _add_param(run_id, "transaction_latency_ns", - options.transaction_latency_ns()); - - for (auto const &p : options.parameters()) { - _add_param(run_id, p.first, p.second); - } + auto run_id = RunID{last_insert_rowid()}; + me->log_entry_order[run_id] = 1; return run_id; } @@ -115,14 +159,14 @@ void SimDB::end_run(RunID run_id, double realized_pandl, auto update_run = "update run " "set " - "end_time=:end_time, " + "end_run_time=:end_run_time, " "realized_pandl=:realized_pandl," "position_pandl=:position_pandl " "where " "run_id=:run_id"; SQLite::Statement statement(me->db, update_run); - statement.bind(":end_time", datetime::time_string(datetime::now())); + statement.bind(":end_run_time", datetime::time_string(datetime::now())); statement.bind(":run_id", run_id.value()); statement.bind(":realized_pandl", realized_pandl); statement.bind(":position_pandl", position_pandl); @@ -147,22 +191,24 @@ void SimDB::_log(bool flush) { auto now = l.ts; auto &entry_type = l.log_entry_type; auto &log_args = l.args; - + auto &log_entry_order = me->log_entry_order[rid]; std::ostringstream insert_log_entry; insert_log_entry << "insert into log(" "run_id," + "log_entry_order," "t_time," "h_t_time," "entry_type"; for (size_t i = 0; i < log_args.size(); i++) insert_log_entry << ",val" << i + 1; - insert_log_entry << ") values (:run_id,:t_time,:h_t_time,:entry_type"; + insert_log_entry << ") values (:run_id,:log_entry_order,:t_time,:h_t_time,:entry_type"; for (size_t i = 0; i < log_args.size(); i++) insert_log_entry << ",:val" << i + 1; insert_log_entry << ")"; SQLite::Statement statement(me->db, insert_log_entry.str()); statement.bind(":run_id", rid.value()); + statement.bind(":log_entry_order",(long long) log_entry_order++); statement.bind(":t_time", (long long)now.value()); statement.bind(":h_t_time", to_time_string(now)); statement.bind(":entry_type", entry_type); @@ -178,25 +224,6 @@ void SimDB::_log(bool flush) { } } -void SimDB::_add_param(RunID run_id, std::string const ¶m_name, - std::string const ¶m_value) { - auto insert_param = "insert into param(" - "run_id," - "param_name," - "param_value" - ") values (" - ":run_id," - ":param_name," - ":param_value" - ")"; - - SQLite::Statement statement(me->db, insert_param); - statement.bind(":run_id", run_id.value()); - statement.bind(":param_name", param_name); - statement.bind(":param_value", param_value); - statement.exec(); -} - int SimDB::last_insert_rowid() { SQLite::Statement statement(me->db, "select last_insert_rowid()"); statement.executeStep(); diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h index 88b5602fb..c7c9a0a81 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h @@ -17,9 +17,11 @@ class SimDB { SimDB(std::string const &location = persistence::storage_location("sim_db")); ~SimDB(); - using SessionID = utils::ValueWrapper; - SessionID create_session(std::string const &algo, - std::string const &title = ""); + struct SessionID { + size_t id; + std::string algo; + }; + SessionID create_session(std::string const &algo); using RunID = utils::ValueWrapper; RunID begin_run(SessionID, protos::SimRunOptions const &); @@ -32,15 +34,6 @@ class SimDB { std::string const &location = persistence::storage_location("sim_db")); private: - - template - void _add_param(RunID rid, std::string const ¶m_name, T v) { - _add_param(rid, param_name, std::to_string(v)); - } - - void _add_param(RunID, std::string const ¶m_name, - std::string const ¶m_value); - void _log(bool flush = false); int last_insert_rowid(); struct PendingLog;