Streamlined sim db schema for easier querying.

This commit is contained in:
Pedro Pinto committed 2020-11-10 10:49:14 -05:00
1 parent f9b2f413d7
commit 524e83c595
5 files changed
+177 -257

No files matched your search

+84 -181
View File
@@ -1,59 +1,38 @@
/* A session is a set of simulation runs for the same algo, covering
multiple combinations of days and parameters. */
drop table if exists session;
CREATE TABLE session(
session_id integer primary key,
/* the date when this session was run */
session_run_date text,
/* the algo under test */
algo text,
/* an optional descriptor for this session */
title text
);
/* A run is a simulation covering a single algo, in a single day, with
a fixed set of parameters. */
drop table if exists run;
create table run(
run_id integer primary key,
session_id numeric,
/* Clock time when we started / ended this run */
begin_time text,
end_time text,
session_id text,
algo text,
begin_run_time text,
end_run_time text,
market_date text,
channel text,
base_algo_latency_ns numeric,
order_update_delay_ns numeric,
mbp_update_delay_ns numeric,
transaction_latency_ns numeric,
realized_pandl real,
position_pandl real,
foreign key(session_id) references session(session_id)
);
/*
A run param is the value of a specific parameter on a specific
simulation run. Some parameters are standardized across algos, while
others are algo specific. Some of the standardized parameters include:
- market_date -> market_date
- channel->* channel
- base_algo_latency_ns -> base_algo_latency_ns
- order_update_delay_ns -> order_update_delay_ns
- mbp_update_delay_ns -> mbp_update_delay_ns
- transaction_latency_ns -> transaction_latency_ns
*/
drop table if exists param;
create table param(
param_id integer primary_key,
run_id numeric,
param_name text,
param_value text,
foreign key(run_id) references run(run_id)
/* algo specific parameters */
param1 text,
param2 text,
param3 text,
param4 text,
param5 text,
param6 text,
param7 text,
param8 text,
param9 text,
param10 text
);
/*
A log entry is an entry emited during a simulation run. Some of
these entries are standardized across algos, while others are algo
specific. Some of the standard entries are:
- pnl -> realized pnl, position pnl
- symbol_pnl ->* symbol, pos, realized_pnl
- outright_pos->* contract, pos
@@ -63,8 +42,8 @@ specific. Some of the standard entries are:
drop table if exists log;
create table log(
log_entry_id integer primary_key,
run_id numeric,
log_entry_order numeric,
t_time numeric,
h_t_time TEXT,
entry_type TEXT,
@@ -80,24 +59,78 @@ create table log(
val10 TEXT,
foreign key(run_id) references run(run_id)
);
CREATE INDEX log_idx ON log(run_id,entry_type);
CREATE INDEX log_idx_0 ON log(run_id,entry_type);
CREATE INDEX log_idx_1 ON log(val1);
CREATE INDEX log_idx_2 ON log(val2);
CREATE INDEX log_idx_3 ON log(val3);
/* sniper specific views */
drop view if exists sniper_run;
create view sniper_run as
select
run_id,
session_id ,
begin_run_time ,
end_run_time ,
market_date ,
channel ,
base_algo_latency_ns,
order_update_delay_ns,
mbp_update_delay_ns,
transaction_latency_ns,
realized_pandl ,
position_pandl ,
cast(param1 as numeric) MinDepth,
cast(param2 as numeric) MinOrdersAtTop,
cast(param3 as numeric) MinParamue,
cast(param4 as numeric) Qty,
cast(param5 as numeric) MaxTransactionLatency,
cast(param6 as numeric) DoubleDip,
cast(param7 as numeric) MsToWorkShot
from
run
order by run_id ;
drop view if exists shot_beginnings;
create view shot_beginnings as
select
run_id,
t_time,
val3 polygon,
cast (val1 as numeric) o_eid,
cast (val2 as numeric) o_eid_2,
cast (val4 as real) target,
cast (val5 as numeric) quantity,
cast (val6 as numeric) streak_id
from
log
where
entry_type='shot_begin';
drop view if exists shot_outcomes;
create view shot_outcomes as
select
run_id,
t_time,
log.run_id,
log.t_time,
val1 polygon,
cast(val2 as numeric) observed_eid,
cast(val4 as numeric) executed_eid,
cast(val2 as numeric) o_eid,
cast(val3 as numeric) o_eid_2,
cast(val4 as numeric) e_eid,
cast(val5 as numeric) e_eid_2,
cast(val6 as real) target,
cast(val7 as real) realized
cast(val7 as real) realized,
cast(val8 as numeric) streak_id,
(log.t_time - (lag(log.t_time) over (partition by log.entry_type order by log.t_time)))/1000 t_since_last_shot,
(log.t_time - shot_beginnings.t_time)/1000 cycle_t
from
log
log, shot_beginnings
where
entry_type='shot_outcome' ;
log.entry_type='shot_outcome' and
cast(val2 as numeric) = shot_beginnings.o_eid and
cast(val3 as numeric) = shot_beginnings.o_eid_2 and
log.run_id = shot_beginnings.run_id;
drop view if exists shot_counts;
create view shot_counts as
@@ -112,133 +145,3 @@ from
group by
run_id ;
drop view if exists opportunity_counts;
create view opportunity_counts as
select
run_id, count(*) n_opportunities
from
log
where
entry_type='opportunity'
group by
run_id ;
drop view if exists sniper_profits;
CREATE VIEW sniper_profits as
select
session.algo algo,
session.session_id,
run.run_id,
realized_pandl,
position_pandl,
n_shots,
good_shots,
lucky_shots,
unlucky_shots,
param_latency.param_value latency,
param_date.param_value date,
param_min_depth.param_value min_depth,
param_min_orders.param_value min_orders,
param_min_value.param_value min_value,
param_qty.param_value qty,
param_max_transaction_latency.param_value max_t_latency,
param_double_dip.param_value double_dip,
param_ms_to_work_shot.param_value ms_to_work
from
session,
run,
shot_counts,
(select * from param where param_name='base_algo_latency_ns') param_latency,
(select * from param where param_name='market_date') param_date,
(select * from param where param_name='MinDepth') param_min_depth,
(select * from param where param_name='MinOrdersAtTop') param_min_orders,
(select * from param where param_name='MinValue') param_min_value,
(select * from param where param_name='Qty') param_qty,
(select * from param where param_name='MaxTransactionLatency') param_max_transaction_latency,
(select * from param where param_name='DoubleDip') param_double_dip,
(select * from param where param_name='MsToWorkShot') param_ms_to_work_shot
WHERE
session.session_id = run.session_id and
run.run_id = shot_counts.run_id and
run.run_id = param_latency.run_id and
run.run_id = param_date.run_id and
run.run_id = param_min_depth.run_id and
run.run_id = param_min_orders.run_id and
run.run_id = param_min_value.run_id and
run.run_id = param_qty.run_id and
run.run_id = param_max_transaction_latency.run_id and
run.run_id = param_double_dip.run_id and
run.run_id = param_ms_to_work_shot.run_id
order by
cast(realized_pandl as real) desc;
drop view if exists per_day_ranked_sniper_profits;
create view per_day_ranked_sniper_profits as
select
sniper_profits.*,
row_number() over (partition by date order by realized_pandl desc) rank
from
sniper_profits
where
position_pandl=0 ;
drop view if exists best_sniper_run_of_the_day;
create view best_sniper_run_of_the_day as
select
sniper_profits.*
from
sniper_profits,
per_day_ranked_sniper_profits
where
sniper_profits.run_id=per_day_ranked_sniper_profits.run_id and
per_day_ranked_sniper_profits.rank=1
order by
sniper_profits.date;
drop view if exists fake_shot_outcomes;
create view fake_shot_outcomes as
select
run_id,
val4 symbol,
cast(val1 as numeric) begin_eid,
cast(val2 as numeric) end_eid,
cast(val5 as real) merged_value,
cast(val6 as numeric) merged_qty,
cast(val3 as numeric) begin_ts,
cast(val7 as numeric) end_ts,
(val7-val3)/1000 duration
from
log
where
entry_type='fake_shot_outcome';
drop view if exists top_of_books ;
create view top_of_books as
select
run_id,
val1 symbol,
cast(val2 as numeric) eid,
cast(val3 as real) bid,
cast(val4 as real) bid_qty,
cast(val5 as real) ask,
cast(val6 as real) ask_qty
from
log
where
entry_type = 'book';
drop view if exists poly_observer_opportunities;
create view poly_observer_opportunities as
select
val1 symbol,
cast(val2 as real) merged_value,
cast(val3 as numeric) merged_qty,
cast(val4 as numeric) begin_eid,
cast(val5 as numeric) end_eid,
run_id
from
log
where
entry_type='opportunity' ;
@@ -220,7 +220,7 @@ void SimOrchestrator::run() {
workers[i % workers.size()].assigned_runs.push_back(&me->runs[i]);
auto session_id =
me->db.create_session(me->options.algo(), me->options.session_title());
me->db.create_session(me->options.algo());
std::atomic<size_t> run_count = me->runs.size();
for (auto &w : workers)
@@ -36,9 +36,6 @@ message SimSessionOptions {
// grouped and used as a unit for each simulation run.
repeated string file = 2;
// An optional user friendly title for this session.
string session_title = 3;
// The max number of cores to use for the simulation. By default all
// cores will be used.
uint32 n_cores = 4 ;
@@ -27,6 +27,7 @@ struct SimDB::PImpl {
SQLite::Database db;
std::mutex lock;
std::vector<PendingLog> pending;
std::unordered_map<RunID,size_t> log_entry_order;
};
SimDB::SimDB(std::string const &loc) {
@@ -52,26 +53,18 @@ void SimDB::destroy_db(std::string const &loc) {
SimDB::~SimDB() {}
SimDB::SessionID SimDB::create_session(std::string const &algo,
std::string const &title) {
SimDB::SessionID SimDB::create_session(std::string const &algo) {
std::lock_guard<std::mutex> l(me->lock);
auto insert_session = "insert into session ("
"session_run_date,"
"algo,"
"title"
") values ("
":session_run_date,"
":algo,"
":title"
")";
auto max_session_query = "select max(rowid) from run";
SQLite::Statement statement(me->db, insert_session);
statement.bind(":session_run_date", datetime::date_string(datetime::now()));
statement.bind(":algo", algo);
statement.bind(":title", title);
statement.exec();
return SessionID{last_insert_rowid()};
SQLite::Statement statement(me->db, max_session_query);
size_t session_id=0;
while (statement.executeStep()) {
session_id = (long long)statement.getColumn(0);
}
return SessionID{session_id, algo};
}
SimDB::RunID SimDB::begin_run(SessionID sid,
@@ -80,32 +73,83 @@ SimDB::RunID SimDB::begin_run(SessionID sid,
auto insert_run = "insert into run("
"session_id,"
"begin_time"
"algo,"
"begin_run_time,"
"end_run_time,"
"market_date,"
"channel,"
"base_algo_latency_ns,"
"order_update_delay_ns,"
"mbp_update_delay_ns,"
"transaction_latency_ns,"
"realized_pandl,"
"position_pandl,"
"param1,"
"param2,"
"param3,"
"param4,"
"param5,"
"param6,"
"param7,"
"param8,"
"param9,"
"param10"
") values ("
":session_id,"
":begin_time"
":algo,"
":begin_run_time,"
":end_run_time,"
":market_date,"
":channel,"
":base_algo_latency_ns,"
":order_update_delay_ns,"
":mbp_update_delay_ns,"
":transaction_latency_ns,"
":realized_pandl,"
":position_pandl,"
":param1,"
":param2,"
":param3,"
":param4,"
":param5,"
":param6,"
":param7,"
":param8,"
":param9,"
":param10"
")";
SQLite::Statement statement(me->db, insert_run);
statement.bind(":session_id", sid.value());
statement.bind(":begin_time", datetime::time_string(datetime::now()));
statement.bind(":session_id", (long long)sid.id);
statement.bind(":algo", sid.algo);
statement.bind(":begin_run_time", datetime::time_string(datetime::now()));
statement.bind(":end_run_time", "");
statement.bind(":market_date", options.market_date());
statement.bind(":channel", options.channels(0));
statement.bind(":base_algo_latency_ns",
(long long)options.base_algo_latency_ns());
statement.bind(":order_update_delay_ns",
(long long)options.order_update_delay_ns());
statement.bind(":mbp_update_delay_ns",
(long long)options.mbp_update_delay_ns());
statement.bind(":transaction_latency_ns",
(long long)options.transaction_latency_ns());
statement.bind(":realized_pandl", 0);
statement.bind(":position_pandl", 0);
std::map<std::string, std::string> sorted_params(
options.parameters().begin(), options.parameters().end());
size_t p_idx = 1;
for (auto const &p : sorted_params)
statement.bind(fmt::format(":param{}", p_idx++), p.second);
for (auto i = p_idx; i <= 10; i++)
statement.bind(fmt::format(":param{}", i), "");
statement.exec();
auto run_id = RunID{last_insert_rowid()};
// Log parameters
_add_param(run_id, "market_date", options.market_date());
for (auto const &c : options.channels())
_add_param(run_id, "channel", c);
_add_param(run_id, "base_algo_latency_ns", options.base_algo_latency_ns());
_add_param(run_id, "order_update_delay_ns", options.order_update_delay_ns());
_add_param(run_id, "mbp_update_delay_ns", options.mbp_update_delay_ns());
_add_param(run_id, "transaction_latency_ns",
options.transaction_latency_ns());
for (auto const &p : options.parameters()) {
_add_param(run_id, p.first, p.second);
}
auto run_id = RunID{last_insert_rowid()};
me->log_entry_order[run_id] = 1;
return run_id;
}
@@ -115,14 +159,14 @@ void SimDB::end_run(RunID run_id, double realized_pandl,
auto update_run = "update run "
"set "
"end_time=:end_time, "
"end_run_time=:end_run_time, "
"realized_pandl=:realized_pandl,"
"position_pandl=:position_pandl "
"where "
"run_id=:run_id";
SQLite::Statement statement(me->db, update_run);
statement.bind(":end_time", datetime::time_string(datetime::now()));
statement.bind(":end_run_time", datetime::time_string(datetime::now()));
statement.bind(":run_id", run_id.value());
statement.bind(":realized_pandl", realized_pandl);
statement.bind(":position_pandl", position_pandl);
@@ -147,22 +191,24 @@ void SimDB::_log(bool flush) {
auto now = l.ts;
auto &entry_type = l.log_entry_type;
auto &log_args = l.args;
auto &log_entry_order = me->log_entry_order[rid];
std::ostringstream insert_log_entry;
insert_log_entry << "insert into log("
"run_id,"
"log_entry_order,"
"t_time,"
"h_t_time,"
"entry_type";
for (size_t i = 0; i < log_args.size(); i++)
insert_log_entry << ",val" << i + 1;
insert_log_entry << ") values (:run_id,:t_time,:h_t_time,:entry_type";
insert_log_entry << ") values (:run_id,:log_entry_order,:t_time,:h_t_time,:entry_type";
for (size_t i = 0; i < log_args.size(); i++)
insert_log_entry << ",:val" << i + 1;
insert_log_entry << ")";
SQLite::Statement statement(me->db, insert_log_entry.str());
statement.bind(":run_id", rid.value());
statement.bind(":log_entry_order",(long long) log_entry_order++);
statement.bind(":t_time", (long long)now.value());
statement.bind(":h_t_time", to_time_string(now));
statement.bind(":entry_type", entry_type);
@@ -178,25 +224,6 @@ void SimDB::_log(bool flush) {
}
}
void SimDB::_add_param(RunID run_id, std::string const &param_name,
std::string const &param_value) {
auto insert_param = "insert into param("
"run_id,"
"param_name,"
"param_value"
") values ("
":run_id,"
":param_name,"
":param_value"
")";
SQLite::Statement statement(me->db, insert_param);
statement.bind(":run_id", run_id.value());
statement.bind(":param_name", param_name);
statement.bind(":param_value", param_value);
statement.exec();
}
int SimDB::last_insert_rowid() {
SQLite::Statement statement(me->db, "select last_insert_rowid()");
statement.executeStep();
@@ -17,9 +17,11 @@ class SimDB {
SimDB(std::string const &location = persistence::storage_location("sim_db"));
~SimDB();
using SessionID = utils::ValueWrapper<struct SessionIDTag, int>;
SessionID create_session(std::string const &algo,
std::string const &title = "");
struct SessionID {
size_t id;
std::string algo;
};
SessionID create_session(std::string const &algo);
using RunID = utils::ValueWrapper<struct RunIDTag, int>;
RunID begin_run(SessionID, protos::SimRunOptions const &);
@@ -32,15 +34,6 @@ class SimDB {
std::string const &location = persistence::storage_location("sim_db"));
private:
template <class T>
void _add_param(RunID rid, std::string const &param_name, T v) {
_add_param(rid, param_name, std::to_string(v));
}
void _add_param(RunID, std::string const &param_name,
std::string const &param_value);
void _log(bool flush = false);
int last_insert_rowid();
struct PendingLog;