Streamlined sim db schema for easier querying.
This commit is contained in:
1 parent
f9b2f413d7
commit
524e83c595
5 files changed
+177
-257
No files matched your search
@@ -1,59 +1,38 @@
|
||||
/* A session is a set of simulation runs for the same algo, covering
|
||||
multiple combinations of days and parameters. */
|
||||
|
||||
drop table if exists session;
|
||||
CREATE TABLE session(
|
||||
session_id integer primary key,
|
||||
/* the date when this session was run */
|
||||
session_run_date text,
|
||||
/* the algo under test */
|
||||
algo text,
|
||||
/* an optional descriptor for this session */
|
||||
title text
|
||||
);
|
||||
|
||||
/* A run is a simulation covering a single algo, in a single day, with
|
||||
a fixed set of parameters. */
|
||||
|
||||
drop table if exists run;
|
||||
create table run(
|
||||
run_id integer primary key,
|
||||
session_id numeric,
|
||||
/* Clock time when we started / ended this run */
|
||||
begin_time text,
|
||||
end_time text,
|
||||
session_id text,
|
||||
algo text,
|
||||
begin_run_time text,
|
||||
end_run_time text,
|
||||
market_date text,
|
||||
channel text,
|
||||
base_algo_latency_ns numeric,
|
||||
order_update_delay_ns numeric,
|
||||
mbp_update_delay_ns numeric,
|
||||
transaction_latency_ns numeric,
|
||||
realized_pandl real,
|
||||
position_pandl real,
|
||||
foreign key(session_id) references session(session_id)
|
||||
);
|
||||
|
||||
/*
|
||||
A run param is the value of a specific parameter on a specific
|
||||
simulation run. Some parameters are standardized across algos, while
|
||||
others are algo specific. Some of the standardized parameters include:
|
||||
|
||||
- market_date -> market_date
|
||||
- channel->* channel
|
||||
- base_algo_latency_ns -> base_algo_latency_ns
|
||||
- order_update_delay_ns -> order_update_delay_ns
|
||||
- mbp_update_delay_ns -> mbp_update_delay_ns
|
||||
- transaction_latency_ns -> transaction_latency_ns
|
||||
*/
|
||||
|
||||
drop table if exists param;
|
||||
create table param(
|
||||
param_id integer primary_key,
|
||||
run_id numeric,
|
||||
param_name text,
|
||||
param_value text,
|
||||
foreign key(run_id) references run(run_id)
|
||||
/* algo specific parameters */
|
||||
param1 text,
|
||||
param2 text,
|
||||
param3 text,
|
||||
param4 text,
|
||||
param5 text,
|
||||
param6 text,
|
||||
param7 text,
|
||||
param8 text,
|
||||
param9 text,
|
||||
param10 text
|
||||
);
|
||||
|
||||
/*
|
||||
A log entry is an entry emited during a simulation run. Some of
|
||||
these entries are standardized across algos, while others are algo
|
||||
specific. Some of the standard entries are:
|
||||
|
||||
- pnl -> realized pnl, position pnl
|
||||
- symbol_pnl ->* symbol, pos, realized_pnl
|
||||
- outright_pos->* contract, pos
|
||||
@@ -63,8 +42,8 @@ specific. Some of the standard entries are:
|
||||
|
||||
drop table if exists log;
|
||||
create table log(
|
||||
log_entry_id integer primary_key,
|
||||
run_id numeric,
|
||||
log_entry_order numeric,
|
||||
t_time numeric,
|
||||
h_t_time TEXT,
|
||||
entry_type TEXT,
|
||||
@@ -80,24 +59,78 @@ create table log(
|
||||
val10 TEXT,
|
||||
foreign key(run_id) references run(run_id)
|
||||
);
|
||||
CREATE INDEX log_idx ON log(run_id,entry_type);
|
||||
CREATE INDEX log_idx_0 ON log(run_id,entry_type);
|
||||
CREATE INDEX log_idx_1 ON log(val1);
|
||||
CREATE INDEX log_idx_2 ON log(val2);
|
||||
CREATE INDEX log_idx_3 ON log(val3);
|
||||
|
||||
/* sniper specific views */
|
||||
drop view if exists sniper_run;
|
||||
create view sniper_run as
|
||||
select
|
||||
run_id,
|
||||
session_id ,
|
||||
begin_run_time ,
|
||||
end_run_time ,
|
||||
market_date ,
|
||||
channel ,
|
||||
base_algo_latency_ns,
|
||||
order_update_delay_ns,
|
||||
mbp_update_delay_ns,
|
||||
transaction_latency_ns,
|
||||
realized_pandl ,
|
||||
position_pandl ,
|
||||
cast(param1 as numeric) MinDepth,
|
||||
cast(param2 as numeric) MinOrdersAtTop,
|
||||
cast(param3 as numeric) MinParamue,
|
||||
cast(param4 as numeric) Qty,
|
||||
cast(param5 as numeric) MaxTransactionLatency,
|
||||
cast(param6 as numeric) DoubleDip,
|
||||
cast(param7 as numeric) MsToWorkShot
|
||||
from
|
||||
run
|
||||
order by run_id ;
|
||||
|
||||
|
||||
drop view if exists shot_beginnings;
|
||||
create view shot_beginnings as
|
||||
select
|
||||
run_id,
|
||||
t_time,
|
||||
val3 polygon,
|
||||
cast (val1 as numeric) o_eid,
|
||||
cast (val2 as numeric) o_eid_2,
|
||||
cast (val4 as real) target,
|
||||
cast (val5 as numeric) quantity,
|
||||
cast (val6 as numeric) streak_id
|
||||
from
|
||||
log
|
||||
where
|
||||
entry_type='shot_begin';
|
||||
|
||||
drop view if exists shot_outcomes;
|
||||
create view shot_outcomes as
|
||||
select
|
||||
run_id,
|
||||
t_time,
|
||||
log.run_id,
|
||||
log.t_time,
|
||||
val1 polygon,
|
||||
cast(val2 as numeric) observed_eid,
|
||||
cast(val4 as numeric) executed_eid,
|
||||
cast(val2 as numeric) o_eid,
|
||||
cast(val3 as numeric) o_eid_2,
|
||||
cast(val4 as numeric) e_eid,
|
||||
cast(val5 as numeric) e_eid_2,
|
||||
cast(val6 as real) target,
|
||||
cast(val7 as real) realized
|
||||
cast(val7 as real) realized,
|
||||
cast(val8 as numeric) streak_id,
|
||||
(log.t_time - (lag(log.t_time) over (partition by log.entry_type order by log.t_time)))/1000 t_since_last_shot,
|
||||
(log.t_time - shot_beginnings.t_time)/1000 cycle_t
|
||||
from
|
||||
log
|
||||
log, shot_beginnings
|
||||
where
|
||||
entry_type='shot_outcome' ;
|
||||
log.entry_type='shot_outcome' and
|
||||
cast(val2 as numeric) = shot_beginnings.o_eid and
|
||||
cast(val3 as numeric) = shot_beginnings.o_eid_2 and
|
||||
log.run_id = shot_beginnings.run_id;
|
||||
|
||||
|
||||
drop view if exists shot_counts;
|
||||
create view shot_counts as
|
||||
@@ -112,133 +145,3 @@ from
|
||||
group by
|
||||
run_id ;
|
||||
|
||||
drop view if exists opportunity_counts;
|
||||
create view opportunity_counts as
|
||||
select
|
||||
run_id, count(*) n_opportunities
|
||||
from
|
||||
log
|
||||
where
|
||||
entry_type='opportunity'
|
||||
group by
|
||||
run_id ;
|
||||
|
||||
drop view if exists sniper_profits;
|
||||
CREATE VIEW sniper_profits as
|
||||
select
|
||||
session.algo algo,
|
||||
session.session_id,
|
||||
run.run_id,
|
||||
realized_pandl,
|
||||
position_pandl,
|
||||
n_shots,
|
||||
good_shots,
|
||||
lucky_shots,
|
||||
unlucky_shots,
|
||||
param_latency.param_value latency,
|
||||
param_date.param_value date,
|
||||
param_min_depth.param_value min_depth,
|
||||
param_min_orders.param_value min_orders,
|
||||
param_min_value.param_value min_value,
|
||||
param_qty.param_value qty,
|
||||
param_max_transaction_latency.param_value max_t_latency,
|
||||
param_double_dip.param_value double_dip,
|
||||
param_ms_to_work_shot.param_value ms_to_work
|
||||
from
|
||||
session,
|
||||
run,
|
||||
shot_counts,
|
||||
(select * from param where param_name='base_algo_latency_ns') param_latency,
|
||||
(select * from param where param_name='market_date') param_date,
|
||||
(select * from param where param_name='MinDepth') param_min_depth,
|
||||
(select * from param where param_name='MinOrdersAtTop') param_min_orders,
|
||||
(select * from param where param_name='MinValue') param_min_value,
|
||||
(select * from param where param_name='Qty') param_qty,
|
||||
(select * from param where param_name='MaxTransactionLatency') param_max_transaction_latency,
|
||||
(select * from param where param_name='DoubleDip') param_double_dip,
|
||||
(select * from param where param_name='MsToWorkShot') param_ms_to_work_shot
|
||||
WHERE
|
||||
session.session_id = run.session_id and
|
||||
run.run_id = shot_counts.run_id and
|
||||
run.run_id = param_latency.run_id and
|
||||
run.run_id = param_date.run_id and
|
||||
run.run_id = param_min_depth.run_id and
|
||||
run.run_id = param_min_orders.run_id and
|
||||
run.run_id = param_min_value.run_id and
|
||||
run.run_id = param_qty.run_id and
|
||||
run.run_id = param_max_transaction_latency.run_id and
|
||||
run.run_id = param_double_dip.run_id and
|
||||
run.run_id = param_ms_to_work_shot.run_id
|
||||
order by
|
||||
cast(realized_pandl as real) desc;
|
||||
|
||||
|
||||
drop view if exists per_day_ranked_sniper_profits;
|
||||
create view per_day_ranked_sniper_profits as
|
||||
select
|
||||
sniper_profits.*,
|
||||
row_number() over (partition by date order by realized_pandl desc) rank
|
||||
from
|
||||
sniper_profits
|
||||
where
|
||||
position_pandl=0 ;
|
||||
|
||||
|
||||
drop view if exists best_sniper_run_of_the_day;
|
||||
create view best_sniper_run_of_the_day as
|
||||
select
|
||||
sniper_profits.*
|
||||
from
|
||||
sniper_profits,
|
||||
per_day_ranked_sniper_profits
|
||||
where
|
||||
sniper_profits.run_id=per_day_ranked_sniper_profits.run_id and
|
||||
per_day_ranked_sniper_profits.rank=1
|
||||
order by
|
||||
sniper_profits.date;
|
||||
|
||||
drop view if exists fake_shot_outcomes;
|
||||
create view fake_shot_outcomes as
|
||||
select
|
||||
run_id,
|
||||
val4 symbol,
|
||||
cast(val1 as numeric) begin_eid,
|
||||
cast(val2 as numeric) end_eid,
|
||||
cast(val5 as real) merged_value,
|
||||
cast(val6 as numeric) merged_qty,
|
||||
cast(val3 as numeric) begin_ts,
|
||||
cast(val7 as numeric) end_ts,
|
||||
(val7-val3)/1000 duration
|
||||
from
|
||||
log
|
||||
where
|
||||
entry_type='fake_shot_outcome';
|
||||
|
||||
drop view if exists top_of_books ;
|
||||
create view top_of_books as
|
||||
select
|
||||
run_id,
|
||||
val1 symbol,
|
||||
cast(val2 as numeric) eid,
|
||||
cast(val3 as real) bid,
|
||||
cast(val4 as real) bid_qty,
|
||||
cast(val5 as real) ask,
|
||||
cast(val6 as real) ask_qty
|
||||
from
|
||||
log
|
||||
where
|
||||
entry_type = 'book';
|
||||
|
||||
drop view if exists poly_observer_opportunities;
|
||||
create view poly_observer_opportunities as
|
||||
select
|
||||
val1 symbol,
|
||||
cast(val2 as real) merged_value,
|
||||
cast(val3 as numeric) merged_qty,
|
||||
cast(val4 as numeric) begin_eid,
|
||||
cast(val5 as numeric) end_eid,
|
||||
run_id
|
||||
from
|
||||
log
|
||||
where
|
||||
entry_type='opportunity' ;
|
||||
+1
-1
@@ -220,7 +220,7 @@ void SimOrchestrator::run() {
|
||||
workers[i % workers.size()].assigned_runs.push_back(&me->runs[i]);
|
||||
|
||||
auto session_id =
|
||||
me->db.create_session(me->options.algo(), me->options.session_title());
|
||||
me->db.create_session(me->options.algo());
|
||||
|
||||
std::atomic<size_t> run_count = me->runs.size();
|
||||
for (auto &w : workers)
|
||||
|
||||
-3
@@ -36,9 +36,6 @@ message SimSessionOptions {
|
||||
// grouped and used as a unit for each simulation run.
|
||||
repeated string file = 2;
|
||||
|
||||
// An optional user friendly title for this session.
|
||||
string session_title = 3;
|
||||
|
||||
// The max number of cores to use for the simulation. By default all
|
||||
// cores will be used.
|
||||
uint32 n_cores = 4 ;
|
||||
|
||||
@@ -27,6 +27,7 @@ struct SimDB::PImpl {
|
||||
SQLite::Database db;
|
||||
std::mutex lock;
|
||||
std::vector<PendingLog> pending;
|
||||
std::unordered_map<RunID,size_t> log_entry_order;
|
||||
};
|
||||
|
||||
SimDB::SimDB(std::string const &loc) {
|
||||
@@ -52,26 +53,18 @@ void SimDB::destroy_db(std::string const &loc) {
|
||||
|
||||
SimDB::~SimDB() {}
|
||||
|
||||
SimDB::SessionID SimDB::create_session(std::string const &algo,
|
||||
std::string const &title) {
|
||||
SimDB::SessionID SimDB::create_session(std::string const &algo) {
|
||||
std::lock_guard<std::mutex> l(me->lock);
|
||||
|
||||
auto insert_session = "insert into session ("
|
||||
"session_run_date,"
|
||||
"algo,"
|
||||
"title"
|
||||
") values ("
|
||||
":session_run_date,"
|
||||
":algo,"
|
||||
":title"
|
||||
")";
|
||||
auto max_session_query = "select max(rowid) from run";
|
||||
|
||||
SQLite::Statement statement(me->db, insert_session);
|
||||
statement.bind(":session_run_date", datetime::date_string(datetime::now()));
|
||||
statement.bind(":algo", algo);
|
||||
statement.bind(":title", title);
|
||||
statement.exec();
|
||||
return SessionID{last_insert_rowid()};
|
||||
SQLite::Statement statement(me->db, max_session_query);
|
||||
size_t session_id=0;
|
||||
while (statement.executeStep()) {
|
||||
session_id = (long long)statement.getColumn(0);
|
||||
}
|
||||
|
||||
return SessionID{session_id, algo};
|
||||
}
|
||||
|
||||
SimDB::RunID SimDB::begin_run(SessionID sid,
|
||||
@@ -80,32 +73,83 @@ SimDB::RunID SimDB::begin_run(SessionID sid,
|
||||
|
||||
auto insert_run = "insert into run("
|
||||
"session_id,"
|
||||
"begin_time"
|
||||
"algo,"
|
||||
"begin_run_time,"
|
||||
"end_run_time,"
|
||||
"market_date,"
|
||||
"channel,"
|
||||
"base_algo_latency_ns,"
|
||||
"order_update_delay_ns,"
|
||||
"mbp_update_delay_ns,"
|
||||
"transaction_latency_ns,"
|
||||
"realized_pandl,"
|
||||
"position_pandl,"
|
||||
"param1,"
|
||||
"param2,"
|
||||
"param3,"
|
||||
"param4,"
|
||||
"param5,"
|
||||
"param6,"
|
||||
"param7,"
|
||||
"param8,"
|
||||
"param9,"
|
||||
"param10"
|
||||
") values ("
|
||||
":session_id,"
|
||||
":begin_time"
|
||||
":algo,"
|
||||
":begin_run_time,"
|
||||
":end_run_time,"
|
||||
":market_date,"
|
||||
":channel,"
|
||||
":base_algo_latency_ns,"
|
||||
":order_update_delay_ns,"
|
||||
":mbp_update_delay_ns,"
|
||||
":transaction_latency_ns,"
|
||||
":realized_pandl,"
|
||||
":position_pandl,"
|
||||
":param1,"
|
||||
":param2,"
|
||||
":param3,"
|
||||
":param4,"
|
||||
":param5,"
|
||||
":param6,"
|
||||
":param7,"
|
||||
":param8,"
|
||||
":param9,"
|
||||
":param10"
|
||||
")";
|
||||
|
||||
SQLite::Statement statement(me->db, insert_run);
|
||||
statement.bind(":session_id", sid.value());
|
||||
statement.bind(":begin_time", datetime::time_string(datetime::now()));
|
||||
statement.bind(":session_id", (long long)sid.id);
|
||||
statement.bind(":algo", sid.algo);
|
||||
statement.bind(":begin_run_time", datetime::time_string(datetime::now()));
|
||||
statement.bind(":end_run_time", "");
|
||||
statement.bind(":market_date", options.market_date());
|
||||
statement.bind(":channel", options.channels(0));
|
||||
statement.bind(":base_algo_latency_ns",
|
||||
(long long)options.base_algo_latency_ns());
|
||||
statement.bind(":order_update_delay_ns",
|
||||
(long long)options.order_update_delay_ns());
|
||||
statement.bind(":mbp_update_delay_ns",
|
||||
(long long)options.mbp_update_delay_ns());
|
||||
statement.bind(":transaction_latency_ns",
|
||||
(long long)options.transaction_latency_ns());
|
||||
statement.bind(":realized_pandl", 0);
|
||||
statement.bind(":position_pandl", 0);
|
||||
|
||||
std::map<std::string, std::string> sorted_params(
|
||||
options.parameters().begin(), options.parameters().end());
|
||||
|
||||
size_t p_idx = 1;
|
||||
for (auto const &p : sorted_params)
|
||||
statement.bind(fmt::format(":param{}", p_idx++), p.second);
|
||||
|
||||
for (auto i = p_idx; i <= 10; i++)
|
||||
statement.bind(fmt::format(":param{}", i), "");
|
||||
|
||||
statement.exec();
|
||||
|
||||
auto run_id = RunID{last_insert_rowid()};
|
||||
|
||||
// Log parameters
|
||||
_add_param(run_id, "market_date", options.market_date());
|
||||
for (auto const &c : options.channels())
|
||||
_add_param(run_id, "channel", c);
|
||||
_add_param(run_id, "base_algo_latency_ns", options.base_algo_latency_ns());
|
||||
_add_param(run_id, "order_update_delay_ns", options.order_update_delay_ns());
|
||||
_add_param(run_id, "mbp_update_delay_ns", options.mbp_update_delay_ns());
|
||||
_add_param(run_id, "transaction_latency_ns",
|
||||
options.transaction_latency_ns());
|
||||
|
||||
for (auto const &p : options.parameters()) {
|
||||
_add_param(run_id, p.first, p.second);
|
||||
}
|
||||
auto run_id = RunID{last_insert_rowid()};
|
||||
me->log_entry_order[run_id] = 1;
|
||||
return run_id;
|
||||
}
|
||||
|
||||
@@ -115,14 +159,14 @@ void SimDB::end_run(RunID run_id, double realized_pandl,
|
||||
|
||||
auto update_run = "update run "
|
||||
"set "
|
||||
"end_time=:end_time, "
|
||||
"end_run_time=:end_run_time, "
|
||||
"realized_pandl=:realized_pandl,"
|
||||
"position_pandl=:position_pandl "
|
||||
"where "
|
||||
"run_id=:run_id";
|
||||
|
||||
SQLite::Statement statement(me->db, update_run);
|
||||
statement.bind(":end_time", datetime::time_string(datetime::now()));
|
||||
statement.bind(":end_run_time", datetime::time_string(datetime::now()));
|
||||
statement.bind(":run_id", run_id.value());
|
||||
statement.bind(":realized_pandl", realized_pandl);
|
||||
statement.bind(":position_pandl", position_pandl);
|
||||
@@ -147,22 +191,24 @@ void SimDB::_log(bool flush) {
|
||||
auto now = l.ts;
|
||||
auto &entry_type = l.log_entry_type;
|
||||
auto &log_args = l.args;
|
||||
|
||||
auto &log_entry_order = me->log_entry_order[rid];
|
||||
std::ostringstream insert_log_entry;
|
||||
insert_log_entry << "insert into log("
|
||||
"run_id,"
|
||||
"log_entry_order,"
|
||||
"t_time,"
|
||||
"h_t_time,"
|
||||
"entry_type";
|
||||
for (size_t i = 0; i < log_args.size(); i++)
|
||||
insert_log_entry << ",val" << i + 1;
|
||||
insert_log_entry << ") values (:run_id,:t_time,:h_t_time,:entry_type";
|
||||
insert_log_entry << ") values (:run_id,:log_entry_order,:t_time,:h_t_time,:entry_type";
|
||||
for (size_t i = 0; i < log_args.size(); i++)
|
||||
insert_log_entry << ",:val" << i + 1;
|
||||
insert_log_entry << ")";
|
||||
|
||||
SQLite::Statement statement(me->db, insert_log_entry.str());
|
||||
statement.bind(":run_id", rid.value());
|
||||
statement.bind(":log_entry_order",(long long) log_entry_order++);
|
||||
statement.bind(":t_time", (long long)now.value());
|
||||
statement.bind(":h_t_time", to_time_string(now));
|
||||
statement.bind(":entry_type", entry_type);
|
||||
@@ -178,25 +224,6 @@ void SimDB::_log(bool flush) {
|
||||
}
|
||||
}
|
||||
|
||||
void SimDB::_add_param(RunID run_id, std::string const ¶m_name,
|
||||
std::string const ¶m_value) {
|
||||
auto insert_param = "insert into param("
|
||||
"run_id,"
|
||||
"param_name,"
|
||||
"param_value"
|
||||
") values ("
|
||||
":run_id,"
|
||||
":param_name,"
|
||||
":param_value"
|
||||
")";
|
||||
|
||||
SQLite::Statement statement(me->db, insert_param);
|
||||
statement.bind(":run_id", run_id.value());
|
||||
statement.bind(":param_name", param_name);
|
||||
statement.bind(":param_value", param_value);
|
||||
statement.exec();
|
||||
}
|
||||
|
||||
int SimDB::last_insert_rowid() {
|
||||
SQLite::Statement statement(me->db, "select last_insert_rowid()");
|
||||
statement.executeStep();
|
||||
|
||||
@@ -17,9 +17,11 @@ class SimDB {
|
||||
SimDB(std::string const &location = persistence::storage_location("sim_db"));
|
||||
~SimDB();
|
||||
|
||||
using SessionID = utils::ValueWrapper<struct SessionIDTag, int>;
|
||||
SessionID create_session(std::string const &algo,
|
||||
std::string const &title = "");
|
||||
struct SessionID {
|
||||
size_t id;
|
||||
std::string algo;
|
||||
};
|
||||
SessionID create_session(std::string const &algo);
|
||||
|
||||
using RunID = utils::ValueWrapper<struct RunIDTag, int>;
|
||||
RunID begin_run(SessionID, protos::SimRunOptions const &);
|
||||
@@ -32,15 +34,6 @@ class SimDB {
|
||||
std::string const &location = persistence::storage_location("sim_db"));
|
||||
|
||||
private:
|
||||
|
||||
template <class T>
|
||||
void _add_param(RunID rid, std::string const ¶m_name, T v) {
|
||||
_add_param(rid, param_name, std::to_string(v));
|
||||
}
|
||||
|
||||
void _add_param(RunID, std::string const ¶m_name,
|
||||
std::string const ¶m_value);
|
||||
|
||||
void _log(bool flush = false);
|
||||
int last_insert_rowid();
|
||||
struct PendingLog;
|
||||
|
||||
Reference in new issue
Block a user