Added better logging to simulator
This commit is contained in:
1 parent
116131f0a8
commit
c6c0dc21e2
11 files changed
+3091
-107
No files matched your search
@@ -59,13 +59,29 @@ create table log(
|
||||
val10 TEXT,
|
||||
foreign key(run_id) references run(run_id)
|
||||
);
|
||||
CREATE INDEX log_idx_0 ON log(run_id,entry_type);
|
||||
CREATE INDEX log_idx_1 ON log(val1);
|
||||
CREATE INDEX log_idx_2 ON log(val2);
|
||||
CREATE INDEX log_idx_3 ON log(val3);
|
||||
CREATE INDEX log_idx_0 ON log(run_id);
|
||||
|
||||
/* Gateway log */
|
||||
drop view if exists gateway_log;
|
||||
create view gateway_log as
|
||||
select
|
||||
log_entry_order,
|
||||
t_time,h_t_time,entry_type,
|
||||
cast (val1 as numeric) eid1,
|
||||
cast (val2 as numeric) eid2,
|
||||
val3,val4,val5,val6,val7,
|
||||
val8,val9
|
||||
from
|
||||
log
|
||||
where
|
||||
entry_type='gw_order' or
|
||||
entry_type='gw_algo_order' or
|
||||
entry_type='gw_fill' or
|
||||
entry_type='gw_algo_fill'
|
||||
order by
|
||||
log_entry_order;
|
||||
|
||||
/* sniper specific views */
|
||||
|
||||
drop view if exists shot_beginnings;
|
||||
create view shot_beginnings as
|
||||
select
|
||||
|
||||
+1234
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import pandas as pd
|
||||
import os
|
||||
import sqlite3
|
||||
import pathlib
|
||||
|
||||
store_root = pathlib.Path.home() / pathlib.Path("sparkdata/data")
|
||||
|
||||
def set_pcap_store(store):
|
||||
store_root = store
|
||||
|
||||
def get_db_path(channel, date):
|
||||
return store_root / pathlib.Path(channel) / pathlib.Path(date+"_"+channel+"_24h.sqlite")
|
||||
|
||||
def get_pcap_db(channel,date):
|
||||
path = get_db_path(channel,date)
|
||||
assert os.path.exists(path)
|
||||
return sqlite3.connect(path)
|
||||
|
||||
def get_sim_db():
|
||||
from pathlib import Path
|
||||
path = store_root / Path("databases") / Path("sim_db") /Path("sim_db.sqlite")
|
||||
assert os.path.exists(path)
|
||||
return sqlite3.connect(path)
|
||||
|
||||
def qsim(query):
|
||||
with get_sim_db() as conn:
|
||||
return pd.read_sql_query(query,conn)
|
||||
|
||||
def get_db(channel,store):
|
||||
def querier(query):
|
||||
with get_pcap_db(channel,store) as conn:
|
||||
return pd.read_sql_query(query,conn)
|
||||
return querier
|
||||
|
||||
def clean_sim_books(books):
|
||||
books = books.rename(columns={'val1':'eid1', 'val2':'eid2','val4':'symbol',
|
||||
'val5':'bid','val6':'bid_qty', 'val7':'ask','val8':'ask_qty'})
|
||||
books = books.drop(['log_entry_order','t_time','h_t_time',
|
||||
'run_id','val3','entry_type','val9','val10'],axis=1)
|
||||
|
||||
books = books.astype({'eid1':'int32','eid2':'int32',
|
||||
'bid':'float64','bid_qty':'float64',
|
||||
'ask':'float64','ask_qty':'float64'})
|
||||
return books
|
||||
|
||||
def get_310_lined_prices(books):
|
||||
# create an idx with all events in the range (this is so that we
|
||||
# can have a row for each exchange state even if nothing visibly
|
||||
# changed in those events)
|
||||
idx = pd.DataFrame({'eid1': range(books.eid1.min(),books.eid1.max()+1)})
|
||||
idx['eid2'] = 0
|
||||
|
||||
# Filter for the symbols we care
|
||||
u=books[books.symbol=='ESU0'].drop(['symbol'],axis=1)
|
||||
z=books[books.symbol=='ESZ0'].drop(['symbol'],axis=1)
|
||||
u_z=books[books.symbol=='ESU0-ESZ0'].drop(['symbol'],axis=1)
|
||||
|
||||
# Line up symbols so we have an event time line that includes all symbols
|
||||
prices = idx
|
||||
|
||||
prices = prices.merge(u,
|
||||
how='outer',
|
||||
left_on=['eid1','eid2'],
|
||||
right_on=['eid1','eid2'])
|
||||
prices = prices.rename({'bid':'bid_u','ask':'ask_u',
|
||||
'bid_qty':'bid_qty_u','ask_qty':'ask_qty_u'},axis=1)
|
||||
|
||||
prices = prices.merge(z,
|
||||
how='outer',
|
||||
left_on=['eid1','eid2'],
|
||||
right_on=['eid1','eid2'])
|
||||
prices = prices.rename({'bid':'bid_z','ask':'ask_z',
|
||||
'bid_qty':'bid_qty_z','ask_qty':'ask_qty_z'},axis=1)
|
||||
|
||||
prices = prices.merge(u_z,
|
||||
how='outer',
|
||||
left_on=['eid1','eid2'],
|
||||
right_on=['eid1','eid2'])
|
||||
prices = prices.rename({'bid':'bid_u_z','ask':'ask_u_z',
|
||||
'bid_qty':'bid_qty_u_z','ask_qty':'ask_qty_u_z'},axis=1)
|
||||
|
||||
# set eid1, eid2 as indexes and then sort by them so that ffill
|
||||
# below will propagate the last price in event order
|
||||
prices = prices.set_index(['eid1','eid2'],drop=False)
|
||||
prices = prices.sort_index()
|
||||
|
||||
# Fill holes (every event only updates one symbol) with previous value of book
|
||||
prices = prices.ffill()
|
||||
|
||||
# Add columns to calculate polygon values for buy and sell
|
||||
prices['buy_p_value'] = (-prices.ask_z+prices.bid_u+prices.bid_u_z)/2
|
||||
prices['sell_p_value'] = (prices.bid_z-prices.ask_u-prices.ask_u_z)/2
|
||||
return prices
|
||||
|
||||
def get_shot_outcomes(run_id=1):
|
||||
with get_sim_db() as conn:
|
||||
return pd.read_sql_query("select * from shot_outcome where run_id="+str(run_id),conn)
|
||||
|
||||
|
||||
def get_opp(*dates):
|
||||
result = pd.DataFrame()
|
||||
|
||||
for date in dates:
|
||||
with sqlite3.connect("/home/ppinto/310/"+date+"_310_24h.sqlite") as conn:
|
||||
result = result.append(pd.read_sql_query(("select '{}' date,\"sum(merged_qty*merged_value)\" "
|
||||
"opportunity from total_opportunity").format(date),conn))
|
||||
return result
|
||||
|
||||
|
||||
book_cache = None;
|
||||
def books():
|
||||
def books_from_db():
|
||||
with get_md_conn() as conn:
|
||||
df = pd.read_sql_query("select top_of_books.*,events.t_time "
|
||||
"from top_of_books, events "
|
||||
"where top_of_books.eid=events.eid",conn)
|
||||
df = df.fillna(0)
|
||||
df = df.astype({'eid':'int32','sid':'int32',
|
||||
't_time':'uint64',
|
||||
'bid':'int32','bid_qty':'int32',
|
||||
'ask':'int32','ask_qty':'int32'})
|
||||
df = df.set_index('eid')
|
||||
df = df.sort_index()
|
||||
return df
|
||||
|
||||
global book_cache
|
||||
if not book_cache is None:
|
||||
return book_cache
|
||||
elif os.path.exists("/tmp/books.parquet"):
|
||||
book_cache = pd.read_parquet("/tmp/books.parquet")
|
||||
return book_cache;
|
||||
else:
|
||||
book_cache = books_from_db()
|
||||
book_cache.to_parquet("/tmp/books.parquet")
|
||||
return book_cache
|
||||
|
||||
def events_from_db():
|
||||
with get_md_conn() as conn:
|
||||
df = pd.read_sql_query("select * from events",conn)
|
||||
df = df.fillna(0)
|
||||
df = df.astype({'eid':'int32','t_time':'uint64',
|
||||
'first_sn':'uint64','last_sn':'uint64',
|
||||
'first_st':'uint64','last_st':'uint64'})
|
||||
return df
|
||||
|
||||
def securities():
|
||||
with get_md_conn() as conn:
|
||||
return pd.read_sql_query("select * from securities",conn)
|
||||
|
||||
def get_event_range():
|
||||
with get_md_conn() as conn:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute("select min(eid),max(eid) from events")
|
||||
min,max =cursor.fetchall()[0]
|
||||
return int(min),int(max)
|
||||
|
||||
def events_from_parquet():
|
||||
return pd.read_parquet("/home/ppinto/events.parquet")
|
||||
|
||||
def get_sec_book(symbol):
|
||||
b = books()
|
||||
s = b[b.symbol==symbol]
|
||||
s = s.reindex(range(0,b.shape[0]+1),method='ffill')
|
||||
s = s.fillna(0)
|
||||
s = s.astype({'bid':'int32','bid_qty':'int32','ask_qty':'int32',
|
||||
'ask':'int32','t_time':'uint64'})
|
||||
s = s.drop(columns=['symbol','sid'])
|
||||
return s
|
||||
|
||||
def line_up_books(*symbols):
|
||||
result = None
|
||||
for s in symbols:
|
||||
df = get_sec_book(s)
|
||||
df = df.add_suffix('_'+s)
|
||||
if result is None:
|
||||
result = df
|
||||
else:
|
||||
result = result.join(df)
|
||||
return result
|
||||
|
||||
|
||||
def upload_lined_books(df):
|
||||
book_f = "/tmp/lined_up_books.csv"
|
||||
df.to_csv(book_f);
|
||||
with get_md_conn() as conn:
|
||||
with conn.cursor() as cursor:
|
||||
with open(book_f) as f:
|
||||
cursor.copy_from(f,"lined_up_books",sep=",")
|
||||
|
||||
def price_series(symbol):
|
||||
b = books()
|
||||
s=b[b.symbol==symbol]
|
||||
s = s[(s.shift(1).bid != s.bid) | (s.shift(1).ask != s.ask)]
|
||||
s=s.copy()
|
||||
s['delta'] = s.t_time - s.shift(1).t_time
|
||||
return s
|
||||
|
||||
def get_polygon_valuation():
|
||||
lined_up = line_up_books('ESU0','ESZ0','ESU0-ESZ0')
|
||||
lined_up['buy']=-lined_up.ask_ESZ0+lined_up.bid_ESU0+lined_up['bid_ESU0-ESZ0']
|
||||
lined_up['sell']=lined_up.bid_ESZ0-lined_up.ask_ESU0-lined_up['ask_ESU0-ESZ0']
|
||||
return lined_up
|
||||
|
||||
def filter_for_opportunities(p):
|
||||
p = p[ ((p.buy>0) | (p.shift(1).buy>0) | (p.shift(-1).buy>0)) |
|
||||
((p.sell>0) | (p.shift(1).sell>0) | (p.shift(-1).sell>0)) ]
|
||||
return p
|
||||
|
||||
|
||||
|
||||
@@ -35,19 +35,17 @@ struct PolyObserver::Opportunity {
|
||||
|
||||
struct PolyObserver::PImpl {
|
||||
ISimServices *sim = nullptr;
|
||||
std::unordered_map<MDP3SecurityID,MBPUpdate> books;
|
||||
std::unordered_map<MDP3SecurityID, MBPUpdate> books;
|
||||
std::vector<std::unique_ptr<Target>> targets;
|
||||
AlgoEventID event_id;
|
||||
MDP3TimeStamp transact_time;
|
||||
MDP3TimeStamp observable_time;
|
||||
size_t max_events;
|
||||
};
|
||||
|
||||
PolyObserver::PolyObserver() : me(new PImpl()) {}
|
||||
PolyObserver::~PolyObserver() {}
|
||||
|
||||
|
||||
void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) {
|
||||
void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &) {
|
||||
me->sim = &s;
|
||||
me->books.clear();
|
||||
me->targets.clear();
|
||||
@@ -55,8 +53,6 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) {
|
||||
me->event_id = {0, 0};
|
||||
me->transact_time = MDP3TimeStamp{0};
|
||||
me->observable_time = MDP3TimeStamp{0};
|
||||
me->max_events =
|
||||
param_or_default(options, "MaxEvents", std::numeric_limits<int>::max());
|
||||
|
||||
for (auto &p : s.polygons()) {
|
||||
auto buy_t =
|
||||
@@ -73,25 +69,21 @@ void PolyObserver::init(ISimServices &s, protos::SimRunOptions const &options) {
|
||||
}
|
||||
|
||||
void PolyObserver::on_notification(AlgoNotification const &n) {
|
||||
if (n.event_id.first >= me->max_events)
|
||||
me->sim->terminate_simulation();
|
||||
else {
|
||||
if (me->event_id.first) {
|
||||
for (auto i = me->event_id.first+1;i < n.event_id.first;i++) {
|
||||
me->event_id.first = i;
|
||||
me->event_id.second = 0;
|
||||
me->transact_time = n.transact_time;
|
||||
me->observable_time = n.observable_time;
|
||||
eval();
|
||||
rank();
|
||||
log();
|
||||
}
|
||||
if (me->event_id.first) {
|
||||
for (auto i = me->event_id.first + 1; i < n.event_id.first; i++) {
|
||||
me->event_id.first = i;
|
||||
me->event_id.second = 0;
|
||||
me->transact_time = n.transact_time;
|
||||
me->observable_time = n.observable_time;
|
||||
eval();
|
||||
rank();
|
||||
log();
|
||||
}
|
||||
me->event_id = n.event_id;
|
||||
me->transact_time = n.transact_time;
|
||||
me->observable_time = n.observable_time;
|
||||
std::visit(*this, n.event);
|
||||
}
|
||||
me->event_id = n.event_id;
|
||||
me->transact_time = n.transact_time;
|
||||
me->observable_time = n.observable_time;
|
||||
std::visit(*this, n.event);
|
||||
}
|
||||
|
||||
void PolyObserver::eval() {
|
||||
@@ -134,16 +126,17 @@ void PolyObserver::rank() {
|
||||
void PolyObserver::log() {
|
||||
for (auto const &t : me->targets)
|
||||
if (t->has_value())
|
||||
me->sim->log("opportunity", me->event_id.first, me->event_id.second,
|
||||
t->descriptor, t->value, t->quantity.value());
|
||||
me->sim->log("poly_opportunity", me->event_id.first,
|
||||
me->event_id.second, t->descriptor, t->value,
|
||||
t->quantity.value());
|
||||
}
|
||||
|
||||
std::pair<md::IntPrice, MDP3Quantity> available(Book const&b) {
|
||||
std::pair<md::IntPrice, MDP3Quantity> result ;
|
||||
std::pair<md::IntPrice, MDP3Quantity> available(Book const &b) {
|
||||
std::pair<md::IntPrice, MDP3Quantity> result;
|
||||
result.first = md::IntPrice{};
|
||||
result.second= MDP3Quantity{0};
|
||||
|
||||
for (size_t i = 0; i < b.size();i++) {
|
||||
result.second = MDP3Quantity{0};
|
||||
|
||||
for (size_t i = 0; i < b.size(); i++) {
|
||||
if (b[i].quantity.value() == 0)
|
||||
break;
|
||||
else {
|
||||
@@ -154,22 +147,15 @@ std::pair<md::IntPrice, MDP3Quantity> available(Book const&b) {
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void PolyObserver::operator()(const MBPUpdate &mbp) {
|
||||
|
||||
auto avb = available(mbp.mbp_buy);
|
||||
auto avs = available(mbp.mbp_sell);
|
||||
|
||||
me->sim->log("book", me->sim->get_sec(mbp.security_id).symbol(),
|
||||
me->event_id.first,
|
||||
mbp.mbp_buy[0].price.price(),
|
||||
mbp.mbp_buy[0].quantity.value(),
|
||||
mbp.mbp_sell[0].price.price(),
|
||||
mbp.mbp_sell[0].quantity.value(),
|
||||
avb.first.price(),
|
||||
avb.second.value(),
|
||||
avs.first.price(),
|
||||
avs.second.value());
|
||||
me->sim->log(
|
||||
"poly_book",
|
||||
me->event_id.first,
|
||||
me->event_id.second,
|
||||
me->transact_time.value(),
|
||||
me->sim->get_sec(mbp.security_id).symbol(),
|
||||
mbp.mbp_buy[0].price.price(), mbp.mbp_buy[0].quantity.value(),
|
||||
mbp.mbp_sell[0].price.price(), mbp.mbp_sell[0].quantity.value());
|
||||
|
||||
me->books[mbp.security_id] = mbp;
|
||||
eval();
|
||||
|
||||
@@ -324,6 +324,15 @@ void Sniper::operator()(const MBPUpdate &mbp) {
|
||||
if (it == me->books.end())
|
||||
return;
|
||||
|
||||
// me->sim->log("book",
|
||||
// me->current_event.first,
|
||||
// me->current_event.second,
|
||||
// me->sim->get_sec(mbp.security_id).symbol(),
|
||||
// mbp.mbp_buy[0].price.price(),
|
||||
// mbp.mbp_buy[0].quantity.value(),
|
||||
// mbp.mbp_sell[0].price.price(),
|
||||
// mbp.mbp_sell[0].quantity.value());
|
||||
|
||||
auto &book = it->second;
|
||||
book.book = mbp;
|
||||
|
||||
|
||||
+10
-9
@@ -1,12 +1,12 @@
|
||||
#include "SimOrchestrator.h"
|
||||
#include "../simulator/SimInput.h"
|
||||
#include "lib/utils/PBUtils.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h"
|
||||
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/PolyObserver.h"
|
||||
|
||||
#include <regex>
|
||||
|
||||
@@ -20,8 +20,9 @@ class SimOrchestrator::SimInputManager {
|
||||
if (auto ptr = maybe_input.lock())
|
||||
return ptr;
|
||||
else {
|
||||
LOG(Spartan,Warn, "Loading input from {}", path);
|
||||
auto new_ptr = std::make_shared<SimInput>(make_sim_input_from_proto(path));
|
||||
LOG(Spartan, Warn, "Loading input from {}", path);
|
||||
auto new_ptr =
|
||||
std::make_shared<SimInput>(make_sim_input_from_proto(path));
|
||||
maybe_input = new_ptr;
|
||||
return new_ptr;
|
||||
}
|
||||
@@ -123,7 +124,8 @@ SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options)
|
||||
auto &these_inputs = this_run.inputs;
|
||||
these_options.set_algo(options.algo());
|
||||
these_options.set_market_date(day);
|
||||
|
||||
these_options.set_log_all_transactions(options.log_all_transactions());
|
||||
these_options.set_max_events(options.max_events());
|
||||
// Set inputs
|
||||
for (auto const &[channel, sim_input] : day_inputs) {
|
||||
*these_options.add_channels() = channel;
|
||||
@@ -170,7 +172,7 @@ std::unique_ptr<IAlgoModel> make_algo(std::string const &algo) {
|
||||
return std::make_unique<Sniper>();
|
||||
if (algo == "PolyObserver")
|
||||
return std::make_unique<PolyObserver>();
|
||||
|
||||
|
||||
bts_assert(false);
|
||||
return {};
|
||||
}
|
||||
@@ -189,8 +191,8 @@ void SimOrchestrator::run() {
|
||||
auto algo = make_algo(r->options.algo());
|
||||
auto rid = db.begin_run(sid, r->options);
|
||||
int rr = remaining_runs;
|
||||
LOG(Spartan, Warn, "Starting run {} ({} remaining) {}", rid,
|
||||
rr, utils::pb::to_string(r->options));
|
||||
LOG(Spartan, Warn, "Starting run {} ({} remaining) {}", rid, rr,
|
||||
utils::pb::to_string(r->options));
|
||||
auto logger = [rid, &db](auto ts, auto const &entry_type,
|
||||
auto const &args) {
|
||||
db.log(rid, ts, entry_type, args);
|
||||
@@ -219,8 +221,7 @@ void SimOrchestrator::run() {
|
||||
for (size_t i = 0; i < me->runs.size(); i++)
|
||||
workers[i % workers.size()].assigned_runs.push_back(&me->runs[i]);
|
||||
|
||||
auto session_id =
|
||||
me->db.create_session(me->options.algo());
|
||||
auto session_id = me->db.create_session(me->options.algo());
|
||||
|
||||
std::atomic<size_t> run_count = me->runs.size();
|
||||
for (auto &w : workers)
|
||||
|
||||
+15
-3
@@ -3,7 +3,6 @@ package bts.spartan.protos;
|
||||
|
||||
/*
|
||||
algo: "Sniper"
|
||||
session_title: "A memorable title"
|
||||
n_cores: 4
|
||||
|
||||
file: "/home/ppinto/360/2020-09-07_360_24h.sim"
|
||||
@@ -31,6 +30,17 @@ message SimSessionOptions {
|
||||
// The algo we are testing
|
||||
string algo = 1;
|
||||
|
||||
// Wheter the simulator will record every order, order report and
|
||||
// price update. This produces fantastic ammounts of information so
|
||||
// it is off by default.
|
||||
bool log_all_transactions = 9;
|
||||
|
||||
// The maximum number of events we will process before we terminate
|
||||
// the simulation. This is usefull to speed initial prototyping of
|
||||
// algos or analysis tools. Use 0 (the default) to signifiy all
|
||||
// events.
|
||||
uint64 max_events = 11;
|
||||
|
||||
// The sim input files we are using. File names should end with
|
||||
// "<YYYY-MM-DD>_<channel>_24h.sim". Files for the same day will
|
||||
// grouped and used as a unit for each simulation run.
|
||||
@@ -54,7 +64,7 @@ message SimSessionOptions {
|
||||
|
||||
// An estimate for the average transaction latency at the
|
||||
// exchange. We fallback on this value when we cannot use the send
|
||||
// time of a packet to estimate for that purpose.
|
||||
// time of a packet to estimate transaction duration.
|
||||
repeated uint64 transaction_latency_ns = 8;
|
||||
|
||||
// Algo specific parameters used for this session. For each named
|
||||
@@ -63,12 +73,14 @@ message SimSessionOptions {
|
||||
message ParameterDomain {
|
||||
repeated string item = 1;
|
||||
}
|
||||
map<string,ParameterDomain> parameter = 9;
|
||||
map<string,ParameterDomain> parameter = 10;
|
||||
}
|
||||
|
||||
// The parameters for a single simulation run.
|
||||
message SimRunOptions {
|
||||
string algo = 1;
|
||||
bool log_all_transactions = 10;
|
||||
uint64 max_events = 11;
|
||||
uint64 base_algo_latency_ns = 2;
|
||||
uint64 order_update_delay_ns = 3;
|
||||
uint64 mbp_update_delay_ns = 4;
|
||||
|
||||
@@ -48,6 +48,10 @@ SimDB::SimDB(std::string const &loc) {
|
||||
db.exec(str);
|
||||
}
|
||||
me = std::make_unique<PImpl>(loc);
|
||||
SQLite::Statement sync_off(me->db,"PRAGMA synchronous = OFF");
|
||||
sync_off.executeStep();
|
||||
SQLite::Statement journal_mem(me->db,"PRAGMA journal_mode = MEMORY");
|
||||
journal_mem.executeStep();
|
||||
me->logger_thread = std::thread{[this]() { do_work(); }};
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,8 @@ void ExchangeSim::prepare_run(std::vector<IAlgoModel *> const &algos,
|
||||
|
||||
// Init gateways, add them to the simulator
|
||||
for (auto &i : me->inputs) {
|
||||
auto &gw = me->run_state->gateways.emplace_back(me->logger,i->instruments, options);
|
||||
auto &gw = me->run_state->gateways.emplace_back(me->logger,
|
||||
i->instruments, options);
|
||||
me->run_state->simulator.add_agent(gw);
|
||||
}
|
||||
|
||||
@@ -133,38 +134,45 @@ bool ExchangeSim::has_work() const {
|
||||
void ExchangeSim::do_work() {
|
||||
bts_assert(me->run_state);
|
||||
me->run_state->simulator.run_once();
|
||||
auto now = Timer::now();
|
||||
const auto log_period_s = 10;
|
||||
if (ms_diff(now, me->run_state->last_time_log) > 1000 * log_period_s) {
|
||||
if (me->run_state->options.max_events() &&
|
||||
me->run_state->last_completed_count >=
|
||||
me->run_state->options.max_events()) {
|
||||
me->run_state->simulator.end_simulation();
|
||||
} else {
|
||||
auto now = Timer::now();
|
||||
const auto log_period_s = 10;
|
||||
if (ms_diff(now, me->run_state->last_time_log) > 1000 * log_period_s) {
|
||||
|
||||
auto sim_time = me->run_state->simulator.simulation_time();
|
||||
auto sim_time = me->run_state->simulator.simulation_time();
|
||||
|
||||
size_t total_events = me->run_state->starting_event_count;
|
||||
size_t total_events = me->run_state->starting_event_count;
|
||||
|
||||
size_t remaining_events = me->run_state->simulator.pending_events_count();
|
||||
size_t remaining_events =
|
||||
me->run_state->simulator.pending_events_count();
|
||||
|
||||
size_t completed_events = total_events > remaining_events
|
||||
? total_events - remaining_events
|
||||
: total_events;
|
||||
size_t completed_events = total_events > remaining_events
|
||||
? total_events - remaining_events
|
||||
: total_events;
|
||||
|
||||
size_t p_complete = (double)completed_events / total_events * 100;
|
||||
size_t p_complete = (double)completed_events / total_events * 100;
|
||||
|
||||
size_t completed_this_cycle =
|
||||
completed_events - me->run_state->last_completed_count;
|
||||
size_t completed_this_cycle =
|
||||
completed_events - me->run_state->last_completed_count;
|
||||
|
||||
size_t elapsed_secs = sec_diff(now, me->run_state->run_start);
|
||||
auto eta = (remaining_events * elapsed_secs / completed_events);
|
||||
auto TPS = completed_this_cycle / log_period_s;
|
||||
size_t elapsed_secs = sec_diff(now, me->run_state->run_start);
|
||||
auto eta = (remaining_events * elapsed_secs / completed_events);
|
||||
auto TPS = completed_this_cycle / log_period_s;
|
||||
|
||||
me->run_state->last_time_log = now;
|
||||
me->run_state->last_sim_time = sim_time;
|
||||
me->run_state->last_completed_count = completed_events;
|
||||
me->run_state->last_time_log = now;
|
||||
me->run_state->last_sim_time = sim_time;
|
||||
me->run_state->last_completed_count = completed_events;
|
||||
|
||||
LOG(Spartan, Diag,
|
||||
"Run {} -> Time: {} TPS: {} Complete: {}% ETA: {} "
|
||||
"secs",
|
||||
me->run_id, to_time_string(MDP3TimeStamp{sim_time}), TPS, p_complete,
|
||||
eta);
|
||||
LOG(Spartan, Diag,
|
||||
"Run {} -> Time: {} TPS: {} Complete: {}% ETA: {} "
|
||||
"secs",
|
||||
me->run_id, to_time_string(MDP3TimeStamp{sim_time}), TPS,
|
||||
p_complete, eta);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+75
-21
@@ -117,6 +117,16 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) {
|
||||
else
|
||||
forward_clock_by(me->options.transaction_latency_ns());
|
||||
|
||||
if (me->options.log_all_transactions())
|
||||
for (auto const &change : ev->order_changes) {
|
||||
log(me->logger, time(), "gw_order", me->current_event.first,
|
||||
me->current_event.second, ev->transact_time.value(),
|
||||
change_name(change.change),
|
||||
get_sec(change.order.security).symbol(),
|
||||
change.order.price.price(), change.order.quantity.value(),
|
||||
api::Side_Name(change.order.side), change.order.key.value());
|
||||
}
|
||||
|
||||
// Now match the recorded orders and send out their effects as
|
||||
// algo notification events
|
||||
std::vector<Matcher::MatchEffect> effects;
|
||||
@@ -126,12 +136,34 @@ void GatewayAgent::_handle(ExchangeTransactionEvent const &e) {
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](MBPUpdate &mbp) {
|
||||
if (me->options.log_all_transactions()) {
|
||||
log(me->logger, time(),
|
||||
"gw_book",
|
||||
me->current_event.first,
|
||||
me->current_event.second,
|
||||
ev->transact_time.value(),
|
||||
get_sec(mbp.security_id).symbol(),
|
||||
mbp.mbp_buy[0].price.price(),
|
||||
mbp.mbp_buy[0].quantity.value(),
|
||||
mbp.mbp_sell[0].price.price(),
|
||||
mbp.mbp_sell[0].quantity.value());
|
||||
}
|
||||
emit_delayed_event<AlgoNotificationEvent>(
|
||||
me->options.mbp_update_delay_ns(),
|
||||
AlgoNotification{me->current_event, ev->transact_time,
|
||||
MDP3TimeStamp{time()}, mbp});
|
||||
},
|
||||
[&](OrderUpdate const &order_update) {
|
||||
if (me->options.log_all_transactions()) {
|
||||
log(me->logger, time(), "gw_fill",
|
||||
me->current_event.first, me->current_event.second,
|
||||
ev->transact_time.value(), order_update.status_str(),
|
||||
api::Side_Name(order_update.side),
|
||||
get_sec(order_update.security_id).symbol(),
|
||||
order_update.last_price.price(),
|
||||
order_update.last_fills.value(),
|
||||
order_update.key.value());
|
||||
}
|
||||
emit_delayed_event<AlgoNotificationEvent>(
|
||||
me->options.order_update_delay_ns(),
|
||||
AlgoNotification{me->current_event, ev->transact_time,
|
||||
@@ -148,33 +180,55 @@ void GatewayAgent::_handle(AlgoActionEvent const &e) {
|
||||
auto const &algo_action = e.ev;
|
||||
auto const &change = algo_action.change;
|
||||
|
||||
|
||||
log(me->logger, time(), "gw_algo_order", algo_action.transact_time.value(),
|
||||
change_name(change.change), get_sec(change.order.security).symbol(),
|
||||
change.order.price.price(), change.order.quantity.value(),
|
||||
api::Side_Name(change.order.side), change.order.key.value(),
|
||||
change.order.security.value(), me->current_event.first,
|
||||
me->current_event.second);
|
||||
if (me->options.log_all_transactions()) {
|
||||
log(me->logger, time(), "gw_algo_order", me->current_event.first,
|
||||
me->current_event.second, algo_action.transact_time.value(),
|
||||
change_name(change.change), get_sec(change.order.security).symbol(),
|
||||
change.order.price.price(), change.order.quantity.value(),
|
||||
api::Side_Name(change.order.side), change.order.key.value());
|
||||
}
|
||||
|
||||
forward_clock_by(me->options.transaction_latency_ns());
|
||||
std::vector<Matcher::MatchEffect> effects;
|
||||
me->matcher.match(Matcher::ALGO, {algo_action.change}, &effects);
|
||||
for (auto &ef : effects)
|
||||
std::visit(
|
||||
overloaded{[&](MBPUpdate &mbp) {
|
||||
emit_delayed_event<AlgoNotificationEvent>(
|
||||
me->options.mbp_update_delay_ns(),
|
||||
AlgoNotification{me->current_event,
|
||||
algo_action.transact_time,
|
||||
MDP3TimeStamp{time()}, mbp});
|
||||
},
|
||||
[&](OrderUpdate const &order_update) {
|
||||
emit_delayed_event<AlgoNotificationEvent>(
|
||||
me->options.order_update_delay_ns(),
|
||||
AlgoNotification{
|
||||
me->current_event, algo_action.transact_time,
|
||||
MDP3TimeStamp{time()}, order_update});
|
||||
}},
|
||||
overloaded{
|
||||
[&](MBPUpdate &mbp) {
|
||||
if (me->options.log_all_transactions()) {
|
||||
log(me->logger, time(), "gw_book", me->current_event.first,
|
||||
me->current_event.second,
|
||||
algo_action.transact_time.value(),
|
||||
get_sec(mbp.security_id).symbol(),
|
||||
mbp.mbp_buy[0].price.price(),
|
||||
mbp.mbp_buy[0].quantity.value(),
|
||||
mbp.mbp_sell[0].price.price(),
|
||||
mbp.mbp_sell[0].quantity.value());
|
||||
}
|
||||
emit_delayed_event<AlgoNotificationEvent>(
|
||||
me->options.mbp_update_delay_ns(),
|
||||
AlgoNotification{me->current_event,
|
||||
algo_action.transact_time,
|
||||
MDP3TimeStamp{time()}, mbp});
|
||||
},
|
||||
[&](OrderUpdate const &order_update) {
|
||||
if (me->options.log_all_transactions()) {
|
||||
log(me->logger, time(), "gw_algo_fill", me->current_event.first,
|
||||
me->current_event.second,
|
||||
algo_action.transact_time.value(),
|
||||
order_update.status_str(),
|
||||
api::Side_Name(order_update.side),
|
||||
get_sec(order_update.security_id).symbol(),
|
||||
order_update.last_price.price(),
|
||||
order_update.last_fills.value(),
|
||||
order_update.key.value());
|
||||
}
|
||||
emit_delayed_event<AlgoNotificationEvent>(
|
||||
me->options.order_update_delay_ns(),
|
||||
AlgoNotification{me->current_event,
|
||||
algo_action.transact_time,
|
||||
MDP3TimeStamp{time()}, order_update});
|
||||
}},
|
||||
ef);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user