Sim2 and polyobserver agree.

This commit is contained in:
Pedro Pinto committed 2020-12-01 07:44:59 -05:00
1 parent 398397ca7f
commit 2de4ec6ebe
10 files changed
+154 -138

No files matched your search

@@ -1,5 +1,6 @@
#include "PolyObserver2.h"
#include "lib/utils/bts_debugger.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
namespace bts::spartan::mdp3 {
@@ -7,8 +8,10 @@ namespace bts::spartan::mdp3 {
static auto constexpr Worthless = -1.0;
struct PolyObserver2::Target {
size_t polygon_id;
Polygon const polygon;
std::string const descriptor;
api::Side side;
std::string const product;
double value = Worthless;
MDP3Quantity quantity = MDP3Quantity{0};
@@ -40,10 +43,11 @@ void PolyObserver2::on_simulation_reset() {
me.reset(new PImpl());
for (auto &p : polygons()) {
auto buy_t = Target{p.second, polygon_descriptor(p.second, instruments()),
auto descriptor = polygon_descriptor(p.second, instruments());
auto buy_t = Target{p.first, p.second, descriptor, api::SIDE_BUY,
polygon_product(p.second, instruments())};
auto sell_t = Target{flip(p.second),
polygon_descriptor(flip(p.second), instruments()),
auto sell_t = Target{p.first, flip(p.second), descriptor, api::SIDE_SELL,
polygon_product(p.second, instruments())};
me->targets.emplace_back(std::make_unique<Target>(buy_t));
@@ -82,17 +86,19 @@ void PolyObserver2::eval() {
}
void PolyObserver2::rank() {
std::sort(me->targets.begin(), me->targets.end(),
[](std::unique_ptr<Target> const &ptr1,
std::unique_ptr<Target> const &ptr2) {
auto val1 = ptr1->value * ptr1->quantity.value();
auto val2 = ptr2->value * ptr2->quantity.value();
if (std::abs(val1 - val2) > securities::market_epsilon())
return val1 > val2;
else
return ptr1->polygon.parts.size() <
ptr2->polygon.parts.size();
});
std::sort(
me->targets.begin(), me->targets.end(),
[](std::unique_ptr<Target> const &ptr1,
std::unique_ptr<Target> const &ptr2) {
auto val1 = ptr1->value; // * ptr1->quantity.value();
auto val2 = ptr2->value; // * ptr2->quantity.value();
if (std::abs(val1 - val2) > securities::market_epsilon())
return val1 > val2;
else if (ptr1->polygon.parts.size() != ptr2->polygon.parts.size())
return ptr1->polygon.parts.size() < ptr2->polygon.parts.size();
else
return ptr1->polygon_id < ptr2->polygon_id;
});
}
void PolyObserver2::try_start_new_opportunity() {
@@ -103,10 +109,13 @@ void PolyObserver2::try_start_new_opportunity() {
me->current_opportunity =
Opportunity{best, me->event_id, best->value,
best->quantity, me->transact_time, me->observable_time};
log("opportunity_begin", best->descriptor, best->value,
log("opportunity_begin",
best->polygon_id,
best->descriptor,
api::Side_Name(best->side),
best->value,
best->quantity.value(), 0, 0,
us_diff(me->observable_time, me->transact_time), 1,
me->event_id.first, me->event_id.first);
us_diff(me->observable_time, me->transact_time), me->event_id.first, me->event_id.first);
}
}
@@ -116,24 +125,28 @@ void PolyObserver2::analyze() {
auto &target = *opportunity.target;
if (target.has_value()) {
// Add value and still has value, opportunity is still ongoing
log(
"opportunity_continue", target.descriptor, target.value,
log("opportunity_continue",
target.polygon_id,
target.descriptor,
api::Side_Name(target.side),
target.value,
target.quantity.value(),
us_diff(me->transact_time,opportunity.observable_time),
us_diff(me->transact_time,opportunity.transact_time),
us_diff(opportunity.observable_time,opportunity.transact_time),
me->event_id.first - opportunity.event_id.first,
us_diff(me->transact_time, opportunity.observable_time),
us_diff(me->transact_time, opportunity.transact_time),
us_diff(opportunity.observable_time, opportunity.transact_time),
opportunity.event_id.first, me->event_id.first);
} else {
// Had value but not anymore, opportunity is concluded
log(
"opportunity_end", target.descriptor, opportunity.value,
log("opportunity_end",
target.polygon_id,
target.descriptor,
api::Side_Name(target.side),
opportunity.value,
opportunity.quantity.value(),
us_diff(me->transact_time,opportunity.observable_time),
us_diff(me->transact_time,opportunity.transact_time),
us_diff(opportunity.observable_time,opportunity.transact_time),
me->event_id.first - opportunity.event_id.first,
opportunity.event_id.first, me->event_id.first-1);
us_diff(me->transact_time, opportunity.observable_time),
us_diff(me->transact_time, opportunity.transact_time),
us_diff(opportunity.observable_time, opportunity.transact_time),
opportunity.event_id.first, me->event_id.first - 1);
me->current_opportunity.reset();
try_start_new_opportunity();
}
@@ -125,7 +125,7 @@ SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options)
these_options.set_algo(options.algo());
these_options.set_market_date(day);
these_options.set_log_all_transactions(options.log_all_transactions());
these_options.set_max_events(options.max_events());
these_options.set_last_eid(options.last_eid());
// Set inputs
for (auto const &[channel, sim_input] : day_inputs) {
*these_options.add_channels() = channel;
@@ -35,11 +35,11 @@ message SimSessionOptions {
// it is off by default.
bool log_all_transactions = 9;
// The maximum number of events we will process before we terminate
// the simulation. This is usefull to speed initial prototyping of
// algos or analysis tools. Use 0 (the default) to signifiy all
// events.
uint64 max_events = 11;
// The latest pcap event we will process before we terminate the
// simulation. This is usefull to speed initial prototyping of algos
// or analysis tools. Use 0 (the default) to indicate all events
// should be processed.
uint64 last_eid = 11;
// The sim input files we are using. File names should end with
// "<YYYY-MM-DD>_<channel>_24h.sim". Files for the same day will
@@ -80,7 +80,7 @@ message SimSessionOptions {
message SimRunOptions {
string algo = 1;
bool log_all_transactions = 10;
uint64 max_events = 11;
uint64 last_eid = 11;
uint64 base_algo_latency_ns = 2;
uint64 order_update_delay_ns = 3;
uint64 mbp_update_delay_ns = 4;
@@ -134,9 +134,9 @@ bool ExchangeSim::has_work() const {
void ExchangeSim::do_work() {
bts_assert(me->run_state);
me->run_state->simulator.run_once();
if (me->run_state->options.max_events() &&
if (me->run_state->options.last_eid() &&
me->run_state->last_completed_count >=
me->run_state->options.max_events()) {
me->run_state->options.last_eid()) {
me->run_state->simulator.end_simulation();
} else {
auto now = Timer::now();
@@ -23,166 +23,163 @@ struct AlgoModel::PImpl {
MDP3OrderKey{std::numeric_limits<MDP3OrderKey::Value>::max()};
std::unordered_set<MDP3OrderKey> my_orders = {};
std::vector<OrderUpdate> my_fills = {};
std::unordered_map<MDP3SecurityID, MBPUpdate> books = {};
mutable std::unordered_map<MDP3SecurityID, MBPUpdate> books = {};
};
AlgoModel::AlgoModel() : me() {}
AlgoModel::AlgoModel() : self() {}
AlgoModel::~AlgoModel() {}
void AlgoModel::init_algo_model(protos::SimRunOptions const &options,
PolygonMap const &pm, InstrumentMap const &im,
SimLogger const &logger) {
bts_assert(!me);
me.reset(new PImpl{pm, im, logger, options});
bts_assert(!self);
self.reset(new PImpl{pm, im, logger, options});
}
std::string AlgoModel::param(std::string const &name,
std::string const &default_value) const {
bts_assert(me);
bts_assert(me->sim);
auto it = me->options.parameters().find(name);
if (it != me->options.parameters().end())
bts_assert(self);
bts_assert(self->sim);
auto it = self->options.parameters().find(name);
if (it != self->options.parameters().end())
return it->second;
else
return default_value;
}
int AlgoModel::param(std::string const &name, int default_value) const {
bts_assert(me);
bts_assert(me->sim);
auto it = me->options.parameters().find(name);
if (it != me->options.parameters().end())
bts_assert(self);
bts_assert(self->sim);
auto it = self->options.parameters().find(name);
if (it != self->options.parameters().end())
return std::stoi(it->second);
else
return default_value;
}
PolygonMap const &AlgoModel::polygons() const {
bts_assert(me);
bts_assert(me->sim);
return me->polygons;
bts_assert(self);
bts_assert(self->sim);
return self->polygons;
}
InstrumentMap const &AlgoModel::instruments() const {
bts_assert(me);
bts_assert(me->sim);
return me->instruments;
bts_assert(self);
bts_assert(self->sim);
return self->instruments;
}
MDP3TimeStamp AlgoModel::time() const {
bts_assert(me);
bts_assert(me->sim);
return MDP3TimeStamp(me->sim->time());
bts_assert(self);
bts_assert(self->sim);
return MDP3TimeStamp(self->sim->time());
}
InstrumentDefinition const &AlgoModel::get_sec(MDP3SecurityID id) const {
bts_assert(me);
bts_assert(me->sim);
bts_assert(self);
bts_assert(self->sim);
auto it = me->instruments.find(id);
bts_assert(it != me->instruments.end());
auto it = self->instruments.find(id);
bts_assert(it != self->instruments.end());
return *it->second;
}
MBPUpdate const& AlgoModel::books(MDP3SecurityID id) const {
bts_assert(me);
bts_assert(me->sim);
bts_assert(me->mode != Mode::RESETTING);
auto it = me->books.find(id);
bts_assert(it != me->books.end());
return it->second;
bts_assert(self);
bts_assert(self->sim);
bts_assert(self->mode != Mode::RESETTING);
return self->books[id];
}
MDP3OrderKey AlgoModel::send(SmallOrderChange const &change,
size_t propagation_delay) {
bts_assert(me);
bts_assert(me->sim);
bts_assert(me->mode != Mode::RESETTING);
bts_assert(self);
bts_assert(self->sim);
bts_assert(self->mode != Mode::RESETTING);
if (me->mode == Mode::REPLAY)
me->mode = Mode::SIMULATION;
if (self->mode == Mode::REPLAY)
self->mode = Mode::SIMULATION;
auto total_delay = me->options.base_algo_latency_ns() + propagation_delay;
auto t_time = me->sim->time() + total_delay;
auto total_delay = self->options.base_algo_latency_ns() + propagation_delay;
auto t_time = self->sim->time() + total_delay;
if (change.change == OrderChange::added) {
auto cpy = change;
cpy.order.key = generate_order_key();
me->my_orders.insert(cpy.order.key);
me->sim->emit_delayed<AlgoActionEvent>(
self->my_orders.insert(cpy.order.key);
self->sim->emit_delayed<AlgoActionEvent>(
total_delay, AlgoAction{MDP3TimeStamp{t_time}, cpy});
return cpy.order.key;
} else {
bts_assert(change.order.key != MDP3OrderKey{});
me->sim->emit_delayed<AlgoActionEvent>(
self->sim->emit_delayed<AlgoActionEvent>(
total_delay, AlgoAction{MDP3TimeStamp{t_time}, change});
return change.order.key;
}
}
void AlgoModel::log_pandl() {
bts_assert(me);
bts_assert(me->sim);
bts_assert(self);
bts_assert(self->sim);
auto mkt_pricer = [this](MDP3SecurityID id, MDP3Quantity qty, api::Side s) {
return price_at_market(id, qty, s);
};
PandL pl{me->instruments, mkt_pricer};
auto outright_pos = pl.outright_position(me->my_fills);
PandL pl{self->instruments, mkt_pricer};
auto outright_pos = pl.outright_position(self->my_fills);
log("pnl", pl.net_cash(me->my_fills),
log("pnl", pl.net_cash(self->my_fills),
pl.position_liquidation_value(outright_pos));
}
void AlgoModel::sleep(size_t ns) {
bts_assert(me);
bts_assert(me->sim);
me->sim->sleep(ns);
bts_assert(self);
bts_assert(self->sim);
self->sim->sleep(ns);
}
void AlgoModel::reset_simulation() {
bts_assert(me);
bts_assert(me->sim);
bts_assert(me->mode == Mode::SIMULATION);
me->sim->emit<ResetToPCapRequestEvent>();
me->mode = Mode::RESETTING;
bts_assert(self);
bts_assert(self->sim);
bts_assert(self->mode == Mode::SIMULATION);
self->sim->emit<ResetToPCapRequestEvent>();
self->mode = Mode::RESETTING;
}
void AlgoModel::init(sim::ISimulator2 &sim) {
bts_assert(me);
bts_assert(!me->sim);
me->sim = &sim;
bts_assert(self);
bts_assert(!self->sim);
self->sim = &sim;
on_simulation_reset();
}
void AlgoModel::handle(sim::Event const &ev) {
bts_assert(me);
bts_assert(me->sim);
bts_assert(self);
bts_assert(self->sim);
if (dynamic_cast<TimerEvent const *>(&ev)) {
if (me->mode != Mode::RESETTING)
if (self->mode != Mode::RESETTING)
on_time(MDP3TimeStamp{time()});
} else if (auto n = dynamic_cast<AlgoNotificationEvent const *>(&ev); n) {
std::visit(overloaded{[&](const MBPUpdate &mbp) {
me->books[mbp.security_id] = mbp;
if (me->mode != Mode::RESETTING)
self->books[mbp.security_id] = mbp;
if (self->mode != Mode::RESETTING)
on_notification(n->ev);
},
[&](const OrderUpdate &update) {
if (me->my_orders.count(update.key)) {
if (self->my_orders.count(update.key)) {
if (update.last_fills.value()) {
me->my_fills.push_back(update);
self->my_fills.push_back(update);
}
if (me->mode != Mode::RESETTING)
if (self->mode != Mode::RESETTING)
on_notification(n->ev);
}
}},
n->ev.event);
} else if (dynamic_cast<ResetToPCapReplyEvent const *>(&ev)) {
bts_assert(me->mode == Mode::RESETTING);
me->mode = Mode::REPLAY;
me->books.clear();
bts_assert(self->mode == Mode::RESETTING);
self->mode = Mode::REPLAY;
self->books.clear();
on_simulation_reset();
}
}
@@ -190,16 +187,16 @@ void AlgoModel::handle(sim::Event const &ev) {
void AlgoModel::finit() { log_pandl(); }
MDP3OrderKey AlgoModel::generate_order_key() {
return MDP3OrderKey{me->last_order_key.value()--};
return MDP3OrderKey{self->last_order_key.value()--};
}
double AlgoModel::price_at_market(MDP3SecurityID id, MDP3Quantity qty,
api::Side s) const {
bts_assert(me);
bts_assert(me->sim);
bts_assert(self);
bts_assert(self->sim);
auto it = me->books.find(id);
if (it != me->books.end())
auto it = self->books.find(id);
if (it != self->books.end())
return os::nan();
else {
auto &book = it->second.mbp_side(s);
@@ -224,15 +221,15 @@ double AlgoModel::price_at_market(MDP3SecurityID id, MDP3Quantity qty,
void AlgoModel::_log(std::string const &log_entry_type,
std::initializer_list<std::string> const &log_args) {
bts_assert(me);
bts_assert(me->sim);
bts_assert(self);
bts_assert(self->sim);
std::ostringstream str;
str << to_time_string(time()) << " " << log_entry_type << " ";
for (auto const &a : log_args)
str << a << " ";
LOG(Spartan, Info, "{}", str.str());
me->logger(time(), log_entry_type, log_args);
self->logger(time(), log_entry_type, log_args);
}
} // namespace bts::spartan::mdp3
@@ -60,7 +60,7 @@ class AlgoModel : public sim::IAgent2 {
private:
enum class Mode;
struct PImpl;
std::unique_ptr<PImpl> me;
std::unique_ptr<PImpl> self;
MDP3OrderKey generate_order_key();
double price_at_market(MDP3SecurityID, MDP3Quantity, api::Side s) const;
@@ -38,7 +38,7 @@ struct ExchangeGateway::PImpl {
protos::SimRunOptions const options;
AlgoEventID current_event = {0, 0};
sim::ISimulator2 *sim = nullptr;
std::unordered_map<MDP3SecurityID,InstrumentSnapshot> snapshots = {};
std::unordered_map<MDP3SecurityID, InstrumentSnapshot> snapshots = {};
Mode mode = Mode::REPLAY;
std::optional<SimulationState> simulation = {};
};
@@ -269,8 +269,10 @@ void ExchangeGateway::clear_simulated_order_book(MDP3TimeStamp transact_time,
void ExchangeGateway::enqueue_next_pcap_event() {
auto event = me->event_generator.next_event();
if (!event)
if (!event ||
(me->options.last_eid() && me->options.last_eid() < event->event_id))
return;
auto now = me->sim->time();
auto delay = now < event->transact_time.value()
? event->transact_time.value() - now
@@ -34,9 +34,6 @@ class ExchangeGateway : public sim::IAgent2 {
size_t transaction_delay, MDP3SecurityID);
SmallOrderChange correct_pcap_order(OrderChange const &pcap_order);
void enqueue_next_pcap_event();
void emit_next_pcap_event();
void emit_next_simulation_event();
void handle(AlgoActionEvent const &);
void handle(ResetToPCapRequestEvent const &);
@@ -45,8 +45,7 @@ void poly_values_to_csv(std::vector<PolygonValuation> const &vals,
<< to_time_string(e.send_ts) << "," << e.send_ts.value()
<< "," << api::Side_Name(e.side) << "," << e.direct_value
<< "," << e.direct_quantity.value() << "," << e.merged_value
<< "," << e.merged_quantity.value()
<< "," << e.min_n_orders;
<< "," << e.merged_quantity.value() << "," << e.min_n_orders;
for (size_t i = 0; i < 5; i++) {
if (i < e.books_used.size()) {
auto &br = e.books_used[i];
@@ -69,9 +68,8 @@ void poly_opportunities_to_csv(std::vector<PolygonOpportunity> const &ops,
<< api::Side_Name(op.side) << "," << op.merged_value << ","
<< op.merged_quantity.value() << "," << op.direct_value << ","
<< op.direct_quantity.value() << "," << op.begin_vid << ","
<< op.end_vid << "," << op.begin_eid << "," << op.end_eid
<< "," << op.min_n_orders
<< std::endl;
<< op.end_vid << "," << op.begin_eid << "," << op.end_eid << ","
<< op.min_n_orders << std::endl;
}
}
@@ -103,7 +101,12 @@ PolygonValuation get_best_valuation(std::vector<PolygonValuation> const &vals) {
PolygonValuation best_so_far = vals[0];
for (size_t i = 1; i < vals.size(); i++) {
auto &v = vals[i];
if (v.merged_value > best_so_far.merged_value)
if (v.merged_value > best_so_far.merged_value ||
(v.merged_value == best_so_far.merged_value &&
v.polygon.parts.size() < best_so_far.polygon.parts.size()) ||
(v.merged_value == best_so_far.merged_value &&
v.polygon.parts.size() == best_so_far.polygon.parts.size() &&
v.polygon_id < best_so_far.polygon_id))
best_so_far = v;
}
return best_so_far;
@@ -206,7 +209,6 @@ struct PolygonOpportunityFinder::PImpl {
std::unordered_set<size_t> active_polygons = {};
std::unordered_map<MDP3SecurityID, SnapshotRec> all_snapshots = {};
std::unordered_map<MDP3SecurityID, std::vector<size_t>> dependencies = {};
};
PolygonOpportunityFinder::PolygonOpportunityFinder(PolygonMap const &polys)
@@ -234,12 +236,11 @@ void PolygonOpportunityFinder::on_real_time_event(MDP3Event const &ev) {
// their value calculated
std::unordered_set<size_t> to_recalc = me->active_polygons;
me->active_polygons.clear();
for (auto &s : ev.affected_securities) {
me->all_snapshots[s->security_id()] = {s, ev.event_id};
auto &these_dependencies = me->dependencies[s->security_id()];
to_recalc.insert(these_dependencies.begin(),
these_dependencies.end());
to_recalc.insert(these_dependencies.begin(), these_dependencies.end());
}
auto direct_pricer = [this](MDP3SecurityID sec_id, MDP3Quantity,
@@ -266,9 +267,13 @@ void PolygonOpportunityFinder::on_real_time_event(MDP3Event const &ev) {
if (side == api::SIDE_SELL)
polygon = flip(polygon);
PolygonValuation val{
0, pid, ev.event_id, ev.transact_time, ev.last_packet_send_time,
side};
PolygonValuation val{polygon,
0,
pid,
ev.event_id,
ev.transact_time,
ev.last_packet_send_time,
side};
val.direct_value = eval_poly(polygon, direct_pricer);
val.merged_value = eval_poly(polygon, merged_pricer);
@@ -305,9 +310,10 @@ void PolygonOpportunityFinder::on_real_time_event(MDP3Event const &ev) {
merged_available_qty = this_merged_ratio;
// Orders available
val.min_n_orders = std::min((size_t)snap.snapshot->book(mkt_side)[0].n_orders,
val.min_n_orders);
val.min_n_orders = std::min(
(size_t)snap.snapshot->book(mkt_side)[0].n_orders,
val.min_n_orders);
val.books_used.push_back(book_ref);
}
}
@@ -15,6 +15,7 @@ struct BookReference {
};
struct PolygonValuation {
Polygon polygon;
size_t vid;
size_t polygon_id;
size_t event_id;