Driving the simulator across days and parameter variations.

This commit is contained in:
Pedro Pinto committed 2020-10-06 16:08:33 -04:00
1 parent 6984676e7f
commit 24df444584
17 files changed
+307 -67

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/md.sql)
set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/md.sql;files/spartan/sim_db.sql)
+28
View File
@@ -0,0 +1,28 @@
drop table if exists session;
CREATE TABLE session(
session_id integer primary key,
date text,
algo text,
title text
);
drop table if exists run;
create table run(
run_id integer primary key,
session_id numeric,
start_time text,
end_time text,
file text,
channel text,
data_date text,
realized_pandl float,
position_pandl float
);
drop table if exists run_param;
create table run_param(
run_id numeric,
param text,
param_value text
);
+1 -1
View File
@@ -1 +1 @@
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/tas/RunTAS.cpp;cmds/spartan/algo/tas/TASOrderReactor.cpp;cmds/spartan/algo/tas/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto;cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IExchangeServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.proto;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/TimerAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/AgentBase.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/tests/test_simulator_core.cpp;cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.cpp;cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/md/mdp3/utils/polygon_utils.cpp;cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.cpp;cmds/spartan/md/mdp3/utils/queries/FillCounter.cpp;cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonFinder.cpp;cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp;cmds/spartan/md/mdp3/utils/tests/test_polygon_finder.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/tas/RunTAS.cpp;cmds/spartan/algo/tas/TASOrderReactor.cpp;cmds/spartan/algo/tas/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto;cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IExchangeServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.proto;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/TimerAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/AgentBase.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/tests/test_simulator_core.cpp;cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.cpp;cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/md/mdp3/utils/polygon_utils.cpp;cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.cpp;cmds/spartan/md/mdp3/utils/queries/FillCounter.cpp;cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonFinder.cpp;cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp;cmds/spartan/md/mdp3/utils/tests/test_polygon_finder.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+23 -32
View File
@@ -1,44 +1,35 @@
#include "QuickTest.h"
#include "ICmd.h"
#include "lib/config/config.h"
#include "lib/config/protos/orders_fills_cleanup.pb.h"
#include "lib/orderman/orders/algo/IAlgoServices.h"
#include "lib/tracing/PerfLog.h"
#include "lib/utils/datetime.h"
#include "lib/config/protos/logging.pb.h"
#include "lib/domain_model/sod.h"
#include "lib/logging/logging.h"
#include "lib/utils/datetime.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h"
#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h"
#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h"
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
#include "server/sparkd/cmds/spartan/util/SecurityDB.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h"
#include "lib/utils/PBUtils.h"
#include <boost/algorithm/string.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <iostream>
#include <regex>
namespace bts {
namespace cmd {
using namespace spartan;
void gen_sim_conf() {
protos::SimSessionOptions options;
*options.mutable_algo() = "Sniper";
*options.add_input_files() = "/home/ppinto/360/2020-09-07_360_24h.sim";
*options.add_input_files() = "/home/ppinto/360/2020-09-08_360_24h.sim";
*options.add_input_files() = "/home/ppinto/360/2020-09-09_360_24h.sim";
*options.add_input_files() = "/home/ppinto/360/2020-09-10_360_24h.sim";
options.set_session_title("First run");
(*options.mutable_parameter())["MinOrdersAtTop"].add_item("1");
(*options.mutable_parameter())["MinOrdersAtTop"].add_item("2");
(*options.mutable_parameter())["Qty"].add_item("1");
(*options.mutable_parameter())["Qty"].add_item("2");
std::cout << utils::pb::to_string(options) << std::endl;
}
int QuickTest::execute(std::vector<std::string> const &) {
SecurityDB db;
for (auto &s : db.query()) {
std::cout << utils::pb::to_string(*s) << std::endl;
}
return 0 ;
mdp3::SimDB db;
return 0;
}
std::string QuickTest::help() {
+12 -11
View File
@@ -19,6 +19,9 @@
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/FillCounter.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h"
#include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonFinder.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
#include "lib/utils/PBUtils.h"
#include <Pcap++/PcapFileDevice.h>
#include <boost/algorithm/string/predicate.hpp>
@@ -67,15 +70,13 @@ int load_sim_input(std::string const &sim_input) {
}
}
int sim_sniper(std::string const &input_file) {
if (boost::algorithm::ends_with(input_file, ".xz")) {
gen_sim_input(input_file);
return sim_sniper(drop_extensions(drop_extensions(input_file)) + ".sim");
}
auto input = make_sim_input_from_proto(input_file);
ExchangeSim sim({input});
Sniper sniper;
std::cout << sim.run(sniper);
int run_sim(std::string const &input_file) {
spartan::protos::SimSessionOptions options;
std::ifstream proto_in(input_file);
std::string str((std::istreambuf_iterator<char>(proto_in)),
std::istreambuf_iterator<char>());
bts_assert(utils::pb::from_string(str,&options));
SimOrchestrator(options).run();
return 0;
}
@@ -261,9 +262,9 @@ void validate_fifo(std::string const &pcap) {
std::map<std::string, std::function<int(std::vector<std::string> const &)>>
extensions = {
{"sim_sniper",
{"run_sim",
[](auto const &args) {
sim_sniper(args[1]);
run_sim(args[1]);
return 0;
}},
@@ -5,6 +5,7 @@
#include "lib/logging/Logger.h"
#include "lib/os/os.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
#include <queue>
#include <sstream>
@@ -15,6 +16,7 @@ enum class Sniper::State { WATCH, WAIT };
struct Sniper::PImpl {
ISimServices *sim = nullptr;
protos::SimRunOptions options;
State state = State::WATCH;
std::unordered_map<size_t, Polygon> const *polygons;
std::unordered_map<MDP3SecurityID, MBPUpdate> books;
@@ -22,6 +24,9 @@ struct Sniper::PImpl {
size_t current_shot;
MDP3TimeStamp transact_time;
MDP3TimeStamp observable_time;
int min_orders_at_top;
int order_qty;
};
struct Sniper::Target {
@@ -39,14 +44,27 @@ struct Sniper::Target {
Sniper::Sniper() : me(new PImpl()) {}
Sniper::~Sniper() {}
void Sniper::init(ISimServices &s) {
void Sniper::init(ISimServices &s, protos::SimRunOptions const&options) {
me->sim = &s;
me->options = options;
me->state = State::WATCH;
me->polygons = &s.polygons();
me->books.clear();
me->in_flight.clear();
me->current_shot = 0;
auto it = options.parameters().find("MinOrdersAtTop");
if (it != options.parameters().end())
me->min_orders_at_top = std::stoi(it->second);
else
me->min_orders_at_top = 1;
it = options.parameters().find("OrderQty");
if (it != options.parameters().end())
me->order_qty = std::stoi(it->second);
else
me->min_orders_at_top = 1;
for (auto &p : *me->polygons)
for (auto &pp : p.second.parts)
me->books[pp.sec_id] = {};
@@ -76,7 +94,7 @@ Sniper::Target Sniper::eval(size_t id, Polygon const &p, api::Side s) {
for (auto &l : poly.parts) {
auto mkt_side = securities::other_side(side_from_ratio(l.ratio));
auto &mkt = me->books[l.sec_id].mbp_side(mkt_side);
if (mkt[0].n_orders < 2)
if (mkt[0].n_orders < me->min_orders_at_top)
return Target{id, poly, 0};
else {
auto it = me->sim->instruments().find(l.sec_id);
@@ -97,7 +115,7 @@ Sniper::MaybeTarget Sniper::pick() {
if (valuation.size() && valuation.rbegin()->value > 0) {
std::ostringstream str;
for (auto it = valuation.rbegin(); it != valuation.rend(); it++) {
if (it->value <= 0.9)
if (it->value <= 1)
break;
str << " (" << it->value << "," << it->id << ")";
}
@@ -130,7 +148,7 @@ void Sniper::fire(Target const &t) {
order_change.order.key = me->sim->generate_order_key();
order_change.order.security = p.sec_id;
order_change.order.price = worst_price;
order_change.order.quantity = MDP3Quantity{std::abs(p.ratio)};
order_change.order.quantity = MDP3Quantity{me->order_qty * std::abs(p.ratio)};
order_change.order.side = part_side;
order_change.change = OrderChange::added;
LOG(Spartan, Info, "Sending for price {} order: {}", best_price.price(),
@@ -13,7 +13,7 @@ public:
Sniper();
~Sniper();
void init(ISimServices &) override;
void init(ISimServices &, protos::SimRunOptions const&) override;
void on_time(MDP3TimeStamp) override;
void on_notification(AlgoNotification const&) override;
@@ -80,7 +80,7 @@ const SpreaderOrder& SimSpreader::spreader() const{
// IAlgoModel /////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
void SimSpreader::init(ISimServices& sim_services){
void SimSpreader::init(ISimServices& sim_services, protos::SimRunOptions const&){
me->sim_services = &sim_services;
me->sim_order_gateway.init(sim_services,me->spreader);
// TODO (no test_ blah)
@@ -24,7 +24,7 @@ public:
Price, Quantity);
const bts::orderman::SpreaderOrder& spreader() const;
// IAlgoModel
void init(ISimServices&) override;
void init(ISimServices&, protos::SimRunOptions const&) override;
void on_time(MDP3TimeStamp) override;
void on_notification(AlgoNotification const&) override;
~SimSpreader(){}
@@ -7,6 +7,7 @@
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.h"
#include "lib/orderman/orders/spreader_orders/tests/tests_common.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
using namespace bts::spartan::mdp3;
@@ -121,7 +122,8 @@ TEST_CASE ("sim_spreader", "[spreader]"){
bts::spartan::mdp3::SimSpreader sim_spreader(order_def,sec,Price(-10),Quantity(1));
//ISimServices* sim_services = new TestSimServices();
TestSimServices sim_services;
sim_spreader.init(*(ISimServices*)&sim_services);
spartan::protos::SimRunOptions options;
sim_spreader.init(*(ISimServices*)&sim_services,options);
SECTION("requoting"){
std::vector<std::tuple<double,uint32_t>> bids =
{
@@ -1,20 +1,145 @@
#include "SimOrchestrator.h"
#include "../simulator/SimInput.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h"
namespace bts::spartan::mdp3 {
struct SimOrchestrator::PImpl {
protos::SimSessionOptions options;
std::vector<SimInput> files;
std::vector<SimInput> inputs;
std::vector<protos::SimRunOptions> run_options;
};
SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options)
: me(new PImpl{options,{}}) {
void generate_param_combinations(std::vector<size_t> const &dimensionality,
std::vector<int> &current_combination,
size_t current_idx,
std::vector<std::vector<int>> *out) {
bts_assert(dimensionality.size() == current_combination.size());
if (current_idx == dimensionality.size())
out->push_back(current_combination);
else {
if (dimensionality[current_idx] == 0)
generate_param_combinations(dimensionality, current_combination,
current_idx + 1, out);
else
for (size_t i = 0; i < dimensionality[current_idx]; i++) {
current_combination[current_idx] = i;
generate_param_combinations(dimensionality, current_combination,
current_idx + 1, out);
}
}
}
for (auto const &file : options.input_files())
me->files.emplace_back (make_sim_input_from_proto(file));
SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options)
: me(new PImpl{}) {
me->options = options;
std::vector<size_t> dimensionality;
dimensionality.push_back(options.base_algo_latency_ns().size());
dimensionality.push_back(options.order_update_delay_ns().size());
dimensionality.push_back(options.mbp_update_delay_ns().size());
dimensionality.push_back(options.transaction_latency_ns().size());
for (auto &p : options.parameter())
dimensionality.push_back(p.second.item().size());
std::vector<int> combination(dimensionality.size(), -1);
std::vector<std::vector<int>> all_combinations;
generate_param_combinations(dimensionality, combination, 0,
&all_combinations);
for (auto &c : all_combinations) {
auto &these_options = me->run_options.emplace_back();
if (c[0] == -1)
these_options.set_base_algo_latency_ns(0);
else
these_options.set_base_algo_latency_ns(
options.base_algo_latency_ns(c[0]));
if (c[1] == -1)
these_options.set_order_update_delay_ns(0);
else
these_options.set_order_update_delay_ns(
options.order_update_delay_ns(c[1]));
if (c[2] == -1)
these_options.set_mbp_update_delay_ns(0);
else
these_options.set_mbp_update_delay_ns(
options.mbp_update_delay_ns(c[2]));
if (c[3] == -1)
these_options.set_transaction_latency_ns(0);
else
these_options.set_transaction_latency_ns(
options.transaction_latency_ns(c[3]));
size_t n = 4;
for (auto &p : options.parameter()) {
auto this_parameter_name = p.first;
auto this_parameter_value = p.second.item(c[n]);
these_options.mutable_parameters()->insert(
{this_parameter_name, this_parameter_value});
n++;
}
}
for (auto const &file : options.input_files())
me->inputs.emplace_back(make_sim_input_from_proto(file));
}
SimOrchestrator::~SimOrchestrator() {}
std::unique_ptr<IAlgoModel> make_algo(std::string const &) {
return std::make_unique<Sniper>();
}
void SimOrchestrator::run() {
struct Run {
std::string algo;
SimInput const *input;
protos::SimRunOptions *options;
};
struct Worker {
std::vector<Run> assigned_runs;
std::thread thread;
void start() {
thread = std::thread([this]() {
for (auto const &r : assigned_runs) {
auto algo = make_algo(r.algo);
ExchangeSim sim({*r.input});
sim.run(*algo,*r.options);
}
});
}
void join() { thread.join(); }
};
std::vector<Worker> workers;
auto n_workers = me->options.n_cores();
if (n_workers == 0)
n_workers = os::num_cores() ;
workers.resize(n_workers);
size_t idx = 0;
for (auto &i : me->inputs)
for (auto &o : me->run_options) {
workers[idx % n_workers].assigned_runs.push_back(
{me->options.algo(), &i, &o});
idx++;
}
for (auto &w : workers)
w.start();
for (auto &w : workers)
w.join();
}
} // namespace bts::spartan::mdp3
@@ -9,9 +9,7 @@ class SimOrchestrator {
public:
SimOrchestrator(protos::SimSessionOptions const&);
~SimOrchestrator();
void run();
private:
struct PImpl;
std::unique_ptr<PImpl> me;
@@ -13,7 +13,7 @@ message SimSessionOptions {
string algo = 1;
repeated string input_files = 2;
string session_title = 3;
uint32 n_processes = 4 ;
uint32 n_cores = 4 ;
repeated uint64 base_algo_latency_ns = 5;
repeated uint64 order_update_delay_ns = 6;
@@ -21,7 +21,7 @@ message SimSessionOptions {
repeated uint64 transaction_latency_ns = 8;
message ParameterDomain {
repeated string element = 1;
repeated string item = 1;
}
map<string,ParameterDomain> parameters = 9;
map<string,ParameterDomain> parameter = 9;
}
@@ -0,0 +1,40 @@
#include "SimDB.h"
#include <SQLiteCpp/SQLiteCpp.h>
#include <boost/filesystem.hpp>
#include <mutex>
#include "lib/utils/bts_assert.h"
namespace bts::spartan::mdp3 {
struct
struct SimDB::PImpl {
PImpl(std::string const &loc)
: db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE),
lock() {}
SQLite::Database db;
std::mutex lock;
};
SimDB::SimDB(std::string const &loc) {
auto parent = boost::filesystem::path(loc).parent_path();
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
if (!boost::filesystem::is_regular_file(loc)) {
auto sql_path = os::sparkd_wd() + "/data_files/spartan/sim_db.sql";
std::ifstream f(sql_path);
std::string str((std::istreambuf_iterator<char>(f)),
(std::istreambuf_iterator<char>()));
SQLite::Database db (loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE);
db.exec(str);
}
me = std::make_unique<PImpl> (loc);
}
SimDB::~SimDB() {}
}
@@ -0,0 +1,31 @@
#pragma once
#include <memory>
#include "lib/persistence/persistence_utils.h"
#include "lib/utils/ValueWrapper.h"
namespace bts::spartan::protos {
class SimRunOptions;
}
namespace bts::spartan::mdp3 {
class SimDB {
public:
SimDB(std::string const &location = persistence::storage_location("sim_db"));
~SimDB();
using SessionID = utils::ValueWrapper<struct SessionIDTag,uint64_t>;
SessionID create_session(std::string const &algo,
std::string const &title = "");
using RunID = utils::ValueWrapper<struct RunIDTag,uint64_t>;
RunID create_run(SessionID, protos::SimRunOptions const &);
void finish_run(RunID, double realized_pandl, double position_pandl);
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -3,14 +3,16 @@
#include "../../common/mdp3_common.h"
#include "AlgoEvent.h"
namespace bts::spartan::protos {
class SimRunOptions;
}
namespace bts::spartan::mdp3 {
class ISimServices;
class IAlgoModel {
public:
virtual ~IAlgoModel();
virtual void init(ISimServices &) = 0;
virtual void init(ISimServices &,protos::SimRunOptions const&) = 0;
virtual void on_time(MDP3TimeStamp) {}
virtual void on_notification(AlgoNotification const&) = 0;
};
@@ -2,6 +2,7 @@
#include "../IAlgoModel.h"
#include "../IExchangeServices.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h"
#include "lib/utils/PBUtils.h"
namespace bts::spartan::mdp3 {
@@ -30,7 +31,10 @@ AlgoAgent &AlgoAgent::operator=(AlgoAgent &&src) {
return *this;
}
void AlgoAgent::init() { me->algo.init(*this); }
void AlgoAgent::init() {
LOG(Spartan,Info, "Starting algo with options {}", utils::pb::to_string(me->options));
me->algo.init(*this, me->options);
}
int mkt_to_cash_flow(api::Side s) {
if (s == api::SIDE_BUY)