diff --git a/src/data_files/CMakeDeps.cmake b/src/data_files/CMakeDeps.cmake index 767e98d74..ffef95174 100644 --- a/src/data_files/CMakeDeps.cmake +++ b/src/data_files/CMakeDeps.cmake @@ -1 +1 @@ -set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/md.sql) \ No newline at end of file +set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js;files/small_es.pcap.xz;files/spartan/md.sql;files/spartan/sim_db.sql) \ No newline at end of file diff --git a/src/data_files/files/spartan/sim_db.sql b/src/data_files/files/spartan/sim_db.sql new file mode 100644 index 000000000..ab30e5e93 --- /dev/null +++ b/src/data_files/files/spartan/sim_db.sql @@ -0,0 +1,28 @@ + +drop table if exists session; +CREATE TABLE session( + session_id integer primary key, + date text, + algo text, + title text +); + +drop table if exists run; +create table run( + run_id integer primary key, + session_id numeric, + start_time text, + end_time text, + file text, + channel text, + data_date text, + realized_pandl float, + position_pandl float +); + +drop table if exists run_param; +create table run_param( + run_id numeric, + param text, + param_value text +); diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 627837dea..6e02d3bee 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/tas/RunTAS.cpp;cmds/spartan/algo/tas/TASOrderReactor.cpp;cmds/spartan/algo/tas/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto;cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IExchangeServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.proto;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/TimerAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/AgentBase.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/tests/test_simulator_core.cpp;cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.cpp;cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/md/mdp3/utils/polygon_utils.cpp;cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.cpp;cmds/spartan/md/mdp3/utils/queries/FillCounter.cpp;cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonFinder.cpp;cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp;cmds/spartan/md/mdp3/utils/tests/test_polygon_finder.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/tas/RunTAS.cpp;cmds/spartan/algo/tas/TASOrderReactor.cpp;cmds/spartan/algo/tas/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/Matcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp;cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto;cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IExchangeServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimInput.proto;cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/GatewayAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/agents/TimerAgent.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/AgentBase.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/core/tests/test_simulator_core.cpp;cmds/spartan/md/mdp3/back_testing/simulator/events/SimEvents.cpp;cmds/spartan/md/mdp3/common/InstrumentDefinition.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/md/mdp3/utils/polygon_utils.cpp;cmds/spartan/md/mdp3/utils/queries/DefinitionCollector.cpp;cmds/spartan/md/mdp3/utils/queries/FillCounter.cpp;cmds/spartan/md/mdp3/utils/queries/IcebergFinder.cpp;cmds/spartan/md/mdp3/utils/queries/PolygonFinder.cpp;cmds/spartan/md/mdp3/utils/queries/TransactionDeducer.cpp;cmds/spartan/md/mdp3/utils/tests/test_polygon_finder.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index b5c0f19b9..57259fbe0 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -1,44 +1,35 @@ #include "QuickTest.h" -#include "ICmd.h" -#include "lib/config/config.h" -#include "lib/config/protos/orders_fills_cleanup.pb.h" -#include "lib/orderman/orders/algo/IAlgoServices.h" -#include "lib/tracing/PerfLog.h" -#include "lib/utils/datetime.h" -#include "lib/config/protos/logging.pb.h" -#include "lib/domain_model/sod.h" -#include "lib/logging/logging.h" -#include "lib/utils/datetime.h" -#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h" -#include "server/sparkd/cmds/spartan/md/mdp3/common/mdp3_common.h" -#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h" -#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h" -#include "server/sparkd/cmds/spartan/md/mdp3/handler/ChannelHandler.h" -#include "server/sparkd/cmds/spartan/md/mdp3/handler/IChannelObserver.h" -#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h" -#include "server/sparkd/cmds/spartan/md/mdp3/utils/mdp3_utils.h" -#include "server/sparkd/cmds/spartan/md/mdp3/utils/polygon_utils.h" -#include "server/sparkd/cmds/spartan/util/PCapReader.h" -#include "server/sparkd/cmds/spartan/util/SecurityDB.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h" #include "lib/utils/PBUtils.h" -#include -#include -#include -#include -#include - namespace bts { namespace cmd { using namespace spartan; +void gen_sim_conf() { + protos::SimSessionOptions options; + *options.mutable_algo() = "Sniper"; + *options.add_input_files() = "/home/ppinto/360/2020-09-07_360_24h.sim"; + *options.add_input_files() = "/home/ppinto/360/2020-09-08_360_24h.sim"; + *options.add_input_files() = "/home/ppinto/360/2020-09-09_360_24h.sim"; + *options.add_input_files() = "/home/ppinto/360/2020-09-10_360_24h.sim"; + + options.set_session_title("First run"); + + (*options.mutable_parameter())["MinOrdersAtTop"].add_item("1"); + (*options.mutable_parameter())["MinOrdersAtTop"].add_item("2"); + + (*options.mutable_parameter())["Qty"].add_item("1"); + (*options.mutable_parameter())["Qty"].add_item("2"); + + std::cout << utils::pb::to_string(options) << std::endl; +} + int QuickTest::execute(std::vector const &) { - SecurityDB db; - for (auto &s : db.query()) { - std::cout << utils::pb::to_string(*s) << std::endl; - } - return 0 ; + mdp3::SimDB db; + return 0; } std::string QuickTest::help() { diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 91f596cfd..496473959 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -19,6 +19,9 @@ #include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/FillCounter.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/IcebergFinder.h" #include "server/sparkd/cmds/spartan/md/mdp3/utils/queries/PolygonFinder.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "lib/utils/PBUtils.h" #include #include @@ -67,15 +70,13 @@ int load_sim_input(std::string const &sim_input) { } } -int sim_sniper(std::string const &input_file) { - if (boost::algorithm::ends_with(input_file, ".xz")) { - gen_sim_input(input_file); - return sim_sniper(drop_extensions(drop_extensions(input_file)) + ".sim"); - } - auto input = make_sim_input_from_proto(input_file); - ExchangeSim sim({input}); - Sniper sniper; - std::cout << sim.run(sniper); +int run_sim(std::string const &input_file) { + spartan::protos::SimSessionOptions options; + std::ifstream proto_in(input_file); + std::string str((std::istreambuf_iterator(proto_in)), + std::istreambuf_iterator()); + bts_assert(utils::pb::from_string(str,&options)); + SimOrchestrator(options).run(); return 0; } @@ -261,9 +262,9 @@ void validate_fifo(std::string const &pcap) { std::map const &)>> extensions = { - {"sim_sniper", + {"run_sim", [](auto const &args) { - sim_sniper(args[1]); + run_sim(args[1]); return 0; }}, diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp index c342e4984..2c9d0e876 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.cpp @@ -5,6 +5,7 @@ #include "lib/logging/Logger.h" #include "lib/os/os.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" #include #include @@ -15,6 +16,7 @@ enum class Sniper::State { WATCH, WAIT }; struct Sniper::PImpl { ISimServices *sim = nullptr; + protos::SimRunOptions options; State state = State::WATCH; std::unordered_map const *polygons; std::unordered_map books; @@ -22,6 +24,9 @@ struct Sniper::PImpl { size_t current_shot; MDP3TimeStamp transact_time; MDP3TimeStamp observable_time; + + int min_orders_at_top; + int order_qty; }; struct Sniper::Target { @@ -39,14 +44,27 @@ struct Sniper::Target { Sniper::Sniper() : me(new PImpl()) {} Sniper::~Sniper() {} -void Sniper::init(ISimServices &s) { +void Sniper::init(ISimServices &s, protos::SimRunOptions const&options) { me->sim = &s; + me->options = options; me->state = State::WATCH; me->polygons = &s.polygons(); me->books.clear(); me->in_flight.clear(); me->current_shot = 0; + auto it = options.parameters().find("MinOrdersAtTop"); + if (it != options.parameters().end()) + me->min_orders_at_top = std::stoi(it->second); + else + me->min_orders_at_top = 1; + + it = options.parameters().find("OrderQty"); + if (it != options.parameters().end()) + me->order_qty = std::stoi(it->second); + else + me->min_orders_at_top = 1; + for (auto &p : *me->polygons) for (auto &pp : p.second.parts) me->books[pp.sec_id] = {}; @@ -76,7 +94,7 @@ Sniper::Target Sniper::eval(size_t id, Polygon const &p, api::Side s) { for (auto &l : poly.parts) { auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); auto &mkt = me->books[l.sec_id].mbp_side(mkt_side); - if (mkt[0].n_orders < 2) + if (mkt[0].n_orders < me->min_orders_at_top) return Target{id, poly, 0}; else { auto it = me->sim->instruments().find(l.sec_id); @@ -97,7 +115,7 @@ Sniper::MaybeTarget Sniper::pick() { if (valuation.size() && valuation.rbegin()->value > 0) { std::ostringstream str; for (auto it = valuation.rbegin(); it != valuation.rend(); it++) { - if (it->value <= 0.9) + if (it->value <= 1) break; str << " (" << it->value << "," << it->id << ")"; } @@ -130,7 +148,7 @@ void Sniper::fire(Target const &t) { order_change.order.key = me->sim->generate_order_key(); order_change.order.security = p.sec_id; order_change.order.price = worst_price; - order_change.order.quantity = MDP3Quantity{std::abs(p.ratio)}; + order_change.order.quantity = MDP3Quantity{me->order_qty * std::abs(p.ratio)}; order_change.order.side = part_side; order_change.change = OrderChange::added; LOG(Spartan, Info, "Sending for price {} order: {}", best_price.price(), diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h index 394c26e73..f4ab78323 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h @@ -13,7 +13,7 @@ public: Sniper(); ~Sniper(); - void init(ISimServices &) override; + void init(ISimServices &, protos::SimRunOptions const&) override; void on_time(MDP3TimeStamp) override; void on_notification(AlgoNotification const&) override; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp index 6fa5e30e0..a9004b568 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.cpp @@ -80,7 +80,7 @@ const SpreaderOrder& SimSpreader::spreader() const{ // IAlgoModel ///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// -void SimSpreader::init(ISimServices& sim_services){ +void SimSpreader::init(ISimServices& sim_services, protos::SimRunOptions const&){ me->sim_services = &sim_services; me->sim_order_gateway.init(sim_services,me->spreader); // TODO (no test_ blah) diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.h index bf3f9e048..3cffd3b08 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/SimSpreader.h @@ -24,7 +24,7 @@ public: Price, Quantity); const bts::orderman::SpreaderOrder& spreader() const; // IAlgoModel - void init(ISimServices&) override; + void init(ISimServices&, protos::SimRunOptions const&) override; void on_time(MDP3TimeStamp) override; void on_notification(AlgoNotification const&) override; ~SimSpreader(){} diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp index db49553ba..b14191749 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/spreader/tests/sim_spreader_tests.cpp @@ -7,6 +7,7 @@ #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" #include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ISimServices.h" #include "lib/orderman/orders/spreader_orders/tests/tests_common.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" using namespace bts::spartan::mdp3; @@ -121,7 +122,8 @@ TEST_CASE ("sim_spreader", "[spreader]"){ bts::spartan::mdp3::SimSpreader sim_spreader(order_def,sec,Price(-10),Quantity(1)); //ISimServices* sim_services = new TestSimServices(); TestSimServices sim_services; - sim_spreader.init(*(ISimServices*)&sim_services); + spartan::protos::SimRunOptions options; + sim_spreader.init(*(ISimServices*)&sim_services,options); SECTION("requoting"){ std::vector> bids = { diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp index 8634666a8..4895d8640 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.cpp @@ -1,20 +1,145 @@ #include "SimOrchestrator.h" #include "../simulator/SimInput.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/algos/Sniper.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.pb.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/ExchangeSim.h" +#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h" namespace bts::spartan::mdp3 { - struct SimOrchestrator::PImpl { protos::SimSessionOptions options; - std::vector files; + std::vector inputs; + std::vector run_options; }; -SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options) - : me(new PImpl{options,{}}) { +void generate_param_combinations(std::vector const &dimensionality, + std::vector ¤t_combination, + size_t current_idx, + std::vector> *out) { + bts_assert(dimensionality.size() == current_combination.size()); + if (current_idx == dimensionality.size()) + out->push_back(current_combination); + else { + if (dimensionality[current_idx] == 0) + generate_param_combinations(dimensionality, current_combination, + current_idx + 1, out); + else + for (size_t i = 0; i < dimensionality[current_idx]; i++) { + current_combination[current_idx] = i; + generate_param_combinations(dimensionality, current_combination, + current_idx + 1, out); + } + } +} - for (auto const &file : options.input_files()) - me->files.emplace_back (make_sim_input_from_proto(file)); +SimOrchestrator::SimOrchestrator(protos::SimSessionOptions const &options) + : me(new PImpl{}) { + me->options = options; + + std::vector dimensionality; + dimensionality.push_back(options.base_algo_latency_ns().size()); + dimensionality.push_back(options.order_update_delay_ns().size()); + dimensionality.push_back(options.mbp_update_delay_ns().size()); + dimensionality.push_back(options.transaction_latency_ns().size()); + for (auto &p : options.parameter()) + dimensionality.push_back(p.second.item().size()); + std::vector combination(dimensionality.size(), -1); + std::vector> all_combinations; + generate_param_combinations(dimensionality, combination, 0, + &all_combinations); + + for (auto &c : all_combinations) { + auto &these_options = me->run_options.emplace_back(); + + if (c[0] == -1) + these_options.set_base_algo_latency_ns(0); + else + these_options.set_base_algo_latency_ns( + options.base_algo_latency_ns(c[0])); + + if (c[1] == -1) + these_options.set_order_update_delay_ns(0); + else + these_options.set_order_update_delay_ns( + options.order_update_delay_ns(c[1])); + + if (c[2] == -1) + these_options.set_mbp_update_delay_ns(0); + else + these_options.set_mbp_update_delay_ns( + options.mbp_update_delay_ns(c[2])); + + if (c[3] == -1) + these_options.set_transaction_latency_ns(0); + else + these_options.set_transaction_latency_ns( + options.transaction_latency_ns(c[3])); + + size_t n = 4; + for (auto &p : options.parameter()) { + auto this_parameter_name = p.first; + auto this_parameter_value = p.second.item(c[n]); + these_options.mutable_parameters()->insert( + {this_parameter_name, this_parameter_value}); + n++; + } + } + for (auto const &file : options.input_files()) + me->inputs.emplace_back(make_sim_input_from_proto(file)); } SimOrchestrator::~SimOrchestrator() {} + +std::unique_ptr make_algo(std::string const &) { + return std::make_unique(); +} + +void SimOrchestrator::run() { + + struct Run { + std::string algo; + SimInput const *input; + protos::SimRunOptions *options; + }; + + struct Worker { + std::vector assigned_runs; + std::thread thread; + + void start() { + thread = std::thread([this]() { + for (auto const &r : assigned_runs) { + auto algo = make_algo(r.algo); + ExchangeSim sim({*r.input}); + sim.run(*algo,*r.options); + } + }); + } + + void join() { thread.join(); } + }; + + std::vector workers; + + auto n_workers = me->options.n_cores(); + if (n_workers == 0) + n_workers = os::num_cores() ; + workers.resize(n_workers); + + size_t idx = 0; + for (auto &i : me->inputs) + for (auto &o : me->run_options) { + workers[idx % n_workers].assigned_runs.push_back( + {me->options.algo(), &i, &o}); + idx++; + } + + for (auto &w : workers) + w.start(); + + for (auto &w : workers) + w.join(); +} + } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.h index 582669ecb..57bd6a462 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimOrchestrator.h @@ -9,9 +9,7 @@ class SimOrchestrator { public: SimOrchestrator(protos::SimSessionOptions const&); ~SimOrchestrator(); - void run(); - private: struct PImpl; std::unique_ptr me; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto index 4e18aa952..8ce7a7f60 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/orchestrator/SimSessionOptions.proto @@ -13,7 +13,7 @@ message SimSessionOptions { string algo = 1; repeated string input_files = 2; string session_title = 3; - uint32 n_processes = 4 ; + uint32 n_cores = 4 ; repeated uint64 base_algo_latency_ns = 5; repeated uint64 order_update_delay_ns = 6; @@ -21,7 +21,7 @@ message SimSessionOptions { repeated uint64 transaction_latency_ns = 8; message ParameterDomain { - repeated string element = 1; + repeated string item = 1; } - map parameters = 9; + map parameter = 9; } \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp new file mode 100644 index 000000000..6afa0d3ad --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.cpp @@ -0,0 +1,40 @@ +#include "SimDB.h" + +#include +#include +#include +#include "lib/utils/bts_assert.h" + +namespace bts::spartan::mdp3 { + +struct +struct SimDB::PImpl { + PImpl(std::string const &loc) + : db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE), + lock() {} + + SQLite::Database db; + std::mutex lock; +}; + + +SimDB::SimDB(std::string const &loc) { + auto parent = boost::filesystem::path(loc).parent_path(); + if (!boost::filesystem::is_directory(parent)) + bts_assert(boost::filesystem::create_directories(parent)); + + if (!boost::filesystem::is_regular_file(loc)) { + auto sql_path = os::sparkd_wd() + "/data_files/spartan/sim_db.sql"; + std::ifstream f(sql_path); + std::string str((std::istreambuf_iterator(f)), + (std::istreambuf_iterator())); + SQLite::Database db (loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); + db.exec(str); + } + me = std::make_unique (loc); +} + +SimDB::~SimDB() {} + +} + diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h new file mode 100644 index 000000000..65126880f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/sim_db/SimDB.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include "lib/persistence/persistence_utils.h" +#include "lib/utils/ValueWrapper.h" + +namespace bts::spartan::protos { +class SimRunOptions; +} + +namespace bts::spartan::mdp3 { + +class SimDB { + public: + SimDB(std::string const &location = persistence::storage_location("sim_db")); + ~SimDB(); + + using SessionID = utils::ValueWrapper; + SessionID create_session(std::string const &algo, + std::string const &title = ""); + using RunID = utils::ValueWrapper; + RunID create_run(SessionID, protos::SimRunOptions const &); + void finish_run(RunID, double realized_pandl, double position_pandl); + + private: + struct PImpl; + std::unique_ptr me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h index 83d6db882..819c35f28 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.h @@ -3,14 +3,16 @@ #include "../../common/mdp3_common.h" #include "AlgoEvent.h" +namespace bts::spartan::protos { +class SimRunOptions; +} namespace bts::spartan::mdp3 { - class ISimServices; class IAlgoModel { public: virtual ~IAlgoModel(); - virtual void init(ISimServices &) = 0; + virtual void init(ISimServices &,protos::SimRunOptions const&) = 0; virtual void on_time(MDP3TimeStamp) {} virtual void on_notification(AlgoNotification const&) = 0; }; diff --git a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp index 53e982586..8adff4c9f 100644 --- a/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp +++ b/src/server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/agents/AlgoAgent.cpp @@ -2,6 +2,7 @@ #include "../IAlgoModel.h" #include "../IExchangeServices.h" #include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentDefinition.h" +#include "lib/utils/PBUtils.h" namespace bts::spartan::mdp3 { @@ -30,7 +31,10 @@ AlgoAgent &AlgoAgent::operator=(AlgoAgent &&src) { return *this; } -void AlgoAgent::init() { me->algo.init(*this); } +void AlgoAgent::init() { + LOG(Spartan,Info, "Starting algo with options {}", utils::pb::to_string(me->options)); + me->algo.init(*this, me->options); +} int mkt_to_cash_flow(api::Side s) { if (s == api::SIDE_BUY)