From 25e07d0a607922a1976221577b3b03b70e396368 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 7 Jan 2021 16:45:07 -0600 Subject: [PATCH 01/13] refactor: Move BookStatRecorder into 'feed_observers' dir --- src/server/sparkd/cmds/Spartan.cpp | 2 +- .../{utils/queries => feed_observers}/BookStatRecorder.cpp | 0 .../{utils/queries => feed_observers}/BookStatRecorder.h | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename src/server/sparkd/cmds/spartan/market_data/{utils/queries => feed_observers}/BookStatRecorder.cpp (100%) rename src/server/sparkd/cmds/spartan/market_data/{utils/queries => feed_observers}/BookStatRecorder.h (100%) diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 7fb41f89c..a50c50c99 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -12,13 +12,13 @@ #include "lib/orderman/orderman_utils/Formatter.h" #include "lib/utils/PBUtils.h" #include "server/sparkd/cmds/spartan/back_testing/run/run_sim.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.h" #include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" #include "server/sparkd/cmds/spartan/market_data/feed_observers/FillCounter.h" #include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h" #include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.h" #include "server/sparkd/cmds/spartan/market_data/parsing/MDP3Parser.h" #include "server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h" -#include "server/sparkd/cmds/spartan/market_data/utils/queries/BookStatRecorder.h" #include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h" #include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h" diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/queries/BookStatRecorder.cpp b/src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp similarity index 100% rename from src/server/sparkd/cmds/spartan/market_data/utils/queries/BookStatRecorder.cpp rename to src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/queries/BookStatRecorder.h b/src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.h similarity index 100% rename from src/server/sparkd/cmds/spartan/market_data/utils/queries/BookStatRecorder.h rename to src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.h From 66e1b94d1bb9b7ac1a613b4456c1430d83fdb327 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 11 Jan 2021 23:47:23 -0600 Subject: [PATCH 02/13] refactor: Revamp cli with CLI11 App --- src/server/sparkd/cmds/AdminReport.cpp | 25 +- src/server/sparkd/cmds/AdminReport.h | 16 +- src/server/sparkd/cmds/AutoCert.cpp | 20 +- src/server/sparkd/cmds/AutoCert.h | 15 +- src/server/sparkd/cmds/Benchmark.cpp | 213 +++++----- src/server/sparkd/cmds/Benchmark.h | 15 +- .../cmds/{CMECertCmd.cpp => CMECert.cpp} | 15 +- src/server/sparkd/cmds/CMECert.h | 10 + src/server/sparkd/cmds/CMECertCmd.h | 15 - src/server/sparkd/cmds/CmdRegistry.cpp | 57 --- src/server/sparkd/cmds/CmdRegistry.h | 36 -- src/server/sparkd/cmds/ConfigCheck.cpp | 17 +- src/server/sparkd/cmds/ConfigCheck.h | 15 +- src/server/sparkd/cmds/FIXSim.cpp | 14 +- src/server/sparkd/cmds/FIXSim.h | 15 +- src/server/sparkd/cmds/Help.cpp | 29 -- src/server/sparkd/cmds/Help.h | 15 - src/server/sparkd/cmds/ICmd.h | 17 - src/server/sparkd/cmds/KeyCode.cpp | 359 ++++++---------- src/server/sparkd/cmds/KeyCode.h | 32 +- src/server/sparkd/cmds/Md.cpp | 20 +- src/server/sparkd/cmds/Md.h | 15 +- src/server/sparkd/cmds/ParseHistoricalFIX.cpp | 29 +- src/server/sparkd/cmds/ParseHistoricalFIX.h | 15 +- src/server/sparkd/cmds/PrintHedgeOrders.cpp | 19 +- src/server/sparkd/cmds/PrintHedgeOrders.h | 15 +- src/server/sparkd/cmds/QuickTest.cpp | 28 +- src/server/sparkd/cmds/QuickTest.h | 15 +- src/server/sparkd/cmds/Run.cpp | 15 +- src/server/sparkd/cmds/Run.h | 15 +- src/server/sparkd/cmds/Span.cpp | 14 +- src/server/sparkd/cmds/Span.h | 15 +- src/server/sparkd/cmds/Spartan.cpp | 395 ++---------------- src/server/sparkd/cmds/Spartan.h | 22 +- src/server/sparkd/cmds/Test.cpp | 26 +- src/server/sparkd/cmds/Test.h | 15 +- src/server/sparkd/cmds/Web.cpp | 27 +- src/server/sparkd/cmds/Web.h | 16 +- .../sparkd/cmds/spartan/cmds/BookStats.cpp | 32 ++ .../sparkd/cmds/spartan/cmds/BookStats.h | 10 + src/server/sparkd/cmds/spartan/cmds/CSV.cpp | 20 + src/server/sparkd/cmds/spartan/cmds/CSV.h | 10 + .../sparkd/cmds/spartan/cmds/Compress.cpp | 22 + .../sparkd/cmds/spartan/cmds/Compress.h | 10 + .../sparkd/cmds/spartan/cmds/FindPolys.cpp | 52 +++ .../sparkd/cmds/spartan/cmds/FindPolys.h | 10 + .../sparkd/cmds/spartan/cmds/Icebergs.cpp | 51 +++ .../sparkd/cmds/spartan/cmds/Icebergs.h | 10 + src/server/sparkd/cmds/spartan/cmds/Json.cpp | 26 ++ src/server/sparkd/cmds/spartan/cmds/Json.h | 10 + .../sparkd/cmds/spartan/cmds/ListSec.cpp | 40 ++ src/server/sparkd/cmds/spartan/cmds/ListSec.h | 10 + .../sparkd/cmds/spartan/cmds/PolygonBooks.cpp | 120 ++++++ .../sparkd/cmds/spartan/cmds/PolygonBooks.h | 10 + src/server/sparkd/cmds/spartan/cmds/SQL.cpp | 20 + src/server/sparkd/cmds/spartan/cmds/SQL.h | 10 + src/server/sparkd/cmds/spartan/cmds/Sim.cpp | 23 + src/server/sparkd/cmds/spartan/cmds/Sim.h | 10 + src/server/sparkd/cmds/spartan/cmds/Stats.cpp | 42 ++ src/server/sparkd/cmds/spartan/cmds/Stats.h | 10 + .../sparkd/cmds/spartan/cmds/ValidateFifo.cpp | 28 ++ .../sparkd/cmds/spartan/cmds/ValidateFifo.h | 10 + .../sparkd/cmds/spartan/cmds/ValuePolys.cpp | 47 +++ .../sparkd/cmds/spartan/cmds/ValuePolys.h | 10 + src/server/sparkd/main.cpp | 42 +- 65 files changed, 1175 insertions(+), 1186 deletions(-) rename src/server/sparkd/cmds/{CMECertCmd.cpp => CMECert.cpp} (98%) create mode 100644 src/server/sparkd/cmds/CMECert.h delete mode 100644 src/server/sparkd/cmds/CMECertCmd.h delete mode 100644 src/server/sparkd/cmds/CmdRegistry.cpp delete mode 100644 src/server/sparkd/cmds/CmdRegistry.h delete mode 100644 src/server/sparkd/cmds/Help.cpp delete mode 100644 src/server/sparkd/cmds/Help.h delete mode 100644 src/server/sparkd/cmds/ICmd.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/BookStats.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/BookStats.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/CSV.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/CSV.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/Compress.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/Compress.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/FindPolys.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/Icebergs.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/Json.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/Json.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/ListSec.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/ListSec.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/SQL.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/SQL.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/Sim.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/Sim.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/Stats.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/Stats.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/ValuePolys.h diff --git a/src/server/sparkd/cmds/AdminReport.cpp b/src/server/sparkd/cmds/AdminReport.cpp index eab98c01b..ce9fa83c7 100644 --- a/src/server/sparkd/cmds/AdminReport.cpp +++ b/src/server/sparkd/cmds/AdminReport.cpp @@ -14,23 +14,13 @@ #include #include -namespace bts { -namespace cmd { +namespace bts::cmds { -int AdminReport::execute(const std::vector &args) { +void md_report() { auto cfg = config::get(); cfg.mutable_console()->set_default_level("Fatal"); log::configure_logging(cfg); - if (args[0] == "md") - return md_report(); - else { - std::cout << std::endl << std::endl << help() << std::endl << std::endl; - return -1; - } -} - -int AdminReport::md_report() { std::map mic_map; mic_map[api::CMEMics::CME] = "XCME"; mic_map[api::CMEMics::COMEX] = "XCEC"; @@ -93,13 +83,12 @@ int AdminReport::md_report() { } } } - return 0; } -std::string AdminReport::help() { - return "\t sparkd adminreport [md] \n" - "\t Write a market data entitlement report csv to stdout \n"; +void AdminReport::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand( + "admin-report", "Write a market data entitlement report csv to stdout"); + cmd->add_subcommand("md")->callback(md_report); } -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/AdminReport.h b/src/server/sparkd/cmds/AdminReport.h index 37ca912f6..54cb00bc2 100644 --- a/src/server/sparkd/cmds/AdminReport.h +++ b/src/server/sparkd/cmds/AdminReport.h @@ -1,16 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class AdminReport : public ICmd { - public: - int execute(std::vector const &args); - int md_report(); - std::string help(); +struct AdminReport { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/AutoCert.cpp b/src/server/sparkd/cmds/AutoCert.cpp index 015d87991..7aed1e700 100644 --- a/src/server/sparkd/cmds/AutoCert.cpp +++ b/src/server/sparkd/cmds/AutoCert.cpp @@ -1,6 +1,5 @@ #include "AutoCert.h" #include "lib/utils/PBUtils.h" -#include "server/sparkd/cmds/CmdRegistry.h" #include "lib/jetfix/common/InMsgBuffer.h" #include "lib/utils/bts_chrono.h" @@ -16,8 +15,7 @@ #include #include -namespace bts { -namespace cmd { +namespace bts::cmds { unsigned int their_sn; unsigned int last_received_sn; @@ -1275,13 +1273,13 @@ void iLinkCGWMidWeekKeyRotation() { } } -int AutoCert::execute(std::vector const &) { - CGWFailoverFromPrimaryToBackupComponents(); - return 0; +static void execute() { CGWFailoverFromPrimaryToBackupComponents(); } + +void AutoCert::register_cmd(CLI::App &app) { + app.add_subcommand( + "auto-cert", + "Provide control over JetFIX so that we can go through AutoCert") + ->callback(execute); } -std::string AutoCert::help() { - return "Provide control over JetFIX so that we can go through AutoCert"; -} -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/AutoCert.h b/src/server/sparkd/cmds/AutoCert.h index 81a80d7ea..5f4b6e8e5 100644 --- a/src/server/sparkd/cmds/AutoCert.h +++ b/src/server/sparkd/cmds/AutoCert.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class AutoCert : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct AutoCert { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Benchmark.cpp b/src/server/sparkd/cmds/Benchmark.cpp index 971d4acb8..b5985aa12 100644 --- a/src/server/sparkd/cmds/Benchmark.cpp +++ b/src/server/sparkd/cmds/Benchmark.cpp @@ -15,7 +15,6 @@ #include "lib/utils/TimeStamp.h" #include "lib/utils/Uuid.h" #include "lib/utils/mpmc_bounded_queue.h" -#include "server/sparkd/cmds/CmdRegistry.h" #include "lib/tracing/PerfLog.h" #include @@ -42,63 +41,63 @@ #include #include #include -namespace bts { -namespace cmd { + +namespace bts::cmds { using namespace orderman; -static void test_date_conversion(benchmark::State &state) { +static void test_date_conversion(::benchmark::State &state) { auto lldt = datetime::ll_now(); for (auto _ : state) { - benchmark::DoNotOptimize(datetime::utc_ll_to_timepoint(lldt)); + ::benchmark::DoNotOptimize(datetime::utc_ll_to_timepoint(lldt)); } } BENCHMARK(test_date_conversion); -static void gen_id(benchmark::State &state) { +static void gen_id(::benchmark::State &state) { std::string scope = "benchmark"; for (auto _ : state) { auto id = utils::reserve_ids(scope, 1); - benchmark::DoNotOptimize(id + 1 > 0); + ::benchmark::DoNotOptimize(id + 1 > 0); } } BENCHMARK(gen_id); -static void mutex_lock_unlock(benchmark::State &state) { +static void mutex_lock_unlock(::benchmark::State &state) { int i = 0; std::mutex lock; for (auto _ : state) { std::lock_guard l(lock); - benchmark::DoNotOptimize(i++); + ::benchmark::DoNotOptimize(i++); } } BENCHMARK(mutex_lock_unlock); -static void spin_lock_unlock(benchmark::State &state) { +static void spin_lock_unlock(::benchmark::State &state) { int i = 0; std::atomic_flag _mutex = ATOMIC_FLAG_INIT; for (auto _ : state) { while (_mutex.test_and_set(std::memory_order_acquire)) ; - benchmark::DoNotOptimize(i++); + ::benchmark::DoNotOptimize(i++); _mutex.clear(std::memory_order_release); } } BENCHMARK(spin_lock_unlock); -static void bts_spinlock_lock_unlock(benchmark::State &state) { +static void bts_spinlock_lock_unlock(::benchmark::State &state) { int i = 0; utils::SpinLock l; for (auto _ : state) { std::lock_guard g(l); - benchmark::DoNotOptimize(i++); + ::benchmark::DoNotOptimize(i++); } } BENCHMARK(bts_spinlock_lock_unlock); -static void serialize_update(benchmark::State &state) { +static void serialize_update(::benchmark::State &state) { api::orderman::Update update; update.set_order_key(1); update.set_price(1.2); @@ -121,7 +120,7 @@ struct IntId { uint32_t value; }; -static void binary_search_of_guid(benchmark::State &state) { +static void binary_search_of_guid(::benchmark::State &state) { using Id = utils::Uuid; using Map = boost::container::flat_map; auto sz = state.range(0); @@ -136,12 +135,12 @@ static void binary_search_of_guid(benchmark::State &state) { auto it = m.find(target); if (it != m.end()) found_count++; - benchmark::DoNotOptimize(found_count); + ::benchmark::DoNotOptimize(found_count); } } BENCHMARK(binary_search_of_guid)->Range(8, 8 << 10); -static void binary_search_of_int64(benchmark::State &state) { +static void binary_search_of_int64(::benchmark::State &state) { using Id = IntId; using Map = boost::container::flat_map; auto sz = state.range(0); @@ -156,12 +155,12 @@ static void binary_search_of_int64(benchmark::State &state) { auto it = m.find(target); if (it != m.end()) found_count++; - benchmark::DoNotOptimize(found_count); + ::benchmark::DoNotOptimize(found_count); } } BENCHMARK(binary_search_of_int64)->Range(8, 8 << 10); -static void binary_search_of_int32(benchmark::State &state) { +static void binary_search_of_int32(::benchmark::State &state) { using Id = IntId; using Map = boost::container::flat_map; auto sz = state.range(0); @@ -176,14 +175,14 @@ static void binary_search_of_int32(benchmark::State &state) { auto it = m.find(target); if (it != m.end()) found_count++; - benchmark::DoNotOptimize(found_count); + ::benchmark::DoNotOptimize(found_count); } } BENCHMARK(binary_search_of_int32)->Range(8, 8 << 10); -static void guid_gen(benchmark::State &state) { +static void guid_gen(::benchmark::State &state) { for (auto _ : state) { - benchmark::DoNotOptimize(utils::Uuid()); + ::benchmark::DoNotOptimize(utils::Uuid()); } } BENCHMARK(guid_gen); @@ -192,56 +191,56 @@ struct A { int x; }; -static void shared_ptr_cpy(benchmark::State &state) { +static void shared_ptr_cpy(::benchmark::State &state) { auto a = std::make_shared(); for (auto _ : state) { std::shared_ptr b; - benchmark::DoNotOptimize(b = a); + ::benchmark::DoNotOptimize(b = a); } } BENCHMARK(shared_ptr_cpy); -static void alloc_free(benchmark::State &state) { +static void alloc_free(::benchmark::State &state) { for (auto _ : state) { auto a = std::make_shared(); - benchmark::DoNotOptimize(a.get()); + ::benchmark::DoNotOptimize(a.get()); } } BENCHMARK(alloc_free); -static void new_delete(benchmark::State &state) { +static void new_delete(::benchmark::State &state) { for (auto _ : state) { auto a = new A(); delete a; - benchmark::DoNotOptimize(a); + ::benchmark::DoNotOptimize(a); } } BENCHMARK(new_delete); -static void boost_pool(benchmark::State &state) { +static void boost_pool(::benchmark::State &state) { boost::pool<> p(sizeof(A)); for (auto _ : state) { auto a = p.malloc(); p.free(a); - benchmark::DoNotOptimize(a); + ::benchmark::DoNotOptimize(a); } } BENCHMARK(boost_pool); -static void boost_object_pool(benchmark::State &state) { +static void boost_object_pool(::benchmark::State &state) { boost::object_pool p; for (auto _ : state) { auto a = p.malloc(); p.free(a); - benchmark::DoNotOptimize(a); + ::benchmark::DoNotOptimize(a); } } BENCHMARK(boost_object_pool); -static void protected_boost_pool(benchmark::State &state) { +static void protected_boost_pool(::benchmark::State &state) { boost::pool<> p(sizeof(A)); std::atomic_flag _mutex = ATOMIC_FLAG_INIT; for (auto _ : state) { @@ -253,31 +252,31 @@ static void protected_boost_pool(benchmark::State &state) { ; p.free(a); _mutex.clear(std::memory_order_release); - benchmark::DoNotOptimize(a); + ::benchmark::DoNotOptimize(a); } } BENCHMARK(protected_boost_pool); -static void boost_singleton_pool(benchmark::State &state) { +static void boost_singleton_pool(::benchmark::State &state) { using Pool = boost::singleton_pool; for (auto _ : state) { auto a = Pool::malloc(); Pool::free(a); - benchmark::DoNotOptimize(a); + ::benchmark::DoNotOptimize(a); } } BENCHMARK(boost_singleton_pool); -static void atomic_inc(benchmark::State &state) { +static void atomic_inc(::benchmark::State &state) { std::atomic i(0); for (auto _ : state) { - benchmark::DoNotOptimize(i++); + ::benchmark::DoNotOptimize(i++); } } BENCHMARK(atomic_inc); -static void mem_copy(benchmark::State &state) { +static void mem_copy(::benchmark::State &state) { char *src = new char[state.range(0)]; char *dst = new char[state.range(0)]; memset(src, 'x', state.range(0)); @@ -291,39 +290,41 @@ static void mem_copy(benchmark::State &state) { BENCHMARK(mem_copy)->Arg(8)->Arg(64)->Arg(128)->Arg(512)->Arg(1 << 10)->Arg( 8 << 10); -static void alloc_and_copy_from_pool(benchmark::State &state) { +static void alloc_and_copy_from_pool(::benchmark::State &state) { boost::pool<> p(sizeof(A)); for (auto _ : state) { auto a = new (p.malloc()) A(); // NOLINT a->~A(); // NOLINT p.free(a); - benchmark::DoNotOptimize(a); + ::benchmark::DoNotOptimize(a); } } BENCHMARK(alloc_and_copy_from_pool); -static void lambda_copy(benchmark::State &state) { +static void lambda_copy(::benchmark::State &state) { auto p = std::make_shared(); std::function f = [p]() { return 1; }; std::array, 1> buffer; for (auto _ : state) { - benchmark::DoNotOptimize(buffer[0] = f); + ::benchmark::DoNotOptimize(buffer[0] = f); } } BENCHMARK(lambda_copy); -static void boost_pool_allocator(benchmark::State &state) { +static void boost_pool_allocator(::benchmark::State &state) { boost::fast_pool_allocator p; for (auto _ : state) { auto ptr = std::allocate_shared(p); - benchmark::DoNotOptimize(ptr.get()); + ::benchmark::DoNotOptimize(ptr.get()); } } BENCHMARK(boost_pool_allocator); -int Benchmark::execute(std::vector const &args) { +static std::vector args; + +static void execute() { bts_assert(args.size() < 99); int argc = args.size() + 1; @@ -332,52 +333,51 @@ int Benchmark::execute(std::vector const &args) { for (size_t i = 0; i < args.size(); i++) argv[i + 1] = (char *)args[i].c_str(); - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); - return 0; + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); } -static void boost_lockless_queue(benchmark::State &state) { +static void boost_lockless_queue(::benchmark::State &state) { using FreeList = boost::lockfree::queue; FreeList queue(1000000); for (auto _ : state) { void *result; queue.push(nullptr); - benchmark::DoNotOptimize(queue.pop(result)); + ::benchmark::DoNotOptimize(queue.pop(result)); } } BENCHMARK(boost_lockless_queue); -static void tbb_concurrent_queue(benchmark::State &state) { +static void tbb_concurrent_queue(::benchmark::State &state) { using FreeList = tbb::concurrent_queue; FreeList queue; for (auto _ : state) { void *result; queue.push(nullptr); - benchmark::DoNotOptimize(queue.try_pop(result)); + ::benchmark::DoNotOptimize(queue.try_pop(result)); } } BENCHMARK(tbb_concurrent_queue); -static void boost_spsc(benchmark::State &state) { +static void boost_spsc(::benchmark::State &state) { boost::lockfree::spsc_queue q(10000); int i = 0; for (auto _ : state) { q.push(++i); - benchmark::DoNotOptimize(q.pop(i)); + ::benchmark::DoNotOptimize(q.pop(i)); } } BENCHMARK(boost_spsc); -static void boost_lockless_stack(benchmark::State &state) { +static void boost_lockless_stack(::benchmark::State &state) { using FreeList = boost::lockfree::stack; FreeList stack; for (auto _ : state) { void *result; stack.push(nullptr); - benchmark::DoNotOptimize(stack.pop(result)); + ::benchmark::DoNotOptimize(stack.pop(result)); } } BENCHMARK(boost_lockless_stack); @@ -407,44 +407,44 @@ struct PPQueue { } }; -static void get_thread_id(benchmark::State &state) { +static void get_thread_id(::benchmark::State &state) { for (auto _ : state) { auto tid = std::this_thread::get_id(); - benchmark::DoNotOptimize(tid == std::this_thread::get_id()); + ::benchmark::DoNotOptimize(tid == std::this_thread::get_id()); } } BENCHMARK(get_thread_id); -static void native_order_fix_data_create(benchmark::State &state) { +static void native_order_fix_data_create(::benchmark::State &state) { auto sz = state.range(0); for (auto _ : state) { auto p = new uint8_t[sz]; delete[] p; - benchmark::DoNotOptimize(p); + ::benchmark::DoNotOptimize(p); } } BENCHMARK(native_order_fix_data_create)->Range(8, 8 << 14); -static void native_order_fix_data_fragment(benchmark::State &state) { +static void native_order_fix_data_fragment(::benchmark::State &state) { for (auto _ : state) { auto p = new jetfix::FIXFragment(); delete p; - benchmark::DoNotOptimize(p); + ::benchmark::DoNotOptimize(p); } } BENCHMARK(native_order_fix_data_fragment); -static void small_vec(benchmark::State &state) { +static void small_vec(::benchmark::State &state) { auto i = 0; for (auto _ : state) { boost::container::small_vector vec; vec.emplace_back(i++); - benchmark::DoNotOptimize(vec[0]); + ::benchmark::DoNotOptimize(vec[0]); } } BENCHMARK(small_vec); -static void inter_thread_spsc(benchmark::State &state) { +static void inter_thread_spsc(::benchmark::State &state) { boost::lockfree::spsc_queue q1(10); boost::lockfree::spsc_queue q2(10); @@ -479,7 +479,7 @@ static void inter_thread_spsc(benchmark::State &state) { } BENCHMARK(inter_thread_spsc); -static void inter_thread_mpmc(benchmark::State &state) { +static void inter_thread_mpmc(::benchmark::State &state) { using Queue = boost::lockfree::queue; Queue q1(100), q2(100); @@ -514,7 +514,7 @@ static void inter_thread_mpmc(benchmark::State &state) { } BENCHMARK(inter_thread_mpmc); -static void inter_thread_mpmc_spdlog(benchmark::State &state) { +static void inter_thread_mpmc_spdlog(::benchmark::State &state) { using Queue = spdlog::details::mpmc_bounded_queue; Queue q1(8), q2(8); @@ -549,7 +549,7 @@ static void inter_thread_mpmc_spdlog(benchmark::State &state) { } BENCHMARK(inter_thread_mpmc_spdlog); -static void same_thread_mpmc_spdlog(benchmark::State &state) { +static void same_thread_mpmc_spdlog(::benchmark::State &state) { using Queue = spdlog::details::mpmc_bounded_queue; Queue q1(8); int i = 0; @@ -561,26 +561,26 @@ static void same_thread_mpmc_spdlog(benchmark::State &state) { } BENCHMARK(same_thread_mpmc_spdlog); -static void moodycamel_concurrent_queue(benchmark::State &state) { +static void moodycamel_concurrent_queue(::benchmark::State &state) { using Q = moodycamel::ConcurrentQueue; Q queue(8); int i = 0; for (auto _ : state) { queue.enqueue(++i); // NOLINT - benchmark::DoNotOptimize(queue.try_dequeue(i)); + ::benchmark::DoNotOptimize(queue.try_dequeue(i)); } } BENCHMARK(moodycamel_concurrent_queue); -static void moodycamel_spsc_queue(benchmark::State &state) { +static void moodycamel_spsc_queue(::benchmark::State &state) { using Queue = moodycamel::ReaderWriterQueue; Queue q1(8); int i = 0; for (auto _ : state) { q1.enqueue(++i); - benchmark::DoNotOptimize(q1.try_dequeue(i)); + ::benchmark::DoNotOptimize(q1.try_dequeue(i)); } } BENCHMARK(moodycamel_spsc_queue); @@ -596,28 +596,28 @@ struct MyDirect { int inc(int &x) { return x += 1; }; }; -static void derived_call(benchmark::State &state) { +static void derived_call(::benchmark::State &state) { int x = 0; MyDerived d; MyBase &b = d; for (auto _ : state) { - benchmark::DoNotOptimize(b.inc(x)); + ::benchmark::DoNotOptimize(b.inc(x)); } } BENCHMARK(derived_call); -static void direct_call(benchmark::State &state) { +static void direct_call(::benchmark::State &state) { int x = 0; MyDerived b; for (auto _ : state) { - benchmark::DoNotOptimize(b.inc(x)); + ::benchmark::DoNotOptimize(b.inc(x)); } } BENCHMARK(direct_call); -static void get_time_of_day(benchmark::State &state) { +static void get_time_of_day(::benchmark::State &state) { std::string result(strlen("20121204-14:44:02.999"), ' '); struct timeval tv; for (auto _ : state) { @@ -630,7 +630,7 @@ static void get_time_of_day(benchmark::State &state) { } BENCHMARK(get_time_of_day); -static void fix_time_stamp(benchmark::State &state) { +static void fix_time_stamp(::benchmark::State &state) { std::string r(128, 32); auto now = datetime::now(); for (auto _ : state) { @@ -639,7 +639,7 @@ static void fix_time_stamp(benchmark::State &state) { } BENCHMARK(fix_time_stamp); -static void perf_log(benchmark::State &state) { +static void perf_log(::benchmark::State &state) { trace::PerfLog l; for (auto _ : state) { l.log(trace::CorID{"sim", 0}, "this", 1, "txt", 3); @@ -647,7 +647,7 @@ static void perf_log(benchmark::State &state) { } BENCHMARK(perf_log); -void int_price_less(benchmark::State &state) { +void int_price_less(::benchmark::State &state) { for (auto _ : state) { state.PauseTiming(); int64_t count = state.range(0); @@ -662,7 +662,7 @@ void int_price_less(benchmark::State &state) { } BENCHMARK(int_price_less)->Range(8, 8 << 11); -void set_hashset_insert(benchmark::State &state) { +void set_hashset_insert(::benchmark::State &state) { std::unordered_set s; size_t x = 0; for (auto _ : state) { @@ -670,7 +670,7 @@ void set_hashset_insert(benchmark::State &state) { s.insert(x); } } -void set_stdset_insert(benchmark::State &state) { +void set_stdset_insert(::benchmark::State &state) { std::set s; size_t x = 0; for (auto _ : state) { @@ -678,7 +678,7 @@ void set_stdset_insert(benchmark::State &state) { s.insert(x); } } -void set_btreeset_insert(benchmark::State &state) { +void set_btreeset_insert(::benchmark::State &state) { btree::btree_set s; size_t x = 0; for (auto _ : state) { @@ -687,36 +687,36 @@ void set_btreeset_insert(benchmark::State &state) { } } -void set_hashset_iterate(benchmark::State &state) { +void set_hashset_iterate(::benchmark::State &state) { std::unordered_set s; size_t x = 0; for (auto _ : state) { x = std::rand(); s.insert(x); for (const auto &p : s) { - benchmark::DoNotOptimize(p); + ::benchmark::DoNotOptimize(p); } } } -void set_stdset_iterate(benchmark::State &state) { +void set_stdset_iterate(::benchmark::State &state) { std::set s; size_t x = 0; for (auto _ : state) { x = std::rand(); s.insert(x); for (const auto &p : s) { - benchmark::DoNotOptimize(p); + ::benchmark::DoNotOptimize(p); } } } -void set_btreeset_iterate(benchmark::State &state) { +void set_btreeset_iterate(::benchmark::State &state) { btree::btree_set s; size_t x = 0; for (auto _ : state) { x = std::rand(); s.insert(x); for (const auto &p : s) { - benchmark::DoNotOptimize(p); + ::benchmark::DoNotOptimize(p); } } } @@ -732,7 +732,7 @@ struct Chunk { size_t a, b, c, d; }; -void tbb_concurrent_vector_insert(benchmark::State &state) { +void tbb_concurrent_vector_insert(::benchmark::State &state) { tbb::concurrent_vector vec; size_t x = 0; for (auto _ : state) { @@ -740,32 +740,32 @@ void tbb_concurrent_vector_insert(benchmark::State &state) { } } BENCHMARK(tbb_concurrent_vector_insert); -void tbb_concurrent_vector_iterate(benchmark::State &state) { +void tbb_concurrent_vector_iterate(::benchmark::State &state) { tbb::concurrent_vector vec; for (size_t x = 0; x < 9999; ++x) { vec.push_back({++x, ++x, ++x, ++x}); } for (auto _ : state) { for (const auto &v : vec) { - benchmark::DoNotOptimize(v); + ::benchmark::DoNotOptimize(v); } } } BENCHMARK(tbb_concurrent_vector_iterate); -void tbb_concurrent_vector_index(benchmark::State &state) { +void tbb_concurrent_vector_index(::benchmark::State &state) { tbb::concurrent_vector vec; for (size_t x = 0; x < 9999; ++x) { vec.push_back({++x, ++x, ++x, ++x}); } for (auto _ : state) { for (size_t x = 0; x < vec.size(); ++x) { - benchmark::DoNotOptimize(vec[x]); + ::benchmark::DoNotOptimize(vec[x]); } } } BENCHMARK(tbb_concurrent_vector_index); -void concurrent_append_collection_insert(benchmark::State &state) { +void concurrent_append_collection_insert(::benchmark::State &state) { ConcurrentAppendCollection vec; size_t x = 0; for (auto _ : state) { @@ -773,34 +773,37 @@ void concurrent_append_collection_insert(benchmark::State &state) { } } BENCHMARK(concurrent_append_collection_insert); -void concurrent_append_collection_iterate(benchmark::State &state) { +void concurrent_append_collection_iterate(::benchmark::State &state) { ConcurrentAppendCollection vec; for (size_t x = 0; x < 9999; ++x) { vec.push_back({++x, ++x, ++x, ++x}); } for (auto _ : state) { for (const auto &v : vec) { - benchmark::DoNotOptimize(v); + ::benchmark::DoNotOptimize(v); } } } BENCHMARK(concurrent_append_collection_iterate); -void concurrent_append_collection_index(benchmark::State &state) { +void concurrent_append_collection_index(::benchmark::State &state) { ConcurrentAppendCollection vec; for (size_t x = 0; x < 9999; ++x) { vec.push_back({++x, ++x, ++x, ++x}); } for (auto _ : state) { for (size_t x = 0; x < vec.size(); ++x) { - benchmark::DoNotOptimize(vec[x]); + ::benchmark::DoNotOptimize(vec[x]); } } } BENCHMARK(concurrent_append_collection_index); -std::string Benchmark::help() { - return "Run micro-benchmarks. Use --benchmark_filter=regex to run only " - "specific benchmarks."; +void Benchmark::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand( + "benchmark", "Run micro-benchmarks. Use --benchmark_filter=regex to run " + "only specific benchmarks."); + cmd->add_option("args", args); + cmd->callback(execute); } -} // namespace cmd -} // namespace bts + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Benchmark.h b/src/server/sparkd/cmds/Benchmark.h index e9d0d5583..497637a50 100644 --- a/src/server/sparkd/cmds/Benchmark.h +++ b/src/server/sparkd/cmds/Benchmark.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Benchmark : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct Benchmark { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/CMECertCmd.cpp b/src/server/sparkd/cmds/CMECert.cpp similarity index 98% rename from src/server/sparkd/cmds/CMECertCmd.cpp rename to src/server/sparkd/cmds/CMECert.cpp index 6db25c014..f8f335715 100644 --- a/src/server/sparkd/cmds/CMECertCmd.cpp +++ b/src/server/sparkd/cmds/CMECert.cpp @@ -1,4 +1,4 @@ -#include "CMECertCmd.h" +#include "CMECert.h" #include "lib/logging/logging.h" #include "lib/orderman/actors/ilink3/IL3Dispatcher.h" @@ -17,8 +17,7 @@ #include #include -namespace bts { -namespace cmd { +namespace bts::cmds { using namespace orderman::ilink3; using namespace api::orderman; @@ -566,7 +565,7 @@ struct TestRunner { } }; -int CMECert::execute(const std::vector &) { +static void execute() { // Typically a timestamp SessionSaver ss; ss.load(); @@ -605,10 +604,10 @@ int CMECert::execute(const std::vector &) { } gw.send_terminate(); gw.flush(); - return 0; } -std::string CMECert::help() { return "Tool to drive CME Cert"; } +void CMECert::register_cmd(CLI::App &app) { + app.add_subcommand("cme-cert", "Tool to drive CME Cert")->callback(execute); +} -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/CMECert.h b/src/server/sparkd/cmds/CMECert.h new file mode 100644 index 000000000..8e9c8a045 --- /dev/null +++ b/src/server/sparkd/cmds/CMECert.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::cmds { + +struct CMECert { + void register_cmd(CLI::App &); +}; + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/CMECertCmd.h b/src/server/sparkd/cmds/CMECertCmd.h deleted file mode 100644 index 54c96452a..000000000 --- a/src/server/sparkd/cmds/CMECertCmd.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "ICmd.h" - -namespace bts { -namespace cmd { - -class CMECert : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); -}; - -} // namespace cmd -} // namespace bts diff --git a/src/server/sparkd/cmds/CmdRegistry.cpp b/src/server/sparkd/cmds/CmdRegistry.cpp deleted file mode 100644 index ae9fbfda8..000000000 --- a/src/server/sparkd/cmds/CmdRegistry.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include "CmdRegistry.h" - -#include "ICmd.h" - -#include -#include -#include -#include -#include - -namespace bts { -namespace cmd { - -std::map> CmdRegistry::_cmds; - -int CmdRegistry::dispatch(int argc, char **argv) { - if (argc < 2) - return _cmds["help"]->execute(std::vector()); - else { - std::vector args; - for (int i = 1; i < argc; i++) { - std::string arg(argv[i]); - if (boost::starts_with(boost::to_lower_copy(arg), "-configserver")) { - // skip configserver flag as well as url - i++; - } else - args.push_back(arg); - } - - ICmd *cmd = retrieve(std::string(args[0])); - if (cmd == 0) { - return _cmds["help"]->execute(std::vector()); - } else { - args.erase(args.begin()); - return cmd->execute(args); - } - } -} - -ICmd *CmdRegistry::retrieve(std::string const &name) { - std::string lname(name); - boost::to_lower(lname); - if (_cmds.find(lname) == _cmds.end()) - return 0; - else - return _cmds[lname].get(); -} - -std::vector CmdRegistry::cmds() { - std::vector result; - for (auto &p : _cmds) - result.push_back(p.first); - return result; -} - -} // namespace cmd -} // namespace bts diff --git a/src/server/sparkd/cmds/CmdRegistry.h b/src/server/sparkd/cmds/CmdRegistry.h deleted file mode 100644 index ec169f158..000000000 --- a/src/server/sparkd/cmds/CmdRegistry.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include "lib/utils/bts_assert.h" -#include -#include -#include -#include -#include - -#define BEGIN_CMD -#define REGISTER(CMD) bts::cmd::CmdRegistry::register_cmd(#CMD) -#define END_CMD return bts::cmd::CmdRegistry::dispatch(argc, argv) - -namespace bts { -namespace cmd { - -class ICmd; - -class CmdRegistry { - public: - template static void register_cmd(std::string const &name) { - std::string lname = name; - boost::to_lower(lname); - bts_assert(_cmds.find(lname) == _cmds.end()); - _cmds[lname].reset(new CMD()); - } - static std::vector cmds(); - static int dispatch(int argc, char **argv); - static ICmd *retrieve(std::string const &name); - - private: - static std::map> _cmds; -}; - -} // namespace cmd -} // namespace bts diff --git a/src/server/sparkd/cmds/ConfigCheck.cpp b/src/server/sparkd/cmds/ConfigCheck.cpp index 7e3c1c13f..78be083d0 100644 --- a/src/server/sparkd/cmds/ConfigCheck.cpp +++ b/src/server/sparkd/cmds/ConfigCheck.cpp @@ -1,18 +1,21 @@ #include "ConfigCheck.h" #include "lib/config/ConfigCheck.h" -namespace bts { -namespace cmd { +namespace bts::cmds { -int ConfigCheck::execute(std::vector const &args) { +static std::vector args; + +static void execute() { std::vector ch_args; for (auto const &s : args) ch_args.push_back(s.c_str()); configcheck(ch_args.size(), &ch_args[0]); - return 0; } -std::string ConfigCheck::help() { return "Config check help."; } +void ConfigCheck::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand("config-check"); + cmd->add_option("args", args); + cmd->callback(execute); +} -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ConfigCheck.h b/src/server/sparkd/cmds/ConfigCheck.h index 956719cfd..f5d4296c7 100644 --- a/src/server/sparkd/cmds/ConfigCheck.h +++ b/src/server/sparkd/cmds/ConfigCheck.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class ConfigCheck : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct ConfigCheck { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/FIXSim.cpp b/src/server/sparkd/cmds/FIXSim.cpp index adcb61cf9..4dd9d5063 100644 --- a/src/server/sparkd/cmds/FIXSim.cpp +++ b/src/server/sparkd/cmds/FIXSim.cpp @@ -1,17 +1,15 @@ #include "server/sparkd/cmds/FIXSim.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" -namespace bts { -namespace cmd { +namespace bts::cmds { -int FIXSim::execute(std::vector const &) { +static void execute() { bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM); - return 0; } -std::string FIXSim::help() { - return "FIXSim - Starts a standalone FIX simulator "; +void FIXSim::register_cmd(CLI::App &app) { + app.add_subcommand("fix-sim", "Starts a standalone FIX simulator") + ->callback(execute); } -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/FIXSim.h b/src/server/sparkd/cmds/FIXSim.h index c368594b0..76a37d354 100644 --- a/src/server/sparkd/cmds/FIXSim.h +++ b/src/server/sparkd/cmds/FIXSim.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class FIXSim : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct FIXSim { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Help.cpp b/src/server/sparkd/cmds/Help.cpp deleted file mode 100644 index 615de4b0c..000000000 --- a/src/server/sparkd/cmds/Help.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "Help.h" -#include "server/sparkd/cmds/CmdRegistry.h" -#include - -namespace bts { -namespace cmd { - -int Help::execute(std::vector const &args) { - auto commands = CmdRegistry::cmds(); - if (args.size() != 1 || CmdRegistry::retrieve(args[0]) == 0) { - std::cerr << "Syntax: sparkd or sparkd help , where " - "cmd is one of: " - << std::endl; - - for (auto &cmd : commands) - std::cerr << "\t" << cmd << std::endl; - return 0; - } else { - auto cmd = CmdRegistry::retrieve(args[0]); - std::cerr << "sparkd " << args[0] << ":" << std::endl - << cmd->help() << std::endl; - return 0; - } -} - -std::string Help::help() { return "Display help."; } - -} // namespace cmd -} // namespace bts diff --git a/src/server/sparkd/cmds/Help.h b/src/server/sparkd/cmds/Help.h deleted file mode 100644 index a88bfbd57..000000000 --- a/src/server/sparkd/cmds/Help.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "ICmd.h" - -namespace bts { -namespace cmd { - -class Help : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); -}; - -} // namespace cmd -} // namespace bts diff --git a/src/server/sparkd/cmds/ICmd.h b/src/server/sparkd/cmds/ICmd.h deleted file mode 100644 index b58f09e9c..000000000 --- a/src/server/sparkd/cmds/ICmd.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -namespace bts { -namespace cmd { - -class ICmd { - public: - virtual int execute(std::vector const &args) = 0; - virtual std::string help() = 0; - virtual ~ICmd() {} -}; - -} // namespace cmd -} // namespace bts diff --git a/src/server/sparkd/cmds/KeyCode.cpp b/src/server/sparkd/cmds/KeyCode.cpp index b2d92d179..0ad59dd7b 100644 --- a/src/server/sparkd/cmds/KeyCode.cpp +++ b/src/server/sparkd/cmds/KeyCode.cpp @@ -35,8 +35,7 @@ #include #include -namespace bts { -namespace cmd { +namespace bts::cmds { using namespace std; using namespace datetime; @@ -44,130 +43,58 @@ using namespace bts::api::auth; using namespace bts::crypto; using namespace bts::config; -std::string KeyCode::helpstr() { - std::ostringstream oss; - oss << "\n" - << "ENCODE -keyfile:[keyfile] -pass:password -users: " - "-expiration:<2020-05-18> [-ip:] [-port:] " - "[-allow_concurrent]\n " - << "DECODE -keyfile:[keyfile] -pass:password -source:[file] \n " - << "KEYGEN -keyfile:[keyfile_prefix]\n " - << "CHECKIPS \n "; +struct EncodeArgs { + std::string keyfile; + std::string pass; + std::string users; + std::string expiration; + std::string source; + std::string ip; + uint16_t port; + bool allow_concurrent; +}; - return oss.str(); -} +struct DecryptArgs { + std::string keyfile; + std::string pass; + std::string users; + std::string expiration; + std::string source; +}; -int checkips() { - auto ipmap = bts::os::host_ips(); - for (const auto &x : ipmap) - std::cout << x.first << " => " << x.second << std::endl; - return 0; -} +struct KeygenArgs { + std::string keyfile; + std::string pass; + std::string users; + std::string expiration; +}; + +struct Args { + EncodeArgs encrypt; + DecryptArgs decrypt; + KeygenArgs keygen; +}; std::string parse_date(std::string const &date) { auto the_date = bts::datetime::parse_iso_date_time_string(date); return bts::datetime::iso_date_time_string(the_date); } -void get_cmd( - const std::vector &args, - std::string &keyfile, - std::string &pass, - std::string &users, - std::string &expiration, - std::string *source = nullptr, - std::string *ip = nullptr, - std::string *port = nullptr, - bool *allow_concurrent = nullptr) { - for (size_t i = 1; i < args.size(); i++) { - size_t pass_pos = args[i].find("-pass:"); - if (pass_pos != args[i].npos) - pass = args[i].substr(pass_pos + 6); +void eraseAllSubStr(std::string &mainStr, const std::string &toErase) { + size_t pos = std::string::npos; - size_t file_pos = args[i].find("-keyfile:"); - if (file_pos != args[i].npos) - keyfile = args[i].substr(file_pos + 9); - - size_t u_pos = args[i].find("-users:"); - if (u_pos != args[i].npos) - users = args[i].substr(u_pos + 7); - - size_t e_pos = args[i].find("-expiration:"); - if (e_pos != args[i].npos) - expiration = parse_date(args[i].substr(e_pos + 12)); - - size_t pay_pos = args[i].find("-source:"); - if (source && pay_pos != args[i].npos) - *source = args[i].substr(pay_pos + 8); - - size_t i_pos = args[i].find("-ip:"); - if (ip && i_pos != args[i].npos) - *ip = args[i].substr(i_pos + 4); - - size_t p_pos = args[i].find("-port:"); - if (port && p_pos != args[i].npos) - *port = args[i].substr(p_pos + 6); - - size_t c_pos = args[i].find("-allow_concurrent"); - if (allow_concurrent && c_pos != args[i].npos) - *allow_concurrent = true; + // Search for the substring in string in a loop untill nothing is found + while ((pos = mainStr.find(toErase)) != std::string::npos) { + // If found then erase it from string + mainStr.erase(pos, toErase.length()); } } -int keygen(const std::vector &args) { - std::string keyfile, pass, users, expiration; - get_cmd(args, keyfile, pass, users, expiration); - if (keyfile != "") { - PrivateKeyPtr priv = PrivateKey::create(2048, AlgoType::RSA); - - std::string n1 = (keyfile + "_private.pem"), - n2 = (keyfile + "_public.pem"); - // I can't get anything but DES to work here. - priv->write_file(n1, CipherType::DES); - priv->public_key()->write_file(n2); - - std::cout << "Key pair generated " << n1 << " and " << n2 << std::endl; - return 0; - } else { - std::cout << KeyCode::helpstr(); - return -1; - } -} - -// int Logon::digest(const std::vector& args) -// { -// std::string pass, keyfile, payload, user, expiration, extra_data; -// get_cmd(args, pass, keyfile, payload, user, expiration, extra_data); -// if(extra_data != "") -// { -// unsigned int len; -// //unsigned char* digest; -// //Digest::allocate_array(&digest, &len); -// //Digest::compute_digest(reinterpret_cast(payload.c_str()), payload.size(), digest, &len); - -// std::string digest = Digest::compute_digest(extra_data); -// len = digest.size(); -// UNUSED(len); -// std::cout << std::endl << std::endl; -// //for(size_t i = 0; i < len; i++) -// // printf("%02x", digest[i]); -// std::cout << digest; -// std::cout << std::endl << std::endl; -// return 0; -// } -// else -// { -// std::cout << help(); -// return -1; -// } -// } - void write_keycode( const std::string &users, const std::string &expiration, const std::string &ip, - const std::string &port, + uint16_t port, bool concurrent, bts::config::KeyCode &kc) { std::ostringstream out_stream; @@ -184,102 +111,71 @@ void write_keycode( ip != "127.0.0.1", "you can't generate a keycode for loopback, just " "leave the parameter blank"); kc.set_ip(ip); - kc.set_port(port); + kc.set_port(std::to_string(port)); kc.set_allow_concurrent(concurrent); } -void eraseAllSubStr(std::string &mainStr, const std::string &toErase) { - size_t pos = std::string::npos; +static Args args; - // Search for the substring in string in a loop untill nothing is found - while ((pos = mainStr.find(toErase)) != std::string::npos) { - // If found then erase it from string - mainStr.erase(pos, toErase.length()); - } +void check_ips() { + CryptoInit::initialize(); + auto ipmap = bts::os::host_ips(); + for (const auto &x : ipmap) + std::cout << x.first << " => " << x.second << std::endl; } -int encrypt( - const std::string &keyfile, - const std::string &pass, - const std::string &users, - const std::string &expiration, - const std::string &ip, - const std::string &port, - bool concurrent) { +void encrypt() { + CryptoInit::initialize(); std::string protobuf; + std::string kc_string; + std::stringstream oss, encrypted; + CachedPassword cache(args.encrypt.pass); + PrivateKeyPtr key = PrivateKey::read_file(args.encrypt.keyfile, &cache); - if (keyfile == "") { - std::cout << KeyCode::helpstr(); - return -1; - } else { - std::string kc_string; - std::stringstream oss, encrypted; - CachedPassword cache(pass); - PrivateKeyPtr key = PrivateKey::read_file(keyfile, &cache); - - bts::config::KeyCode kc; - write_keycode(users, expiration, ip, port, concurrent, kc); - if (google::protobuf::TextFormat::PrintToString(kc, &kc_string)) { - oss << kc_string; - key->encrypt(oss, encrypted); - } else - return -1; - // int ret = openssl::RSA_private_encrypt(sz-1, pay, &buffer[0], - // key->pkey.rsa, RSA_PKCS1_PADDING); - - EncryptedKeyCode ekc; - auto kcstr = encrypted.str(); - eraseAllSubStr(kcstr, "-----BEGIN BTS-----\n"); - eraseAllSubStr(kcstr, "\n-----END BTS-----\n"); - kcstr.erase(std::remove(kcstr.begin(), kcstr.end(), '\n'), kcstr.end()); - // kcstr = boost::replace_all_copy(kcstr, "\n", "\\n"); - ekc.set_keycode(kcstr); - std::string pb; - google::protobuf::TextFormat::PrintToString(ekc, &pb); - std::cout << std::endl << std::endl; - std::cout << "#nusers: " << users << std::endl; - std::cout << "#allow concurrent : " - << (kc.allow_concurrent() ? "true" : "false") << std::endl; - std::cout << "#expires: " - << datetime::simple_date_time_string( - datetime::ll_to_ptime(kc.expiration())) - << " " << std::to_string(kc.expiration()) << std::endl; - std::cout << "#generation_time: " - << datetime::simple_date_time_string( - datetime::ll_to_ptime(kc.generation_time())) - << " " << std::to_string(kc.generation_time()) << std::endl; - std::cout << "EncryptedKeyCode {" << std::endl; - std::cout << " value {" << std::endl; - std::cout << " " << pb; - std::cout << " }" << std::endl; - std::cout << "}" << std::endl; - std::cout << std::endl << std::endl; - return 0; - } + bts::config::KeyCode kc; + write_keycode( + args.encrypt.users, args.encrypt.expiration, args.encrypt.ip, + args.encrypt.port, args.encrypt.allow_concurrent, kc); + if (!google::protobuf::TextFormat::PrintToString(kc, &kc_string)) + return; + oss << kc_string; + key->encrypt(oss, encrypted); + EncryptedKeyCode ekc; + auto kcstr = encrypted.str(); + eraseAllSubStr(kcstr, "-----BEGIN BTS-----\n"); + eraseAllSubStr(kcstr, "\n-----END BTS-----\n"); + kcstr.erase(std::remove(kcstr.begin(), kcstr.end(), '\n'), kcstr.end()); + ekc.set_keycode(kcstr); + std::string pb; + google::protobuf::TextFormat::PrintToString(ekc, &pb); + std::cout << std::endl << std::endl; + std::cout << "#nusers: " << args.encrypt.users << std::endl; + std::cout << "#allow concurrent : " + << (kc.allow_concurrent() ? "true" : "false") << std::endl; + std::cout << "#expires: " + << datetime::simple_date_time_string( + datetime::ll_to_ptime(kc.expiration())) + << " " << std::to_string(kc.expiration()) << std::endl; + std::cout << "#generation_time: " + << datetime::simple_date_time_string( + datetime::ll_to_ptime(kc.generation_time())) + << " " << std::to_string(kc.generation_time()) << std::endl; + std::cout << "EncryptedKeyCode {" << std::endl; + std::cout << " value {" << std::endl; + std::cout << " " << pb; + std::cout << " }" << std::endl; + std::cout << "}" << std::endl; + std::cout << std::endl << std::endl; } -int encode(const std::vector &args) { - std::string keyfile, pass, users, expiration, ip, port; - bool concurrent = false; - get_cmd( - args, keyfile, pass, users, expiration, nullptr, &ip, &port, - &concurrent); - - return encrypt(keyfile, pass, users, expiration, ip, port, concurrent); -} - -int decode(const std::vector &args) { - std::string keyfile, pass, users, expiration, source; - get_cmd(args, keyfile, pass, users, expiration, &source); - - if (keyfile != "") { - PublicKeyPtr pub = bts::crypto::PublicKey::read_file(keyfile); - if (pub && source != "") { - // KeycodePayloadHelper helper; +void decrypt() { + CryptoInit::initialize(); + if (args.decrypt.keyfile != "") { + PublicKeyPtr pub = + bts::crypto::PublicKey::read_file(args.decrypt.keyfile); + if (pub && args.decrypt.source != "") { bts::config::EncryptedKeyCode ekc; - std::fstream fs(source); - // google::protobuf::io::IstreamInputStream pay_stream(&fs); - // google::protobuf::TextFormat::Parse(&pay_stream, &ekc); + std::fstream fs(args.decrypt.source); std::string str( (std::istreambuf_iterator(fs)), std::istreambuf_iterator()); @@ -293,39 +189,56 @@ int decode(const std::vector &args) { google::protobuf::TextFormat::PrintToString(kc, &string_out); std::cout << string_out << std::endl; } - - // int ret = openssl::RSA_public_decrypt(sz, data, &buffer[0], - // key->pkey.rsa, RSA_PKCS1_PADDING); - std::cout << std::endl << std::endl; - return 0; + std::cout << "\n\n"; } else { - std::cout << KeyCode::helpstr(); - return -1; + std::cout << "\nFailed\n"; } } -int KeyCode::execute(const std::vector &args) { +void keygen() { CryptoInit::initialize(); - if (args.size() > 0) { - if (boost::to_lower_copy(args[0]) == "keycode" || - boost::to_lower_copy(args[0]) == "encode") { - return encode(args); - } - if (boost::to_lower_copy(args[0]) == "keygen") { - return keygen(args); - } - if (boost::to_lower_copy(args[0]) == "decode") { - return decode(args); - } - if (boost::to_lower_copy(args[0]) == "checkips") { - return checkips(); - } - } - std::cout << help() << std::endl; - return -1; + PrivateKeyPtr priv = PrivateKey::create(2048, AlgoType::RSA); + + std::string n1 = (args.keygen.keyfile + "_private.pem"), + n2 = (args.keygen.keyfile + "_public.pem"); + // I can't get anything but DES to work here. + priv->write_file(n1, CipherType::DES); + priv->public_key()->write_file(n2); + + std::cout << "Key pair generated " << n1 << " and " << n2 << std::endl; } -std::string KeyCode::help() { return helpstr(); } +void KeyCode::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand("key-code"); + cmd->require_subcommand(); + // check ips + cmd->add_subcommand("check-ips")->callback(check_ips); + // decrypt + auto *decrypt_cmd = cmd->add_subcommand("decrypt"); + decrypt_cmd->add_option("--expiration", args.decrypt.expiration); + decrypt_cmd->add_option("--key-file", args.decrypt.keyfile)->required(); + decrypt_cmd->add_option("--pass", args.decrypt.pass); + decrypt_cmd->add_option("--source", args.decrypt.source); + decrypt_cmd->add_option("--users", args.decrypt.users); + decrypt_cmd->callback(decrypt); + // encrypt + auto *encrypt_cmd = cmd->add_subcommand("encrypt"); + encrypt_cmd->add_flag("--allow-concurrent", args.encrypt.allow_concurrent); + encrypt_cmd->add_option("--expiration", args.encrypt.expiration); + encrypt_cmd->add_option("--ip", args.encrypt.ip); + encrypt_cmd->add_option("--key-file", args.encrypt.keyfile)->required(); + encrypt_cmd->add_option("--pass", args.encrypt.pass); + encrypt_cmd->add_option("--port", args.encrypt.port); + encrypt_cmd->add_option("--source", args.encrypt.source); + encrypt_cmd->add_option("--users", args.encrypt.users); + encrypt_cmd->callback(encrypt); + // key-gen + auto *keygen_cmd = cmd->add_subcommand("key-gen"); + keygen_cmd->add_option("--expiration", args.keygen.expiration); + keygen_cmd->add_option("--key-file", args.keygen.keyfile)->required(); + keygen_cmd->add_option("--pass", args.keygen.pass); + keygen_cmd->add_option("--users", args.keygen.users); + keygen_cmd->callback(keygen); +} -} // namespace cmd -} // namespace bts \ No newline at end of file +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/KeyCode.h b/src/server/sparkd/cmds/KeyCode.h index 9307bfcbe..ae3aa6f43 100644 --- a/src/server/sparkd/cmds/KeyCode.h +++ b/src/server/sparkd/cmds/KeyCode.h @@ -1,29 +1,23 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::crypto { -namespace bts { -namespace crypto { class PublicKey; -} -namespace config { + +} // namespace bts::crypto + +namespace bts::config { + class EncryptedKeyCode; class KeyCode; -} // namespace config -namespace cmd { -bool decrypt( - const bts::crypto::PublicKey &public_key, - const bts::config::EncryptedKeyCode &kc, - bts::config::KeyCode *dk); -class KeyCode : public ICmd { - public: - enum class PublicPrivate { Public, Private }; - int execute(std::vector const &args); - std::string help(); +} // namespace bts::config - static std::string helpstr(); +namespace bts::cmds { + +struct KeyCode { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts \ No newline at end of file +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Md.cpp b/src/server/sparkd/cmds/Md.cpp index a6f3c583b..10a561e92 100644 --- a/src/server/sparkd/cmds/Md.cpp +++ b/src/server/sparkd/cmds/Md.cpp @@ -1,17 +1,17 @@ #include "Md.h" -#include "server/sparkd/cmds/CmdRegistry.h" #include "server/sparkd/md/run_md.h" #include -namespace bts { -namespace cmd { +namespace bts::cmds { -int Md::execute(std::vector const &args) { - bts::md::md_tool(args); - return 0; +static std::vector args; + +static void execute() { bts::md::md_tool(args); } + +void Md::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand("md"); + cmd->add_option("args", args); + cmd->callback(execute); } -std::string Md::help() { return "Md command"; } - -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Md.h b/src/server/sparkd/cmds/Md.h index 26c3aee86..9679166c1 100644 --- a/src/server/sparkd/cmds/Md.h +++ b/src/server/sparkd/cmds/Md.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Md : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct Md { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp index 1a5418e1e..c7b836a26 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp @@ -9,8 +9,7 @@ #include #include -namespace bts { -namespace cmd { +namespace bts::cmds { using namespace jetfix; @@ -98,9 +97,13 @@ typedef FIXMsg< std::string to_str(std::string const &s) { return s; } -template std::string to_str(V const &v) { return std::to_string(v); } +template +std::string to_str(V const &v) { + return std::to_string(v); +} -template std::string get_if_valid(G const &update) { +template +std::string get_if_valid(G const &update) { if (update.template has()) return to_str(update.template get()); else @@ -393,7 +396,9 @@ void print_orders( } } -int ParseHistoricalFIX::execute(std::vector const &args) { +static std::vector args; + +static void execute() { std::ifstream input(args[0]); std::ofstream books_stream(args[0] + "_books.csv"); std::ofstream msgs_stream(args[0] + "_msgs.csv"); @@ -434,13 +439,15 @@ int ParseHistoricalFIX::execute(std::vector const &args) { auto sec_duration = sec_diff(Timer::now(), start); std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok << " out of " << c << " time: " << sec_duration << " secs\n"; - return 0; } -std::string ParseHistoricalFIX::help() { - return "ParseHistoricalFIX will produce 2 files, " - "_mdentries.csv and order_entries.csv"; +void ParseHistoricalFix::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand( + "parse-historical-fix", + "ParseHistoricalFIX will produce 2 files, " + "_mdentries.csv and order_entries.csv"); + cmd->add_option("args", args); + cmd->callback(execute); } -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.h b/src/server/sparkd/cmds/ParseHistoricalFIX.h index b4da9c57c..df610647b 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.h +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class ParseHistoricalFIX : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct ParseHistoricalFix { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/PrintHedgeOrders.cpp b/src/server/sparkd/cmds/PrintHedgeOrders.cpp index 8598405fa..79c791267 100644 --- a/src/server/sparkd/cmds/PrintHedgeOrders.cpp +++ b/src/server/sparkd/cmds/PrintHedgeOrders.cpp @@ -3,10 +3,9 @@ #include "lib/persistence/SqliteStore.h" #include "lib/utils/PBUtils.h" -namespace bts { -namespace cmd { +namespace bts::cmds { -int PrintHedgeOrders::execute(std::vector const &) { +static void execute() { persistence::SqliteStore order_store("orders"); auto all_orders = order_store.retrieve_all()["orders"]; @@ -45,14 +44,14 @@ int PrintHedgeOrders::execute(std::vector const &) { << "\n"; } } - - return 0; } -std::string PrintHedgeOrders::help() { - return "Print out order ids for all hedge orders" - "which were completed at least one second being created"; +void PrintHedgeOrders::register_cmd(CLI::App &app) { + app.add_subcommand( + "print-hedge-orders", + "Print out order ids for all hedge orders" + "which were completed at least one second being created") + ->callback(execute); } -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/PrintHedgeOrders.h b/src/server/sparkd/cmds/PrintHedgeOrders.h index 2ff15a33f..bf5289e7a 100644 --- a/src/server/sparkd/cmds/PrintHedgeOrders.h +++ b/src/server/sparkd/cmds/PrintHedgeOrders.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class PrintHedgeOrders : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct PrintHedgeOrders { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index a57b19441..cee27988e 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -2,11 +2,9 @@ #include "server/sparkd/cmds/spartan/market_data/handler/ReactorMDP3FeedHandler.h" #include "server/sparkd/cmds/spartan/system/reactor/Reactor.h" -namespace bts { -namespace cmd { - -int test_md(std::vector const &) { +namespace bts::cmds { +void test_md() { using namespace bts::spartan; using namespace bts::spartan::mdp3; @@ -26,8 +24,7 @@ int test_md(std::vector const &) { void on_recovery_complete(InitialSnapshot const &) override { std::cout << "on_recovery_complete\n"; } - void on_event_end(MDP3Event const &) override { - } + void on_event_end(MDP3Event const &) override {} void on_packet_loss() override { std::cout << "on_real_time_channel_loss\n"; } @@ -38,18 +35,17 @@ int test_md(std::vector const &) { handler.start(); for (;;) reactor.do_work(); - - return 0; } -int QuickTest::execute(std::vector const &) { - return 0; +static void execute() { + // put test code here } -std::string QuickTest::help() { - return "This is a place holder to make it easy to quickly write " - "discardable " - "test code"; +void QuickTest::register_cmd(CLI::App &app) { + app.add_subcommand( + "quick-test", "This is a place holder to make it easy to quickly " + "write discardable test code") + ->callback(execute); } -} // namespace cmd -} // namespace bts + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/QuickTest.h b/src/server/sparkd/cmds/QuickTest.h index 1cc797b07..cdf5df0eb 100644 --- a/src/server/sparkd/cmds/QuickTest.h +++ b/src/server/sparkd/cmds/QuickTest.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class QuickTest : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct QuickTest { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index ccd3c2a67..1991b302f 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -1,5 +1,4 @@ #include "Run.h" -#include "ICmd.h" #include "lib/api/public/ServiceContext.h" #include "lib/config/config.h" @@ -46,10 +45,9 @@ #include -namespace bts { -namespace cmd { +namespace bts::cmds { -int Run::execute(std::vector const &) { +static void execute() { using namespace bts; LOG(Gen, Warn, "------------------------------------ Sparkd launched " @@ -123,9 +121,10 @@ int Run::execute(std::vector const &) { subprocess::run_all(); quickbus.run(); - return 0; } -std::string Run::help() { return "\t run default sparkd services"; } -} // namespace cmd -} // namespace bts +void Run::register_cmd(CLI::App &app) { + app.add_subcommand("run", "Run default sparkd services")->callback(execute); +} + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Run.h b/src/server/sparkd/cmds/Run.h index 1439b68c5..7e2bae1f9 100644 --- a/src/server/sparkd/cmds/Run.h +++ b/src/server/sparkd/cmds/Run.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Run : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct Run { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Span.cpp b/src/server/sparkd/cmds/Span.cpp index 0dd34d8a6..dfe3e0aad 100644 --- a/src/server/sparkd/cmds/Span.cpp +++ b/src/server/sparkd/cmds/Span.cpp @@ -24,10 +24,9 @@ using namespace bts::position; using namespace bts::api::position; -namespace bts { -namespace cmd { +namespace bts::cmds { -int Span::execute(const std::vector &) { +static void execute() { srand(time(NULL)); auto &disp = utils::Dispatcher::current(); MarginServiceCore core(disp); @@ -55,10 +54,11 @@ int Span::execute(const std::vector &) { std::cout << "Waiting for margin results, press any key to exit" << std::endl; disp.run(); - return 0; } -std::string Span::help() { return "\t test CME CORE Span integration \n"; } +void Span::register_cmd(CLI::App &app) { + app.add_subcommand("span", "Test CME CORE Span integration") + ->callback(execute); +} -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Span.h b/src/server/sparkd/cmds/Span.h index b504dfad7..d6ae46089 100644 --- a/src/server/sparkd/cmds/Span.h +++ b/src/server/sparkd/cmds/Span.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Span : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct Span { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index a50c50c99..7f2549819 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -1,368 +1,37 @@ #include "Spartan.h" -#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h" -#include "server/sparkd/cmds/spartan/market_data/feed_control/AB_Arbitrageur.h" -#include "server/sparkd/cmds/spartan/market_data/feed_control/IFeedController.h" -#include "server/sparkd/cmds/spartan/market_data/feed_control/PickedFeed.h" -#include "server/sparkd/cmds/spartan/market_data/handler/MDP3FeedHandler.h" -#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h" -#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" -#include "server/sparkd/cmds/spartan/system/PCapReader.h" +#include "server/sparkd/cmds/spartan/cmds/BookStats.h" +#include "server/sparkd/cmds/spartan/cmds/CSV.h" +#include "server/sparkd/cmds/spartan/cmds/Compress.h" +#include "server/sparkd/cmds/spartan/cmds/FindPolys.h" +#include "server/sparkd/cmds/spartan/cmds/Icebergs.h" +#include "server/sparkd/cmds/spartan/cmds/Json.h" +#include "server/sparkd/cmds/spartan/cmds/ListSec.h" +#include "server/sparkd/cmds/spartan/cmds/PolygonBooks.h" +#include "server/sparkd/cmds/spartan/cmds/SQL.h" +#include "server/sparkd/cmds/spartan/cmds/Sim.h" +#include "server/sparkd/cmds/spartan/cmds/Stats.h" +#include "server/sparkd/cmds/spartan/cmds/ValidateFifo.h" +#include "server/sparkd/cmds/spartan/cmds/ValuePolys.h" -#include "lib/jetfix/codec/f_toa.h" -#include "lib/orderman/orderman_utils/Formatter.h" -#include "lib/utils/PBUtils.h" -#include "server/sparkd/cmds/spartan/back_testing/run/run_sim.h" -#include "server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.h" -#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" -#include "server/sparkd/cmds/spartan/market_data/feed_observers/FillCounter.h" -#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h" -#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.h" -#include "server/sparkd/cmds/spartan/market_data/parsing/MDP3Parser.h" -#include "server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h" +namespace bts::cmds { -#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h" -#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h" -#include "server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.h" -#include "server/sparkd/cmds/spartan/util/args_parser.h" -#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" -#include "server/sparkd/cmds/spartan/util/sql/adapters/polygon_defs.h" -#include "server/sparkd/cmds/spartan/util/sql/sql_utils.h" - -#include -#include -#include -#include -#include - -namespace bts { -namespace cmd { - -#define INFO(...) LOG(OrderMan, Info, __VA_ARGS__) -#define WARN(...) LOG(OrderMan, Warn, __VA_ARGS__) -#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__) - -using namespace spartan::mdp3; - -auto drop_extensions = [](auto const &str) { - size_t lastindex = str.find_last_of("."); - return str.substr(0, lastindex); -}; - -inline std::string output_path( - const std::string &input_path, - const char *output_ext, - const char *input_ext = "") { - return bts::spartan::file::change_file_ext( - input_path, output_ext, input_ext); +void Spartan::register_cmd(CLI::App &app) { + auto *spartan_app = + app.add_subcommand("spartan", "Futures trading system for devs"); + // register commands + bts::spartan::cmds::BookStats::register_cmd(*spartan_app); + bts::spartan::cmds::Compress::register_cmd(*spartan_app); + bts::spartan::cmds::CSV::register_cmd(*spartan_app); + bts::spartan::cmds::FindPolys::register_cmd(*spartan_app); + bts::spartan::cmds::Icebergs::register_cmd(*spartan_app); + bts::spartan::cmds::Json::register_cmd(*spartan_app); + bts::spartan::cmds::ListSec::register_cmd(*spartan_app); + bts::spartan::cmds::PolygonBooks::register_cmd(*spartan_app); + bts::spartan::cmds::Sim::register_cmd(*spartan_app); + bts::spartan::cmds::SQL::register_cmd(*spartan_app); + bts::spartan::cmds::Stats::register_cmd(*spartan_app); + bts::spartan::cmds::ValidateFifo::register_cmd(*spartan_app); + bts::spartan::cmds::ValuePolys::register_cmd(*spartan_app); } -void find_polys(std::vector const &args) { - PolygonFinder::Options options; - if (args.size() > 2) - options.active_threshold_fills = std::stoi(args[2]); - if (args.size() > 3) - options.max_vertices = std::stoi(args[3]); - if (args.size() > 4) - options.max_securities = std::stoi(args[4]); - - DefinitionCollector def_collector; - PolygonFinder poly_finder(options); - observe_channel(args[1], {&poly_finder, &def_collector}); - for (size_t id = 1; id <= poly_finder.result.size(); id++) - std::cout << id << " -> " - << polygon_descriptor( - poly_finder.result[id], def_collector.ptr_result) - << std::endl; -} - -void list_sec(std::string const &input_file) { - DefinitionCollector def; - FillCounter fill_counter; - observe_channel(input_file, {&def, &fill_counter}); - auto fill_counts = fill_counter.result_map(); - for (auto const &s : def.result) { - std::cout << s.second.asset() << "|" << s.second.symbol() << "|" - << fill_counts[s.second.security_id()]; - for (auto const &l : s.second.legs()) { - std::cout << "|"; - if (def.ptr_result[l.security_id]) - std::cout << def.ptr_result[l.security_id]->symbol(); - else - std::cout << "?" << l.security_id; - } - std::cout << std::endl; - } -} - -template -inline int run(const std::string &pcap, T obs) { - observe_channel(pcap, obs); - return 0; -} - -void value_polys(std::vector const &args) { - DefinitionCollector def; - PolygonFinder poly_finder; - - // first pass, just identify the polygons - observe_channel(args[1], {&poly_finder, &def}); - - // second pass, value the polygons - PolygonOpportunityFinder op_finder{poly_finder.result, def.ptr_result}; - observe_channel(args[1], {&op_finder}); - - auto poly_v = - std::ofstream(drop_extensions(args[1]) + ".polygon_values.csv"); - - for (auto &v : op_finder.result) - poly_v << v << std::endl; - - auto poly_op = - std::ofstream(drop_extensions(args[1]) + ".polygon_opportunities.csv"); - spartan::mdp3::poly_opportunities_to_csv( - aggregate(op_finder.result), poly_finder.result, def.ptr_result, - poly_op); -} - -template -void print_book(std::ostream &books, B const &b) { - char buffer[128]; - for (auto level : b) { - d_toa(level.price.price(), buffer, sizeof(buffer), 0); - books << "," << buffer << "," << level.quantity.value() << "," - << level.n_orders; - } -} - -int icebergs_to_csv( - const std::unordered_map &ibs, - const std::string &out_csv) { - const char *ok_col = "ok"; - const char *eid_col = "eid"; - const char *total_col = "total_qty"; - const char *show_qty_col = "show_qty"; - const char *fills_col = "fill_qty"; - // validate args - if (out_csv.empty()) - return 1; - // write icebergs to csv - auto ofs = std::ofstream(out_csv); - // write cols - ofs << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col - << "," << fills_col; - // write rows - for (auto &p : ibs) { - for (auto [eid, show_qty] : p.second.show_quantities()) { - ofs << std::endl - << p.first << "," << eid << "," << p.second.total_qty() << "," - << show_qty << "," << p.second.count_fills_by(eid); - } - } - ofs.flush(); - return 0; -} - -int polygon_books( - const std::string &pcap, - const std::string &db, - const std::string &id_string, - const std::string &out_csv) { - const char *eid_col = "eid"; - const char *pid_col = "pid"; - const char *log_header = "PolygonBookCSV"; - // the nth leg col denotes the last event in which the nth leg's book changed - const int num_leg_cols = 10; - // validate args - if (out_csv.empty()) - return 1; - auto ids = bts::cmd::args::parse_set( - id_string, [](const std::string &s) { return std::stoi(s); }); - if (ids.empty()) - return 1; - INFO( - "generating polygon books for pids: {}", - orderman::Formatter::fmt_values(ids)); - SyntheticBookRecorder sbr( - spartan::sql::adapters::polygon_defs::read_sql(db.c_str(), ids)); - observe_channel(pcap, sbr); - // don't prepend a comma before calling 'append_book' (it does that - // internally) - auto append_book = [](std::ofstream &out, - const SmallBook &bk) -> std::ofstream & { - uint32_t i = 0; - bts_assert(bk.size() <= 10); - for (; i < bk.size(); i++) - out << "," << bk[i].price << "," << bk[i].quantity.value(); - while (i++ < Book::max_depth()) - out << ",NaN,0"; - return out; - }; - auto append_leg_book_eids = - [&](std::ostream &out, const SyntheticMBPUpdate &upd) -> std::ostream & { - int lcount = 0; - for (const auto &p : upd.leg_book_eids) { - out << "," << p.second; - if (++lcount == num_leg_cols) { - WARN( - "{} processing book update for polygon with more than {} legs " - "({} " - "'leg-book-event-id' columns are accounted for in the csv, " - "event " - "ids for susbequent legs will be discarded).", - *log_header, num_leg_cols, num_leg_cols); - break; - } - } - while (lcount++ < num_leg_cols) - out << ",0"; - return out; - }; - // write polygon books to csv - auto ofs = std::ofstream(out_csv); - // add cols - ofs << eid_col << "," << pid_col; - for (int i = 0; i < num_leg_cols; i++) - ofs << ",leg_" << i << "_eid"; - auto add_bk_cols = [&](const char *side_prefix) { - for (int i = 0; i < 10; i++) - ofs << "," << side_prefix << "_p_" << i << "," << side_prefix << "_q_" - << i; - }; - add_bk_cols("bid"); - add_bk_cols("ask"); - for (const auto &p : sbr.result()) { - for (const auto &upd : p.second) { - ofs << std::endl; - ofs << p.first << "," << upd.synthetic_id.value(); - append_leg_book_eids(ofs, upd); - append_book(ofs, upd.bids); - append_book(ofs, upd.asks); - } - } - ofs.flush(); - return 0; -} - -size_t total_orders = 0; -void stats(std::string const &pcap) { - auto out_path = drop_extensions(drop_extensions(pcap)) + ".order_counts.csv"; - auto out_file = std::ofstream{out_path}; - out_file << "sid,n_orders" << std::endl; - struct StatPrinter : IMDP3FeedObserver { - StatPrinter(std::ostream &out) : out{out} {} - std::ostream &out; - void on_event_end(MDP3Event const &ev) override { - for (auto s : ev) { - total_orders += s->orders().size(); - } - } - } stats{out_file}; - - observe_channel(pcap, stats); - std::cout << total_orders << std::endl; -} - -std::map const &)>> - extensions = { - - {"stats", - [](auto const &args) { - stats(args[1]); - return 0; - }}, - - {"value_polys", - [](auto const &args) { - value_polys(args); - return 0; - }}, - - {"sim", - [](auto const &args) { - run_sim(args[1]); - return 0; - }}, - {"list_sec", - [](auto const &args) { - list_sec(args[1]); - return 0; - }}, - {"book_stats", - [](const auto &args) { - auto out_path = args.size() > 2 - ? args[2] - : spartan::file::change_file_ext( - args[1], "book_stats.csv", "pcap.xz"); - std::ofstream out_csv(out_path); - int res = run(args[1], BookStatRecorder(out_csv)); - return res; - }}, - {"find_polys", - [](auto const &args) { - find_polys(args); - return 0; - }}, - {"value_polys", - [](auto const &args) { - value_polys(args); - return 0; - }}, - {"validate_fifo", - [](auto const &args) { - auto ibs = iceberg_file_cache::find_icebergs(args[1]); - return run(args[1], MatcherValidationObserver(ibs)); - }}, -}; - -int Spartan::execute(std::vector const &args) { - auto error = 1; - if (args.size() > 0) { - if (args[0] == "json" && args.size() >= 2) { - if (args.size() == 2) - error = spartan::mdp3::to_json(args[1]); - else if (args.size() == 4) - error = spartan::mdp3::to_json( - args[1], std::atoi(args[2].c_str()), - std::atoi(args[3].c_str())); - } else if (args[0] == "compress" && args.size() == 3) - error = spartan::mdp3::compress(args[1], args[2]); - else if (args[0] == "icebergs_csv" && args.size() >= 2) { - auto out_csv = args.size() > 2 - ? args[2] - : bts::spartan::file::change_file_ext( - args[1], "icebergs.csv", "pcap.xz"); - auto ibs = IcebergFinder::find_all(args[1]); - return icebergs_to_csv(ibs, out_csv); - } else if (args[0] == "csv" && args.size() == 2) - error = spartan::mdp3::to_csv(args[1]); - else if (args[0] == "sql" && args.size() == 2) - error = spartan::mdp3::to_sql(args[1]); - else if (args[0] == "polygon_books" && args.size() >= 4) { - auto out_csv = args.size() > 4 - ? args[4] - : bts::spartan::file::change_file_ext( - args[1], "polygon_books.csv", "pcap.xz"); - error = polygon_books(args[1], args[2], args[3], out_csv); - } else if (extensions.count(args[0]) && args.size() >= 2) { - return extensions[args[0]](args); - } - } - if (error) - std::cout << help(); - return error; -} - -std::string Spartan::help() { - auto str = fmt::format("\tjson [.xz] [start_packet packet_count]\n" - "\tcompress [.xz] \n" - "\tpolygon_books [.xz] [out-csv]\n" - "\tsql [.xz]\n" - "\tcsv [.xz]\n"); - for (auto const &p : extensions) - str += fmt::format("\t{} [.xz] ...\n", p.first); - return str; -} - -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Spartan.h b/src/server/sparkd/cmds/Spartan.h index 81f69aa5e..add33ffec 100644 --- a/src/server/sparkd/cmds/Spartan.h +++ b/src/server/sparkd/cmds/Spartan.h @@ -1,22 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Spartan : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); - - private: - int run_from_pcap(std::vector const &); - int run_from_mcast(std::vector const &); - int parse_pcap(std::vector const &); - int analyze_pcap(std::vector const &); - void run(std::vector const &); +struct Spartan { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Test.cpp b/src/server/sparkd/cmds/Test.cpp index a42870493..f78f46f28 100644 --- a/src/server/sparkd/cmds/Test.cpp +++ b/src/server/sparkd/cmds/Test.cpp @@ -1,19 +1,19 @@ #include "Test.h" -#include "ICmd.h" #include "lib/testing/catch_test_runner.h" -namespace bts { -namespace cmd { +namespace bts::cmds { -int Test::execute(std::vector const &args) { - return run_tests(args); +static std::vector args; + +static void execute() { run_tests(args); } + +void Test::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand( + "test", + "See " + "https://github.com/philsquared/Catch/blob/master/docs/command-line.md"); + cmd->add_option("args", args); + cmd->callback(execute); } -std::string Test::help() { - return "\t see " - "https://github.com/philsquared/Catch/blob/master/docs/" - "command-line.md"; -} - -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Test.h b/src/server/sparkd/cmds/Test.h index 71d79fd9e..cfec12a5f 100644 --- a/src/server/sparkd/cmds/Test.h +++ b/src/server/sparkd/cmds/Test.h @@ -1,15 +1,10 @@ #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Test : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct Test { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index caf021cae..92450b92b 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -20,10 +20,15 @@ #include #include -namespace bts { -namespace cmd { +namespace bts::cmds { -int Web::execute(const std::vector &args) { +struct Args { + bool md; +}; + +static Args args; + +static void execute() { config::QuickbusNodeCfg qb_cfg; auto &ports = config::get(); qb_cfg.set_my_port(ports.qbus_port()); @@ -36,7 +41,7 @@ int Web::execute(const std::vector &args) { http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port()); user::AuthServiceServer login_server(sc); - if (std::find(args.begin(), args.end(), "-md") != args.end()) { + if (args.md) { md::run_market_datapublishers(nullptr, nullptr); } @@ -48,14 +53,14 @@ int Web::execute(const std::vector &args) { admin::AdminServiceServer admin_service(sc); utils::Dispatcher::current().run(); - return 0; } -std::string Web::help() { - return "\t run only the services required for the web admin. This cannot " - "be run concurrently with 'sparkd run'\n\tAdd -md to start market " - "data too."; +void Web::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand( + "web", "Run only the services required for the web admin. This cannot " + "be run concurrently with 'sparkd run.'"); + cmd->add_flag("--md", args.md, "Start market data"); + cmd->callback(execute); } -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Web.h b/src/server/sparkd/cmds/Web.h index 23272259e..f1d4320e7 100644 --- a/src/server/sparkd/cmds/Web.h +++ b/src/server/sparkd/cmds/Web.h @@ -1,16 +1,10 @@ - #pragma once +#include -#include "ICmd.h" +namespace bts::cmds { -namespace bts { -namespace cmd { - -class Web : public ICmd { - public: - int execute(std::vector const &args); - std::string help(); +struct Web { + static void register_cmd(CLI::App &); }; -} // namespace cmd -} // namespace bts +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp b/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp new file mode 100644 index 000000000..388a8e18e --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp @@ -0,0 +1,32 @@ +#include "BookStats.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include +#include + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; + std::string out; +}; + +static Args args; + +static void execute() { + if (args.out.empty()) + args.out = file::change_file_ext(args.pcap, "book_stats.csv", "pcap.xz"); + std::ofstream out(args.out); + mdp3::BookStatRecorder bsr(out); + mdp3::observe_channel(args.pcap, bsr); +} + +void BookStats::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("book-stats"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option("out,--out", args.out); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/BookStats.h b/src/server/sparkd/cmds/spartan/cmds/BookStats.h new file mode 100644 index 000000000..6b64eef8f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/BookStats.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct BookStats { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/CSV.cpp b/src/server/sparkd/cmds/spartan/cmds/CSV.cpp new file mode 100644 index 000000000..5cc2d0cd5 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/CSV.cpp @@ -0,0 +1,20 @@ +#include "CSV.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; +}; + +static Args args; + +static void execute() { mdp3::to_csv(args.pcap); } + +void CSV::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand("csv"); + cmd->add_option("pcap,--pcap", args.pcap)->required(); + cmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/CSV.h b/src/server/sparkd/cmds/spartan/cmds/CSV.h new file mode 100644 index 000000000..653070ad1 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/CSV.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct CSV { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Compress.cpp b/src/server/sparkd/cmds/spartan/cmds/Compress.cpp new file mode 100644 index 000000000..8e53526eb --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Compress.cpp @@ -0,0 +1,22 @@ +#include "Compress.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; + std::string out; +}; + +static Args args; + +static void execute() { spartan::mdp3::compress(args.pcap, args.out); } + +void Compress::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("compress"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option("out,--out", args.out)->required(); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Compress.h b/src/server/sparkd/cmds/spartan/cmds/Compress.h new file mode 100644 index 000000000..89a2a31d7 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Compress.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct Compress { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp b/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp new file mode 100644 index 000000000..4305aa58b --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp @@ -0,0 +1,52 @@ +#include "FindPolys.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h" +#include + +namespace bts::spartan::cmds { + +const int UNSET = -1; + +struct Args { + std::string pcap; + int active_threshold_fills = UNSET; + int max_vertices = UNSET; + int max_securities = UNSET; +}; + +static Args args; + +static void execute() { + // init options (from args) + mdp3::PolygonFinder::Options options; + if (args.active_threshold_fills != UNSET) + options.active_threshold_fills = args.active_threshold_fills; + if (args.max_vertices != UNSET) + options.max_vertices = args.max_vertices; + if (args.max_securities != UNSET) + options.max_securities = args.max_securities; + + mdp3::DefinitionCollector def_collector; + mdp3::PolygonFinder poly_finder(options); + mdp3::observe_channel(args.pcap, {&poly_finder, &def_collector}); + for (size_t id = 1; id <= poly_finder.result.size(); id++) + std::cout << id << " -> " + << mdp3::polygon_descriptor( + poly_finder.result[id], def_collector.ptr_result) + << std::endl; +} + +void FindPolys::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("find-polys"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option( + "active-threshold-fills,--active-threshold-fills", + args.active_threshold_fills); + subcmd->add_option("max-vertices,--max-vertices", args.max_vertices); + subcmd->add_option("max-securities,--max-securities", args.max_securities); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/FindPolys.h b/src/server/sparkd/cmds/spartan/cmds/FindPolys.h new file mode 100644 index 000000000..82eb133ba --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/FindPolys.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct FindPolys { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp b/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp new file mode 100644 index 000000000..bc6249cc4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp @@ -0,0 +1,51 @@ +#include "Icebergs.h" +#include "server/sparkd/cmds/spartan/domain/IcebergQuantity.h" +#include "server/sparkd/cmds/spartan/domain/OrderKey.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h" +#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include +#include + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; + std::string out; +}; + +static Args args; + +static void execute() { + const char *ok_col = "ok"; + const char *eid_col = "eid"; + const char *total_col = "total_qty"; + const char *show_qty_col = "show_qty"; + const char *fills_col = "fill_qty"; + if (args.out.empty()) + args.out = file::change_file_ext(args.pcap, "icebergs.csv", "pcap.xz"); + std::ofstream out(args.out); + auto ibs = mdp3::IcebergFinder::find_all(args.pcap); + // write cols + out << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col + << "," << fills_col; + // write rows + for (auto &i : ibs) { + auto relevent_eids = i.second.all_eids(); + for (auto eid : relevent_eids) + out << i.first.value() << "," << eid << "," + << i.second.total_qty().value() << "," + << std::get<1>(i.second.show_qty_at(eid)) << "," + << i.second.count_fills_by(eid) << "\n"; + } +} + +void Icebergs::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("icebergs"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option("out,--out", args.out); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Icebergs.h b/src/server/sparkd/cmds/spartan/cmds/Icebergs.h new file mode 100644 index 000000000..d03675cfb --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Icebergs.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct Icebergs { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Json.cpp b/src/server/sparkd/cmds/spartan/cmds/Json.cpp new file mode 100644 index 000000000..b379e1794 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Json.cpp @@ -0,0 +1,26 @@ +#include "Json.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" + +namespace bts::spartan::cmds { + +struct Args { + uint32_t start_packet = 0; + uint32_t packet_count = 0; + std::string pcap{}; +}; + +static Args args; + +static void execute() { + spartan::mdp3::to_json(args.pcap, args.start_packet, args.packet_count); +} + +void Json::register_cmd(CLI::App &spartan_app) { + auto *subcmd = spartan_app.add_subcommand("json"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option("start-packet,--start-packet", args.start_packet); + subcmd->add_option("packet-count,--packet-count", args.packet_count); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Json.h b/src/server/sparkd/cmds/spartan/cmds/Json.h new file mode 100644 index 000000000..5337cf92f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Json.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct Json { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp b/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp new file mode 100644 index 000000000..12295b2a8 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp @@ -0,0 +1,40 @@ +#include "ListSec.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/FillCounter.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; +}; + +static Args args; + +static void execute() { + mdp3::DefinitionCollector def; + mdp3::FillCounter fill_counter; + mdp3::observe_channel(args.pcap, {&def, &fill_counter}); + auto fill_counts = fill_counter.result_map(); + for (auto const &s : def.result) { + std::cout << s.second.asset() << "|" << s.second.symbol() << "|" + << fill_counts[s.second.security_id()]; + for (auto const &l : s.second.legs()) { + std::cout << "|"; + if (def.ptr_result[l.security_id]) + std::cout << def.ptr_result[l.security_id]->symbol(); + else + std::cout << "?" << l.security_id; + } + std::cout << std::endl; + } +} + +void ListSec::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("list-sec"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ListSec.h b/src/server/sparkd/cmds/spartan/cmds/ListSec.h new file mode 100644 index 000000000..4bc679da0 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/ListSec.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct ListSec { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp new file mode 100644 index 000000000..64f38b0c2 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp @@ -0,0 +1,120 @@ +#include "PolygonBooks.h" +#include "lib/orderman/orderman_utils/Formatter.h" +#include "server/sparkd/cmds/spartan/domain/book/Book.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include "server/sparkd/cmds/spartan/util/sql/adapters/polygon_defs.h" +#include "server/sparkd/cmds/spartan/util/sql/sql_utils.h" +#include +#include + +namespace bts::spartan::cmds { + +#define INFO(...) LOG(OrderMan, Info, __VA_ARGS__) +#define INFO_FN(...) \ + LOG(OrderMan, Info, "[{}] {}", std::string(__FUNCTION__), \ + fmt::format(__VA_ARGS__)) +#define WARN(...) LOG(OrderMan, Warn, __VA_ARGS__) +#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__) + +struct Args { + std::string pcap; + std::string db; + std::vector security_ids; + std::string out_csv; +}; + +static Args args; + +static void execute() { + const char *eid_col = "eid"; + const char *pid_col = "pid"; + const char *log_header = "PolygonBookCSV"; + // the nth leg col denotes the last event in which the nth leg's book changed + const int num_leg_cols = 10; + + // validate args + if (args.out_csv.empty()) + args.out_csv = + file::change_file_ext(args.pcap, "polygon_books.csv", "pcap.xz"); + if (args.out_csv.empty() || args.security_ids.empty()) + return; + std::unordered_set ids; + for (auto sid : args.security_ids) + ids.insert(sid); + INFO( + "generating polygon books for pids: {}", + orderman::Formatter::fmt_values(ids)); + mdp3::SyntheticBookRecorder sbr( + spartan::sql::adapters::polygon_defs::read_sql(args.db.c_str(), ids)); + mdp3::observe_channel(args.pcap, sbr); + // don't prepend a comma before calling 'append_book' (it does that + // internally) + auto append_book = [](std::ofstream &out, + const mdp3::SmallBook &bk) -> std::ofstream & { + uint32_t i = 0; + bts_assert(bk.size() <= 10); + for (; i < bk.size(); i++) + out << "," << bk[i].price << "," << bk[i].quantity.value(); + while (i++ < mdp3::BookSide::max_depth()) + out << ",NaN,0"; + return out; + }; + auto append_leg_book_eids = + [&](std::ostream &out, + const mdp3::SyntheticMBPUpdate &upd) -> std::ostream & { + int lcount = 0; + for (const auto &p : upd.leg_book_eids) { + out << "," << p.second; + if (++lcount == num_leg_cols) { + WARN( + "{} processing book update for polygon with more than {} legs " + "({} " + "'leg-book-event-id' columns are accounted for in the csv, " + "event " + "ids for susbequent legs will be discarded).", + *log_header, num_leg_cols, num_leg_cols); + break; + } + } + while (lcount++ < num_leg_cols) + out << ",0"; + return out; + }; + // write polygon books to csv + auto ofs = std::ofstream(args.out_csv); + // add cols + ofs << eid_col << "," << pid_col; + for (int i = 0; i < num_leg_cols; i++) + ofs << ",leg_" << i << "_eid"; + auto add_bk_cols = [&](const char *side_prefix) { + for (int i = 0; i < 10; i++) + ofs << "," << side_prefix << "_p_" << i << "," << side_prefix << "_q_" + << i; + }; + add_bk_cols("bid"); + add_bk_cols("ask"); + for (const auto &p : sbr.result()) { + for (const auto &upd : p.second) { + ofs << std::endl; + ofs << p.first << "," << upd.synthetic_id.value(); + append_leg_book_eids(ofs, upd); + append_book(ofs, upd.bids); + append_book(ofs, upd.asks); + } + } + ofs.flush(); +} + +void PolygonBooks::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand("polygon-books"); + cmd->add_option("pcap,--pcap", args.pcap)->required(); + cmd->add_option("db,--db", args.db)->required(); + cmd->add_option("security-ids,--security-ids", args.security_ids) + ->required(); + cmd->add_option("out,--out", args.out_csv); + cmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h new file mode 100644 index 000000000..4900850e2 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct PolygonBooks { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SQL.cpp b/src/server/sparkd/cmds/spartan/cmds/SQL.cpp new file mode 100644 index 000000000..5c779a131 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/SQL.cpp @@ -0,0 +1,20 @@ +#include "SQL.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; +}; + +static Args args; + +static void execute() { mdp3::to_sql(args.pcap); } + +void SQL::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("sql"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SQL.h b/src/server/sparkd/cmds/spartan/cmds/SQL.h new file mode 100644 index 000000000..f3832f520 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/SQL.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct SQL { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Sim.cpp b/src/server/sparkd/cmds/spartan/cmds/Sim.cpp new file mode 100644 index 000000000..eea42b957 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Sim.cpp @@ -0,0 +1,23 @@ +#include "Sim.h" +#include "server/sparkd/cmds/spartan/back_testing/run/run_sim.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include + +namespace bts::spartan::cmds { + +struct Args { + std::string config_path; +}; + +static Args args; + +static void execute() { mdp3::run_sim(args.config_path); } + +void Sim::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("sim"); + subcmd->add_option("config-file,--config-file", args.config_path) + ->required(); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Sim.h b/src/server/sparkd/cmds/spartan/cmds/Sim.h new file mode 100644 index 000000000..80a7ea1db --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Sim.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct Sim { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Stats.cpp b/src/server/sparkd/cmds/spartan/cmds/Stats.cpp new file mode 100644 index 000000000..c7238b34f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Stats.cpp @@ -0,0 +1,42 @@ +#include "Stats.h" +#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h" +#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; + std::string out; +}; + +static Args args; + +size_t total_orders = 0; +static void execute() { + if (args.out.empty()) + args.out = spartan::file::change_file_ext(args.pcap, "order_counts.csv"); + auto out_file = std::ofstream(args.out); + out_file << "sid,n_orders" << std::endl; + struct StatPrinter : spartan::mdp3::IMDP3FeedObserver { + StatPrinter(std::ostream &out) : out{out} {} + std::ostream &out; + void on_event_end(spartan::mdp3::MDP3Event const &ev) override { + for (const auto *s : ev) + total_orders += s->orders().size(); + } + } stats{out_file}; + + observe_channel(args.pcap, stats); + std::cout << total_orders << std::endl; +} + +void Stats::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("stats"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option("out,--out", args.out); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Stats.h b/src/server/sparkd/cmds/spartan/cmds/Stats.h new file mode 100644 index 000000000..26381b4b8 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/Stats.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct Stats { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp new file mode 100644 index 000000000..a0625e42f --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp @@ -0,0 +1,28 @@ +#include "ValidateFifo.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; +}; + +static Args args; + +static void execute() { + auto ibs = mdp3::IcebergFinder::find_all(args.pcap); + mdp3::MatcherValidationObserver obs(ibs); + mdp3::observe_channel(args.pcap, obs); +} + +void ValidateFifo::register_cmd(CLI::App &app) { + auto *cmd = app.add_subcommand("validate-fifo"); + cmd->add_option("pcap,--pcap", args.pcap)->required(); + cmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h new file mode 100644 index 000000000..ffb4fea46 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct ValidateFifo { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp new file mode 100644 index 000000000..8f6d0389a --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp @@ -0,0 +1,47 @@ +#include "ValuePolys.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include + +namespace bts::spartan::cmds { + +struct Args { + std::string pcap; +}; + +static Args args; + +static void execute() { + mdp3::DefinitionCollector def; + mdp3::PolygonFinder poly_finder; + + // first pass, just identify the polygons + mdp3::observe_channel(args.pcap, {&poly_finder, &def}); + + // second pass, value the polygons + mdp3::PolygonOpportunityFinder op_finder{poly_finder.result, def.ptr_result}; + mdp3::observe_channel(args.pcap, {&op_finder}); + + auto poly_v = + std::ofstream(file::change_file_ext(args.pcap, "polygon_values.csv")); + + for (auto &v : op_finder.result) + poly_v << v << std::endl; + + auto poly_op = std::ofstream( + file::change_file_ext(args.pcap, "polygon_opportunities.csv")); + spartan::mdp3::poly_opportunities_to_csv( + aggregate(op_finder.result), poly_finder.result, def.ptr_result, + poly_op); +} + +void ValuePolys::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand("value-polys"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h new file mode 100644 index 000000000..688dc2815 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct ValuePolys { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/main.cpp b/src/server/sparkd/main.cpp index faaa1e5d6..84351772f 100644 --- a/src/server/sparkd/main.cpp +++ b/src/server/sparkd/main.cpp @@ -1,6 +1,6 @@ #include "lib/utils/init_spark_lib.h" #include "lib/utils/thread_manager.h" -#include "server/sparkd/cmds/CmdRegistry.h" +#include // to add a command add the header here and the the REGISTER call below // don't touch this file except to add commands. @@ -9,7 +9,6 @@ #include "server/sparkd/cmds/Benchmark.h" #include "server/sparkd/cmds/ConfigCheck.h" #include "server/sparkd/cmds/FIXSim.h" -#include "server/sparkd/cmds/Help.h" #include "server/sparkd/cmds/KeyCode.h" #include "server/sparkd/cmds/Md.h" #include "server/sparkd/cmds/ParseHistoricalFIX.h" @@ -25,24 +24,27 @@ using namespace bts; int main(int argc, char **argv) { utils::init_spark_lib(); - BEGIN_CMD; - REGISTER(Test); - REGISTER(Help); - REGISTER(Run); - REGISTER(Md); - REGISTER(ConfigCheck); - REGISTER(KeyCode); - REGISTER(Benchmark); - REGISTER(QuickTest); - REGISTER(AutoCert); - REGISTER(FIXSim); - REGISTER(Span); - REGISTER(Web); - REGISTER(ParseHistoricalFIX); - REGISTER(PrintHedgeOrders); - REGISTER(AdminReport); - REGISTER(Spartan); - END_CMD; + + CLI::App app("Futures trading system", "sparkd"); + app.require_subcommand(); + // register subcommands + bts::cmds::AdminReport::register_cmd(app); + bts::cmds::AutoCert::register_cmd(app); + bts::cmds::Benchmark::register_cmd(app); + bts::cmds::ConfigCheck::register_cmd(app); + bts::cmds::FIXSim::register_cmd(app); + bts::cmds::KeyCode::register_cmd(app); + bts::cmds::Md::register_cmd(app); + bts::cmds::ParseHistoricalFix::register_cmd(app); + bts::cmds::PrintHedgeOrders::register_cmd(app); + bts::cmds::QuickTest::register_cmd(app); + bts::cmds::Run::register_cmd(app); + bts::cmds::Span::register_cmd(app); + bts::cmds::Spartan::register_cmd(app); + bts::cmds::Test::register_cmd(app); + bts::cmds::Web::register_cmd(app); + + CLI11_PARSE(app, argc, argv); utils::finit_spark_lib(); return 0; } From c0a7f1b3fe39faa53b928afdb9c7f3e2f7b3db52 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Jan 2021 00:46:13 -0600 Subject: [PATCH 03/13] refactor: Use c++ 'char_traits::length' instead of c 'strlen' --- src/server/sparkd/cmds/spartan/util/file_stream_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/sparkd/cmds/spartan/util/file_stream_utils.h b/src/server/sparkd/cmds/spartan/util/file_stream_utils.h index 76e8e4917..aff740512 100644 --- a/src/server/sparkd/cmds/spartan/util/file_stream_utils.h +++ b/src/server/sparkd/cmds/spartan/util/file_stream_utils.h @@ -57,7 +57,7 @@ inline std::string change_file_ext( bool require_expected_ext = false) { std::size_t exp_ext_pos = path.find_last_of(expected_ext); if (exp_ext_pos < path.size()) - return path.substr(0, exp_ext_pos + 1 - std::strlen(expected_ext)) + + return path.substr(0, exp_ext_pos + 1 - std::char_traits::length(expected_ext)) + new_ext; else if (require_expected_ext) return {}; From 2a04341b3b3fc219465ff04ce87eb8192a02e922 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Jan 2021 01:01:13 -0600 Subject: [PATCH 04/13] Allow for non-null, or non-empty effects vector in 'match' method --- .../spartan/back_testing/matcher/Matcher.cpp | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp index 5413d2c6e..21844566c 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp @@ -111,6 +111,11 @@ void MergedMatcher::match( std::vector *res) { _match_count_this_per++; _match_count++; + // The caller is not obligated to pass an empty vector. Consider an event + // that involves multiple securities; perhaps the caller wants to put the + // effects for each security in the same vector. The elements that were + // already present in the effects vector are skipped/ignored. + uint32_t effects_origin = res == nullptr ? 0 : res->size(); // This could be better optimized, but the idea is to // 1) take a snapshot of every security's book/implied book, // 2) process the order changes, @@ -152,18 +157,20 @@ void MergedMatcher::match( _asset_matcher(c.order.security).on_order_change(c, res, fills); } // log/validate order updates - for (const auto &u : *res) - _trace.log_outgoing_update(std::get(u)); - for (const auto &u : *res) - MatcherUtils::validate_outgoing_update(std::get(u)); - // add mbp updates - for (auto &ass_kv : mbps) { - for (auto &sec_kv : ass_kv.second) { - auto new_mbp = sec_kv.second.get_new_mbp(); - if (new_mbp.has_value()) { - res->emplace_back(MBPUpdate{ - sec_kv.first, new_mbp.value().bids, new_mbp.value().asks, - new_mbp.value().ibids, new_mbp.value().iasks}); + if (res != nullptr) { + for (auto it = res->begin() + effects_origin; it != res->end(); it++) + _trace.log_outgoing_update(std::get(*it)); + for (auto it = res->begin() + effects_origin; it != res->end(); it++) + MatcherUtils::validate_outgoing_update(std::get(*it)); + // add mbp updates + for (auto &ass_kv : mbps) { + for (auto &sec_kv : ass_kv.second) { + auto new_mbp = sec_kv.second.get_new_mbp(); + if (new_mbp.has_value()) { + res->emplace_back(MBPUpdate{ + sec_kv.first, new_mbp.value().bids, new_mbp.value().asks, + new_mbp.value().ibids, new_mbp.value().iasks}); + } } } } From 36bfc9e770e51182b1aa3d504baf735e10cf0fa3 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Jan 2021 09:59:54 -0600 Subject: [PATCH 05/13] Add SecurityCriteria struct to filter instrument sets Inspired by the c# 'SecurityCriteria' class from spark's front end. This may come in handy as a cli option for spartan subcommands, among other things. --- .../cmds/spartan/domain/SecurityCriteria.cpp | 41 +++++++++++++++++++ .../cmds/spartan/domain/SecurityCriteria.h | 37 +++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 src/server/sparkd/cmds/spartan/domain/SecurityCriteria.cpp create mode 100644 src/server/sparkd/cmds/spartan/domain/SecurityCriteria.h diff --git a/src/server/sparkd/cmds/spartan/domain/SecurityCriteria.cpp b/src/server/sparkd/cmds/spartan/domain/SecurityCriteria.cpp new file mode 100644 index 000000000..7d99cf223 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/domain/SecurityCriteria.cpp @@ -0,0 +1,41 @@ +#include "SecurityCriteria.h" +#include + +namespace bts::spartan::mdp3 { + +#define CONTAINS(container, value) \ + (std::find(container.begin(), container.end(), value) != container.end()) + +bool SecurityCriteriaClause::is_defined() const { + return exchanges.size() || security_types.size() || assets.size() || + security_ids.size(); +} + +bool SecurityCriteriaClause::includes(const SecurityDefinition &def) const { + if (!is_defined()) + return false; + if (security_types.size() && !CONTAINS(security_types, def.security_type())) + return false; + if (assets.size() && !CONTAINS(assets, def.asset())) + return false; + if (security_ids.size() && !security_ids.count(def.security_id())) + return false; + return true; +} + +bool SecurityCriteria::is_defined() const { + for (const auto &c : clauses) + if (c.is_defined()) + return true; + return false; +} + +bool SecurityCriteria::includes(const SecurityDefinition &def) const { + for (const auto &c : clauses) { + if (c.includes(def)) + return true; + } + return false; +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/domain/SecurityCriteria.h b/src/server/sparkd/cmds/spartan/domain/SecurityCriteria.h new file mode 100644 index 000000000..ac7800323 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/domain/SecurityCriteria.h @@ -0,0 +1,37 @@ +#pragma once +#include "SecurityDefinition.h" +#include "SecurityID.h" +#include "lib/domain_model/Security.pb.h" +#include +#include +#include + +namespace bts::spartan::mdp3 { + +/** + * A group of properties that define a subset of securities. Each field + * (exchanges, security_types, etc.) is a discrete set of potential values for a + * particular property of the security. The subset of securities defined by the + * SecurityCriteriaClause consists of all those with properties that are + * contained in the [non-empty] discrete set(s). + */ +struct SecurityCriteriaClause { + boost::container::small_vector exchanges; + boost::container::small_vector security_types; + boost::container::small_vector assets; + std::unordered_set security_ids; + + bool is_defined() const; + bool includes(const SecurityDefinition &instrument) const; +}; + +/** + * Represents a subset of securities as the union of [SecurityCriteriaClause] subsets. + */ +struct SecurityCriteria { + bool is_defined() const; + bool includes(const SecurityDefinition &instrument) const; + boost::container::small_vector clauses; +}; + +} // namespace bts::spartan::mdp3 From 9b49c661367de2ff1c98275ba356c330ec012b2e Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 13 Jan 2021 11:48:17 -0600 Subject: [PATCH 06/13] fix: Require valid side param in 'has_min_qty' method Although the side can be inferred by comparing the first two price levels, the book may not have two price levels to compare. --- src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h b/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h index 7b5d77493..bc0d08ccb 100644 --- a/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h +++ b/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h @@ -161,7 +161,8 @@ inline bool has_min_qty( Iter last, Quantity qty, Price limit_price = {}, - api::Side side = api::Side::SIDE_UNDEFINED) { + api::Side side = api::Side::SIDE_BUY) { + bts_assert(side == api::Side::SIDE_BUY || side == api::Side::SIDE_SELL); if (qty.value() == 0) return true; bts_assert(qty.value() > 0); From 25395fcac3fe9754a87abed4775ce24f20e8217c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 13 Jan 2021 12:58:32 -0600 Subject: [PATCH 07/13] refactor: Replace api::Side w/ spartan::Side --- .../back_testing/algos/PolyObserver.cpp | 2 +- .../back_testing/algos/SniperModel.cpp | 20 +-- .../matcher/FifoImpliedSecurityMatcher.cpp | 19 ++- .../matcher/FifoImpliedSecurityMatcher.h | 8 +- .../matcher/FifoMatcherContext.cpp | 14 +- .../back_testing/matcher/FifoMatcherContext.h | 17 ++- .../matcher/FifoMergedSecurityMatcher.cpp | 28 ++-- .../matcher/FifoMergedSecurityMatcher.h | 10 +- .../matcher/FifoPriceLevelMatcher.cpp | 13 +- .../matcher/FifoPriceLevelMatcher.h | 8 +- .../matcher/FifoSecurityMatcher.cpp | 38 +++--- .../matcher/FifoSecurityMatcher.h | 24 ++-- .../spartan/back_testing/matcher/Matcher.cpp | 18 +-- .../spartan/back_testing/matcher/Matcher.h | 8 +- .../back_testing/matcher/MatchingCommon.h | 11 +- .../back_testing/matcher/SecurityMatcher.h | 4 +- .../back_testing/matcher/fill_factory.h | 8 +- .../matcher/matcher_diagnostics.h | 28 ++-- .../matcher/tests/matcher_tests.cpp | 124 +++++++++--------- .../cmds/spartan/back_testing/sim/Sim.cpp | 9 +- .../spartan/back_testing/sim/pandl/PandL.cpp | 12 +- .../spartan/back_testing/sim/pandl/PandL.h | 4 +- .../spartan/back_testing/sim/sim_common.h | 16 +-- .../spartan/domain/SecurityDefinition.cpp | 3 +- .../cmds/spartan/domain/SecurityDefinition.h | 2 +- .../cmds/spartan/domain/SecuritySnapshot.cpp | 12 +- src/server/sparkd/cmds/spartan/domain/Side.h | 5 + .../spartan/market_data/common/mdp3_common.h | 14 +- .../feed_observers/BookStatRecorder.cpp | 4 +- .../feed_observers/PolygonFinder.cpp | 2 +- .../PolygonOpportunityFinder.cpp | 10 +- .../feed_observers/SyntheticBookRecorder.cpp | 4 +- .../market_data/realtime/RealTimeHandler.cpp | 4 +- .../market_data/recovery/RecoveryHandler.cpp | 2 +- .../spartan/market_data/utils/book_utils.h | 15 ++- .../spartan/market_data/utils/mdp3_utils.cpp | 13 +- .../spartan/market_data/utils/order_utils.h | 4 +- .../spartan/market_data/utils/polygon_utils.h | 6 +- .../utils/tests/synthetic_book_tests.cpp | 27 ++-- 39 files changed, 272 insertions(+), 298 deletions(-) diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp index a085d95b1..7caa15782 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/PolyObserver.cpp @@ -72,7 +72,7 @@ void PolyObserver::eval() { t->value = 0; int qty = std::numeric_limits::max(); for (auto &l : t->polygon.parts) { - auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); + auto mkt_side = side_from_ratio(l.ratio); auto &mkt = me->books[l.sec_id].mbp_side(mkt_side); auto this_qty = mkt[0].quantity.value(); auto this_ratio = this_qty / std::abs(l.ratio); diff --git a/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp b/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp index 21fb63eea..e7553d017 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/algos/SniperModel.cpp @@ -294,7 +294,7 @@ void SniperModel::maybe_force_fills() { } if (order.order.quantity.value() > total_fills) { Price worst_price = order.order.price; - if (order.order.side == api::SIDE_BUY) + if (order.order.side == Side::Buy) worst_price += me->sim->get_security(order.order.security) .min_price_increment_int_price() * 100; @@ -315,7 +315,7 @@ void SniperModel::eval(std::vector const &targets) { int qty = me->max_qty; size_t p_idx = 0; for (auto &l : t->polygon.parts) { - auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); + auto mkt_side = other_side(side_from_ratio(l.ratio)); auto &leg_sec_info = t->leg_sec_info[p_idx++]; auto &mkt = leg_sec_info->book.mbp_side(mkt_side); auto this_qty = mkt[0].quantity.value(); @@ -344,7 +344,7 @@ bool SniperModel::within_safety_margin(Target const &t) const { auto &leg_mkt = leg_mkts.emplace_back(); std::get<0>(leg_mkt) = Quantity{leg.ratio}; std::get<1>(leg_mkt) = {}; - auto leg_book_side = securities::other_side(side_from_ratio(leg.ratio)); + auto leg_book_side = other_side(side_from_ratio(leg.ratio)); std::get<2>(leg_mkt) = me->books[leg.sec_id].book.mbp_side(leg_book_side); } @@ -439,8 +439,8 @@ void SniperModel::fire(Shot const &s) { if (me->ms_to_work_shot == 0) { shot.forced_completion = true; for (auto &p : shot.target.polygon.parts) { - auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL; - auto mkt_side = securities::other_side(part_side); + auto part_side = from_ratio(p.ratio); + auto mkt_side = other_side(part_side); auto &book = me->books[p.sec_id].book.mbp_side(mkt_side); Price worst_price; @@ -453,7 +453,7 @@ void SniperModel::fire(Shot const &s) { break; } } - if (part_side == api::SIDE_BUY) + if (part_side == Side::Buy) worst_price += me->sim->get_security(p.sec_id).min_price_increment_int_price() * 100; @@ -476,8 +476,8 @@ void SniperModel::fire(Shot const &s) { } } else { for (auto &p : shot.target.polygon.parts) { - auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL; - auto mkt_side = securities::other_side(part_side); + auto part_side = from_ratio(p.ratio); + auto mkt_side = other_side(part_side); auto &book = me->books[p.sec_id].book.mbp_side(mkt_side); auto limit_price = book[0].price; @@ -485,7 +485,7 @@ void SniperModel::fire(Shot const &s) { me->sim->get_security(p.sec_id).min_price_increment_int_price() * me->pay_up_ticks; - if (part_side == api::SIDE_BUY) + if (part_side == Side::Buy) limit_price += pay_up_adj; else limit_price -= pay_up_adj; @@ -498,7 +498,7 @@ void SniperModel::fire(Shot const &s) { order_change.change = OrderChange::added; order_change.order.key = me->sim->send(order_change); me->sim->log( - "sent_order", api::Side_Name(part_side), + "sent_order", Side_Name(part_side), me->sim->get_security(p.sec_id).symbol(), book[0].price.to_double(), book[0].price.to_double(), book[0].quantity.value()); shot.working[order_change.order.key].order = order_change; diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp index 534cb478a..7518fc13c 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp @@ -95,8 +95,8 @@ void FifoImpliedSecurityMatcher::print_legs(std::ostream &out) const { << " " << leg_size; if (append_bbo) { append_bbo = false; - auto bid = top_of_book(api::Side::SIDE_BUY); - auto ask = top_of_book(api::Side::SIDE_SELL); + auto bid = top_of_book(Side::Buy); + auto ask = top_of_book(Side::Sell); out << std::setw(8) << FMT("(", bid.quantity, ") ") << std::setw(6) << bid.price << std::setw(6) << ask.price << FMT("(", ask.quantity, ")"); @@ -108,7 +108,7 @@ void FifoImpliedSecurityMatcher::print_legs(std::ostream &out) const { /////////////////////// // book related /////////////////////// -ImpliedBookSide FifoImpliedSecurityMatcher::book(api::Side side) const { +ImpliedBookSide FifoImpliedSecurityMatcher::book(Side side) const { // this isn't the best way to do this (copying), // but the ImpliedBook only has 2 levels in practice, // so it should be fine. @@ -122,7 +122,7 @@ ImpliedBookSide FifoImpliedSecurityMatcher::book(api::Side side) const { return ib; } -SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(api::Side side) const { +SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(Side side) const { SmallBook b; _book(side, b, 1); if (b.empty()) @@ -131,13 +131,13 @@ SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(api::Side side) const { } void FifoImpliedSecurityMatcher::_book( - api::Side side, SmallBook &dest, uint32_t depth) const { + Side side, SmallBook &dest, uint32_t depth) const { std::vector> im_vec; for (const auto &p : _me->implied_mkt) { SmallBook bk; auto [sec_matcher, leg_qty] = p.second; bts_assert(leg_qty.value() != 0); - auto leg_side = leg_qty.value() > 0 ? securities::other_side(side) : side; + auto leg_side = leg_qty.value() > 0 ? other_side(side) : side; for (auto bl : sec_matcher->book(leg_side)) bk.emplace_back(bl.price, bl.quantity); im_vec.emplace_back(leg_qty, Price{}, bk); @@ -156,12 +156,12 @@ void FifoImpliedSecurityMatcher::add_leg( Quantity FifoImpliedSecurityMatcher::fill( OrderKey key, - api::Side side, + Side side, Quantity qty, std::vector *effects, FillDepot &fills) { SmallBook b; - auto oside = securities::other_side(side); + auto oside = other_side(side); _book(oside, b, ImpliedBookSide::max_depth()); auto orig_qty = qty; // can't fill more than the volume in the book @@ -174,8 +174,7 @@ Quantity FifoImpliedSecurityMatcher::fill( auto leg_size = std::get<1>(kv.second); // todo: support other leg sizes (keeping it simple for now) bts_assert(leg_size.value() == 1 || leg_size.value() == -1); - api::Side leg_side = - leg_size.value() > 0 ? side : securities::other_side(side); + Side leg_side = leg_size.value() > 0 ? side : other_side(side); FifoSecurityMatcher &m = *std::get<0>(kv.second); if (local_fills) effects = new std::vector(); diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h index ba62240b8..cf67d1d08 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.h @@ -20,10 +20,10 @@ struct FifoImpliedSecurityMatcher { FifoImpliedSecurityMatcher(SecurityID security_id, std::string symbol); ~FifoImpliedSecurityMatcher(); - ImpliedBookSide book(api::Side) const; + ImpliedBookSide book(Side) const; Quantity fill( OrderKey, - api::Side, + Side, Quantity, std::vector *, FillDepot &); @@ -42,7 +42,7 @@ struct FifoImpliedSecurityMatcher { std::string leg_sym); SecurityID security_id() const; std::string symbol() const; - SmallBookLevel top_of_book(api::Side side) const; + SmallBookLevel top_of_book(Side side) const; void print_legs(std::ostream &out = std::cout) const; private: @@ -52,7 +52,7 @@ struct FifoImpliedSecurityMatcher { FifoImpliedSecurityMatcher(const FifoImpliedSecurityMatcher &); std::string _log_header() const; - void _book(api::Side side, SmallBook &dest, uint32_t depth) const; + void _book(Side side, SmallBook &dest, uint32_t depth) const; uint32_t _fill_order(const MDP3Order &order); }; } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp index 5d7455d66..ce6fb1582 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp @@ -23,11 +23,11 @@ namespace bts::spartan::mdp3 { /////////////////////// // static inline std::string _log_header() { return "FifoMatcherContext"; } -static inline Quantity _norm_qty(api::Side side, Quantity qty) { +static inline Quantity _norm_qty(Side side, Quantity qty) { switch (side) { - case api::Side::SIDE_BUY: + case Side::Buy: return qty; - case api::Side::SIDE_SELL: + case Side::Sell: return -qty; default: bts_assert_msg(false, FMT("unexpected side '", side, "'")); @@ -130,12 +130,12 @@ void FifoMatcherContext::implied_matching_status( /////////////////////// // book /////////////////////// -BookSide FifoMatcherContext::book(SecurityID sec_id, api::Side side) { +BookSide FifoMatcherContext::book(SecurityID sec_id, Side side) { return _get_matcher(sec_id)->book(side); } ImpliedBookSide -FifoMatcherContext::implied_book(SecurityID sec_id, api::Side side) { +FifoMatcherContext::implied_book(SecurityID sec_id, Side side) { return _get_matcher(sec_id)->implied_book(side); } @@ -222,8 +222,8 @@ using BBORow = std::tuple< void FifoMatcherContext::print_all_bbos(std::ostream &out) { std::vector rows; for (auto kv : _me->matchers) { - auto bid = kv.second->matcher()->book_level(api::Side::SIDE_BUY, 0); - auto ask = kv.second->matcher()->book_level(api::Side::SIDE_SELL, 0); + auto bid = kv.second->matcher()->book_level(Side::Buy, 0); + auto ask = kv.second->matcher()->book_level(Side::Sell, 0); rows.emplace_back(std::make_tuple( _me->symbols[kv.first], std::get<0>(bid), std::get<1>(bid), std::get<0>(ask), std::get<1>(ask))); diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h index e4fc83fd1..74e3fcb60 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMatcherContext.h @@ -31,8 +31,8 @@ struct FifoMatcherContext { void implied_matching_status(SecurityID sec_id, bool value); bool implied_matching_status(SecurityID sec_id) const; void clear(); - BookSide book(SecurityID sec_id, api::Side side); - ImpliedBookSide implied_book(SecurityID sec_id, api::Side side); + BookSide book(SecurityID sec_id, Side side); + ImpliedBookSide implied_book(SecurityID sec_id, Side side); // the Fills param records individual fills (potentially more than one per // order) @@ -85,11 +85,10 @@ struct FifoMatcherContext { struct Comp { bool operator()(const MDP3Order &a, const MDP3Order &b) { if (a.side == b.side) { - return bts::orderman::price_comp::is_better( - a.side, a.price, b.price) || + return a.price.is_better(a.side, b.price) || (a.price == b.price && a.priority < b.priority); } - return a.side == api::Side::SIDE_BUY; + return a.side == Side::Buy; } }; std::set sorted_orders; @@ -102,10 +101,10 @@ struct FifoMatcherContext { effects->emplace_back(MBPUpdate()); MBPUpdate &mbp = std::get(effects->back()); mbp.security_id = sec_id; - mbp.mbp_buy = matcher->book(api::Side::SIDE_BUY); - mbp.mbp_sell = matcher->book(api::Side::SIDE_SELL); - mbp.implied_mbp_buy = matcher->implied_book(api::Side::SIDE_BUY); - mbp.implied_mbp_sell = matcher->implied_book(api::Side::SIDE_SELL); + mbp.mbp_buy = matcher->book(Side::Buy); + mbp.mbp_sell = matcher->book(Side::Sell); + mbp.implied_mbp_buy = matcher->implied_book(Side::Buy); + mbp.implied_mbp_sell = matcher->implied_book(Side::Sell); } } }; diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp index c222d4cee..eb85cdb3b 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp @@ -91,15 +91,14 @@ FifoMergedSecurityMatcher::add_implied_matcher(SecurityID sec) { return m; } FifoImpliedSecurityMatcher * -FifoMergedSecurityMatcher::_best_implied_matcher(api::Side side) { +FifoMergedSecurityMatcher::_best_implied_matcher(Side side) { if (!_me->implied_matching_status) return nullptr; - bts::orderman::SidePriceComp cmp(side); FifoImpliedSecurityMatcher *rv = nullptr; Price best_price; for (auto &im : _me->implied_matchers) { auto itop = im->top_of_book(side); - if (itop.quantity.value() > 0 && cmp(itop.price, best_price)) { + if (itop.quantity.value() > 0 && itop.price.is_better(side, best_price)) { rv = im; best_price = itop.price; } @@ -137,8 +136,7 @@ void FifoMergedSecurityMatcher::on_order_change( "unable to process order-update event: failed to find order."); break; } - if (orderman::price_comp::is_better( - ch.order.side, ch.order.price, prev_ord->price)) { + if (ch.order.price.is_better(ch.order.side, prev_ord->price)) { // price improved, try to fill the order auto new_ch = ch; _fill(new_ch.order, effects, fills, false); @@ -174,10 +172,10 @@ void FifoMergedSecurityMatcher::on_order_change( ///////////////////////// // books ///////////////////////// -BookSide FifoMergedSecurityMatcher::book(api::Side side) { +BookSide FifoMergedSecurityMatcher::book(Side side) { return matcher()->book(side); } -ImpliedBookSide FifoMergedSecurityMatcher::implied_book(api::Side side) { +ImpliedBookSide FifoMergedSecurityMatcher::implied_book(Side side) { if (_me->implied_matchers.empty()) return ImpliedBookSide(); // In theory, computing the quantity of any price level beyond the top of the @@ -195,8 +193,8 @@ ImpliedBookSide FifoMergedSecurityMatcher::implied_book(api::Side side) { // the only functionality needed is forward iteration, // and erase (the latter for optimization). std::vector> ibooks; - auto oside = bts::securities::other_side(side); - bts::orderman::SidePriceComp cmp(oside); + auto oside = other_side(side); + bts::orderman::SidePriceComp cmp(map_side(oside)); for (auto it = _me->implied_matchers.begin(); it != _me->implied_matchers.end(); it++) { auto res = ibooks.emplace_back((*it)->book(oside), 0); @@ -239,11 +237,11 @@ ImpliedBookSide FifoMergedSecurityMatcher::implied_book(api::Side side) { } void FifoMergedSecurityMatcher::merged_book( - api::Side side, std::vector &dest) { + Side side, std::vector &dest) { // std::vector> ibooks; - auto oside = bts::securities::other_side(side); - bts::orderman::SidePriceComp cmp(oside); + auto oside = other_side(side); + bts::orderman::SidePriceComp cmp(map_side(oside)); for (auto it = _me->implied_matchers.begin(); it != _me->implied_matchers.end(); it++) { auto res = ibooks.emplace_back((*it)->book(oside), 0); @@ -322,14 +320,14 @@ tuple_to_small_book_level(std::tuple t) { Quantity FifoMergedSecurityMatcher::_fill( OrderKey key, - api::Side side, + Side side, Price price, Quantity quantity, std::vector *effects, FillDepot &fills) { // - auto oside = bts::securities::other_side(side); - bts::orderman::SidePriceComp oside_comp(oside); + auto oside = other_side(side); + bts::orderman::SidePriceComp oside_comp(map_side(oside)); while (quantity.value() > 0) { // Fill one price level at a time. // While one market may have the best 1st price level, diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h index 68deb9ce4..61aa67052 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.h @@ -29,9 +29,9 @@ struct FifoMergedSecurityMatcher { const FifoSecurityMatcher *matcheri() const; const std::vector &implied_matchers(); // books - BookSide book(api::Side); - ImpliedBookSide implied_book(api::Side); - void merged_book(api::Side, std::vector &); + BookSide book(Side); + ImpliedBookSide implied_book(Side); + void merged_book(Side, std::vector &); private: struct PImpl; @@ -39,7 +39,7 @@ struct FifoMergedSecurityMatcher { // hide copy constr FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher &); - FifoImpliedSecurityMatcher *_best_implied_matcher(api::Side); + FifoImpliedSecurityMatcher *_best_implied_matcher(Side); // the order's params (quantity, and iceberg_quantity if applicable) // will be appropriately adjusted according to the number of fills. void _fill( @@ -49,7 +49,7 @@ struct FifoMergedSecurityMatcher { bool reset_show_qty); Quantity _fill( OrderKey, - api::Side, + Side, Price, Quantity, std::vector *, diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp index 30e224f7d..a0f58d54e 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp @@ -15,10 +15,6 @@ inline bool in_this_level(MatcherOrder const *o_ptr) { return o_ptr != nullptr && o_ptr->is_active; } -api::Side _opp_side(api::Side side) { - return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL - : api::Side::SIDE_BUY; -} // O(n) linear search MatcherOrder *FifoPriceLevelMatcher::_find(OrderKey key) { for (MatcherOrder *op : _orders) @@ -93,7 +89,7 @@ FifoPriceLevelMatcher::_one_pass_cross_order( ? OrderUpdate::Status::PARTIALLY_FILLED : OrderUpdate::Status::FILLED, _sec, - _opp_side(_side), + other_side(_side), Quantity(filled), _price, OrderUpdate::FillType::AGGRESSIVE}; @@ -141,7 +137,7 @@ FifoPriceLevelMatcher::FifoPriceLevelMatcher( std::map &master_order_map, SecurityID sec, Price price, - api::Side side, + Side side, const Trace &trace) : _master_order_map(&master_order_map), _sec(sec), _price(price), _side(side), _orders(), _aux_usage(), _priority(0), _trace(trace) {} @@ -177,8 +173,7 @@ FifoPriceLevelMatcher::operator=(FifoPriceLevelMatcher &&o) { } bool FifoPriceLevelMatcher::operator<(const FifoPriceLevelMatcher &o) const { bts_assert(_side == o._side); - // bid book is sorted high to low (since highest is the top of the book) - return _side == api::Side::SIDE_SELL ? _price < o._price : o._price < _price; + return _price.is_better(_side, o._price); } // simple accessors /////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// @@ -296,7 +291,7 @@ void FifoPriceLevelMatcher::cleanup() { std::string FifoPriceLevelMatcher::print_mbo(size_t n_orders) const { std::ostringstream buff; auto [n_ords, qty] = n_orders_qty(); - buff << (side() == api::Side::SIDE_BUY ? "B" : "S") << "," << price(); + buff << (side() == Side::Buy ? "B" : "S") << "," << price(); buff << "," << qty << "," << n_ords << ":"; size_t i = 0; for (MatcherOrder *o : _orders) { diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h index 5bd9f8763..95323d435 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.h @@ -19,7 +19,7 @@ class FifoPriceLevelMatcher { std::map *_master_order_map; SecurityID _sec; Price _price; - api::Side _side; + Side _side; std::vector _orders; std::vector _aux_usage; uint32_t _priority; @@ -52,7 +52,7 @@ class FifoPriceLevelMatcher { std::map &, SecurityID sec, Price price, - api::Side side, + Side side, const Trace &); FifoPriceLevelMatcher(const FifoPriceLevelMatcher &); FifoPriceLevelMatcher(FifoPriceLevelMatcher &&); @@ -82,7 +82,7 @@ class FifoPriceLevelMatcher { bool populated() const; Price price() const { return _price; } double price_double() const { return _price.to_double(); } - api::Side side() const { return _side; } + Side side() const { return _side; } MatcherOrder *find_order(OrderKey key) { return _find(key); } BookLevel book_level() const { return BookLevel{_price, quantity(), static_cast(n_orders())}; @@ -90,7 +90,7 @@ class FifoPriceLevelMatcher { // used for testing void copy_values_to(std::vector &out_v) const; // mutators (silly reason) ///////////////////////////////////////////////// - void set_price_side(Price p, api::Side s) { + void set_price_side(Price p, Side s) { _price = p; _side = s; } diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp index ad0d6d123..da50b6f0c 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp @@ -12,9 +12,9 @@ namespace bts::spartan::mdp3 { inline bool _is_cross(const MDP3Order &ord, const FifoPriceLevelMatcher &price_level) { return ord.side != price_level.side() && - ((ord.side == api::Side::SIDE_BUY && + ((ord.side == Side::Buy && ord.price >= price_level.price()) || - (ord.side == api::Side::SIDE_SELL && + (ord.side == Side::Sell && ord.price <= price_level.price())); } @@ -23,14 +23,9 @@ inline Quantity total_quantity(const MDP3Order &ord) { : ord.quantity; } -inline api::Side _opp_side(api::Side side) { - return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL - : api::Side::SIDE_BUY; -} - // O(lg(N)) on the price levels FifoPriceLevelMatcher * -FifoSecurityMatcher::_find_price_level(api::Side side, Price price) { +FifoSecurityMatcher::_find_price_level(Side side, Price price) { auto &bk = _book(side); _search_tok.set_price_side(price, side); auto it = bk.find(_search_tok); @@ -51,7 +46,7 @@ FifoSecurityMatcher::_find_order(OrderKey key) { return std::tuple(nullptr, nullptr); } -FifoPriceLevelMatcher *FifoSecurityMatcher::_top(api::Side side) { +FifoPriceLevelMatcher *FifoSecurityMatcher::_top(Side side) { for (auto &plevel : _book(side)) if (plevel.populated()) return &plevel; @@ -81,7 +76,7 @@ void FifoSecurityMatcher::_cleanup() { } FifoPriceLevelMatcher *FifoSecurityMatcher::_get_or_add_price_level( - api::Side side, Price price) { + Side side, Price price) { auto *price_level = _find_price_level(side, price); if (price_level == nullptr) { auto &same_side_book = _book(side); @@ -100,7 +95,7 @@ void FifoSecurityMatcher::_add_resting_order(const MDP3Order &ord) { void FifoSecurityMatcher::_on_new_order( const MDP3Order &ord, std::vector *res) { _log_trace_order(ord, "FifoSecurityMatcher::_on_new_order begin"); - auto &opp_book = _book(_opp_side(ord.side)); + auto &opp_book = _book(other_side(ord.side)); uint32_t new_ord_rem = total_quantity(ord).value(); for (FifoPriceLevelMatcher &pl : opp_book) { if (_is_cross(ord, pl)) { @@ -220,7 +215,7 @@ void FifoSecurityMatcher::_on_order_not_affecting_books( case OrderChange::updated_price: { // since this does not affect the market all we need to do // is deal with the cross - auto &opp_book = _book(_opp_side(ord.side)); + auto &opp_book = _book(other_side(ord.side)); uint32_t new_ord_rem = total_quantity(ord).value(); for (const FifoPriceLevelMatcher &pl : opp_book) { if (!pl.populated()) @@ -273,7 +268,7 @@ FifoSecurityMatcher::FifoSecurityMatcher( _order_map, SecurityID{}, Price(), - api::Side::SIDE_BUY, + Side::Buy, Trace()), _sec(args.security_id), _event_id(args.event_id), _trace(), _ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders), @@ -325,23 +320,23 @@ const BookSide &FifoSecurityMatcher::BookKeeper::mbp() { } return _mbp; } -BookSide FifoSecurityMatcher::book(api::Side side) { - return (side == api::Side::SIDE_BUY ? _bids : _asks).mbp(); +BookSide FifoSecurityMatcher::book(Side side) { + return (side == Side::Buy ? _bids : _asks).mbp(); } -size_t FifoSecurityMatcher::total_n_orders(api::Side side) const { +size_t FifoSecurityMatcher::total_n_orders(Side side) const { auto rv = 0; for (const auto &p_level : _book(side)) rv += p_level.n_orders(); return rv; } -Quantity FifoSecurityMatcher::total_qty(api::Side side) const { +Quantity FifoSecurityMatcher::total_qty(Side side) const { auto rv = 0; for (const auto &p_level : _book(side)) rv += p_level.total_quantity().value(); return Quantity(rv); } std::tuple -FifoSecurityMatcher::book_level(api::Side side, size_t idx) const { +FifoSecurityMatcher::book_level(Side side, size_t idx) const { size_t i = 0; for (const auto &p_level : _book(side)) if (p_level.populated() && i++ == idx) @@ -384,19 +379,18 @@ void FifoSecurityMatcher::on_order_completed(OrderKey key) { ord != nullptr, "did not find order that was supposedely 'completed.'"); MDP3Order order; order.key = key; - order.side = api::SIDE_UNDEFINED; SmallOrderChange ch(order, OrderChange::ChangeType::removed); on_order_change(ch); } Quantity FifoSecurityMatcher::fill( OrderKey aggr_key, - api::Side side, + Side side, Price agg_price, Quantity qty, std::vector *res, bool add_aggr_fill) { - FifoPriceLevelMatcher *top = _top(_opp_side(side)); + FifoPriceLevelMatcher *top = _top(other_side(side)); if (top != nullptr) { bts_assert(top->price() == agg_price || agg_price.is_nan()); auto t = top->on_new_opp_order(aggr_key, qty, add_aggr_fill, res); @@ -465,7 +459,7 @@ std::string FifoSecurityMatcher::print_book(size_t n_levels) { // auto rem_if = [](const auto& p_lev) { return !p_lev.populated(); }; std::ostringstream buff; auto add_to_buff = [&](const auto &pl) { - std::string side = pl.side() == api::Side::SIDE_BUY ? "B" : "S"; + std::string side = pl.side() == Side::Buy ? "B" : "S"; buff << side << ":" << pl.price_double() << ":" << pl.quantity().value() << ":" << pl.n_orders() << std::endl; }; diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h index 8dec59e43..b9d804c84 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h @@ -59,13 +59,13 @@ class FifoSecurityMatcher : public SecurityMatcher { // helpers ///////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////// // complexity: O(lg(n)) - FifoPriceLevelMatcher *_find_price_level(api::Side, Price price); - boost::container::flat_set &_book(api::Side side) { - return (side == api::Side::SIDE_BUY ? _bids : _asks).book(); + FifoPriceLevelMatcher *_find_price_level(Side, Price price); + boost::container::flat_set &_book(Side side) { + return (side == Side::Buy ? _bids : _asks).book(); } const boost::container::flat_set & - _book(api::Side side) const { - return (side == api::Side::SIDE_BUY ? _bids : _asks).booki(); + _book(Side side) const { + return (side == Side::Buy ? _bids : _asks).booki(); } void _invalidate_mbp() { _bids.invalidate(); @@ -76,7 +76,7 @@ class FifoSecurityMatcher : public SecurityMatcher { // for price level in the book (flat_set) std::tuple _find_order(OrderKey key); - FifoPriceLevelMatcher *_top(api::Side side); + FifoPriceLevelMatcher *_top(Side side); void _cleanup(); // order interaction void _on_new_order(const MDP3Order &, std::vector * = nullptr); @@ -99,7 +99,7 @@ class FifoSecurityMatcher : public SecurityMatcher { ~FifoSecurityMatcher(); // SecurityMatcher implementation ////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////// - BookSide book(api::Side) override; + BookSide book(Side) override; // The qty, and add_aggr_fill params address a couple of issues wrt 'merged // matching:' Particular quantities of passive fills need to be generated // when an aggressor order is filled against an implied market, while the @@ -116,20 +116,20 @@ class FifoSecurityMatcher : public SecurityMatcher { // necessarily have the 2nd best price as well. Quantity fill( OrderKey aggr_key, - api::Side side, + Side side, Price price, Quantity qty, std::vector *res, bool add_aggr_fill); std::tuple - book_level(api::Side side, size_t idx) const override; + book_level(Side side, size_t idx) const override; void on_order_change( SmallOrderChange const &, std::vector * = nullptr) override; void on_order_completed(OrderKey key); // For testing std::optional has_same_orders(Orders const &); - size_t total_n_orders(api::Side) const; - Quantity total_qty(api::Side) const; + size_t total_n_orders(Side) const; + Quantity total_qty(Side) const; // Removes existing orders. void clear(); // Removes existing orders, then populates the book with the @@ -170,6 +170,6 @@ class FifoSecurityMatcher : public SecurityMatcher { // into the matcher, as the order's visible quantity is often less than // the show quantity (e.g., after partial fills, before the next reload). void _add_resting_order(const MDP3Order &); - FifoPriceLevelMatcher *_get_or_add_price_level(api::Side side, Price); + FifoPriceLevelMatcher *_get_or_add_price_level(Side side, Price); }; } // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp index 21844566c..4b71b05e4 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.cpp @@ -50,10 +50,10 @@ template inline bool mbp_equals(const MBP &a, const MBP &b) { struct MBPData { MBPData() {} MBPData(FifoMergedSecurityMatcher &mat) - : bids(mat.book(api::Side::SIDE_BUY)), - asks(mat.book(api::Side::SIDE_SELL)), - ibids(mat.implied_book(api::Side::SIDE_BUY)), - iasks(mat.implied_book(api::Side::SIDE_SELL)) {} + : bids(mat.book(Side::Buy)), + asks(mat.book(Side::Sell)), + ibids(mat.implied_book(Side::Buy)), + iasks(mat.implied_book(Side::Sell)) {} BookSide bids, asks; ImpliedBookSide ibids, iasks; }; @@ -186,11 +186,11 @@ void MergedMatcher::match( } } -BookSide MergedMatcher::book(SecurityID id, api::Side s) { +BookSide MergedMatcher::book(SecurityID id, Side s) { return _asset_matcher(id).book(id, s); } -ImpliedBookSide MergedMatcher::implied_book(SecurityID id, api::Side s) { +ImpliedBookSide MergedMatcher::implied_book(SecurityID id, Side s) { return _asset_matcher(id).implied_book(id, s); } @@ -242,8 +242,8 @@ void DirectMatcher::match( // books for (auto cm : dirty) res->emplace_back(MBPUpdate{ - cm.first, cm.second->book(api::Side::SIDE_BUY), - cm.second->book(api::Side::SIDE_SELL), ImpliedBookSide(), ImpliedBookSide()}); + cm.first, cm.second->book(Side::Buy), + cm.second->book(Side::Sell), ImpliedBookSide(), ImpliedBookSide()}); auto now = TimeUtil::GetCurrentTime(); auto usecs = TimeUtil::TimestampToMicroseconds(now) - TimeUtil::TimestampToMicroseconds(_per_start_time); @@ -256,7 +256,7 @@ void DirectMatcher::match( } } -BookSide DirectMatcher::book(SecurityID id, api::Side s) { +BookSide DirectMatcher::book(SecurityID id, Side s) { auto it = _matcher_map.find(id); bts_assert(it != _matcher_map.end()); return it->second.book(s); diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h index 748f85deb..806a17002 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h @@ -40,8 +40,8 @@ class MergedMatcher { void match(ChangeSrc, SmallOrderChangeVec const &, std::vector *); - BookSide book(SecurityID, api::Side); - ImpliedBookSide implied_book(SecurityID, api::Side); + BookSide book(SecurityID, Side); + ImpliedBookSide implied_book(SecurityID, Side); template void init(TIter first, TIter last) { while (first != last) { @@ -84,8 +84,8 @@ class DirectMatcher { void match(ChangeSrc, SmallOrderChangeVec const &, std::vector *); - BookSide book(SecurityID, api::Side); - ImpliedBookSide implied_book(SecurityID, api::Side); + BookSide book(SecurityID, Side); + ImpliedBookSide implied_book(SecurityID, Side); template void reset(SecurityID sec, Iter first, Iter last) { _matcher_map.try_emplace(sec).first->second.reset(first, last); diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h index 993a32d8d..f990473b8 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/MatchingCommon.h @@ -24,21 +24,18 @@ struct Trace { bool is_ignored_event(size_t eid) const { return ignored_events.find(eid) != ignored_events.end(); } - std::string _side(api::Side s) { - return s == api::Side::SIDE_BUY ? "BUY" : "SELL"; - } void log_incoming_change(const SmallOrderChange &c) { if (traced_orders.find(c.order.key) != traced_orders.end()) LOG(OrderMan, Diag, "Matcher incoming change:{} ok:{} side:{} price:{} qty:{} ib:{}", - c.change, c.order.key, _side(c.order.side), c.order.price, + c.change, c.order.key, c.order.side, c.order.price, c.order.quantity, c.order.iceberg_quantity); } void log_outgoing_update(const OrderUpdate &u) { if (traced_orders.find(u.key) != traced_orders.end()) LOG(OrderMan, Diag, "Matcher outgoing update:{} ok:{} side:{} last_fp:{} last_fq:{}", - u.status, u.key, _side(u.side), u.last_price, u.last_fills); + u.status, u.key, u.side, u.last_price, u.last_fills); } }; @@ -86,7 +83,7 @@ struct MatcherOrder { return std::make_tuple(resting_qty, cross_qty); } OrderKey key; - api::Side side; + Side side; Price price; Quantity quantity; uint32_t priority = 0; @@ -94,7 +91,7 @@ struct MatcherOrder { bool is_active = true; MatcherOrder( OrderKey key, - api::Side side, + Side side, Price price, Quantity qty, // working qty out there uint32_t priority, diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h index 9f5fe8f5c..1d689ae3e 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/SecurityMatcher.h @@ -13,9 +13,9 @@ class SecurityMatcher { // for performance reasons. Its internal state will change // when this method is called if its book is dirty, // and has to be recomputed. - virtual BookSide book(api::Side) = 0; + virtual BookSide book(Side) = 0; virtual std::tuple - book_level(api::Side, size_t level = 0) const = 0; + book_level(Side, size_t level = 0) const = 0; // mutators virtual void on_order_change( SmallOrderChange const &, std::vector * = nullptr) = 0; diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h index 56bfd7a9c..fc455d754 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/fill_factory.h @@ -2,11 +2,11 @@ #include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h" namespace bts::spartan::mdp3 { -inline MDP3Fill::Aggressor _to_aggressor(api::Side s) { +inline MDP3Fill::Aggressor _to_aggressor(Side s) { switch (s) { - case api::Side::SIDE_BUY: + case Side::Buy: return MDP3Fill::Aggressor::THIS_BUY; - case api::Side::SIDE_SELL: + case Side::Sell: return MDP3Fill::Aggressor::THIS_SELL; default: return MDP3Fill::Aggressor::OTHER; @@ -25,7 +25,7 @@ _to_passive_fill(const OrderUpdate &upd, bool implied_aggressor) { } inline MDP3Fill -_to_aggressor_fill(OrderKey k, api::Side side, const OrderUpdate &upd) { +_to_aggressor_fill(OrderKey k, Side side, const OrderUpdate &upd) { MDP3Fill aggr; aggr.key = k; aggr.price = upd.last_price; diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h index 7fc65cb56..51743c218 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/matcher_diagnostics.h @@ -216,8 +216,8 @@ inline void print_detailed_order_changes( template inline void print_instrument_snapshots( TIter first, TIter last, std::ostream &out = std::cout) { - auto book_volume = [](const SecuritySnapshot *snap, api::Side side) { - const auto &b = snap->book_side(map_side(side)); + auto book_volume = [](const SecuritySnapshot *snap, Side side) { + const auto &b = snap->book_side(side); int vol = ranges::accumulate( b | ranges::views::transform( [](const BookLevel &l) { return l.quantity.value(); }), @@ -226,8 +226,8 @@ inline void print_instrument_snapshots( }; auto total_book_volume = [&](const SecuritySnapshot *snap) { return Quantity( - book_volume(snap, api::Side::SIDE_BUY).value() + - book_volume(snap, api::Side::SIDE_SELL).value()); + book_volume(snap, Side::Buy).value() + + book_volume(snap, Side::Sell).value()); }; bts::spartan::table_display::print_table( out, first, last, @@ -253,11 +253,11 @@ inline void print_instrument_snapshots( [](const auto *s) { return FMT(s->cleared_volume()); }}, {" Bid Vol ", 10, [&](const auto *s) { - return FMT(book_volume(s, api::Side::SIDE_BUY)); + return FMT(book_volume(s, Side::Buy)); }}, {" Ask Vol ", 10, [&](const auto *s) { - return FMT(book_volume(s, api::Side::SIDE_SELL)); + return FMT(book_volume(s, Side::Sell)); }}, {" Tot Vol ", 10, [&](const auto *s) { return FMT(total_book_volume(s)); }}, @@ -269,19 +269,19 @@ inline void print_instrument_snapshots( [](const auto *s) { return FMT(s->fills().size()); }}, {" Qty ", 8, [](const auto *s) { - return FMT(s->book(api::Side::SIDE_BUY)[0].quantity); + return FMT(s->book(Side::Buy)[0].quantity); }}, {" Bid ", 8, [](const auto *s) { - return FMT(s->book(api::Side::SIDE_BUY)[0].price); + return FMT(s->book(Side::Buy)[0].price); }}, {" Ask ", 8, [](const auto *s) { - return FMT(s->book(api::Side::SIDE_SELL)[0].price); + return FMT(s->book(Side::Sell)[0].price); }}, {" Qty ", 8, [](const auto *s) { - return FMT(s->book(api::Side::SIDE_SELL)[0].quantity); + return FMT(s->book(Side::Sell)[0].quantity); }}, {"DImp", 4, [](const auto *s) { @@ -295,18 +295,18 @@ inline void print_instrument_snapshots( }}, {" I Qty ", 8, [](const auto *s) { - return FMT(s->implied_book(api::Side::SIDE_BUY)[0].quantity); + return FMT(s->implied_book(Side::Buy)[0].quantity); }}, {" I Bid ", 8, [](const auto *s) { - return FMT(s->implied_book(api::Side::SIDE_BUY)[0].price); + return FMT(s->implied_book(Side::Buy)[0].price); }}, {" I Ask ", 8, [](const auto *s) { - return FMT(s->implied_book(api::Side::SIDE_SELL)[0].price); + return FMT(s->implied_book(Side::Sell)[0].price); }}, {" I Qty ", 8, [](const auto *s) { - return FMT(s->implied_book(api::Side::SIDE_SELL)[0].quantity); + return FMT(s->implied_book(Side::Sell)[0].quantity); }}}); out << std::endl; } diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp index 2dd16fe6b..e2aa5ba23 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp @@ -25,16 +25,16 @@ class RandomGenerator { return static_cast(random_double(min, max)); } - api::Side random_side() { - return random_int(1, 100) % 2 == 0 ? api::Side::SIDE_BUY - : api::Side::SIDE_SELL; + Side random_side() { + return random_int(1, 100) % 2 == 0 ? Side::Buy + : Side::Sell; } }; MDP3Order create_order( uint64_t key, int32_t sec, - api::Side side, + Side side, uint64_t priority, double price, uint32_t qty, @@ -51,13 +51,13 @@ MDP3Order create_order( } MDP3Order create_empty_order(uint64_t key) { - return create_order(key, 0, api::Side::SIDE_BUY, 0, 0.0, 0); + return create_order(key, 0, Side::Buy, 0, 0.0, 0); } TEST_CASE("fifo_matching_engine_single_level", "[matcher]") { auto sec_key = 10; double price = 100.0; - api::Side side = api::Side::SIDE_BUY; + Side side = Side::Buy; Trace trace; std::map master_ord_map; FifoPriceLevelMatcher price_level( @@ -124,7 +124,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher]") { auto &ord_59 = out_ords.at(59); REQUIRE(ord_59.quantity.value() == 60); auto ord_59_mod = create_order( - ord_59.key.value(), 60, api::Side::SIDE_BUY, 0, price, 20); + ord_59.key.value(), 60, Side::Buy, 0, price, 20); price_level.on_order_update(ord_59_mod); // stays at the same level price_level.copy_values_to(out_ords); @@ -140,7 +140,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher]") { auto &ord_59 = out_ords.at(59); REQUIRE(ord_59.quantity.value() == 60); auto ord_59_mod = create_order( - ord_59.key.value(), 60, api::Side::SIDE_BUY, 0, price, 61); + ord_59.key.value(), 60, Side::Buy, 0, price, 61); price_level.on_order_update(ord_59_mod); // goes to the end price_level.copy_values_to(out_ords); @@ -151,19 +151,19 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher]") { SECTION("iceberg_simple") { auto k = 0; auto price = 100.0; - auto o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, 100.0, 1); + auto o = create_order(k++, sec_key, Side::Buy, 0, 100.0, 1); price_level.on_new_same_side_order(o); - o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 2); + o = create_order(k++, sec_key, Side::Buy, 0, price, 2); price_level.on_new_same_side_order(o); - o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 3); + o = create_order(k++, sec_key, Side::Buy, 0, price, 3); price_level.on_new_same_side_order(o); // iceberg, show qty=2, total_qty=100 o = create_order( - k++, sec_key, api::Side::SIDE_BUY, 0, price, 2, + k++, sec_key, Side::Buy, 0, price, 2, IcebergQuantity{Quantity(2), Quantity(100)}); price_level.on_new_same_side_order(o); // non ib, to get the visible qty to 15 - o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 7); + o = create_order(k++, sec_key, Side::Buy, 0, price, 7); price_level.on_new_same_side_order(o); REQUIRE(price_level.quantity().value() == 15); REQUIRE(price_level.total_quantity().value() == 113); @@ -210,7 +210,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { FifoSecurityMatcher sec_me(SecurityID(100)); auto order_key = 0; auto priority = 0; - auto add_to_book = [&](uint32_t n_orders, api::Side side, double price, + auto add_to_book = [&](uint32_t n_orders, Side side, double price, bool affect_book = true) -> void { for (uint32_t i = 1; i <= n_orders; i++) { auto o = @@ -232,7 +232,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + add_to_book((lp.i + 1) * 10, Side::Buy, lp.p); // asks for (struct { int i = 0; @@ -243,10 +243,10 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + add_to_book((lp.i + 1) * 10, Side::Sell, lp.p); // verify bids { - BookSide bid_bk = sec_me.book(api::Side::SIDE_BUY); + BookSide bid_bk = sec_me.book(Side::Buy); double exp_price = 15.09; for (int i = 0; i < n_levels; i++) { auto exp_n_orders = 100 - i * 10; @@ -258,7 +258,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } // verify asks { - BookSide ask_bk = sec_me.book(api::Side::SIDE_SELL); + BookSide ask_bk = sec_me.book(Side::Sell); double exp_price = 15.10; for (int i = 0; i < n_levels; i++) { auto exp_n_orders = (i + 1) * 10; @@ -282,7 +282,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + add_to_book((lp.i + 1) * 10, Side::Buy, lp.p); // asks for (struct { int i = 0; @@ -293,9 +293,9 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + add_to_book((lp.i + 1) * 10, Side::Sell, lp.p); auto verify_bids = [&]() { - BookSide bid_bk = sec_me.book(api::Side::SIDE_BUY); + BookSide bid_bk = sec_me.book(Side::Buy); double exp_price = 15.09; for (int i = 0; i < n_levels; i++) { auto exp_n_orders = 100 - i * 10; @@ -306,7 +306,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } }; auto verify_asks = [&]() { - BookSide ask_bk = sec_me.book(api::Side::SIDE_SELL); + BookSide ask_bk = sec_me.book(Side::Sell); double exp_price = 15.10; for (int i = 0; i < n_levels; i++) { auto exp_n_orders = (i + 1) * 10; @@ -321,11 +321,11 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { // bids: 15.00-15.09 // asks: 15.10-15.19 // add a passive order, should have no effect on the markets - add_to_book(100, api::Side::SIDE_BUY, 15.09, false); + add_to_book(100, Side::Buy, 15.09, false); verify_bids(); verify_asks(); // add an aggressive order - add_to_book(100, api::Side::SIDE_BUY, 15.19, false); + add_to_book(100, Side::Buy, 15.19, false); verify_bids(); verify_asks(); } @@ -341,7 +341,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + add_to_book((lp.i + 1) * 10, Side::Buy, lp.p); // asks for (struct { int i = 0; @@ -352,7 +352,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + add_to_book((lp.i + 1) * 10, Side::Sell, lp.p); // remove one order at a time from the second level // 90 orders starting with id 360 auto b = 360; @@ -368,7 +368,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { remvd += qty; auto now_qty = init_qty - remvd; auto now_n_orders = init_n_orders - (i + 1); - BookSide bk = sec_me.book(api::Side::SIDE_BUY); + BookSide bk = sec_me.book(Side::Buy); if (now_qty == 0) // whole level got removed // now level 1 should be 15.07 @@ -376,7 +376,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { else verify_book_level(bk[1], 15.08, now_qty, now_n_orders); } - BookSide bk = sec_me.book(api::Side::SIDE_BUY); + BookSide bk = sec_me.book(Side::Buy); verify_book_level(bk[1], 15.07, 3240, 80); } SECTION("modify") { @@ -391,7 +391,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + add_to_book((lp.i + 1) * 10, Side::Buy, lp.p); // asks for (struct { int i = 0; @@ -402,28 +402,28 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + add_to_book((lp.i + 1) * 10, Side::Sell, lp.p); // modify qty of all orders to 1 auto b = 360; for (auto i = 0; i < 90; i++) { auto id = i + b; auto ord = - create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.08, 1); + create_order(id, sec_key, Side::Buy, id, 15.08, 1); sec_me.on_order_change(OrderChange{ create_empty_order(ord.key.value()), ord, OrderChange::updated}); } - BookSide bk = sec_me.book(api::Side::SIDE_BUY); + BookSide bk = sec_me.book(Side::Buy); verify_book_level(bk[1], 15.08, 90, 90); // now change all these prices to move to a diff level for (auto i = 0; i < 90; i++) { auto id = i + b; auto ord = - create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.09, 1); + create_order(id, sec_key, Side::Buy, id, 15.09, 1); sec_me.on_order_change(OrderChange{ create_empty_order(ord.key.value()), ord, OrderChange::updated}); } // 15.08 is gone - bk = sec_me.book(api::Side::SIDE_BUY); + bk = sec_me.book(Side::Buy); verify_book_level(bk[1], 15.07, 3240, 80); verify_book_level(bk[0], 15.09, 5140, 190); } @@ -439,7 +439,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + add_to_book((lp.i + 1) * 10, Side::Buy, lp.p); // asks for (struct { int i = 0; @@ -450,24 +450,24 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); + add_to_book((lp.i + 1) * 10, Side::Sell, lp.p); // send a new sell at 15.09 to take the first 10 orders auto ord = - create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.09, 55); + create_order(10000, sec_key, Side::Sell, 10000, 15.09, 55); sec_me.on_order_change(OrderChange{MDP3Order{}, ord, OrderChange::added}); - auto bk = sec_me.book(api::Side::SIDE_BUY); + auto bk = sec_me.book(Side::Buy); verify_book_level(bk[0], 15.09, 4995, 90); verify_book_level(bk[1], 15.08, 4095, 90); verify_book_level(bk[2], 15.07, 3240, 80); // now completely sweep 2 complete levels and make the rest a working // order ord = create_order( - 10001, sec_key, api::Side::SIDE_SELL, 10001, 15.08, 10000); + 10001, sec_key, Side::Sell, 10001, 15.08, 10000); sec_me.on_order_change(OrderChange{MDP3Order{}, ord, OrderChange::added}); - bk = sec_me.book(api::Side::SIDE_BUY); + bk = sec_me.book(Side::Buy); verify_book_level(bk[0], 15.07, 3240, 80); // failing (11/16) - auto bk_ask = sec_me.book(api::Side::SIDE_SELL); + auto bk_ask = sec_me.book(Side::Sell); verify_book_level(bk_ask[0], 15.08, 910, 1); } SECTION("IB_cross") { @@ -484,7 +484,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p); + add_to_book((lp.i + 1) * 10, Side::Buy, lp.p); // asks for (struct { int i = 0; @@ -495,8 +495,8 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { } } lp; lp.i < n_levels; lp.inc()) - add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p); - auto bid_bk = sec_me.book(api::Side::SIDE_BUY); + add_to_book((lp.i + 1) * 10, Side::Sell, lp.p); + auto bid_bk = sec_me.book(Side::Buy); verify_book_level(bid_bk[0], 15.09, 5050, 100); // the bid book is // 15.09 5050 100 @@ -505,13 +505,13 @@ TEST_CASE("fifo_matching_engine_book", "[matcher]") { // 15.06 2485 70 // send a new sell at 15.07 to take the first 3 levels auto cross_ib = create_order( - 10000, sec_key, api::Side::SIDE_SELL, 10000, 15.07, 13, + 10000, sec_key, Side::Sell, 10000, 15.07, 13, IcebergQuantity{Quantity(13), Quantity(12500)}); sec_me.on_order_change( OrderChange{MDP3Order{}, cross_ib, OrderChange::added}); - bid_bk = sec_me.book(api::Side::SIDE_BUY); + bid_bk = sec_me.book(Side::Buy); verify_book_level(bid_bk[0], 15.06, 2485, 70); - auto ask_bk = sec_me.book(api::Side::SIDE_SELL); + auto ask_bk = sec_me.book(Side::Sell); verify_book_level(ask_bk[0], 15.07, 13, 1); } } @@ -532,7 +532,7 @@ TEST_CASE("perf", "[matcher]") { int p_lev = rnd_gen.random_int(0, n_levels - 1); bts_assert(p_lev >= 0); double price; - if (ord.side == api::Side::SIDE_BUY) + if (ord.side == Side::Buy) price = top_bid - (p_lev * tick_size); else price = (top_bid + tick_size) + (p_lev * tick_size); @@ -567,8 +567,8 @@ TEST_CASE("perf", "[matcher]") { TimeUtil::TimestampToNanoseconds(t0); std::cout << "Adding " << aux.size() << " took " << 1.0 * nsecs / 1000000 << " msecs" << std::endl; - size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) + - sec_m.total_n_orders(api::Side::SIDE_SELL); + size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) + + sec_m.total_n_orders(Side::Sell); REQUIRE(comp_n_orders == n_orders); n_orders += inc; } @@ -593,8 +593,8 @@ TEST_CASE("perf", "[matcher]") { std::cout << "Removing " << n_orders << " took " << 1.0 * nsecs / 1000000 << " msecs" << std::endl; n_orders += inc; - size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) + - sec_m.total_n_orders(api::Side::SIDE_SELL); + size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) + + sec_m.total_n_orders(Side::Sell); REQUIRE(comp_n_orders == 0); } } @@ -618,8 +618,8 @@ TEST_CASE("perf", "[matcher]") { TimeUtil::TimestampToNanoseconds(t0); std::cout << "Updaing (qty dec) " << n_orders << " took " << 1.0 * nsecs / 1000000 << " msecs" << std::endl; - size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) + - sec_m.total_n_orders(api::Side::SIDE_SELL); + size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) + + sec_m.total_n_orders(Side::Sell); REQUIRE(comp_n_orders == n_orders); n_orders += inc; } @@ -644,8 +644,8 @@ TEST_CASE("perf", "[matcher]") { TimeUtil::TimestampToNanoseconds(t0); std::cout << "Updaing (qty inc) " << n_orders << " took " << 1.0 * nsecs / 1000000 << " msecs" << std::endl; - size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) + - sec_m.total_n_orders(api::Side::SIDE_SELL); + size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) + + sec_m.total_n_orders(Side::Sell); REQUIRE(comp_n_orders == n_orders); n_orders += inc; } @@ -663,7 +663,7 @@ TEST_CASE("perf", "[matcher]") { for (auto &oc : aux) { oc.change = OrderChange::ChangeType::updated; double inc = tick_size; - if (oc.order.side == api::Side::SIDE_BUY) + if (oc.order.side == Side::Buy) inc = -1 * inc; oc.order.price = Price::from_double<9>(oc.order.price.to_double() + inc), @@ -675,8 +675,8 @@ TEST_CASE("perf", "[matcher]") { std::cout << "Updating (price change no cross) " << n_orders << " took " << 1.0 * nsecs / 1000000 << " msecs" << std::endl; - size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) + - sec_m.total_n_orders(api::Side::SIDE_SELL); + size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) + + sec_m.total_n_orders(Side::Sell); REQUIRE(comp_n_orders == n_orders); n_orders += inc; } @@ -693,13 +693,13 @@ TEST_CASE("perf", "[matcher]") { // send one lot buy orders to completely cross // the offer market size_t buy_orders_to_send = - sec_m.total_qty(api::Side::SIDE_SELL).value(); + sec_m.total_qty(Side::Sell).value(); OrderChange ochange; ochange.change = OrderChange::ChangeType::added; ochange.order = MDP3Order{ OrderKey(0), SecurityID(100), - api::Side::SIDE_BUY, + Side::Buy, MDP3OrderPriority(0), Price::from_double<9>(10000), Quantity(1), @@ -721,7 +721,7 @@ TEST_CASE("perf", "[matcher]") { std::cout << "Crossing (with new orders) " << n_orders << " took " << 1.0 * nsecs / 1000000 << " msecs" << std::endl; // whole sell side should be gone - REQUIRE(sec_m.total_n_orders(api::Side::SIDE_SELL) == 0); + REQUIRE(sec_m.total_n_orders(Side::Sell) == 0); n_orders += inc; } } diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp index 1b083867c..46af2e656 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/Sim.cpp @@ -611,8 +611,8 @@ double Sim::price_at_market(SecurityID id, Quantity qty, Side s) const { } PandLSummary Sim::pnl() const { - auto mkt_pricer = [this](SecurityID id, Quantity qty, api::Side s) { - return price_at_market(id, qty, map_side(s)); + auto mkt_pricer = [this](SecurityID id, Quantity qty, Side s) { + return price_at_market(id, qty, s); }; PandL pl{me->instruments, mkt_pricer}; @@ -710,8 +710,7 @@ void Sim::validate_matcher_books(Side side) { for (auto &[sec_id, snap] : me->snapshots) { if (snap.ready_to_trade()) { - if (snap.book_side(side) != - simulation.matcher.book(sec_id, map_side(side))) { + if (snap.book_side(side) != simulation.matcher.book(sec_id, side)) { LOG(Spartan, Error, "Mismatch between matcher and pcap books on event: {}. " @@ -722,7 +721,7 @@ void Sim::validate_matcher_books(Side side) { LOG(Spartan, Error, "{}", snap.book_side(side)); LOG(Spartan, Error, "Matcher Book:"); LOG(Spartan, Error, "{}", - simulation.matcher.book(sec_id, map_side(side))); + simulation.matcher.book(sec_id, side)); bts_assert(false); } } diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp index 4060d5820..70a0159e0 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.cpp @@ -78,11 +78,7 @@ double PandL::position_liquidation_value( OutrightPosition const &outright_position) const { double liquidation_proceeds = 0; for (auto &[id, this_pos] : outright_position) { - auto mkt_action = api::SIDE_UNDEFINED; - if (this_pos > 0) - mkt_action = api::SIDE_SELL; - else - mkt_action = api::SIDE_BUY; + auto mkt_action = this_pos > 0 ? Side::Sell : Side::Buy; auto cash_flow = me->pricer(id, Quantity{(int)this_pos}, mkt_action); liquidation_proceeds += cash_flow; } @@ -97,14 +93,14 @@ double PandL::price_to_cash(SecurityID id, double p) const { return get_sec(id).to_cash(p); } -int PandL::mkt_to_cash_flow(api::Side s) const { - if (s == api::SIDE_BUY) +int PandL::mkt_to_cash_flow(Side s) const { + if (s == Side::Buy) return -1; else return 1; } -int PandL::mkt_to_contract_flow(api::Side s) const { +int PandL::mkt_to_contract_flow(Side s) const { return -1 * mkt_to_cash_flow(s); } diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h index 993130bb8..4d473a957 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pandl/PandL.h @@ -35,8 +35,8 @@ class PandL { struct PImpl; SecurityDefinition const &get_sec(SecurityID id) const; double price_to_cash(SecurityID id, double p) const; - int mkt_to_cash_flow(api::Side s) const; - int mkt_to_contract_flow(api::Side s) const; + int mkt_to_cash_flow(Side s) const; + int mkt_to_contract_flow(Side s) const; std::unique_ptr me; }; diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h b/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h index c527d217b..6527b744e 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/sim_common.h @@ -25,17 +25,11 @@ struct MBPUpdate { BookSide mbp_sell; ImpliedBookSide implied_mbp_buy; ImpliedBookSide implied_mbp_sell; - BookSide const &mbp_side(api::Side side) const { - if (side == api::SIDE_BUY) - return mbp_buy; - else - return mbp_sell; + BookSide const &mbp_side(Side side) const { + return side == Side::Buy ? mbp_buy : mbp_sell; } - const ImpliedBookSide &implied_mbp_side(api::Side side) const { - if (side == api::SIDE_BUY) - return implied_mbp_buy; - else - return implied_mbp_sell; + const ImpliedBookSide &implied_mbp_side(Side side) const { + return side == Side::Buy ? implied_mbp_buy : implied_mbp_sell; } }; @@ -44,7 +38,7 @@ struct OrderUpdate { enum Status { FILLED, PARTIALLY_FILLED, CANCELED }; Status status; SecurityID security_id; - api::Side side; + Side side; Quantity last_fills; Price last_price; enum FillType { PASSIVE, AGGRESSIVE }; diff --git a/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.cpp b/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.cpp index 529a101d7..0c6990c88 100644 --- a/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.cpp +++ b/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.cpp @@ -89,8 +89,7 @@ SecurityDefinition::SecurityDefinition( spread.noLegs().forEach([this](auto const &l) { LegDef ll{ SecurityID{l.legSecurityID()}, - l.legSide() == ::mdp3::LegSide::BuySide ? api::SIDE_BUY - : api::SIDE_SELL, + l.legSide() == ::mdp3::LegSide::BuySide ? Side::Buy : Side::Sell, Quantity{l.legRatioQty()}}; legs_.push_back(ll); }); diff --git a/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.h b/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.h index 79cd83f0f..b1ef52917 100644 --- a/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.h +++ b/src/server/sparkd/cmds/spartan/domain/SecurityDefinition.h @@ -12,7 +12,7 @@ namespace bts::spartan::mdp3 { struct LegDef { SecurityID security_id; - api::Side side; + Side side; Quantity ratio; }; diff --git a/src/server/sparkd/cmds/spartan/domain/SecuritySnapshot.cpp b/src/server/sparkd/cmds/spartan/domain/SecuritySnapshot.cpp index eebc0a83a..c944366d0 100644 --- a/src/server/sparkd/cmds/spartan/domain/SecuritySnapshot.cpp +++ b/src/server/sparkd/cmds/spartan/domain/SecuritySnapshot.cpp @@ -348,7 +348,7 @@ BookSide SecuritySnapshot::build_book_from_orders(Side side) const { auto invert = side == Side::Buy ? -1 : 1; auto book_idx = std::map(); for (auto &o : orders()) - if (o.second.side == map_side(side)) { + if (o.second.side == side) { auto &order = o.second; auto &level = book_idx[order.price * invert]; level.n_orders++; @@ -479,16 +479,16 @@ OrderChanges SecuritySnapshot::deduced_order_changes() const { deduced_order.previous_order = {}; deduced_order.order.key = agg_order.ok; deduced_order.order.security = security_id(); - deduced_order.order.side = map_side( + deduced_order.order.side = aggressor_fills[0].aggressor == MDP3Fill::THIS_BUY ? Side::Buy - : Side::Sell); + : Side::Sell; deduced_order.order.priority = {}; deduced_order.order.price = aggressor_fills[0].price; deduced_order.order.quantity = Quantity{0}; for (auto &f : aggressor_fills) { if (securities::is_worse( - deduced_order.order.side, + map_side(deduced_order.order.side), bts::Price(deduced_order.order.price.to_double()), bts::Price(f.price.to_double()))) deduced_order.order.price = f.price; @@ -567,11 +567,11 @@ std::ostream &operator<<(std::ostream &out, SecuritySnapshot const &snap) { out << "Order Changes -----------------------------------\n"; for (auto &change : snap.order_changes()) out << change << "\n"; - auto orders = sort_orders(snap.orders(), map_side(Side::Buy)); + auto orders = sort_orders(snap.orders(), Side::Buy); out << "Buy Orders -----------------------------------\n"; for (auto &order : orders) out << " " << order << "\n"; - orders = sort_orders(snap.orders(), map_side(Side::Sell)); + orders = sort_orders(snap.orders(), Side::Sell); out << "Sell Orders ----------------------------------\n"; for (auto &order : orders) out << " " << order << "\n"; diff --git a/src/server/sparkd/cmds/spartan/domain/Side.h b/src/server/sparkd/cmds/spartan/domain/Side.h index aea82207f..b7f069a00 100644 --- a/src/server/sparkd/cmds/spartan/domain/Side.h +++ b/src/server/sparkd/cmds/spartan/domain/Side.h @@ -1,5 +1,6 @@ #pragma once #include "lib/domain_model/Security.pb.h" +#include namespace bts::spartan { @@ -42,5 +43,9 @@ inline constexpr const char *Side_Name(Side s) { return "Sell"; } +inline std::ostream& operator<<(std::ostream& out, const Side& s) { + out << Side_Name(s); + return out; +} } diff --git a/src/server/sparkd/cmds/spartan/market_data/common/mdp3_common.h b/src/server/sparkd/cmds/spartan/market_data/common/mdp3_common.h index 90ae05c6e..e5aec255c 100644 --- a/src/server/sparkd/cmds/spartan/market_data/common/mdp3_common.h +++ b/src/server/sparkd/cmds/spartan/market_data/common/mdp3_common.h @@ -31,7 +31,7 @@ inline std::string str_from_fixed_array(const char *ptr, size_t max_len) { return result; } -using Pricer = std::function; +using Pricer = std::function; using MDP3OrderPriority = utils::ValueWrapper; @@ -72,7 +72,7 @@ struct MDP3Order { OrderKey key = OrderKey{0}; SecurityID security = SecurityID{0}; - api::Side side = api::SIDE_UNDEFINED; + Side side = Side::Buy; MDP3OrderPriority priority = MDP3OrderPriority{0}; Price price; Quantity quantity = Quantity{0}; @@ -131,7 +131,7 @@ using OrderChanges = boost::container::small_vector; inline std::ostream &operator<<(std::ostream &s, MDP3Order const &order) { s << " k: " << order.key.value() << " sk: " << order.security.value() - << " side: " << securities::side_name(order.side) + << " side: " << order.side << " price: " << order.price.to_double() << " qty: " << order.quantity.value() << " pri: " << order.priority.value() << " ice?: " << order.iceberg_quantity; @@ -196,7 +196,7 @@ inline const char *aggressor_name(MDP3Fill::Aggressor ag) { } inline std::vector -sort_orders(Orders const &orders, api::Side side) { +sort_orders(Orders const &orders, Side side) { std::vector result; for (auto &o : orders) { auto &order = o.second; @@ -206,7 +206,7 @@ sort_orders(Orders const &orders, api::Side side) { std::sort( result.begin(), result.end(), [side](auto const &o1, auto const &o2) { if (o1.price.to_double() != o2.price.to_double()) { - if (side == api::SIDE_BUY) + if (side == Side::Buy) return o2.price.to_double() < o1.price.to_double(); else return o1.price.to_double() < o2.price.to_double(); @@ -217,11 +217,11 @@ sort_orders(Orders const &orders, api::Side side) { } inline std::ostream &operator<<(std::ostream &out, Orders const &orders) { - auto sorted = sort_orders(orders, api::SIDE_BUY); + auto sorted = sort_orders(orders, Side::Buy); std::cout << " Buy Orders ------------------" << std::endl; for (auto &o : sorted) out << o << std::endl; - sorted = sort_orders(orders, api::SIDE_SELL); + sorted = sort_orders(orders, Side::Sell); std::cout << " Sell Orders ------------------" << std::endl; for (auto &o : sorted) out << o << std::endl; diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp b/src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp index d8543c44e..335e2aaa6 100644 --- a/src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp @@ -146,8 +146,8 @@ BookStatRecorder::~BookStatRecorder() {} inline BookStatsRow book_stats_row(const SecuritySnapshot *s, size_t event_id) { bts_assert(s != nullptr); return BookStatsRow( - s->security_id(), event_id, s->book_side(map_side(api::Side::SIDE_BUY)), - s->book_side(map_side(api::Side::SIDE_SELL))); + s->security_id(), event_id, s->book_side(Side::Buy), + s->book_side(Side::Sell)); } /** diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.cpp b/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.cpp index 2a45636ab..a769fe1b8 100644 --- a/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.cpp @@ -188,7 +188,7 @@ void PolygonFinder::finit() { for (auto &l : legs) { poly_instrument.parts.push_back(InstrumentPart{ l.security_id, - (l.side == api::SIDE_BUY ? 1 : -1) * l.ratio.value()}); + (l.side == Side::Buy ? 1 : -1) * l.ratio.value()}); all_legs.insert(l.security_id); } } diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.cpp b/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.cpp index 1b849b2c9..22d41d892 100644 --- a/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.cpp @@ -234,7 +234,7 @@ PolygonOpportunityFinder::~PolygonOpportunityFinder() {} double eval_poly(Polygon const &poly, Pricer const &price) { double value = 0; for (auto &l : poly.parts) { - auto mkt_side = securities::other_side(side_from_ratio(l.ratio)); + auto mkt_side = other_side(from_ratio(l.ratio)); auto cash_price = price(l.sec_id, Quantity{std::abs(l.ratio)}, mkt_side); value += -l.ratio * cash_price; @@ -311,16 +311,16 @@ void PolygonOpportunityFinder::on_event_end(MDP3Event const &ev) { // } // gets the direct price in cash terms for the a sec, side auto direct_pricer = - [this](SecurityID sec_id, Quantity, api::Side side) { - auto &mkt = me->all_snapshots[sec_id].snapshot->book_side(map_side(side))[0]; + [this](SecurityID sec_id, Quantity, Side side) { + auto &mkt = me->all_snapshots[sec_id].snapshot->book_side(side)[0]; if (mkt.quantity.value() > 0) return get_sec(sec_id).to_cash(mkt.price.to_double()); else return os::nan(); }; auto merged_pricer = - [this](SecurityID sec_id, Quantity, api::Side side) { - auto mkt = me->all_snapshots[sec_id].snapshot->merged_book_side(map_side(side))[0]; + [this](SecurityID sec_id, Quantity, Side side) { + auto mkt = me->all_snapshots[sec_id].snapshot->merged_book_side(side)[0]; if (mkt.quantity.value() > 0) return get_sec(sec_id).to_cash(mkt.price.to_double()); else diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.cpp b/src/server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.cpp index ebefdce97..0aa384a2c 100644 --- a/src/server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.cpp @@ -67,8 +67,8 @@ void update_books( books.insert_or_assign( s->security_id(), std::make_tuple( - event_id, to_small_book(s->book_side(map_side(api::Side::SIDE_BUY))), - to_small_book(s->book_side(map_side(api::Side::SIDE_SELL))))); + event_id, to_small_book(s->book_side(Side::Buy)), + to_small_book(s->book_side(Side::Sell)))); first++; } } diff --git a/src/server/sparkd/cmds/spartan/market_data/realtime/RealTimeHandler.cpp b/src/server/sparkd/cmds/spartan/market_data/realtime/RealTimeHandler.cpp index dd603f063..87859c8f3 100644 --- a/src/server/sparkd/cmds/spartan/market_data/realtime/RealTimeHandler.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/realtime/RealTimeHandler.cpp @@ -260,7 +260,7 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshBook46 &m) { MDP3Order order; order.key = OrderKey{order_id_entry.orderID()}; order.security = sid; - order.side = map_side(md_entry.side); + order.side = md_entry.side; order.priority = MDP3OrderPriority{order_id_entry.mDOrderPriority()}; order.quantity = Quantity{order_id_entry.mDDisplayQty()}; order.price = md_entry.price; @@ -348,7 +348,7 @@ void RealTimeHandler::operator()(::mdp3::MDIncrementalRefreshOrderBook47 &m) { MDP3Order order; order.key = OrderKey{md_entry.orderID()}; order.security = sid; - order.side = map_side(convert_side(md_entry.mDEntryType())); + order.side = convert_side(md_entry.mDEntryType()); order.priority = MDP3OrderPriority{md_entry.mDOrderPriority()}; order.quantity = Quantity{md_entry.mDDisplayQty()}; order.price = Price{ diff --git a/src/server/sparkd/cmds/spartan/market_data/recovery/RecoveryHandler.cpp b/src/server/sparkd/cmds/spartan/market_data/recovery/RecoveryHandler.cpp index 563b3bd1f..d96b8dcf2 100644 --- a/src/server/sparkd/cmds/spartan/market_data/recovery/RecoveryHandler.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/recovery/RecoveryHandler.cpp @@ -431,7 +431,7 @@ void RecoveryHandler::update_book_chunk_in_snapshot( MDP3Order order; order.key = OrderKey{md_entry.orderID()}; order.security = snap.security_id(); - order.side = map_side(convert_side(md_entry.mDEntryType())); + order.side = convert_side(md_entry.mDEntryType()); order.priority = MDP3OrderPriority{md_entry.mDOrderPriority()}; order.price = Price{ md_entry.mDEntryPx().mantissa(), md_entry.mDEntryPx().exponent()}; diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h b/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h index bc0d08ccb..6359afa13 100644 --- a/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h +++ b/src/server/sparkd/cmds/spartan/market_data/utils/book_utils.h @@ -71,7 +71,7 @@ fmt_price_level(std::ostream &out, Price price, Quantity qty) { } inline std::ostream &fmt_price_level( - std::ostream &out, api::Side side, Price price, Quantity qty) { + std::ostream &out, Side side, Price price, Quantity qty) { out << side << " "; fmt_price_level(out, price, qty); return out; @@ -89,20 +89,20 @@ fmt_price_level(const std::tuple &pl) { } inline std::string -fmt_price_level(api::Side side, Price price, Quantity qty) { +fmt_price_level(Side side, Price price, Quantity qty) { std::stringstream out; fmt_price_level(out, side, price, qty); return out.str(); } -inline std::string fmt_price_level(api::Side side, SmallBookLevel l) { +inline std::string fmt_price_level(Side side, SmallBookLevel l) { std::stringstream out; fmt_price_level(out, side, l.price, l.quantity); return out.str(); } inline std::string fmt_price_level( - api::Side side, const std::tuple &pl) { + Side side, const std::tuple &pl) { return fmt_price_level(side, std::get<0>(pl), std::get<1>(pl)); } @@ -161,14 +161,14 @@ inline bool has_min_qty( Iter last, Quantity qty, Price limit_price = {}, - api::Side side = api::Side::SIDE_BUY) { - bts_assert(side == api::Side::SIDE_BUY || side == api::Side::SIDE_SELL); + Side side = Side::Buy) { + bts_assert(side == Side::Buy || side == Side::Sell); if (qty.value() == 0) return true; bts_assert(qty.value() > 0); while (first != last && (limit_price.is_nan() || - (side == api::Side::SIDE_BUY ? limit_price > first->price : limit_price < first->price))) { + (side == Side::Buy ? limit_price < first->price : limit_price > first->price))) { qty -= first->quantity; if (qty.value() <= first->quantity.value()) return true; @@ -177,6 +177,7 @@ inline bool has_min_qty( return false; } + bool remove_qty(SmallBook &book, Quantity qty); Quantity volume(const SmallBook &book); diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.cpp b/src/server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.cpp index 033f3924d..caa7bc71c 100644 --- a/src/server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.cpp @@ -537,7 +537,7 @@ struct CSVObs : IMDP3FeedObserver { for (auto const &l : s->definition().legs()) spread_legs << s->security_id().value() << "," << l.security_id.value() << "," - << api::Side_Name(l.side) << "," << l.ratio.value() + << Side_Name(l.side) << "," << l.ratio.value() << std::endl; known_securities.insert(s->security_id()); @@ -567,7 +567,7 @@ struct CSVObs : IMDP3FeedObserver { << "order_change," // type << "init," // change << id << "," // order key - << api::Side_Name(order.side) << "," // side + << Side_Name(order.side) << "," // side << order.priority.value() << "," // priority << order.price.to_double() << "," // price << order.quantity.value() << "," // quantity @@ -641,7 +641,7 @@ struct CSVObs : IMDP3FeedObserver { << "order_change," // type << change_name(change.change) << "," // change type << change.order.key.value() << "," // order key - << api::Side_Name(change.order.side) << "," // side + << Side_Name(change.order.side) << "," // side << change.order.priority.value() << "," // priority << change.order.price.to_double() << "," // price << change.order.quantity.value() << "," // quantity @@ -682,7 +682,7 @@ struct CSVObs : IMDP3FeedObserver { << "deduced_order," // type << change_name(deduced_order.change) << "," << deduced_order.order.key.value() << "," // order key - << api::Side_Name(deduced_order.order.side) + << Side_Name(deduced_order.order.side) << "," // side << "0," // priority << deduced_order.order.price.to_double() << "," // price @@ -725,9 +725,8 @@ int to_csv(std::string const &file) { polys << pid << "," << polygon_descriptor(p, imap) << std::endl; for (auto &part : p.parts) { poly_legs << pid << "," << part.sec_id << "," - << (part.ratio > 0 ? api::Side_Name(api::SIDE_BUY) - : api::Side_Name(api::SIDE_SELL)) - << "," << std::abs(part.ratio) << std::endl; + << Side_Name(from_ratio(part.ratio)) << "," + << std::abs(part.ratio) << std::endl; } } diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/order_utils.h b/src/server/sparkd/cmds/spartan/market_data/utils/order_utils.h index 4ba86d2eb..e540e1dd4 100644 --- a/src/server/sparkd/cmds/spartan/market_data/utils/order_utils.h +++ b/src/server/sparkd/cmds/spartan/market_data/utils/order_utils.h @@ -75,10 +75,10 @@ inline bool orders_equal(const MDP3Order &lhs, const MDP3Order &rhs) { } inline std::string fmt_order( - OrderKey key, api::Side side, Price price, Quantity qty) { + OrderKey key, Side side, Price price, Quantity qty) { std::stringstream str; str << "[" << key.value() << "] " - << " " << (side == api::Side::SIDE_BUY ? "Buy" : "Sell") << " " + << " " << (side == Side::Buy ? "Buy" : "Sell") << " " << qty.value() << " @ " << price.to_double(); return str.str(); } diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h b/src/server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h index e9d519c10..5fd994a90 100644 --- a/src/server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h +++ b/src/server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h @@ -33,11 +33,11 @@ using PolygonMap = std::unordered_map; std::string polygon_descriptor(Polygon const &, SecurityMap const &, bool sort = true); std::string polygon_product(Polygon const &, SecurityMap const &); -inline api::Side side_from_ratio(int ratio) { +inline Side side_from_ratio(int ratio) { if (ratio > 0) - return api::Side::SIDE_BUY; + return Side::Buy; else - return api::Side::SIDE_SELL; + return Side::Sell; } bool operator==(Polygon const &, Polygon const &); diff --git a/src/server/sparkd/cmds/spartan/market_data/utils/tests/synthetic_book_tests.cpp b/src/server/sparkd/cmds/spartan/market_data/utils/tests/synthetic_book_tests.cpp index 26caf3f8a..00e06992a 100644 --- a/src/server/sparkd/cmds/spartan/market_data/utils/tests/synthetic_book_tests.cpp +++ b/src/server/sparkd/cmds/spartan/market_data/utils/tests/synthetic_book_tests.cpp @@ -24,20 +24,20 @@ inline SmallBookLevel book_level(int price, int qty) { // and 'undefined' means the sort-order is inferred // from the comparison of the first 2 price levels. inline bool -is_sorted(const SmallBook &book, api::Side side = api::Side::SIDE_UNDEFINED) { +is_sorted(const SmallBook &book, std::optional side = std::nullopt) { if (book.size() < 2) return true; - switch (side) { - case api::Side::SIDE_BUY: - case api::Side::SIDE_SELL: - return std::is_sorted( - book.begin(), book.end(), orderman::SidePriceComp(side)); - case api::Side::SIDE_UNDEFINED: - default: - return is_sorted( - book, book.front().price > book.back().price ? api::Side::SIDE_SELL - : api::Side::SIDE_BUY); + if (side.has_value()) { + switch (side.value()) { + case Side::Buy: + case Side::Sell: + return std::is_sorted( + book.begin(), book.end(), + orderman::SidePriceComp(map_side(side.value()))); + } } + return is_sorted( + book, book.front().price > book.back().price ? Side::Sell : Side::Buy); } // Computes the cost of the specified qty at the top price(s) in the book. @@ -54,8 +54,7 @@ Price qty_cost(const SmallBook &book, Quantity qty) { // book's prices should be in ascending order. // If the qty is negative, it is to be 'sold' to the 'bid' market // so the book's prices should be in descending order. - REQUIRE( - is_sorted(book, qty > 0 ? api::Side::SIDE_SELL : api::Side::SIDE_BUY)); + REQUIRE(is_sorted(book, qty > 0 ? Side::Sell : Side::Buy)); Price cost = Price::zero(); bool negate = qty.value() < 0; qty = abs(qty); @@ -388,7 +387,7 @@ void test_synth_book(LegMarkets synth) { auto max_lp = max_last_price(p); REQUIRE(book_out.front().price >= min_fp); REQUIRE(book_out.back().price <= max_lp); - REQUIRE(is_sorted(book_out, api::Side::SIDE_SELL)); + REQUIRE(is_sorted(book_out, Side::Sell)); if (book_out.size() > 1) { REQUIRE(min_fp < max_lp); REQUIRE(book_out.front().price < book_out.back().price); From 41374d62eadc9295d9c8c11dfd50ef22c52a7e18 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Jan 2021 10:05:59 -0600 Subject: [PATCH 08/13] spartan cli: Add SimBooks command This command is a diagnostic tool that produces a csv file of matcher-generated books for a particular pcap, which can then be imported to sql, and joined with the actual books table for comparison. --- src/server/sparkd/cmds/Spartan.cpp | 2 + .../sim/pcap/PCapOrderChangeBootstrapper.cpp | 112 +++++++++++++ .../sim/pcap/PCapOrderChangeBootstrapper.h | 30 ++++ .../sparkd/cmds/spartan/cmds/SimBooks.cpp | 86 ++++++++++ .../sparkd/cmds/spartan/cmds/SimBooks.h | 10 ++ .../feed_observers/recorders/FeedRecorder.h | 75 +++++++++ .../recorders/MatcherBookRecorder.cpp | 149 ++++++++++++++++++ .../recorders/MatcherBookRecorder.h | 32 ++++ 8 files changed, 496 insertions(+) create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.cpp create mode 100644 src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.h create mode 100644 src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp create mode 100644 src/server/sparkd/cmds/spartan/cmds/SimBooks.h create mode 100644 src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/FeedRecorder.h create mode 100644 src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.cpp create mode 100644 src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.h diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 7f2549819..8501e386a 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -9,6 +9,7 @@ #include "server/sparkd/cmds/spartan/cmds/PolygonBooks.h" #include "server/sparkd/cmds/spartan/cmds/SQL.h" #include "server/sparkd/cmds/spartan/cmds/Sim.h" +#include "server/sparkd/cmds/spartan/cmds/SimBooks.h" #include "server/sparkd/cmds/spartan/cmds/Stats.h" #include "server/sparkd/cmds/spartan/cmds/ValidateFifo.h" #include "server/sparkd/cmds/spartan/cmds/ValuePolys.h" @@ -28,6 +29,7 @@ void Spartan::register_cmd(CLI::App &app) { bts::spartan::cmds::ListSec::register_cmd(*spartan_app); bts::spartan::cmds::PolygonBooks::register_cmd(*spartan_app); bts::spartan::cmds::Sim::register_cmd(*spartan_app); + bts::spartan::cmds::SimBooks::register_cmd(*spartan_app); bts::spartan::cmds::SQL::register_cmd(*spartan_app); bts::spartan::cmds::Stats::register_cmd(*spartan_app); bts::spartan::cmds::ValidateFifo::register_cmd(*spartan_app); diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.cpp b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.cpp new file mode 100644 index 000000000..6365b2df7 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.cpp @@ -0,0 +1,112 @@ +#include "PCapOrderChangeBootstrapper.h" +#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h" +#include "server/sparkd/cmds/spartan/market_data/gen/src/SecurityTradingStatus.h" +#include + +namespace bts::spartan::mdp3 { + +using TradingStatus = ::mdp3::SecurityTradingStatus; + +const size_t NO_EVENT = 0; + +struct SecStatusInfo { + size_t current_event = NO_EVENT; + size_t last_status_change_event = NO_EVENT; + size_t consecutive_events_with_status = 0; + TradingStatus::Value status = TradingStatus::PreOpen; +}; + +template +inline OrderChanges _order_changes( + std::unordered_map &status_info_map, + const SecuritySnapshot *s, + size_t event_id, + GetChanges incremental_changes) { + auto res = status_info_map.emplace( + s->security_id(), SecStatusInfo{event_id, event_id, 1, s->trading_status()}); + if (res.second) + return {}; + SecStatusInfo &status_info = res.first->second; + // This class is designed to be used in conjunction with pcap observation. + // Its purpose is to bootstrap incremental order changes, which occur as time + // moves in one direction: forward. Assuming that event ids increase over + // time, it is reasonable to expect the sequence of event id parameters to be + // passed to this class in (weak) ascending order. + bts_assert(event_id >= status_info.current_event); + if (status_info.current_event == event_id) { + // If event id == NO_EVENT, then this is the first time this security + // has been observed. + if (event_id != NO_EVENT) { + // The security's trading status has already been recorded for this + // event. Assert that it hasn't mysteriously changed. + bts_assert(status_info.status == s->trading_status()); + } + } else { + status_info.current_event = event_id; + if (s->trading_status() == status_info.status) + status_info.consecutive_events_with_status++; + else { + status_info.last_status_change_event = event_id; + status_info.consecutive_events_with_status = 1; + status_info.status = s->trading_status(); + } + } + if (status_info.status != TradingStatus::ReadyToTrade) + return {}; + // If the security's status just changed (on this event), the + // 'consecutive_events_with_status' field would be at its logical minimum: 1. + // Assert that this value is logical. + bts_assert(status_info.consecutive_events_with_status > 0); + switch (status_info.consecutive_events_with_status) { + case 1: + // The security just transitioned to 'ReadyToTrade'. Presumably, all + // preopen trades are flushed in this event, so it is ignored. + return {}; + case 2: + // This is the first event AFTER the security transitioned to + // 'ReadyToTrade'. Presumably, the preopen trades were flushed in the + // last event, so this is the first 'normal' event for the security. + // Bootstrap the incremental change sequence by adding all resting orders. + OrderChanges resting_orders; + for (const auto &p : s->orders()) { + auto &ch = resting_orders.emplace_back(); + ch.order = p.second; + ch.change = OrderChange::ChangeType::added; + } + return resting_orders; + } + // This is a 'normal' update, and the incremental change sequence has already + // been bootstrapped. Simply return the incremental changes. + return incremental_changes(s); +} + +struct PCapOrderChangeBootstrapper::PImpl { + std::unordered_map status_info_map; +}; + +PCapOrderChangeBootstrapper::PCapOrderChangeBootstrapper() + : _me(new PCapOrderChangeBootstrapper::PImpl()) {} + +PCapOrderChangeBootstrapper::~PCapOrderChangeBootstrapper() {} + +OrderChanges PCapOrderChangeBootstrapper::order_changes( + const SecuritySnapshot *s, size_t event_id) { + struct { + inline OrderChanges operator()(const SecuritySnapshot *s) { + return s->order_changes(); + } + } get_changes; + return _order_changes(_me->status_info_map, s, event_id, get_changes); +} + +OrderChanges PCapOrderChangeBootstrapper::deduced_order_changes( + const SecuritySnapshot *s, size_t event_id) { + struct { + inline OrderChanges operator()(const SecuritySnapshot *s) { + return s->deduced_order_changes(); + } + } get_changes; + return _order_changes(_me->status_info_map, s, event_id, get_changes); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.h b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.h new file mode 100644 index 000000000..4d91dadbd --- /dev/null +++ b/src/server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.h @@ -0,0 +1,30 @@ +#pragma once +#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h" +#include + +namespace bts::spartan::mdp3 { + +/** + * IFeedObservers may use this class to ignore (deduced)order changes until all + * preopen trades have been processed. The (deduced_)order_changes methods + * return empty vectors until the second consecutive event in which a security's + * trading status is 'ReadyToTrade' (pending preopen trades are flushed in the + * first event), at which point the returned vectors consist of 'added' changes + * for all resting orders to bootstrap the incremental change sequence. After + * that, 'deduced_order_changes', and 'order_changes' simply return + * SecuritySnapshot::deduced_order_changes and SecuritySnapshot::order_changes, + * respectively. + */ +struct PCapOrderChangeBootstrapper { + PCapOrderChangeBootstrapper(); + ~PCapOrderChangeBootstrapper(); + + OrderChanges order_changes(const SecuritySnapshot *, size_t event_id); + OrderChanges deduced_order_changes(const SecuritySnapshot *, size_t event_id); + + private: + struct PImpl; + std::unique_ptr _me; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp b/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp new file mode 100644 index 000000000..db36181b4 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp @@ -0,0 +1,86 @@ +#include "SimBooks.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h" +#include "server/sparkd/cmds/spartan/domain/SecurityCriteria.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h" +#include "server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.h" +#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h" +#include "server/sparkd/cmds/spartan/util/file_stream_utils.h" +#include + +namespace bts::spartan::cmds { + +#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__) +#define INFO(...) _LOG(Info, __VA_ARGS__) + +struct Args { + std::string pcap; + std::string out; + std::vector assets; + std::vector security_groups; + mdp3::SecurityCriteria security_criteria; +}; + +static Args args; + +void execute() { + // Validate args + bts_assert(args.pcap.size()); + if (args.out.empty()) + args.out = file::change_file_ext(args.pcap, "sim_books.csv", "pcap.xz"); + bts_assert(args.out.size()); + // Sweep pcap for securityies and icebergs + mdp3::DefinitionCollector dc; + mdp3::IcebergFinder ice_finder; + INFO( + "Begin observe '{}' (1/2): detect all securityies and iceberg orders", + args.pcap); + mdp3::observe_channel(args.pcap, {&dc, &ice_finder}); + INFO( + "End observe '{}' (1/2): detected {} securities, and {} iceberg orders", + args.pcap, dc.result.size(), ice_finder.result.size()); + // Init MergedMatcher + mdp3::MergedMatcher matcher; + for (auto p : dc.result) + matcher.add_def(p.second); + // Init MatcherBookRecorder + mdp3::SecurityCriteria security_criteria; + if (args.assets.size() || args.security_groups.size()) { + // assets + auto &assets = security_criteria.clauses.emplace_back().assets; + assets.insert(assets.end(), args.assets.begin(), args.assets.end()); + // security groups + auto &security_groups = security_criteria.clauses.emplace_back().assets; + security_groups.insert( + security_groups.end(), args.security_groups.begin(), + args.security_groups.end()); + } + mdp3::MatcherBookRecorder rec( + {matcher, ice_finder.result, security_criteria, true}); + // Init FeedRecorderWrapper and observe pcap to generate the csv + std::ofstream out(args.out); + mdp3::FeedRecorderWrapper rec_wrap(out, &rec); + INFO("Begin observe '{}' (2/2): record matcher books", args.pcap); + mdp3::observe_channel(args.pcap, rec_wrap); + INFO("End observe '{}' (2/2): recorded matcher book", args.pcap); + out.flush(); +} + +void SimBooks::register_cmd(CLI::App &app) { + auto *subcmd = app.add_subcommand( + "sim-books", "Generate a csv file of book updates by feeding deduced " + "orders from a pcap file to the simulation matcher"); + subcmd->add_option("pcap,--pcap", args.pcap)->required(); + subcmd->add_option("out,--out", args.out); + subcmd->add_option( + "--assets", args.assets, + "Specify asset(s) to filter the set of securities for which books are " + "generated"); + subcmd->add_option( + "--security-groups", args.security_groups, + "Specify security group(s) to filter the set of securities for which " + "books are generated"); + subcmd->callback(execute); +} + +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SimBooks.h b/src/server/sparkd/cmds/spartan/cmds/SimBooks.h new file mode 100644 index 000000000..585c1f1f0 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/cmds/SimBooks.h @@ -0,0 +1,10 @@ +#pragma once +#include + +namespace bts::spartan::cmds { + +struct SimBooks { + static void register_cmd(CLI::App &); +}; + +} // namespace bts::spartan::cmds::sim_books diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/FeedRecorder.h b/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/FeedRecorder.h new file mode 100644 index 000000000..7b7e3fb95 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/FeedRecorder.h @@ -0,0 +1,75 @@ +#pragma once +#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h" +#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h" +#include "server/sparkd/cmds/spartan/market_data/parsing/MDP3Parser.h" +#include + +namespace bts::spartan::mdp3 { + +struct IFeedRecorder { + virtual void init(std::ostream &) {} + virtual void finit(std::ostream &) {} + virtual void on_recovery_complete( + std::ostream &, const std::vector &) {} + virtual void on_packet_begin(std::ostream &, MDP3PacketView const &) {} + virtual void on_packet_end(std::ostream &) {} + virtual void on_event_begin(std::ostream &, MDP3Event const &) {} + virtual void on_books( + std::ostream &, + SecuritySnapshot const &, + Book const &, + ImpliedBook const &) {} + virtual void on_trade_summary_signal( + std::ostream &, SecuritySnapshot const &, TradeSummarySignal const &) {} + virtual void on_event_end(std::ostream &, MDP3Event const &) {} + + virtual void on_packet_loss(std::ostream &) {} +}; + +template +struct FeedRecorderWrapper : public IMDP3FeedObserver { + FeedRecorderWrapper() {} + FeedRecorderWrapper(std::ostream &out, Observer *observer) + : _out(out), _observer(observer) {} + + void init() override { _observer->init(_out); } + void finit() override { _observer->finit(_out); } + void on_recovery_complete( + const std::vector &s) override { + _observer->on_recovery_complete(_out, s); + } + + void on_packet_begin(MDP3PacketView const &mpv) override { + _observer->on_packet_begin(_out, mpv); + } + + void on_packet_end() override { _observer->on_packet_end(_out); } + + void on_event_begin(MDP3Event const &ev) override { + _observer->on_event_begin(_out, ev); + } + + void on_books( + SecuritySnapshot const &sec, + Book const &bk, + ImpliedBook const &ibk) override { + _observer->on_books(_out, sec, bk, ibk); + } + + void on_trade_summary_signal( + SecuritySnapshot const &sec, TradeSummarySignal const &sig) override { + _observer->on_trade_summary_signal(_out, sec, sig); + } + + void on_event_end(MDP3Event const &ev) override { + _observer->on_event_end(_out, ev); + } + + void on_packet_loss() override { _observer->on_packet_loss(_out); } + + private: + std::ostream &_out; + Observer *_observer; +}; + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.cpp b/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.cpp new file mode 100644 index 000000000..3be623d75 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.cpp @@ -0,0 +1,149 @@ +#include "MatcherBookRecorder.h" +#include "server/sparkd/cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.h" +#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h" +#include +#include +#include + +namespace bts::spartan::mdp3 { + +#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__) +#define INFO(...) _LOG(Info, __VA_ARGS__) +#define WARN(...) _LOG(Warn, __VA_ARGS__) + +bool book_equals( + const SecuritySnapshot *snap, MergedMatcher &matcher, Side side) { + auto exch_books = snap->book_side(side); + auto match_books = matcher.book(snap->security_id(), side); + for (uint32_t i = 0; i < exch_books.size(); i++) { + if (match_books[i].quantity != exch_books[i].quantity) + return false; + if (match_books[i].quantity == 0) + return true; + if (match_books[i].price != exch_books[i].price) + return false; + } + return true; +} + +void _write_book(std::ostream &out, const BookSide &book) { + for (const auto &l : book) + out << "," << l.price << "," << l.quantity << "," << l.n_orders; +} + +void _write_row( + std::ostream &out, + size_t event_id, + SecurityID sec_id, + const BookSide &bids, + const BookSide &asks) { + out << event_id << "," << sec_id; + _write_book(out, bids); + _write_book(out, asks); + out << "\n"; +} + +void _write_rows( + std::ostream &out, + size_t event_id, + const std::vector &snaps, + MergedMatcher &matcher) { + for (const auto *s : snaps) { + auto bids = matcher.book(s->security_id(), Side::Buy); + auto asks = matcher.book(s->security_id(), Side::Sell); + _write_row(out, event_id, s->security_id(), bids, asks); + } +} + +SmallOrderChangeVec to_small_change_vec( + const OrderChanges &changes, const IcebergMap &icebergs, size_t event_id) { + SmallOrderChangeVec rv; + for (const auto &ch : changes) { + auto &sch = rv.emplace_back(ch, true); + auto ib_it = icebergs.find(ch.order.key); + if (ib_it != icebergs.end()) + iceberg_helper::add_iceberg_annotation( + sch.order, sch.change, ib_it->second, event_id, false); + } + return rv; +} + +template +inline void _process_changes( + std::ostream &out, + const SnapshotContainer &snaps, + const SecurityCriteria &instrument_criteria, + PCapOrderChangeBootstrapper &bootstrapper, + MergedMatcher &matcher, + const IcebergMap &icebergs, + bool reset_after_mismatch, + size_t event_id) { + std::vector effects; + std::unordered_set processed_secs; + for (const auto *s : snaps) { + if (instrument_criteria.is_defined() && + !instrument_criteria.includes(s->definition())) + continue; + auto changes = bootstrapper.deduced_order_changes(s, event_id); + if (changes.empty()) + continue; + auto schanges = to_small_change_vec(changes, icebergs, event_id); + matcher.match( + MergedMatcher::ChangeSrc::SIMULATOR_INPUT, schanges, &effects); + processed_secs.insert(s->security_id()); + } + for (const auto &eff : effects) { + if (std::holds_alternative(eff)) { + const auto &up = std::get(eff); + _write_row(out, event_id, up.security_id, up.mbp_buy, up.mbp_sell); + } + } + if (reset_after_mismatch) { + for (const auto *s : snaps) { + if (!processed_secs.count(s->security_id())) + continue; + if (!book_equals(s, matcher, Side::Buy) || + !book_equals(s, matcher, Side::Sell)) { + WARN( + "Discrepancy between matcher books and actual books detected " + "for [{}]{} on event {}. Resetting matcher to exchange state", + s->security_id(), s->symbol(), event_id); + std::vector resting_orders; + for (const auto &p : s->orders()) + resting_orders.emplace_back(p.second); + matcher.reset(s->security_id(), resting_orders); + } + } + } +} + +struct MatcherBookRecorder::PImpl { + MergedMatcher &matcher; + const IcebergMap &icebergs; + const SecurityCriteria &instrument_criteria; + bool reset_after_mismatch; + PCapOrderChangeBootstrapper bootstrapper; +}; + +MatcherBookRecorder::MatcherBookRecorder(Args args) + : _me(new MatcherBookRecorder::PImpl{ + args.matcher, args.icebergs, args.instrument_criteria, + args.reset_after_mismatch, PCapOrderChangeBootstrapper()}) {} + +MatcherBookRecorder::~MatcherBookRecorder() {} + +void MatcherBookRecorder::on_recovery_complete( + std::ostream &out, const std::vector &snaps) { + _process_changes( + out, snaps, _me->instrument_criteria, _me->bootstrapper, _me->matcher, + _me->icebergs, _me->reset_after_mismatch, 0); +} + +void MatcherBookRecorder::on_event_end(std::ostream &out, const MDP3Event &ev) { + std::vector effects; + _process_changes( + out, ev.secs(), _me->instrument_criteria, _me->bootstrapper, + _me->matcher, _me->icebergs, _me->reset_after_mismatch, ev.event_id); +} + +} // namespace bts::spartan::mdp3 diff --git a/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.h b/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.h new file mode 100644 index 000000000..4fd744765 --- /dev/null +++ b/src/server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.h @@ -0,0 +1,32 @@ +#pragma once +#include "FeedRecorder.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/iceberg_helper.h" +#include "server/sparkd/cmds/spartan/domain/SecurityCriteria.h" +#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h" +#include +#include + +namespace bts::spartan::mdp3 { + +struct MatcherBookRecorder : public IFeedRecorder { + struct Args { + MergedMatcher &matcher; + const IcebergMap &icebergs; + const SecurityCriteria &instrument_criteria; + bool reset_after_mismatch; + }; + + MatcherBookRecorder(Args); + ~MatcherBookRecorder(); + + void on_recovery_complete( + std::ostream &, const std::vector &) override; + void on_event_end(std::ostream &, const MDP3Event &) override; + + private: + struct PImpl; + std::unique_ptr _me; +}; + +} // namespace bts::spartan::mdp3 From 456177af30ef44f71149bd56356d6f4cb348290c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Jan 2021 23:20:49 -0600 Subject: [PATCH 09/13] typo: Flip ternary op clauses, and delete superfluous 'return' --- .../back_testing/matcher/FifoImpliedSecurityMatcher.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp index 7518fc13c..f8cd9efc2 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp @@ -137,12 +137,12 @@ void FifoImpliedSecurityMatcher::_book( SmallBook bk; auto [sec_matcher, leg_qty] = p.second; bts_assert(leg_qty.value() != 0); - auto leg_side = leg_qty.value() > 0 ? other_side(side) : side; + auto leg_side = leg_qty.value() > 0 ? side : other_side(side); for (auto bl : sec_matcher->book(leg_side)) bk.emplace_back(bl.price, bl.quantity); im_vec.emplace_back(leg_qty, Price{}, bk); } - return book_utils::compute_synthetic_book(dest, im_vec, depth); + book_utils::compute_synthetic_book(dest, im_vec, depth); } void FifoImpliedSecurityMatcher::add_leg( From f42137d86c91458cbd03917255a12660479d5f63 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 12 Jan 2021 23:44:07 -0600 Subject: [PATCH 10/13] Don't assert that an iceberg order's vis qty <= show qty There isn't a compelling reason to enforce this constraint. The show qty is the 'reload' qty, and is orthogonal to the current [visible] qty. --- .../back_testing/matcher/FifoMergedSecurityMatcher.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp index eb85cdb3b..5ccf9785a 100644 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp +++ b/src/server/sparkd/cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp @@ -301,11 +301,6 @@ void FifoMergedSecurityMatcher::_fill( std::vector *effects, FillDepot &fills, bool reset_show_qty) { - if (order.iceberg_quantity.has_value()) { - // the order's quantity cannot exceed the show qty (but it can be - // smaller). - bts_assert(order.quantity <= order.iceberg_quantity.value().show); - } auto rem = _fill( order.key, order.side, order.price, get_total_qty(order), effects, fills); From 17d20bac8bf0bb30436d2968bfe2b7f293f1e0ef Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 13 Jan 2021 12:34:18 -0600 Subject: [PATCH 11/13] Remove unused file 'ImpliedFifoSecurityMatcher.h' This was an old version of the implied fifo matcher, now defined in 'FifoImpliedSecurityMatcher.h'. --- .../matcher/ImpliedFifoSecurityMatcher.h | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h diff --git a/src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h b/src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h deleted file mode 100644 index d4fea24aa..000000000 --- a/src/server/sparkd/cmds/spartan/back_testing/matcher/ImpliedFifoSecurityMatcher.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once -#include "lib/md/IntPrice.h" -#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h" -#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h" -#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h" -#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h" -#include -#include -#include -#include -#include -#include - -namespace bts::spartan::mdp3 { -struct ImpliedFifoSecurityMatcher { - ImpliedFifoSecurityMatcher(SecurityID security_id) - : _security_id(security_id), _implied_mkt() {} - - ImpliedBook book(api::Side side) const; - Quantity - fill(OrderKey key, api::Side side, Quantity qty, FillDepot &fills); - // Note: this class is NOT responsible for deleting any FifoSecurityMatcher - // pointers. The FifoMatcherContext creates one FifoMergedSecurityMatcher per - // each security, and each FifoMergedSecurityMatcher creates/owns a - // FifoSecurityMatcher pointer (and is responsible for deleting it). The - // FifoSecurityMatcher pointer passed to this method should be owned by a - // FifoMergedSecurityMatcher. - // Note the absense of a 'side' parameter: unit_qty should be positive for - // Buys, negative for Sells. - void add_leg( - SecurityID sec_id, FifoSecurityMatcher *m, Quantity unit_qty) { - _implied_mkt.insert({sec_id, {m, unit_qty}}); - } - - SecurityID security_id() const { return _security_id; } - std::tuple top_of_book(api::Side side) const; - - private: - SecurityID _security_id; - std::map> - _implied_mkt; - // hide copy ctr - ImpliedFifoSecurityMatcher(const ImpliedFifoSecurityMatcher &); - - void _book( - api::Side side, - std::vector> &dest) const; - uint32_t _fill_order(const MDP3Order &order); -}; -} // namespace bts::spartan::mdp3 From 45de55a552ee1d7b51b1ea5909d27fb0e85f3723 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 15 Jan 2021 00:08:38 -0600 Subject: [PATCH 12/13] Remove unused files 'args_parser.h/cpp' --- .../sparkd/cmds/spartan/util/args_parser.cpp | 108 ------------------ .../sparkd/cmds/spartan/util/args_parser.h | 64 ----------- 2 files changed, 172 deletions(-) delete mode 100644 src/server/sparkd/cmds/spartan/util/args_parser.cpp delete mode 100644 src/server/sparkd/cmds/spartan/util/args_parser.h diff --git a/src/server/sparkd/cmds/spartan/util/args_parser.cpp b/src/server/sparkd/cmds/spartan/util/args_parser.cpp deleted file mode 100644 index ee810b131..000000000 --- a/src/server/sparkd/cmds/spartan/util/args_parser.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "server/sparkd/cmds/spartan/util/args_parser.h" -#include "server/sparkd/cmds/spartan/util/string_splitter.h" -//#include -//#include -#include -//#include -#include -#include -#include -#include -#include -#include -#include - -namespace bts::cmd::args { -// this container carries the burden of freeing the c string memory -class ArgvContainer { - int _argc; - char **_argv; - - inline static char **from_vect(const std::vector &argv) { - int argc = argv.size(); - char **new_argv = (char **)malloc((argc + 1) * sizeof *new_argv); - for (int i = 0; i < argc; i++) { - size_t length = argv[i].size() + 1; - new_argv[i] = (char *)malloc(length); - memcpy(new_argv[i], argv[i].c_str(), length); - } - new_argv[argc] = NULL; - return new_argv; - } - - public: - ArgvContainer(const std::vector &argv) - : _argc(argv.size()), _argv(from_vect(argv)) {} - - ~ArgvContainer() { - for (int i = 0; i < _argc; i++) - free(_argv[i]); - } - - char **value() { return _argv; } -}; - -// static struct option long_options[] = { -// {"add", required_argument, 0, 0}, -// {"append", no_argument, 0, 0}, -// {"delete", required_argument, 0, 0}, -// {"verbose", no_argument, 0, 0}, -// {"create", required_argument, 0, 'c'}, -// {"file", required_argument, 0, 0}, -// {0, 0, 0, 0} -// }; -void get_opts( - const std::vector &argv, - const char *options, - option *long_options, - std::unordered_map &dest) { - ArgvContainer ac(argv); - return get_opts(argv.size(), ac.value(), options, long_options, dest); -} - -void get_opts( - int argc, - char **argv, - const char *options, - option *long_options, - std::unordered_map &dest) { - // - int c; - while (1) { - int curr_long_index = optind ? optind : 1; - int option_index = 0; - - c = getopt_long( - argc, argv, options, (const option *)long_options, &option_index); - if (c == -1) - break; - - switch (c) { - case 0: - printf("option %s", long_options[option_index].name); - if (optarg) - printf(" with arg %s", optarg); - printf("\n"); - dest.insert( - {std::string(long_options[curr_long_index].name), - optarg ? std::string(optarg) : std::string()}); - break; - case '?': - std::cout << "unrecognized option '" << argv[optind - 1] << "'" - << std::endl; - break; - - default: - break; - } - } - - if (optind < argc) { - printf("non-option ARGV-elements: "); - while (optind < argc) - printf("%s ", argv[optind++]); - printf("\n"); - } -} - -} // namespace bts::cmd::args \ No newline at end of file diff --git a/src/server/sparkd/cmds/spartan/util/args_parser.h b/src/server/sparkd/cmds/spartan/util/args_parser.h deleted file mode 100644 index 50c8e6f66..000000000 --- a/src/server/sparkd/cmds/spartan/util/args_parser.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace bts::cmd::args { -// struct option { -// const char *name; -// int has_arg; -// int *flag; -// int val; -// }; - -struct PrettyOption { - const char *name; - bool has_arg; -}; - -void get_opts( - const std::vector &argv, - const char *options, - option *long_options, - std::unordered_map &dest); -void get_opts( - int argc, - char **argv, - const char *options, - option *long_options, - std::unordered_map &dest); - -template -void parse_items( - std::string str, ParseOp op, const std::string &delim = {","}) { - size_t pos = 0; - std::string token; - while ((pos = str.find(delim)) != std::string::npos) { - token = str.substr(0, pos); - std::cout << token << std::endl; - str.erase(0, pos + delim.length()); - op(token); - } - if (str.size()) - op(str); -} - -template -std::unordered_set parse_set( - const std::string &str, ParseOp op, const std::string &delim = {","}) { - std::unordered_set result; - auto insert_op = [&](const std::string &id_str) { - result.insert(op(id_str)); - }; - parse_items(str, insert_op, delim); - return result; -} -} // namespace bts::cmd::args \ No newline at end of file From ca4e477f56229b9f685ae9eba38e1e15fa254e32 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 15 Jan 2021 21:54:04 -0600 Subject: [PATCH 13/13] fix: CLI11 integration * Run 'sparkd tests' outside of the CLI::App framework (delegate to 'catch', which has a solid cli). This fixes a bug which prevented callers from specifying test tags (in the form of '[]'), as the CLI::App's parse method removes square brackets from cli options. * Bind cli options to instance members of their subcommand's CLI::App derived class. This seems to fix a bug which caused the [static instance] binding source to be destructed twice. * Add help text for several subcommands --- src/server/sparkd/cmds/AdminReport.cpp | 9 +- src/server/sparkd/cmds/AdminReport.h | 4 +- src/server/sparkd/cmds/AutoCert.cpp | 12 +- src/server/sparkd/cmds/AutoCert.h | 4 +- src/server/sparkd/cmds/Benchmark.cpp | 46 ++--- src/server/sparkd/cmds/Benchmark.h | 10 +- src/server/sparkd/cmds/CMECert.cpp | 70 ++++--- src/server/sparkd/cmds/CMECert.h | 4 +- src/server/sparkd/cmds/ConfigCheck.cpp | 25 +-- src/server/sparkd/cmds/ConfigCheck.h | 10 +- src/server/sparkd/cmds/FIXSim.cpp | 9 +- src/server/sparkd/cmds/FIXSim.h | 4 +- src/server/sparkd/cmds/KeyCode.cpp | 93 +++++---- src/server/sparkd/cmds/KeyCode.h | 10 +- src/server/sparkd/cmds/Md.cpp | 15 +- src/server/sparkd/cmds/Md.h | 10 +- src/server/sparkd/cmds/ParseHistoricalFIX.cpp | 92 ++++----- src/server/sparkd/cmds/ParseHistoricalFIX.h | 10 +- src/server/sparkd/cmds/PrintHedgeOrders.cpp | 91 +++++---- src/server/sparkd/cmds/PrintHedgeOrders.h | 4 +- src/server/sparkd/cmds/QuickTest.cpp | 11 +- src/server/sparkd/cmds/QuickTest.h | 4 +- src/server/sparkd/cmds/Run.cpp | 4 +- src/server/sparkd/cmds/Run.h | 4 +- src/server/sparkd/cmds/Span.cpp | 5 +- src/server/sparkd/cmds/Span.h | 4 +- src/server/sparkd/cmds/Spartan.cpp | 36 ++-- src/server/sparkd/cmds/Spartan.h | 4 +- src/server/sparkd/cmds/Test.cpp | 19 -- src/server/sparkd/cmds/Test.h | 10 - src/server/sparkd/cmds/Web.cpp | 60 +++--- src/server/sparkd/cmds/Web.h | 10 +- .../sparkd/cmds/spartan/cmds/BookStats.cpp | 32 ++-- .../sparkd/cmds/spartan/cmds/BookStats.h | 10 +- src/server/sparkd/cmds/spartan/cmds/CSV.cpp | 17 +- src/server/sparkd/cmds/spartan/cmds/CSV.h | 10 +- .../sparkd/cmds/spartan/cmds/Compress.cpp | 17 +- .../sparkd/cmds/spartan/cmds/Compress.h | 10 +- .../sparkd/cmds/spartan/cmds/FindPolys.cpp | 61 +++--- .../sparkd/cmds/spartan/cmds/FindPolys.h | 10 +- .../sparkd/cmds/spartan/cmds/Icebergs.cpp | 62 +++--- .../sparkd/cmds/spartan/cmds/Icebergs.h | 10 +- src/server/sparkd/cmds/spartan/cmds/Json.cpp | 23 ++- src/server/sparkd/cmds/spartan/cmds/Json.h | 10 +- .../sparkd/cmds/spartan/cmds/ListSec.cpp | 45 +++-- src/server/sparkd/cmds/spartan/cmds/ListSec.h | 10 +- .../sparkd/cmds/spartan/cmds/PolygonBooks.cpp | 176 +++++++++--------- .../sparkd/cmds/spartan/cmds/PolygonBooks.h | 10 +- src/server/sparkd/cmds/spartan/cmds/SQL.cpp | 15 +- src/server/sparkd/cmds/spartan/cmds/SQL.h | 10 +- src/server/sparkd/cmds/spartan/cmds/Sim.cpp | 17 +- src/server/sparkd/cmds/spartan/cmds/Sim.h | 10 +- .../sparkd/cmds/spartan/cmds/SimBooks.cpp | 119 ++++++------ .../sparkd/cmds/spartan/cmds/SimBooks.h | 12 +- src/server/sparkd/cmds/spartan/cmds/Stats.cpp | 36 ++-- src/server/sparkd/cmds/spartan/cmds/Stats.h | 10 +- .../sparkd/cmds/spartan/cmds/ValidateFifo.cpp | 22 +-- .../sparkd/cmds/spartan/cmds/ValidateFifo.h | 10 +- .../sparkd/cmds/spartan/cmds/ValuePolys.cpp | 53 +++--- .../sparkd/cmds/spartan/cmds/ValuePolys.h | 10 +- src/server/sparkd/main.cpp | 43 +++-- 61 files changed, 841 insertions(+), 742 deletions(-) delete mode 100644 src/server/sparkd/cmds/Test.cpp delete mode 100644 src/server/sparkd/cmds/Test.h diff --git a/src/server/sparkd/cmds/AdminReport.cpp b/src/server/sparkd/cmds/AdminReport.cpp index ce9fa83c7..f887afad0 100644 --- a/src/server/sparkd/cmds/AdminReport.cpp +++ b/src/server/sparkd/cmds/AdminReport.cpp @@ -85,10 +85,11 @@ void md_report() { } } -void AdminReport::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand( - "admin-report", "Write a market data entitlement report csv to stdout"); - cmd->add_subcommand("md")->callback(md_report); +AdminReport::AdminReport() + : CLI::App( + "Write a market data entitlement report csv to stdout", + "admin-report") { + add_subcommand("md")->callback(md_report); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/AdminReport.h b/src/server/sparkd/cmds/AdminReport.h index 54cb00bc2..1a339bffd 100644 --- a/src/server/sparkd/cmds/AdminReport.h +++ b/src/server/sparkd/cmds/AdminReport.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct AdminReport { - static void register_cmd(CLI::App &); +struct AdminReport : public CLI::App { + AdminReport(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/AutoCert.cpp b/src/server/sparkd/cmds/AutoCert.cpp index 7aed1e700..2e1577f05 100644 --- a/src/server/sparkd/cmds/AutoCert.cpp +++ b/src/server/sparkd/cmds/AutoCert.cpp @@ -1273,13 +1273,11 @@ void iLinkCGWMidWeekKeyRotation() { } } -static void execute() { CGWFailoverFromPrimaryToBackupComponents(); } - -void AutoCert::register_cmd(CLI::App &app) { - app.add_subcommand( - "auto-cert", - "Provide control over JetFIX so that we can go through AutoCert") - ->callback(execute); +AutoCert::AutoCert() + : CLI::App( + "Provide control over JetFIX so that we can go through AutoCert", + "auto-cert") { + callback(CGWFailoverFromPrimaryToBackupComponents); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/AutoCert.h b/src/server/sparkd/cmds/AutoCert.h index 5f4b6e8e5..71abb20c7 100644 --- a/src/server/sparkd/cmds/AutoCert.h +++ b/src/server/sparkd/cmds/AutoCert.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct AutoCert { - static void register_cmd(CLI::App &); +struct AutoCert : public CLI::App { + AutoCert(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Benchmark.cpp b/src/server/sparkd/cmds/Benchmark.cpp index b5985aa12..21829c1d7 100644 --- a/src/server/sparkd/cmds/Benchmark.cpp +++ b/src/server/sparkd/cmds/Benchmark.cpp @@ -322,21 +322,6 @@ static void boost_pool_allocator(::benchmark::State &state) { } BENCHMARK(boost_pool_allocator); -static std::vector args; - -static void execute() { - bts_assert(args.size() < 99); - int argc = args.size() + 1; - - char *argv[100]; - argv[0] = (char *)"spark"; - for (size_t i = 0; i < args.size(); i++) - argv[i + 1] = (char *)args[i].c_str(); - - ::benchmark::Initialize(&argc, argv); - ::benchmark::RunSpecifiedBenchmarks(); -} - static void boost_lockless_queue(::benchmark::State &state) { using FreeList = boost::lockfree::queue; FreeList queue(1000000); @@ -798,12 +783,31 @@ void concurrent_append_collection_index(::benchmark::State &state) { } BENCHMARK(concurrent_append_collection_index); -void Benchmark::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand( - "benchmark", "Run micro-benchmarks. Use --benchmark_filter=regex to run " - "only specific benchmarks."); - cmd->add_option("args", args); - cmd->callback(execute); +struct Benchmark::Args { + std::vector args; +}; + +Benchmark::Benchmark() + : CLI::App( + "Run micro-benchmarks. Use --benchmark_filter=regex to run only " + "specific benchmarks.", + "benchmark"), + _args(new Benchmark::Args()) { + add_option("args", _args->args); + callback([&] { + bts_assert(_args->args.size() < 99); + int argc = _args->args.size() + 1; + + char *argv[100]; + argv[0] = (char *)"spark"; + for (size_t i = 0; i < _args->args.size(); i++) + argv[i + 1] = (char *)_args->args[i].c_str(); + + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); + }); } +Benchmark::~Benchmark() {} + } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Benchmark.h b/src/server/sparkd/cmds/Benchmark.h index 497637a50..757295a5f 100644 --- a/src/server/sparkd/cmds/Benchmark.h +++ b/src/server/sparkd/cmds/Benchmark.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::cmds { -struct Benchmark { - static void register_cmd(CLI::App &); +struct Benchmark : public CLI::App { + Benchmark(); + ~Benchmark(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/CMECert.cpp b/src/server/sparkd/cmds/CMECert.cpp index f8f335715..40c2de4e5 100644 --- a/src/server/sparkd/cmds/CMECert.cpp +++ b/src/server/sparkd/cmds/CMECert.cpp @@ -565,49 +565,47 @@ struct TestRunner { } }; -static void execute() { - // Typically a timestamp - SessionSaver ss; - ss.load(); - Config config{"NGG", "004", false, [](auto, auto, auto) {}}; - IL3Reactor reactor; - IL3MessageFramer framer; - auto &connection = reactor.register_connection( - {// local dev tunnel - TCPEndpoint{"127.0.0.1", 64530}, TCPEndpoint{"127.0.0.1", 64531}, - // Direct connection - TCPEndpoint{"69.50.112.199", 64530}, - TCPEndpoint{"69.50.12.205", 64530}}); +CMECert::CMECert() : CLI::App("Tool to drive CME Cert", "cme-cert") { + callback([] { + // Typically a timestamp + SessionSaver ss; + ss.load(); + Config config{"NGG", "004", false, [](auto, auto, auto) {}}; + IL3Reactor reactor; + IL3MessageFramer framer; + auto &connection = reactor.register_connection( + {// local dev tunnel + TCPEndpoint{"127.0.0.1", 64530}, TCPEndpoint{"127.0.0.1", 64531}, + // Direct connection + TCPEndpoint{"69.50.112.199", 64530}, + TCPEndpoint{"69.50.12.205", 64530}}); - // NGG primary - 69.50.112.199 64530 - // NGG backup - 69.50.112.205 64530 + // NGG primary - 69.50.112.199 64530 + // NGG backup - 69.50.112.205 64530 - IL3GW gw(connection, framer, nullptr, ss.current, config, {}); - gw.subscribe_session([&](auto &s) { ss.session_changed(s); }); - gw.connect(); + IL3GW gw(connection, framer, nullptr, ss.current, config, {}); + gw.subscribe_session([&](auto &s) { ss.session_changed(s); }); + gw.connect(); - TestRunner run(reactor, gw); + TestRunner run(reactor, gw); - /////////////////////////// - // - // The current test - // - /////////////////////////// + /////////////////////////// + // + // The current test + // + /////////////////////////// - run.scratch_space(); + run.scratch_space(); - /////////////////////////// + /////////////////////////// - for (auto x = 0; x < 5; ++x) { - reactor.do_work(); - sleep(1); - } - gw.send_terminate(); - gw.flush(); -} - -void CMECert::register_cmd(CLI::App &app) { - app.add_subcommand("cme-cert", "Tool to drive CME Cert")->callback(execute); + for (auto x = 0; x < 5; ++x) { + reactor.do_work(); + sleep(1); + } + gw.send_terminate(); + gw.flush(); + }); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/CMECert.h b/src/server/sparkd/cmds/CMECert.h index 8e9c8a045..4ca560784 100644 --- a/src/server/sparkd/cmds/CMECert.h +++ b/src/server/sparkd/cmds/CMECert.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct CMECert { - void register_cmd(CLI::App &); +struct CMECert : public CLI::App { + CMECert(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ConfigCheck.cpp b/src/server/sparkd/cmds/ConfigCheck.cpp index 78be083d0..604427e46 100644 --- a/src/server/sparkd/cmds/ConfigCheck.cpp +++ b/src/server/sparkd/cmds/ConfigCheck.cpp @@ -3,19 +3,22 @@ namespace bts::cmds { -static std::vector args; +struct ConfigCheck::Args { + std::vector args; +}; -static void execute() { - std::vector ch_args; - for (auto const &s : args) - ch_args.push_back(s.c_str()); - configcheck(ch_args.size(), &ch_args[0]); +ConfigCheck::ConfigCheck() + : CLI::App("Print current configuration", "config-check"), + _args(new ConfigCheck::Args()) { + add_option("args", _args->args); + callback([&] { + std::vector ch_args; + for (auto const &s : _args->args) + ch_args.push_back(s.c_str()); + configcheck(ch_args.size(), &ch_args[0]); + }); } -void ConfigCheck::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand("config-check"); - cmd->add_option("args", args); - cmd->callback(execute); -} +ConfigCheck::~ConfigCheck() {} } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ConfigCheck.h b/src/server/sparkd/cmds/ConfigCheck.h index f5d4296c7..bba116251 100644 --- a/src/server/sparkd/cmds/ConfigCheck.h +++ b/src/server/sparkd/cmds/ConfigCheck.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::cmds { -struct ConfigCheck { - static void register_cmd(CLI::App &); +struct ConfigCheck : public CLI::App { + ConfigCheck(); + ~ConfigCheck(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/FIXSim.cpp b/src/server/sparkd/cmds/FIXSim.cpp index 4dd9d5063..5f6715e92 100644 --- a/src/server/sparkd/cmds/FIXSim.cpp +++ b/src/server/sparkd/cmds/FIXSim.cpp @@ -3,13 +3,8 @@ namespace bts::cmds { -static void execute() { - bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM); -} - -void FIXSim::register_cmd(CLI::App &app) { - app.add_subcommand("fix-sim", "Starts a standalone FIX simulator") - ->callback(execute); +FIXSim::FIXSim() : CLI::App("Starts a standalone FIX simulator", "fix-sim") { + callback([] { bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM); }); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/FIXSim.h b/src/server/sparkd/cmds/FIXSim.h index 76a37d354..c4cea8149 100644 --- a/src/server/sparkd/cmds/FIXSim.h +++ b/src/server/sparkd/cmds/FIXSim.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct FIXSim { - static void register_cmd(CLI::App &); +struct FIXSim : public CLI::App { + FIXSim(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/KeyCode.cpp b/src/server/sparkd/cmds/KeyCode.cpp index 0ad59dd7b..08d1caff2 100644 --- a/src/server/sparkd/cmds/KeyCode.cpp +++ b/src/server/sparkd/cmds/KeyCode.cpp @@ -43,7 +43,7 @@ using namespace bts::api::auth; using namespace bts::crypto; using namespace bts::config; -struct EncodeArgs { +struct EncryptArgs { std::string keyfile; std::string pass; std::string users; @@ -69,12 +69,6 @@ struct KeygenArgs { std::string expiration; }; -struct Args { - EncodeArgs encrypt; - DecryptArgs decrypt; - KeygenArgs keygen; -}; - std::string parse_date(std::string const &date) { auto the_date = bts::datetime::parse_iso_date_time_string(date); return bts::datetime::iso_date_time_string(the_date); @@ -115,8 +109,6 @@ void write_keycode( kc.set_allow_concurrent(concurrent); } -static Args args; - void check_ips() { CryptoInit::initialize(); auto ipmap = bts::os::host_ips(); @@ -124,18 +116,18 @@ void check_ips() { std::cout << x.first << " => " << x.second << std::endl; } -void encrypt() { +void encrypt(const EncryptArgs &args) { CryptoInit::initialize(); std::string protobuf; std::string kc_string; std::stringstream oss, encrypted; - CachedPassword cache(args.encrypt.pass); - PrivateKeyPtr key = PrivateKey::read_file(args.encrypt.keyfile, &cache); + CachedPassword cache(args.pass); + PrivateKeyPtr key = PrivateKey::read_file(args.keyfile, &cache); bts::config::KeyCode kc; write_keycode( - args.encrypt.users, args.encrypt.expiration, args.encrypt.ip, - args.encrypt.port, args.encrypt.allow_concurrent, kc); + args.users, args.expiration, args.ip, args.port, args.allow_concurrent, + kc); if (!google::protobuf::TextFormat::PrintToString(kc, &kc_string)) return; oss << kc_string; @@ -149,7 +141,7 @@ void encrypt() { std::string pb; google::protobuf::TextFormat::PrintToString(ekc, &pb); std::cout << std::endl << std::endl; - std::cout << "#nusers: " << args.encrypt.users << std::endl; + std::cout << "#nusers: " << args.users << std::endl; std::cout << "#allow concurrent : " << (kc.allow_concurrent() ? "true" : "false") << std::endl; std::cout << "#expires: " @@ -168,14 +160,13 @@ void encrypt() { std::cout << std::endl << std::endl; } -void decrypt() { +void decrypt(const DecryptArgs &args) { CryptoInit::initialize(); - if (args.decrypt.keyfile != "") { - PublicKeyPtr pub = - bts::crypto::PublicKey::read_file(args.decrypt.keyfile); - if (pub && args.decrypt.source != "") { + if (args.keyfile != "") { + PublicKeyPtr pub = bts::crypto::PublicKey::read_file(args.keyfile); + if (pub && args.source != "") { bts::config::EncryptedKeyCode ekc; - std::fstream fs(args.decrypt.source); + std::fstream fs(args.source); std::string str( (std::istreambuf_iterator(fs)), std::istreambuf_iterator()); @@ -195,12 +186,12 @@ void decrypt() { } } -void keygen() { +void keygen(const KeygenArgs &args) { CryptoInit::initialize(); PrivateKeyPtr priv = PrivateKey::create(2048, AlgoType::RSA); - std::string n1 = (args.keygen.keyfile + "_private.pem"), - n2 = (args.keygen.keyfile + "_public.pem"); + std::string n1 = (args.keyfile + "_private.pem"), + n2 = (args.keyfile + "_public.pem"); // I can't get anything but DES to work here. priv->write_file(n1, CipherType::DES); priv->public_key()->write_file(n2); @@ -208,37 +199,45 @@ void keygen() { std::cout << "Key pair generated " << n1 << " and " << n2 << std::endl; } -void KeyCode::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand("key-code"); - cmd->require_subcommand(); +struct KeyCode::Args { + EncryptArgs encrypt; + DecryptArgs decrypt; + KeygenArgs keygen; +}; + +KeyCode::KeyCode() : CLI::App("key-code"), _args(new KeyCode::Args()) { + require_subcommand(); + auto *cmd = this; // check ips cmd->add_subcommand("check-ips")->callback(check_ips); // decrypt auto *decrypt_cmd = cmd->add_subcommand("decrypt"); - decrypt_cmd->add_option("--expiration", args.decrypt.expiration); - decrypt_cmd->add_option("--key-file", args.decrypt.keyfile)->required(); - decrypt_cmd->add_option("--pass", args.decrypt.pass); - decrypt_cmd->add_option("--source", args.decrypt.source); - decrypt_cmd->add_option("--users", args.decrypt.users); - decrypt_cmd->callback(decrypt); + decrypt_cmd->add_option("--expiration", _args->decrypt.expiration); + decrypt_cmd->add_option("--key-file", _args->decrypt.keyfile)->required(); + decrypt_cmd->add_option("--pass", _args->decrypt.pass); + decrypt_cmd->add_option("--source", _args->decrypt.source); + decrypt_cmd->add_option("--users", _args->decrypt.users); + decrypt_cmd->callback([&] { decrypt(_args->decrypt); }); // encrypt auto *encrypt_cmd = cmd->add_subcommand("encrypt"); - encrypt_cmd->add_flag("--allow-concurrent", args.encrypt.allow_concurrent); - encrypt_cmd->add_option("--expiration", args.encrypt.expiration); - encrypt_cmd->add_option("--ip", args.encrypt.ip); - encrypt_cmd->add_option("--key-file", args.encrypt.keyfile)->required(); - encrypt_cmd->add_option("--pass", args.encrypt.pass); - encrypt_cmd->add_option("--port", args.encrypt.port); - encrypt_cmd->add_option("--source", args.encrypt.source); - encrypt_cmd->add_option("--users", args.encrypt.users); - encrypt_cmd->callback(encrypt); + encrypt_cmd->add_flag("--allow-concurrent", _args->encrypt.allow_concurrent); + encrypt_cmd->add_option("--expiration", _args->encrypt.expiration); + encrypt_cmd->add_option("--ip", _args->encrypt.ip); + encrypt_cmd->add_option("--key-file", _args->encrypt.keyfile)->required(); + encrypt_cmd->add_option("--pass", _args->encrypt.pass); + encrypt_cmd->add_option("--port", _args->encrypt.port); + encrypt_cmd->add_option("--source", _args->encrypt.source); + encrypt_cmd->add_option("--users", _args->encrypt.users); + encrypt_cmd->callback([&] { encrypt(_args->encrypt); }); // key-gen auto *keygen_cmd = cmd->add_subcommand("key-gen"); - keygen_cmd->add_option("--expiration", args.keygen.expiration); - keygen_cmd->add_option("--key-file", args.keygen.keyfile)->required(); - keygen_cmd->add_option("--pass", args.keygen.pass); - keygen_cmd->add_option("--users", args.keygen.users); - keygen_cmd->callback(keygen); + keygen_cmd->add_option("--expiration", _args->keygen.expiration); + keygen_cmd->add_option("--key-file", _args->keygen.keyfile)->required(); + keygen_cmd->add_option("--pass", _args->keygen.pass); + keygen_cmd->add_option("--users", _args->keygen.users); + keygen_cmd->callback([&] { keygen(_args->keygen); }); } +KeyCode::~KeyCode() {} + } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/KeyCode.h b/src/server/sparkd/cmds/KeyCode.h index ae3aa6f43..8f9268eda 100644 --- a/src/server/sparkd/cmds/KeyCode.h +++ b/src/server/sparkd/cmds/KeyCode.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace bts::crypto { @@ -16,8 +17,13 @@ class KeyCode; namespace bts::cmds { -struct KeyCode { - static void register_cmd(CLI::App &); +struct KeyCode : public CLI::App { + KeyCode(); + ~KeyCode(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Md.cpp b/src/server/sparkd/cmds/Md.cpp index 10a561e92..6fcd64439 100644 --- a/src/server/sparkd/cmds/Md.cpp +++ b/src/server/sparkd/cmds/Md.cpp @@ -4,14 +4,15 @@ namespace bts::cmds { -static std::vector args; +struct Md::Args { + std::vector args; +}; -static void execute() { bts::md::md_tool(args); } - -void Md::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand("md"); - cmd->add_option("args", args); - cmd->callback(execute); +Md::Md() : CLI::App("", "md"), _args(new Md::Args()) { + add_option("args", _args->args); + callback([&] { md::md_tool(_args->args); }); } +Md::~Md() {} + } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Md.h b/src/server/sparkd/cmds/Md.h index 9679166c1..529711f2d 100644 --- a/src/server/sparkd/cmds/Md.h +++ b/src/server/sparkd/cmds/Md.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::cmds { -struct Md { - static void register_cmd(CLI::App &); +struct Md : public CLI::App { + Md(); + ~Md(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp index c7b836a26..820d88f84 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.cpp +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.cpp @@ -396,58 +396,62 @@ void print_orders( } } -static std::vector args; +struct ParseHistoricalFix::Args { + std::vector args; +}; -static void execute() { - std::ifstream input(args[0]); - std::ofstream books_stream(args[0] + "_books.csv"); - std::ofstream msgs_stream(args[0] + "_msgs.csv"); - std::ofstream trades_stream(args[0] + "_trades.csv"); - std::ofstream orders_stream(args[0] + "_orders.csv"); +ParseHistoricalFix::ParseHistoricalFix() + : CLI::App( + "ParseHistoricalFIX will produce 2 files, " + "_mdentries.csv and order_entries.csv", + "parse-historical-fix"), + _args(new ParseHistoricalFix::Args()) { + add_option("args", _args->args); + callback([&] { + auto &args = _args->args; + std::ifstream input(args[0]); + std::ofstream books_stream(args[0] + "_books.csv"); + std::ofstream msgs_stream(args[0] + "_msgs.csv"); + std::ofstream trades_stream(args[0] + "_trades.csv"); + std::ofstream orders_stream(args[0] + "_orders.csv"); - std::unordered_map books; - TradeSummaryEvent current_trade_summary; + std::unordered_map books; + TradeSummaryEvent current_trade_summary; - MDIncrementalUpdate update; - FIXParser parser; + MDIncrementalUpdate update; + FIXParser parser; - std::string line; - size_t c = 0; - auto ok = 0; - auto start = Timer::now(); + std::string line; + size_t c = 0; + auto ok = 0; + auto start = Timer::now(); - print_book_header(books_stream); - print_msg_headers(msgs_stream); - print_trades_header(trades_stream); - print_orders_header(orders_stream); + print_book_header(books_stream); + print_msg_headers(msgs_stream); + print_trades_header(trades_stream); + print_orders_header(orders_stream); - while (std::getline(input, line)) { - c++; - auto parsed_ptr = parser.parse(InMsgBuffer( - line.data(), line.size(), std::shared_ptr{})); - if (parsed_ptr) { - auto msg = - std::dynamic_pointer_cast(parsed_ptr); - update_and_maybe_print_book(*msg, books, books_stream); - print_msgs(c, *msg, msgs_stream); - update_and_maybe_print_trades( - *msg, current_trade_summary, trades_stream); - print_orders(*msg, orders_stream); + while (std::getline(input, line)) { + c++; + auto parsed_ptr = parser.parse(InMsgBuffer( + line.data(), line.size(), std::shared_ptr{})); + if (parsed_ptr) { + auto msg = std::dynamic_pointer_cast( + parsed_ptr); + update_and_maybe_print_book(*msg, books, books_stream); + print_msgs(c, *msg, msgs_stream); + update_and_maybe_print_trades( + *msg, current_trade_summary, trades_stream); + print_orders(*msg, orders_stream); + } + ok += (bool)(parsed_ptr); } - ok += (bool)(parsed_ptr); - } - auto sec_duration = sec_diff(Timer::now(), start); - std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok - << " out of " << c << " time: " << sec_duration << " secs\n"; + auto sec_duration = sec_diff(Timer::now(), start); + std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok + << " out of " << c << " time: " << sec_duration << " secs\n"; + }); } -void ParseHistoricalFix::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand( - "parse-historical-fix", - "ParseHistoricalFIX will produce 2 files, " - "_mdentries.csv and order_entries.csv"); - cmd->add_option("args", args); - cmd->callback(execute); -} +ParseHistoricalFix::~ParseHistoricalFix() {} } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/ParseHistoricalFIX.h b/src/server/sparkd/cmds/ParseHistoricalFIX.h index df610647b..692c91843 100644 --- a/src/server/sparkd/cmds/ParseHistoricalFIX.h +++ b/src/server/sparkd/cmds/ParseHistoricalFIX.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::cmds { -struct ParseHistoricalFix { - static void register_cmd(CLI::App &); +struct ParseHistoricalFix : public CLI::App { + ParseHistoricalFix(); + ~ParseHistoricalFix(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/PrintHedgeOrders.cpp b/src/server/sparkd/cmds/PrintHedgeOrders.cpp index 79c791267..54b1a7937 100644 --- a/src/server/sparkd/cmds/PrintHedgeOrders.cpp +++ b/src/server/sparkd/cmds/PrintHedgeOrders.cpp @@ -5,53 +5,52 @@ namespace bts::cmds { -static void execute() { - persistence::SqliteStore order_store("orders"); - auto all_orders = order_store.retrieve_all()["orders"]; - - std::map> fill_times; - persistence::SqliteStore fill_store("fills"); - auto all_fills = fill_store.retrieve_all()["fills"]; - for (auto const &kvp : all_fills) { - api::orderman::Fill fill_rec; - if (utils::pb::from_string(kvp.second, &fill_rec)) { - auto ts = utils::pb::to_double_time(fill_rec.timestamp()); - auto &p = fill_times[fill_rec.order_key()]; - auto &min_time = p.second; - if (min_time < ts) { - p.first = kvp.first; - min_time = ts; - } - } - } - - std::cout << "order_key,time_to_first_fill,fill_id" - << "\n"; - api::orderman::OrderRecord rec; - for (auto const &kvp : all_orders) - if (utils::pb::from_string( - kvp.second, &rec)) { - auto &def = rec.def(); - auto &state = rec.state(); - if (rec.parent_key() && def.custom_def().has_limit() && - def.sub_type() == - api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) { - auto create_ts = utils::pb::to_double_time(state.created()); - auto &first_fill = fill_times[rec.key()]; - auto delta = first_fill.second - create_ts; - if (first_fill.first.size()) - std::cout << kvp.first << "," << delta << "," << first_fill.first - << "\n"; - } - } -} - -void PrintHedgeOrders::register_cmd(CLI::App &app) { - app.add_subcommand( - "print-hedge-orders", +PrintHedgeOrders::PrintHedgeOrders() + : CLI::App( "Print out order ids for all hedge orders" - "which were completed at least one second being created") - ->callback(execute); + "which were completed at least one second being created", + "print-hedge-orders") { + callback([&] { + persistence::SqliteStore order_store("orders"); + auto all_orders = order_store.retrieve_all()["orders"]; + + std::map> fill_times; + persistence::SqliteStore fill_store("fills"); + auto all_fills = fill_store.retrieve_all()["fills"]; + for (auto const &kvp : all_fills) { + api::orderman::Fill fill_rec; + if (utils::pb::from_string( + kvp.second, &fill_rec)) { + auto ts = utils::pb::to_double_time(fill_rec.timestamp()); + auto &p = fill_times[fill_rec.order_key()]; + auto &min_time = p.second; + if (min_time < ts) { + p.first = kvp.first; + min_time = ts; + } + } + } + + std::cout << "order_key,time_to_first_fill,fill_id" + << "\n"; + api::orderman::OrderRecord rec; + for (auto const &kvp : all_orders) + if (utils::pb::from_string( + kvp.second, &rec)) { + auto &def = rec.def(); + auto &state = rec.state(); + if (rec.parent_key() && def.custom_def().has_limit() && + def.sub_type() == + api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) { + auto create_ts = utils::pb::to_double_time(state.created()); + auto &first_fill = fill_times[rec.key()]; + auto delta = first_fill.second - create_ts; + if (first_fill.first.size()) + std::cout << kvp.first << "," << delta << "," + << first_fill.first << "\n"; + } + } + }); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/PrintHedgeOrders.h b/src/server/sparkd/cmds/PrintHedgeOrders.h index bf5289e7a..831b66615 100644 --- a/src/server/sparkd/cmds/PrintHedgeOrders.h +++ b/src/server/sparkd/cmds/PrintHedgeOrders.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct PrintHedgeOrders { - static void register_cmd(CLI::App &); +struct PrintHedgeOrders : public CLI::App { + PrintHedgeOrders(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index cee27988e..110b53b1f 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -41,11 +41,12 @@ static void execute() { // put test code here } -void QuickTest::register_cmd(CLI::App &app) { - app.add_subcommand( - "quick-test", "This is a place holder to make it easy to quickly " - "write discardable test code") - ->callback(execute); +QuickTest::QuickTest() + : CLI::App( + "This is a place holder to make it easy to quickly " + "write discardable test code", + "quick-test") { + callback(execute); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/QuickTest.h b/src/server/sparkd/cmds/QuickTest.h index cdf5df0eb..7e9099a7e 100644 --- a/src/server/sparkd/cmds/QuickTest.h +++ b/src/server/sparkd/cmds/QuickTest.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct QuickTest { - static void register_cmd(CLI::App &); +struct QuickTest : public CLI::App { + QuickTest(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 1991b302f..42ed6f405 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -123,8 +123,8 @@ static void execute() { quickbus.run(); } -void Run::register_cmd(CLI::App &app) { - app.add_subcommand("run", "Run default sparkd services")->callback(execute); +Run::Run() : CLI::App("Run default sparkd services", "run") { + callback(execute); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Run.h b/src/server/sparkd/cmds/Run.h index 7e2bae1f9..563fc8a60 100644 --- a/src/server/sparkd/cmds/Run.h +++ b/src/server/sparkd/cmds/Run.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct Run { - static void register_cmd(CLI::App &); +struct Run : public CLI::App { + Run(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Span.cpp b/src/server/sparkd/cmds/Span.cpp index dfe3e0aad..f225076c2 100644 --- a/src/server/sparkd/cmds/Span.cpp +++ b/src/server/sparkd/cmds/Span.cpp @@ -56,9 +56,8 @@ static void execute() { disp.run(); } -void Span::register_cmd(CLI::App &app) { - app.add_subcommand("span", "Test CME CORE Span integration") - ->callback(execute); +Span::Span() : CLI::App("Test CME CORE Span integration", "span") { + callback(execute); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Span.h b/src/server/sparkd/cmds/Span.h index d6ae46089..3d8c418dd 100644 --- a/src/server/sparkd/cmds/Span.h +++ b/src/server/sparkd/cmds/Span.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct Span { - static void register_cmd(CLI::App &); +struct Span : public CLI::App { + Span(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Spartan.cpp b/src/server/sparkd/cmds/Spartan.cpp index 8501e386a..4eee88947 100644 --- a/src/server/sparkd/cmds/Spartan.cpp +++ b/src/server/sparkd/cmds/Spartan.cpp @@ -16,24 +16,24 @@ namespace bts::cmds { -void Spartan::register_cmd(CLI::App &app) { - auto *spartan_app = - app.add_subcommand("spartan", "Futures trading system for devs"); - // register commands - bts::spartan::cmds::BookStats::register_cmd(*spartan_app); - bts::spartan::cmds::Compress::register_cmd(*spartan_app); - bts::spartan::cmds::CSV::register_cmd(*spartan_app); - bts::spartan::cmds::FindPolys::register_cmd(*spartan_app); - bts::spartan::cmds::Icebergs::register_cmd(*spartan_app); - bts::spartan::cmds::Json::register_cmd(*spartan_app); - bts::spartan::cmds::ListSec::register_cmd(*spartan_app); - bts::spartan::cmds::PolygonBooks::register_cmd(*spartan_app); - bts::spartan::cmds::Sim::register_cmd(*spartan_app); - bts::spartan::cmds::SimBooks::register_cmd(*spartan_app); - bts::spartan::cmds::SQL::register_cmd(*spartan_app); - bts::spartan::cmds::Stats::register_cmd(*spartan_app); - bts::spartan::cmds::ValidateFifo::register_cmd(*spartan_app); - bts::spartan::cmds::ValuePolys::register_cmd(*spartan_app); +using namespace spartan::cmds; + +Spartan::Spartan() : CLI::App("Futures trading system for devs", "spartan") { + // subcommands + add_subcommand(CLI::App_p(new BookStats())); + add_subcommand(CLI::App_p(new Compress())); + add_subcommand(CLI::App_p(new CSV())); + add_subcommand(CLI::App_p(new FindPolys())); + add_subcommand(CLI::App_p(new Icebergs())); + add_subcommand(CLI::App_p(new Json())); + add_subcommand(CLI::App_p(new ListSec())); + add_subcommand(CLI::App_p(new PolygonBooks())); + add_subcommand(CLI::App_p(new Sim())); + add_subcommand(CLI::App_p(new SimBooks())); + add_subcommand(CLI::App_p(new SQL())); + add_subcommand(CLI::App_p(new Stats())); + add_subcommand(CLI::App_p(new ValidateFifo())); + add_subcommand(CLI::App_p(new ValuePolys())); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Spartan.h b/src/server/sparkd/cmds/Spartan.h index add33ffec..832c838e8 100644 --- a/src/server/sparkd/cmds/Spartan.h +++ b/src/server/sparkd/cmds/Spartan.h @@ -3,8 +3,8 @@ namespace bts::cmds { -struct Spartan { - static void register_cmd(CLI::App &); +struct Spartan : public CLI::App { + Spartan(); }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Test.cpp b/src/server/sparkd/cmds/Test.cpp deleted file mode 100644 index f78f46f28..000000000 --- a/src/server/sparkd/cmds/Test.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "Test.h" -#include "lib/testing/catch_test_runner.h" - -namespace bts::cmds { - -static std::vector args; - -static void execute() { run_tests(args); } - -void Test::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand( - "test", - "See " - "https://github.com/philsquared/Catch/blob/master/docs/command-line.md"); - cmd->add_option("args", args); - cmd->callback(execute); -} - -} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Test.h b/src/server/sparkd/cmds/Test.h deleted file mode 100644 index cfec12a5f..000000000 --- a/src/server/sparkd/cmds/Test.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once -#include - -namespace bts::cmds { - -struct Test { - static void register_cmd(CLI::App &); -}; - -} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 92450b92b..bad2076f3 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -22,45 +22,45 @@ namespace bts::cmds { -struct Args { +struct Web::Args { bool md; }; -static Args args; +Web::Web() + : CLI::App( + "Run only the services required for the web admin. This cannot be " + "run concurrently with 'sparkd run.'", + "web"), + _args(new Web::Args()) { + add_flag("--md", _args->md, "Start market data"); + callback([&] { + config::QuickbusNodeCfg qb_cfg; + auto &ports = config::get(); + qb_cfg.set_my_port(ports.qbus_port()); + qbus::Quickbus quickbus(qb_cfg); + api::ServiceContext sc(&quickbus); -static void execute() { - config::QuickbusNodeCfg qb_cfg; - auto &ports = config::get(); - qb_cfg.set_my_port(ports.qbus_port()); - qbus::Quickbus quickbus(qb_cfg); - api::ServiceContext sc(&quickbus); + metrics::MetricsContainer stats; + metrics::publish_system_metrics(stats); + trace::PerfLog perf_log; + http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port()); - metrics::MetricsContainer stats; - metrics::publish_system_metrics(stats); - trace::PerfLog perf_log; - http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port()); + user::AuthServiceServer login_server(sc); + if (_args->md) { + md::run_market_datapublishers(nullptr, nullptr); + } - user::AuthServiceServer login_server(sc); - if (args.md) { - md::run_market_datapublishers(nullptr, nullptr); - } + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service); + md::QueryServerPtr md_query_server( + md::run_query_server(sc, mdclient, stats)); - api::synthetic::SyntheticSecurityService synthetic_service(sc); - md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service); - md::QueryServerPtr md_query_server( - md::run_query_server(sc, mdclient, stats)); + admin::AdminServiceServer admin_service(sc); - admin::AdminServiceServer admin_service(sc); - - utils::Dispatcher::current().run(); + utils::Dispatcher::current().run(); + }); } -void Web::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand( - "web", "Run only the services required for the web admin. This cannot " - "be run concurrently with 'sparkd run.'"); - cmd->add_flag("--md", args.md, "Start market data"); - cmd->callback(execute); -} +Web::~Web() {} } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/Web.h b/src/server/sparkd/cmds/Web.h index f1d4320e7..575c2f947 100644 --- a/src/server/sparkd/cmds/Web.h +++ b/src/server/sparkd/cmds/Web.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::cmds { -struct Web { - static void register_cmd(CLI::App &); +struct Web : public CLI::App { + Web(); + ~Web(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp b/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp index 388a8e18e..409fb133a 100644 --- a/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/BookStats.cpp @@ -7,26 +7,28 @@ namespace bts::spartan::cmds { -struct Args { +struct BookStats::Args { std::string pcap; std::string out; }; -static Args args; - -static void execute() { - if (args.out.empty()) - args.out = file::change_file_ext(args.pcap, "book_stats.csv", "pcap.xz"); - std::ofstream out(args.out); - mdp3::BookStatRecorder bsr(out); - mdp3::observe_channel(args.pcap, bsr); +BookStats::BookStats() + : CLI::App( + "Generate a csv file of various statistics about book updates", + "book-stats"), + _args(new BookStats::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("out,--out", _args->out); + callback([&] { + if (_args->out.empty()) + _args->out = + file::change_file_ext(_args->pcap, "book_stats.csv", "pcap.xz"); + std::ofstream out(_args->out); + mdp3::BookStatRecorder bsr(out); + mdp3::observe_channel(_args->pcap, bsr); + }); } -void BookStats::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("book-stats"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option("out,--out", args.out); - subcmd->callback(execute); -} +BookStats::~BookStats() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/BookStats.h b/src/server/sparkd/cmds/spartan/cmds/BookStats.h index 6b64eef8f..ffcc4683d 100644 --- a/src/server/sparkd/cmds/spartan/cmds/BookStats.h +++ b/src/server/sparkd/cmds/spartan/cmds/BookStats.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct BookStats { - static void register_cmd(CLI::App &); +struct BookStats : public CLI::App { + BookStats(); + ~BookStats(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/CSV.cpp b/src/server/sparkd/cmds/spartan/cmds/CSV.cpp index 5cc2d0cd5..64917b7fd 100644 --- a/src/server/sparkd/cmds/spartan/cmds/CSV.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/CSV.cpp @@ -3,18 +3,17 @@ namespace bts::spartan::cmds { -struct Args { +struct CSV::Args { std::string pcap; }; -static Args args; - -static void execute() { mdp3::to_csv(args.pcap); } - -void CSV::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand("csv"); - cmd->add_option("pcap,--pcap", args.pcap)->required(); - cmd->callback(execute); +CSV::CSV() + : CLI::App("Generate csv files for various pcap-derived tables", "csv"), + _args(new CSV::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + callback([&] { mdp3::to_csv(_args->pcap); }); } +CSV::~CSV() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/CSV.h b/src/server/sparkd/cmds/spartan/cmds/CSV.h index 653070ad1..9f26769cc 100644 --- a/src/server/sparkd/cmds/spartan/cmds/CSV.h +++ b/src/server/sparkd/cmds/spartan/cmds/CSV.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct CSV { - static void register_cmd(CLI::App &); +struct CSV : public CLI::App { + CSV(); + ~CSV(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Compress.cpp b/src/server/sparkd/cmds/spartan/cmds/Compress.cpp index 8e53526eb..1c847fd4e 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Compress.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/Compress.cpp @@ -3,20 +3,17 @@ namespace bts::spartan::cmds { -struct Args { +struct Compress::Args { std::string pcap; std::string out; }; -static Args args; - -static void execute() { spartan::mdp3::compress(args.pcap, args.out); } - -void Compress::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("compress"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option("out,--out", args.out)->required(); - subcmd->callback(execute); +Compress::Compress() : CLI::App("", "compress"), _args(new Compress::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("out,--out", _args->out); + callback([&] { mdp3::compress(_args->pcap, _args->out); }); } +Compress::~Compress() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Compress.h b/src/server/sparkd/cmds/spartan/cmds/Compress.h index 89a2a31d7..f851ce89d 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Compress.h +++ b/src/server/sparkd/cmds/spartan/cmds/Compress.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct Compress { - static void register_cmd(CLI::App &); +struct Compress : public CLI::App { + Compress(); + ~Compress(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp b/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp index 4305aa58b..d352dbc1b 100644 --- a/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/FindPolys.cpp @@ -9,44 +9,43 @@ namespace bts::spartan::cmds { const int UNSET = -1; -struct Args { +struct FindPolys::Args { std::string pcap; int active_threshold_fills = UNSET; int max_vertices = UNSET; int max_securities = UNSET; }; -static Args args; - -static void execute() { - // init options (from args) - mdp3::PolygonFinder::Options options; - if (args.active_threshold_fills != UNSET) - options.active_threshold_fills = args.active_threshold_fills; - if (args.max_vertices != UNSET) - options.max_vertices = args.max_vertices; - if (args.max_securities != UNSET) - options.max_securities = args.max_securities; - - mdp3::DefinitionCollector def_collector; - mdp3::PolygonFinder poly_finder(options); - mdp3::observe_channel(args.pcap, {&poly_finder, &def_collector}); - for (size_t id = 1; id <= poly_finder.result.size(); id++) - std::cout << id << " -> " - << mdp3::polygon_descriptor( - poly_finder.result[id], def_collector.ptr_result) - << std::endl; -} - -void FindPolys::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("find-polys"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option( +FindPolys::FindPolys() + : CLI::App("Find polygons in a pcap file", "find-polys"), + _args(new FindPolys::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option( "active-threshold-fills,--active-threshold-fills", - args.active_threshold_fills); - subcmd->add_option("max-vertices,--max-vertices", args.max_vertices); - subcmd->add_option("max-securities,--max-securities", args.max_securities); - subcmd->callback(execute); + _args->active_threshold_fills); + add_option("max-vertices,--max-vertices", _args->max_vertices); + add_option("max-securities,--max-securities", _args->max_securities); + callback([&] { + // init options (from _args-> + mdp3::PolygonFinder::Options options; + if (_args->active_threshold_fills != UNSET) + options.active_threshold_fills = _args->active_threshold_fills; + if (_args->max_vertices != UNSET) + options.max_vertices = _args->max_vertices; + if (_args->max_securities != UNSET) + options.max_securities = _args->max_securities; + + mdp3::DefinitionCollector def_collector; + mdp3::PolygonFinder poly_finder(options); + mdp3::observe_channel(_args->pcap, {&poly_finder, &def_collector}); + for (size_t id = 1; id <= poly_finder.result.size(); id++) + std::cout << id << " -> " + << mdp3::polygon_descriptor( + poly_finder.result[id], def_collector.ptr_result) + << std::endl; + }); } +FindPolys::~FindPolys() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/FindPolys.h b/src/server/sparkd/cmds/spartan/cmds/FindPolys.h index 82eb133ba..4afecce31 100644 --- a/src/server/sparkd/cmds/spartan/cmds/FindPolys.h +++ b/src/server/sparkd/cmds/spartan/cmds/FindPolys.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct FindPolys { - static void register_cmd(CLI::App &); +struct FindPolys : public CLI::App { + FindPolys(); + ~FindPolys(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp b/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp index bc6249cc4..5736103aa 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/Icebergs.cpp @@ -10,42 +10,42 @@ namespace bts::spartan::cmds { -struct Args { +struct Icebergs::Args { std::string pcap; std::string out; }; -static Args args; - -static void execute() { - const char *ok_col = "ok"; - const char *eid_col = "eid"; - const char *total_col = "total_qty"; - const char *show_qty_col = "show_qty"; - const char *fills_col = "fill_qty"; - if (args.out.empty()) - args.out = file::change_file_ext(args.pcap, "icebergs.csv", "pcap.xz"); - std::ofstream out(args.out); - auto ibs = mdp3::IcebergFinder::find_all(args.pcap); - // write cols - out << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col - << "," << fills_col; - // write rows - for (auto &i : ibs) { - auto relevent_eids = i.second.all_eids(); - for (auto eid : relevent_eids) - out << i.first.value() << "," << eid << "," - << i.second.total_qty().value() << "," - << std::get<1>(i.second.show_qty_at(eid)) << "," - << i.second.count_fills_by(eid) << "\n"; - } +Icebergs::Icebergs() + : CLI::App("Generate a csv file of iceberg orders in a pcap", "icebergs"), + _args(new Icebergs::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("out,--out", _args->out); + callback([&] { + const char *ok_col = "ok"; + const char *eid_col = "eid"; + const char *total_col = "total_qty"; + const char *show_qty_col = "show_qty"; + const char *fills_col = "fill_qty"; + if (_args->out.empty()) + _args->out = + file::change_file_ext(_args->pcap, "icebergs.csv", "pcap.xz"); + std::ofstream out(_args->out); + auto ibs = mdp3::IcebergFinder::find_all(_args->pcap); + // write cols + out << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col + << "," << fills_col; + // write rows + for (auto &i : ibs) { + auto relevent_eids = i.second.all_eids(); + for (auto eid : relevent_eids) + out << i.first.value() << "," << eid << "," + << i.second.total_qty().value() << "," + << std::get<1>(i.second.show_qty_at(eid)) << "," + << i.second.count_fills_by(eid) << "\n"; + } + }); } -void Icebergs::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("icebergs"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option("out,--out", args.out); - subcmd->callback(execute); -} +Icebergs::~Icebergs() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Icebergs.h b/src/server/sparkd/cmds/spartan/cmds/Icebergs.h index d03675cfb..90dec7925 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Icebergs.h +++ b/src/server/sparkd/cmds/spartan/cmds/Icebergs.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct Icebergs { - static void register_cmd(CLI::App &); +struct Icebergs : public CLI::App { + Icebergs(); + ~Icebergs(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Json.cpp b/src/server/sparkd/cmds/spartan/cmds/Json.cpp index b379e1794..84c51a68e 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Json.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/Json.cpp @@ -3,24 +3,23 @@ namespace bts::spartan::cmds { -struct Args { +struct Json::Args { uint32_t start_packet = 0; uint32_t packet_count = 0; std::string pcap{}; }; -static Args args; - -static void execute() { - spartan::mdp3::to_json(args.pcap, args.start_packet, args.packet_count); +Json::Json() + : CLI::App("Generate a json file for data packets in a pcap", "json"), + _args(new Json::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("start-packet,--start-packet", _args->start_packet); + add_option("packet-count,--packet-count", _args->packet_count); + callback([&] { + mdp3::to_json(_args->pcap, _args->start_packet, _args->packet_count); + }); } -void Json::register_cmd(CLI::App &spartan_app) { - auto *subcmd = spartan_app.add_subcommand("json"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option("start-packet,--start-packet", args.start_packet); - subcmd->add_option("packet-count,--packet-count", args.packet_count); - subcmd->callback(execute); -} +Json::~Json() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Json.h b/src/server/sparkd/cmds/spartan/cmds/Json.h index 5337cf92f..ec41f258d 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Json.h +++ b/src/server/sparkd/cmds/spartan/cmds/Json.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct Json { - static void register_cmd(CLI::App &); +struct Json : public CLI::App { + Json(); + ~Json(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp b/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp index 12295b2a8..7b64d54e7 100644 --- a/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/ListSec.cpp @@ -6,35 +6,32 @@ namespace bts::spartan::cmds { -struct Args { +struct ListSec::Args { std::string pcap; }; -static Args args; - -static void execute() { - mdp3::DefinitionCollector def; - mdp3::FillCounter fill_counter; - mdp3::observe_channel(args.pcap, {&def, &fill_counter}); - auto fill_counts = fill_counter.result_map(); - for (auto const &s : def.result) { - std::cout << s.second.asset() << "|" << s.second.symbol() << "|" - << fill_counts[s.second.security_id()]; - for (auto const &l : s.second.legs()) { - std::cout << "|"; - if (def.ptr_result[l.security_id]) - std::cout << def.ptr_result[l.security_id]->symbol(); - else - std::cout << "?" << l.security_id; +ListSec::ListSec() : CLI::App("List securities in a pcap file", "list-sec"), _args(new ListSec::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + callback([&] { + mdp3::DefinitionCollector def; + mdp3::FillCounter fill_counter; + mdp3::observe_channel(_args->pcap, {&def, &fill_counter}); + auto fill_counts = fill_counter.result_map(); + for (auto const &s : def.result) { + std::cout << s.second.asset() << "|" << s.second.symbol() << "|" + << fill_counts[s.second.security_id()]; + for (auto const &l : s.second.legs()) { + std::cout << "|"; + if (def.ptr_result[l.security_id]) + std::cout << def.ptr_result[l.security_id]->symbol(); + else + std::cout << "?" << l.security_id; + } + std::cout << std::endl; } - std::cout << std::endl; - } + }); } -void ListSec::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("list-sec"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->callback(execute); -} +ListSec::~ListSec() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ListSec.h b/src/server/sparkd/cmds/spartan/cmds/ListSec.h index 4bc679da0..dafa95d95 100644 --- a/src/server/sparkd/cmds/spartan/cmds/ListSec.h +++ b/src/server/sparkd/cmds/spartan/cmds/ListSec.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct ListSec { - static void register_cmd(CLI::App &); +struct ListSec : public CLI::App { + ListSec(); + ~ListSec(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp index 64f38b0c2..e3b372611 100644 --- a/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.cpp @@ -18,103 +18,105 @@ namespace bts::spartan::cmds { #define WARN(...) LOG(OrderMan, Warn, __VA_ARGS__) #define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__) -struct Args { +struct PolygonBooks::Args { std::string pcap; std::string db; std::vector security_ids; std::string out_csv; }; -static Args args; +PolygonBooks::PolygonBooks() + : CLI::App( + "Generate a csv file of book updates for polygons", "polygon-books"), + _args(new PolygonBooks::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("data-base,--db,--data-base", _args->db)->required(); + add_option("security-ids,--security-ids", _args->security_ids)->required(); + add_option("out,--out", _args->out_csv); + callback([&] { + const char *eid_col = "eid"; + const char *pid_col = "pid"; + const char *log_header = "PolygonBookCSV"; + // the nth leg col denotes the last event in which the nth leg's book + // changed + const int num_leg_cols = 10; -static void execute() { - const char *eid_col = "eid"; - const char *pid_col = "pid"; - const char *log_header = "PolygonBookCSV"; - // the nth leg col denotes the last event in which the nth leg's book changed - const int num_leg_cols = 10; - - // validate args - if (args.out_csv.empty()) - args.out_csv = - file::change_file_ext(args.pcap, "polygon_books.csv", "pcap.xz"); - if (args.out_csv.empty() || args.security_ids.empty()) - return; - std::unordered_set ids; - for (auto sid : args.security_ids) - ids.insert(sid); - INFO( - "generating polygon books for pids: {}", - orderman::Formatter::fmt_values(ids)); - mdp3::SyntheticBookRecorder sbr( - spartan::sql::adapters::polygon_defs::read_sql(args.db.c_str(), ids)); - mdp3::observe_channel(args.pcap, sbr); - // don't prepend a comma before calling 'append_book' (it does that - // internally) - auto append_book = [](std::ofstream &out, - const mdp3::SmallBook &bk) -> std::ofstream & { - uint32_t i = 0; - bts_assert(bk.size() <= 10); - for (; i < bk.size(); i++) - out << "," << bk[i].price << "," << bk[i].quantity.value(); - while (i++ < mdp3::BookSide::max_depth()) - out << ",NaN,0"; - return out; - }; - auto append_leg_book_eids = - [&](std::ostream &out, - const mdp3::SyntheticMBPUpdate &upd) -> std::ostream & { - int lcount = 0; - for (const auto &p : upd.leg_book_eids) { - out << "," << p.second; - if (++lcount == num_leg_cols) { - WARN( - "{} processing book update for polygon with more than {} legs " - "({} " - "'leg-book-event-id' columns are accounted for in the csv, " - "event " - "ids for susbequent legs will be discarded).", - *log_header, num_leg_cols, num_leg_cols); - break; + // validate args + if (_args->out_csv.empty()) + _args->out_csv = + file::change_file_ext(_args->pcap, "polygon_books.csv", "pcap.xz"); + if (_args->out_csv.empty() || _args->security_ids.empty()) + return; + std::unordered_set ids; + for (auto sid : _args->security_ids) + ids.insert(sid); + INFO( + "generating polygon books for pids: {}", + orderman::Formatter::fmt_values(ids)); + mdp3::SyntheticBookRecorder sbr( + spartan::sql::adapters::polygon_defs::read_sql( + _args->db.c_str(), ids)); + mdp3::observe_channel(_args->pcap, sbr); + // don't prepend a comma before calling 'append_book' (it does that + // internally) + auto append_book = [](std::ofstream &out, + const mdp3::SmallBook &bk) -> std::ofstream & { + uint32_t i = 0; + bts_assert(bk.size() <= 10); + for (; i < bk.size(); i++) + out << "," << bk[i].price << "," << bk[i].quantity.value(); + while (i++ < mdp3::BookSide::max_depth()) + out << ",NaN,0"; + return out; + }; + auto append_leg_book_eids = + [&](std::ostream &out, + const mdp3::SyntheticMBPUpdate &upd) -> std::ostream & { + int lcount = 0; + for (const auto &p : upd.leg_book_eids) { + out << "," << p.second; + if (++lcount == num_leg_cols) { + WARN( + "{} processing book update for polygon with more than {} " + "legs " + "({} " + "'leg-book-event-id' columns are accounted for in the csv, " + "event " + "ids for susbequent legs will be discarded).", + *log_header, num_leg_cols, num_leg_cols); + break; + } + } + while (lcount++ < num_leg_cols) + out << ",0"; + return out; + }; + // write polygon books to csv + auto ofs = std::ofstream(_args->out_csv); + // add cols + ofs << eid_col << "," << pid_col; + for (int i = 0; i < num_leg_cols; i++) + ofs << ",leg_" << i << "_eid"; + auto add_bk_cols = [&](const char *side_prefix) { + for (int i = 0; i < 10; i++) + ofs << "," << side_prefix << "_p_" << i << "," << side_prefix + << "_q_" << i; + }; + add_bk_cols("bid"); + add_bk_cols("ask"); + for (const auto &p : sbr.result()) { + for (const auto &upd : p.second) { + ofs << std::endl; + ofs << p.first << "," << upd.synthetic_id.value(); + append_leg_book_eids(ofs, upd); + append_book(ofs, upd.bids); + append_book(ofs, upd.asks); } } - while (lcount++ < num_leg_cols) - out << ",0"; - return out; - }; - // write polygon books to csv - auto ofs = std::ofstream(args.out_csv); - // add cols - ofs << eid_col << "," << pid_col; - for (int i = 0; i < num_leg_cols; i++) - ofs << ",leg_" << i << "_eid"; - auto add_bk_cols = [&](const char *side_prefix) { - for (int i = 0; i < 10; i++) - ofs << "," << side_prefix << "_p_" << i << "," << side_prefix << "_q_" - << i; - }; - add_bk_cols("bid"); - add_bk_cols("ask"); - for (const auto &p : sbr.result()) { - for (const auto &upd : p.second) { - ofs << std::endl; - ofs << p.first << "," << upd.synthetic_id.value(); - append_leg_book_eids(ofs, upd); - append_book(ofs, upd.bids); - append_book(ofs, upd.asks); - } - } - ofs.flush(); + ofs.flush(); + }); } -void PolygonBooks::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand("polygon-books"); - cmd->add_option("pcap,--pcap", args.pcap)->required(); - cmd->add_option("db,--db", args.db)->required(); - cmd->add_option("security-ids,--security-ids", args.security_ids) - ->required(); - cmd->add_option("out,--out", args.out_csv); - cmd->callback(execute); -} +PolygonBooks::~PolygonBooks() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h index 4900850e2..631310e24 100644 --- a/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h +++ b/src/server/sparkd/cmds/spartan/cmds/PolygonBooks.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct PolygonBooks { - static void register_cmd(CLI::App &); +struct PolygonBooks : public CLI::App { + PolygonBooks(); + ~PolygonBooks(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SQL.cpp b/src/server/sparkd/cmds/spartan/cmds/SQL.cpp index 5c779a131..b5e802bd7 100644 --- a/src/server/sparkd/cmds/spartan/cmds/SQL.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/SQL.cpp @@ -3,18 +3,15 @@ namespace bts::spartan::cmds { -struct Args { +struct SQL::Args { std::string pcap; }; -static Args args; - -static void execute() { mdp3::to_sql(args.pcap); } - -void SQL::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("sql"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->callback(execute); +SQL::SQL() : CLI::App("Generate a sqlite database for a pcap file", "sql"), _args(new SQL::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + callback([&] { mdp3::to_sql(_args->pcap); }); } +SQL::~SQL() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SQL.h b/src/server/sparkd/cmds/spartan/cmds/SQL.h index f3832f520..04f43c6ee 100644 --- a/src/server/sparkd/cmds/spartan/cmds/SQL.h +++ b/src/server/sparkd/cmds/spartan/cmds/SQL.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct SQL { - static void register_cmd(CLI::App &); +struct SQL : public CLI::App { + SQL(); + ~SQL(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Sim.cpp b/src/server/sparkd/cmds/spartan/cmds/Sim.cpp index eea42b957..afa9e7ed0 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Sim.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/Sim.cpp @@ -5,19 +5,16 @@ namespace bts::spartan::cmds { -struct Args { +struct Sim::Args { std::string config_path; }; -static Args args; - -static void execute() { mdp3::run_sim(args.config_path); } - -void Sim::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("sim"); - subcmd->add_option("config-file,--config-file", args.config_path) - ->required(); - subcmd->callback(execute); +Sim::Sim() + : CLI::App("Run trading simulation(s)", "sim"), _args(new Sim::Args()) { + add_option("config-file,--config-file", _args->config_path)->required(); + callback([&] { mdp3::run_sim(_args->config_path); }); } +Sim::~Sim() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Sim.h b/src/server/sparkd/cmds/spartan/cmds/Sim.h index 80a7ea1db..e9e9b801f 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Sim.h +++ b/src/server/sparkd/cmds/spartan/cmds/Sim.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct Sim { - static void register_cmd(CLI::App &); +struct Sim : public CLI::App { + Sim(); + ~Sim(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp b/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp index db36181b4..651de86b3 100644 --- a/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/SimBooks.cpp @@ -1,5 +1,6 @@ #include "SimBooks.h" #include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h" +#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h" #include "server/sparkd/cmds/spartan/domain/SecurityCriteria.h" #include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h" #include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h" @@ -13,7 +14,7 @@ namespace bts::spartan::cmds { #define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__) #define INFO(...) _LOG(Info, __VA_ARGS__) -struct Args { +struct SimBooks::Args { std::string pcap; std::string out; std::vector assets; @@ -21,66 +22,70 @@ struct Args { mdp3::SecurityCriteria security_criteria; }; -static Args args; - -void execute() { - // Validate args - bts_assert(args.pcap.size()); - if (args.out.empty()) - args.out = file::change_file_ext(args.pcap, "sim_books.csv", "pcap.xz"); - bts_assert(args.out.size()); - // Sweep pcap for securityies and icebergs - mdp3::DefinitionCollector dc; - mdp3::IcebergFinder ice_finder; - INFO( - "Begin observe '{}' (1/2): detect all securityies and iceberg orders", - args.pcap); - mdp3::observe_channel(args.pcap, {&dc, &ice_finder}); - INFO( - "End observe '{}' (1/2): detected {} securities, and {} iceberg orders", - args.pcap, dc.result.size(), ice_finder.result.size()); - // Init MergedMatcher - mdp3::MergedMatcher matcher; - for (auto p : dc.result) - matcher.add_def(p.second); - // Init MatcherBookRecorder - mdp3::SecurityCriteria security_criteria; - if (args.assets.size() || args.security_groups.size()) { - // assets - auto &assets = security_criteria.clauses.emplace_back().assets; - assets.insert(assets.end(), args.assets.begin(), args.assets.end()); - // security groups - auto &security_groups = security_criteria.clauses.emplace_back().assets; - security_groups.insert( - security_groups.end(), args.security_groups.begin(), - args.security_groups.end()); - } - mdp3::MatcherBookRecorder rec( - {matcher, ice_finder.result, security_criteria, true}); - // Init FeedRecorderWrapper and observe pcap to generate the csv - std::ofstream out(args.out); - mdp3::FeedRecorderWrapper rec_wrap(out, &rec); - INFO("Begin observe '{}' (2/2): record matcher books", args.pcap); - mdp3::observe_channel(args.pcap, rec_wrap); - INFO("End observe '{}' (2/2): recorded matcher book", args.pcap); - out.flush(); -} - -void SimBooks::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand( - "sim-books", "Generate a csv file of book updates by feeding deduced " - "orders from a pcap file to the simulation matcher"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option("out,--out", args.out); - subcmd->add_option( - "--assets", args.assets, +SimBooks::SimBooks() + : CLI::App( + "Generate a csv file of book updates by feeding deduced orders from " + "a pcap file to the simulation matcher", + "sim-books"), + _args(new SimBooks::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("out,--out", _args->out); + add_option( + "--assets", _args->assets, "Specify asset(s) to filter the set of securities for which books are " "generated"); - subcmd->add_option( - "--security-groups", args.security_groups, + add_option( + "--security-groups", _args->security_groups, "Specify security group(s) to filter the set of securities for which " "books are generated"); - subcmd->callback(execute); + callback([&] { + // Validate args + bts_assert(_args->pcap.size()); + if (_args->out.empty()) + _args->out = + file::change_file_ext(_args->pcap, "sim_books.csv", "pcap.xz"); + bts_assert(_args->out.size()); + // Sweep pcap for securityies and icebergs + mdp3::DefinitionCollector dc; + // mdp3::IcebergFinder ice_finder; + INFO( + "Begin observe '{}' (1/2): detect all securityies and iceberg orders", + _args->pcap); + mdp3::observe_channel(_args->pcap, dc); + auto ibs = mdp3::iceberg_file_cache::find_icebergs(_args->pcap); + INFO( + "End observe '{}' (1/2): detected {} securities, and {} iceberg " + "orders", + _args->pcap, dc.result.size(), ibs.size()); + // Init MergedMatcher + mdp3::MergedMatcher matcher; + for (auto p : dc.result) + matcher.add_def(p.second); + // Init MatcherBookRecorder + mdp3::SecurityCriteria security_criteria; + if (_args->assets.size() || _args->security_groups.size()) { + // assets + auto &assets = security_criteria.clauses.emplace_back().assets; + assets.insert( + assets.end(), _args->assets.begin(), _args->assets.end()); + // security groups + auto &security_groups = + security_criteria.clauses.emplace_back().assets; + security_groups.insert( + security_groups.end(), _args->security_groups.begin(), + _args->security_groups.end()); + } + mdp3::MatcherBookRecorder rec({matcher, ibs, security_criteria, true}); + // Init FeedRecorderWrapper and observe pcap to generate the csv + std::ofstream out(_args->out); + mdp3::FeedRecorderWrapper rec_wrap(out, &rec); + INFO("Begin observe '{}' (2/2): record matcher books", _args->pcap); + mdp3::observe_channel(_args->pcap, rec_wrap); + INFO("End observe '{}' (2/2): recorded matcher book", _args->pcap); + out.flush(); + }); } +SimBooks::~SimBooks() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/SimBooks.h b/src/server/sparkd/cmds/spartan/cmds/SimBooks.h index 585c1f1f0..d2fa7c07e 100644 --- a/src/server/sparkd/cmds/spartan/cmds/SimBooks.h +++ b/src/server/sparkd/cmds/spartan/cmds/SimBooks.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct SimBooks { - static void register_cmd(CLI::App &); +struct SimBooks : public CLI::App { + SimBooks(); + ~SimBooks(); + + private: + struct Args; + std::unique_ptr _args; }; -} // namespace bts::spartan::cmds::sim_books +} // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Stats.cpp b/src/server/sparkd/cmds/spartan/cmds/Stats.cpp index c7238b34f..16a139b09 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Stats.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/Stats.cpp @@ -6,18 +6,8 @@ namespace bts::spartan::cmds { -struct Args { - std::string pcap; - std::string out; -}; - -static Args args; - size_t total_orders = 0; -static void execute() { - if (args.out.empty()) - args.out = spartan::file::change_file_ext(args.pcap, "order_counts.csv"); - auto out_file = std::ofstream(args.out); +static void execute(const std::string &pcap, std::ostream &out_file) { out_file << "sid,n_orders" << std::endl; struct StatPrinter : spartan::mdp3::IMDP3FeedObserver { StatPrinter(std::ostream &out) : out{out} {} @@ -28,15 +18,27 @@ static void execute() { } } stats{out_file}; - observe_channel(args.pcap, stats); + observe_channel(pcap, stats); std::cout << total_orders << std::endl; } -void Stats::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("stats"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->add_option("out,--out", args.out); - subcmd->callback(execute); +struct Stats::Args { + std::string pcap; + std::string out; +}; + +Stats::Stats() : CLI::App("", "stats"), _args(new Stats::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + add_option("out,--out", _args->out); + callback([&] { + if (_args->out.empty()) + _args->out = + spartan::file::change_file_ext(_args->pcap, "order_counts.csv"); + std::ofstream out(_args->out); + execute(_args->pcap, out); + }); } +Stats::~Stats() {} + } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/Stats.h b/src/server/sparkd/cmds/spartan/cmds/Stats.h index 26381b4b8..83b770ba2 100644 --- a/src/server/sparkd/cmds/spartan/cmds/Stats.h +++ b/src/server/sparkd/cmds/spartan/cmds/Stats.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct Stats { - static void register_cmd(CLI::App &); +struct Stats : public CLI::App { + Stats(); + ~Stats(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp index a0625e42f..832c17289 100644 --- a/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.cpp @@ -7,22 +7,20 @@ namespace bts::spartan::cmds { -struct Args { +struct ValidateFifo::Args { std::string pcap; }; -static Args args; - -static void execute() { - auto ibs = mdp3::IcebergFinder::find_all(args.pcap); - mdp3::MatcherValidationObserver obs(ibs); - mdp3::observe_channel(args.pcap, obs); +ValidateFifo::ValidateFifo() + : CLI::App("", "validate-fifo"), _args(new ValidateFifo::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + callback([&] { + auto ibs = mdp3::IcebergFinder::find_all(_args->pcap); + mdp3::MatcherValidationObserver obs(ibs); + mdp3::observe_channel(_args->pcap, obs); + }); } -void ValidateFifo::register_cmd(CLI::App &app) { - auto *cmd = app.add_subcommand("validate-fifo"); - cmd->add_option("pcap,--pcap", args.pcap)->required(); - cmd->callback(execute); -} +ValidateFifo::~ValidateFifo() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h index ffb4fea46..d54c7059e 100644 --- a/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h +++ b/src/server/sparkd/cmds/spartan/cmds/ValidateFifo.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct ValidateFifo { - static void register_cmd(CLI::App &); +struct ValidateFifo : public CLI::App { + ValidateFifo(); + ~ValidateFifo(); + +private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp index 8f6d0389a..7b486a39c 100644 --- a/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp +++ b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.cpp @@ -8,40 +8,43 @@ namespace bts::spartan::cmds { -struct Args { +struct ValuePolys::Args { std::string pcap; }; -static Args args; +ValuePolys::ValuePolys() + : CLI::App( + "Generate csv files of polygon values and opportunities. Each of " + "which are calculated from the polygon book updates in a pcap file", + "value-polys"), + _args(new ValuePolys::Args()) { + add_option("pcap,--pcap", _args->pcap)->required(); + callback([&] { + mdp3::DefinitionCollector def; + mdp3::PolygonFinder poly_finder; -static void execute() { - mdp3::DefinitionCollector def; - mdp3::PolygonFinder poly_finder; + // first pass, just identify the polygons + mdp3::observe_channel(_args->pcap, {&poly_finder, &def}); - // first pass, just identify the polygons - mdp3::observe_channel(args.pcap, {&poly_finder, &def}); + // second pass, value the polygons + mdp3::PolygonOpportunityFinder op_finder{ + poly_finder.result, def.ptr_result}; + mdp3::observe_channel(_args->pcap, {&op_finder}); - // second pass, value the polygons - mdp3::PolygonOpportunityFinder op_finder{poly_finder.result, def.ptr_result}; - mdp3::observe_channel(args.pcap, {&op_finder}); + auto poly_v = std::ofstream( + file::change_file_ext(_args->pcap, "polygon_values.csv")); - auto poly_v = - std::ofstream(file::change_file_ext(args.pcap, "polygon_values.csv")); + for (auto &v : op_finder.result) + poly_v << v << std::endl; - for (auto &v : op_finder.result) - poly_v << v << std::endl; - - auto poly_op = std::ofstream( - file::change_file_ext(args.pcap, "polygon_opportunities.csv")); - spartan::mdp3::poly_opportunities_to_csv( - aggregate(op_finder.result), poly_finder.result, def.ptr_result, - poly_op); + auto poly_op = std::ofstream( + file::change_file_ext(_args->pcap, "polygon_opportunities.csv")); + spartan::mdp3::poly_opportunities_to_csv( + aggregate(op_finder.result), poly_finder.result, def.ptr_result, + poly_op); + }); } -void ValuePolys::register_cmd(CLI::App &app) { - auto *subcmd = app.add_subcommand("value-polys"); - subcmd->add_option("pcap,--pcap", args.pcap)->required(); - subcmd->callback(execute); -} +ValuePolys::~ValuePolys() {} } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h index 688dc2815..dbd30aa6a 100644 --- a/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h +++ b/src/server/sparkd/cmds/spartan/cmds/ValuePolys.h @@ -1,10 +1,16 @@ #pragma once #include +#include namespace bts::spartan::cmds { -struct ValuePolys { - static void register_cmd(CLI::App &); +struct ValuePolys : public CLI::App { + ValuePolys(); + ~ValuePolys(); + + private: + struct Args; + std::unique_ptr _args; }; } // namespace bts::spartan::cmds diff --git a/src/server/sparkd/main.cpp b/src/server/sparkd/main.cpp index 84351772f..9d3396f1b 100644 --- a/src/server/sparkd/main.cpp +++ b/src/server/sparkd/main.cpp @@ -1,3 +1,4 @@ +#include "lib/testing/catch_test_runner.h" #include "lib/utils/init_spark_lib.h" #include "lib/utils/thread_manager.h" #include @@ -17,7 +18,6 @@ #include "server/sparkd/cmds/Run.h" #include "server/sparkd/cmds/Span.h" #include "server/sparkd/cmds/Spartan.h" -#include "server/sparkd/cmds/Test.h" #include "server/sparkd/cmds/Web.h" using namespace bts; @@ -25,24 +25,33 @@ using namespace bts; int main(int argc, char **argv) { utils::init_spark_lib(); + // The 'test' cli is delegated to the catch lib/framework. + if (argc > 1 && strcmp(argv[1], "test") == 0) { + std::vector v; + for (int i = 2; i < argc; i++) + v.emplace_back(argv[i]); + return run_tests(v); + } + CLI::App app("Futures trading system", "sparkd"); app.require_subcommand(); - // register subcommands - bts::cmds::AdminReport::register_cmd(app); - bts::cmds::AutoCert::register_cmd(app); - bts::cmds::Benchmark::register_cmd(app); - bts::cmds::ConfigCheck::register_cmd(app); - bts::cmds::FIXSim::register_cmd(app); - bts::cmds::KeyCode::register_cmd(app); - bts::cmds::Md::register_cmd(app); - bts::cmds::ParseHistoricalFix::register_cmd(app); - bts::cmds::PrintHedgeOrders::register_cmd(app); - bts::cmds::QuickTest::register_cmd(app); - bts::cmds::Run::register_cmd(app); - bts::cmds::Span::register_cmd(app); - bts::cmds::Spartan::register_cmd(app); - bts::cmds::Test::register_cmd(app); - bts::cmds::Web::register_cmd(app); + // subcommands + app.add_subcommand(CLI::App_p(new cmds::AdminReport())); + app.add_subcommand(CLI::App_p(new cmds::AutoCert())); + app.add_subcommand(CLI::App_p(new cmds::Benchmark())); + app.add_subcommand(CLI::App_p(new cmds::ConfigCheck())); + app.add_subcommand(CLI::App_p(new cmds::FIXSim())); + app.add_subcommand(CLI::App_p(new cmds::Md())); + app.add_subcommand(CLI::App_p(new cmds::ParseHistoricalFix())); + app.add_subcommand(CLI::App_p(new cmds::PrintHedgeOrders())); + app.add_subcommand(CLI::App_p(new cmds::QuickTest())); + app.add_subcommand(CLI::App_p(new cmds::Run())); + app.add_subcommand(CLI::App_p(new cmds::Span())); + app.add_subcommand(CLI::App_p(new cmds::Spartan())); + app.add_subcommand(CLI::App_p(new cmds::Web())); + // Add a 'test' dummy subcommand, so it will be listed in the cli when the + // user runs 'sparkd --help'. + app.add_subcommand("test", "Run unit test(s)"); CLI11_PARSE(app, argc, argv); utils::finit_spark_lib();