This commit is contained in:
Pedro Pinto committed 2021-01-16 12:32:34 -05:00
commit 2cffcb4e25
122 files changed
+2331 -1922

No files matched your search

+7
View File
@@ -11,6 +11,13 @@
namespace bts {
int run_tests(int argc, char **argv) {
std::vector<std::string> v;
for (int i = 2; i < argc; i++)
v.emplace_back(argv[i]);
return run_tests(v);
}
int run_tests(std::vector<std::string> const &args) {
utils::internal::force_unittest_environment();
+1
View File
@@ -4,5 +4,6 @@
#include <vector>
namespace bts {
int run_tests(int argc, char **argv) ;
int run_tests(std::vector<std::string> const &args);
}
+5
View File
@@ -12,5 +12,10 @@ namespace utils {
extern "C" BTS_EXPORT void init_spark_lib();
extern "C" BTS_EXPORT void finit_spark_lib();
struct SparkLibInit {
SparkLibInit() { init_spark_lib(); }
~SparkLibInit() { finit_spark_lib(); }
};
} // namespace utils
} // namespace bts
+1 -1
View File
@@ -1 +1 @@
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algoman/IAlgo.cpp;cmds/spartan/algoman/IAlgoExecutionServices.cpp;cmds/spartan/algoman/IAlgoSetupServices.cpp;cmds/spartan/algoman/algos/tas/RunTAS.cpp;cmds/spartan/algoman/algos/tas/TASOrderReactor.cpp;cmds/spartan/algoman/algos/tas/TASTriggerDef.cpp;cmds/spartan/back_testing/algos/PolyObserver.cpp;cmds/spartan/back_testing/algos/SniperModel.cpp;cmds/spartan/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/back_testing/db/SimDB.cpp;cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp;cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp;cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp;cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/back_testing/matcher/Matcher.cpp;cmds/spartan/back_testing/matcher/MatchingCommon.cpp;cmds/spartan/back_testing/matcher/fill_depot.cpp;cmds/spartan/back_testing/matcher/iceberg_helper.cpp;cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.cpp;cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.cpp;cmds/spartan/back_testing/run/SimSessionOptions.proto;cmds/spartan/back_testing/run/run_sim.cpp;cmds/spartan/back_testing/sim/ISim.cpp;cmds/spartan/back_testing/sim/Sim.cpp;cmds/spartan/back_testing/sim/pandl/PandL.cpp;cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.cpp;cmds/spartan/back_testing/sim/pcap/PCapEventSource.cpp;cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.cpp;cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.cpp;cmds/spartan/common/EndPoint.cpp;cmds/spartan/common/TimeInterval.cpp;cmds/spartan/common/TimeStamp.cpp;cmds/spartan/domain/IcebergQuantity.cpp;cmds/spartan/domain/OrderKey.cpp;cmds/spartan/domain/Price.cpp;cmds/spartan/domain/Quantity.cpp;cmds/spartan/domain/SecurityDefinition.cpp;cmds/spartan/domain/SecurityID.cpp;cmds/spartan/domain/SecuritySnapshot.cpp;cmds/spartan/domain/TradeSummarySignal.cpp;cmds/spartan/domain/book/BookLevel.cpp;cmds/spartan/market_data/common/MDP3Event.cpp;cmds/spartan/market_data/common/mdp3_common.cpp;cmds/spartan/market_data/common/tests/book_merging_tests.cpp;cmds/spartan/market_data/feed_control/IFeedController.cpp;cmds/spartan/market_data/feed_observers/DefinitionCollector.cpp;cmds/spartan/market_data/feed_observers/FillCounter.cpp;cmds/spartan/market_data/feed_observers/IcebergFinder.cpp;cmds/spartan/market_data/feed_observers/PolygonFinder.cpp;cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.cpp;cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.cpp;cmds/spartan/market_data/handler/MDP3FeedHandler.cpp;cmds/spartan/market_data/handler/ReactorMDP3FeedHandler.cpp;cmds/spartan/market_data/implied/implied_market.cpp;cmds/spartan/market_data/parsing/MDP3MsgBox.cpp;cmds/spartan/market_data/parsing/MDP3Parser.cpp;cmds/spartan/market_data/realtime/RealTimeHandler.cpp;cmds/spartan/market_data/recovery/BookSnapshot.cpp;cmds/spartan/market_data/recovery/DefinitionSnapshot.cpp;cmds/spartan/market_data/recovery/IncrementalQueue.cpp;cmds/spartan/market_data/recovery/RecoveryHandler.cpp;cmds/spartan/market_data/tests/spartan_md_tests.cpp;cmds/spartan/market_data/utils/book_utils.cpp;cmds/spartan/market_data/utils/mdp3_utils.cpp;cmds/spartan/market_data/utils/order_utils.cpp;cmds/spartan/market_data/utils/polygon_utils.cpp;cmds/spartan/market_data/utils/queries/BookStatRecorder.cpp;cmds/spartan/market_data/utils/security_comp.cpp;cmds/spartan/market_data/utils/security_parser.cpp;cmds/spartan/market_data/utils/tests/synthetic_book_tests.cpp;cmds/spartan/market_data/utils/tests/test_polygon_finder.cpp;cmds/spartan/order_execution/gateway/IL3Gateway.cpp;cmds/spartan/order_execution/gateway/states/IL3InitHandler.cpp;cmds/spartan/order_execution/parsing/IL3Parser.cpp;cmds/spartan/system/PCapReader.cpp;cmds/spartan/system/reactor/Reactor.cpp;cmds/spartan/system/sbe/SBEParser.cpp;cmds/spartan/util/action_summarizer.cpp;cmds/spartan/util/args_parser.cpp;cmds/spartan/util/sql/adapters/polygon_defs.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cppLine truncated
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECert.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Web.cpp;cmds/spartan/algoman/IAlgo.cpp;cmds/spartan/algoman/IAlgoExecutionServices.cpp;cmds/spartan/algoman/IAlgoSetupServices.cpp;cmds/spartan/algoman/algos/tas/RunTAS.cpp;cmds/spartan/algoman/algos/tas/TASOrderReactor.cpp;cmds/spartan/algoman/algos/tas/TASTriggerDef.cpp;cmds/spartan/back_testing/algos/PolyObserver.cpp;cmds/spartan/back_testing/algos/SniperModel.cpp;cmds/spartan/back_testing/algos/spreader/SimOrderGateway.cpp;cmds/spartan/back_testing/algos/spreader/SimSpreader.cpp;cmds/spartan/back_testing/algos/spreader/tests/sim_spreader_tests.cpp;cmds/spartan/back_testing/db/SimDB.cpp;cmds/spartan/back_testing/matcher/FifoImpliedSecurityMatcher.cpp;cmds/spartan/back_testing/matcher/FifoMatcherContext.cpp;cmds/spartan/back_testing/matcher/FifoMergedSecurityMatcher.cpp;cmds/spartan/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/back_testing/matcher/Matcher.cpp;cmds/spartan/back_testing/matcher/MatchingCommon.cpp;cmds/spartan/back_testing/matcher/fill_depot.cpp;cmds/spartan/back_testing/matcher/iceberg_helper.cpp;cmds/spartan/back_testing/matcher/tests/matcher_tests.cpp;cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.cpp;cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.cpp;cmds/spartan/back_testing/run/SimSessionOptions.proto;cmds/spartan/back_testing/run/run_sim.cpp;cmds/spartan/back_testing/sim/ISim.cpp;cmds/spartan/back_testing/sim/Sim.cpp;cmds/spartan/back_testing/sim/pandl/PandL.cpp;cmds/spartan/back_testing/sim/pcap/PCapEventGenerator.cpp;cmds/spartan/back_testing/sim/pcap/PCapEventSource.cpp;cmds/spartan/back_testing/sim/pcap/PCapOrderChangeBootstrapper.cpp;cmds/spartan/back_testing/sim/spark_bridge/AlgoServicesBridge.cpp;cmds/spartan/back_testing/sim/spark_bridge/BeforeLaunchBridge.cpp;cmds/spartan/cmds/BookStats.cpp;cmds/spartan/cmds/CSV.cpp;cmds/spartan/cmds/Compress.cpp;cmds/spartan/cmds/FindPolys.cpp;cmds/spartan/cmds/Icebergs.cpp;cmds/spartan/cmds/Json.cpp;cmds/spartan/cmds/ListSec.cpp;cmds/spartan/cmds/PolygonBooks.cpp;cmds/spartan/cmds/SQL.cpp;cmds/spartan/cmds/Sim.cpp;cmds/spartan/cmds/SimBooks.cpp;cmds/spartan/cmds/Stats.cpp;cmds/spartan/cmds/ValidateFifo.cpp;cmds/spartan/cmds/ValuePolys.cpp;cmds/spartan/common/EndPoint.cpp;cmds/spartan/common/TimeInterval.cpp;cmds/spartan/common/TimeStamp.cpp;cmds/spartan/domain/IcebergQuantity.cpp;cmds/spartan/domain/OrderKey.cpp;cmds/spartan/domain/Price.cpp;cmds/spartan/domain/Quantity.cpp;cmds/spartan/domain/SecurityCriteria.cpp;cmds/spartan/domain/SecurityDefinition.cpp;cmds/spartan/domain/SecurityID.cpp;cmds/spartan/domain/SecuritySnapshot.cpp;cmds/spartan/domain/TradeSummarySignal.cpp;cmds/spartan/domain/book/BookLevel.cpp;cmds/spartan/market_data/common/MDP3Event.cpp;cmds/spartan/market_data/common/mdp3_common.cpp;cmds/spartan/market_data/common/tests/book_merging_tests.cpp;cmds/spartan/market_data/feed_control/IFeedController.cpp;cmds/spartan/market_data/feed_observers/BookStatRecorder.cpp;cmds/spartan/market_data/feed_observers/DefinitionCollector.cpp;cmds/spartan/market_data/feed_observers/FillCounter.cpp;cmds/spartan/market_data/feed_observers/IcebergFinder.cpp;cmds/spartan/market_data/feed_observers/PolygonFinder.cpp;cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.cpp;cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.cpp;cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.cpp;cmds/spartan/market_data/handler/MDP3FeedHandler.cpp;cmds/spartan/market_data/handler/ReactorMDP3FeedHandler.cpp;cmds/spartan/market_data/implied/implied_market.cpp;cmds/spartan/market_data/parsing/MDP3MsgBox.cpp;cmds/spartan/market_data/parsing/MDP3Parser.cpp;cmds/spartan/market_data/realtime/RealTimeHandler.cpp;cmds/spartan/market_data/recovery/BookSnapshot.cpp;cmds/spartan/market_data/recovery/DefinitionSnapshot.cpp;cmds/spartan/market_data/recovery/IncrementalQueue.cpp;cmds/spartan/market_data/recovery/RecoveryHandler.cpp;cmds/spartan/market_data/tests/spartan_md_tests.cpp;cmds/spartan/market_data/utils/book_utils.cpp;cmds/spartan/market_data/utils/mdp3_utils.cpp;cmds/spartan/market_data/utils/order_utils.cpp;cmds/spartan/market_data/utils/polygon_utils.cpp;cmds/spartan/market_data/utils/security_comp.cpp;cmds/spartan/market_data/utils/security_parser.cpp;cmds/spartan/market_data/utils/tests/synthetic_book_tests.cpp;cmds/spartan/market_data/utils/tests/test_polygon_finder.cpp;cmds/spartan/order_execution/gateway/IL3Gateway.cpp;cmds/spartan/order_execution/gateway/states/IL3InitHandler.cpp;cmds/spartan/order_execution/parsing/IL3Parser.cpp;cmds/spartan/system/PCapReader.cpp;cmds/spartan/system/reactor/ReaLine truncated
+8 -18
View File
@@ -14,23 +14,13 @@
#include <iostream>
#include <map>
namespace bts {
namespace cmd {
namespace bts::cmds {
int AdminReport::execute(const std::vector<std::string> &args) {
void md_report() {
auto cfg = config::get<config::Logging>();
cfg.mutable_console()->set_default_level("Fatal");
log::configure_logging(cfg);
if (args[0] == "md")
return md_report();
else {
std::cout << std::endl << std::endl << help() << std::endl << std::endl;
return -1;
}
}
int AdminReport::md_report() {
std::map<api::CMEMics, std::string> mic_map;
mic_map[api::CMEMics::CME] = "XCME";
mic_map[api::CMEMics::COMEX] = "XCEC";
@@ -93,13 +83,13 @@ int AdminReport::md_report() {
}
}
}
return 0;
}
std::string AdminReport::help() {
return "\t sparkd adminreport [md] \n"
"\t Write a market data entitlement report csv to stdout \n";
AdminReport::AdminReport()
: CLI::App(
"Write a market data entitlement report csv to stdout",
"admin-report") {
add_subcommand("md")->callback(md_report);
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -11
View File
@@ -1,16 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class AdminReport : public ICmd {
public:
int execute(std::vector<std::string> const &args);
int md_report();
std::string help();
struct AdminReport : public CLI::App {
AdminReport();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+7 -11
View File
@@ -1,6 +1,5 @@
#include "AutoCert.h"
#include "lib/utils/PBUtils.h"
#include "server/sparkd/cmds/CmdRegistry.h"
#include "lib/jetfix/common/InMsgBuffer.h"
#include "lib/utils/bts_chrono.h"
@@ -16,8 +15,7 @@
#include <iostream>
#include <sys/select.h>
namespace bts {
namespace cmd {
namespace bts::cmds {
unsigned int their_sn;
unsigned int last_received_sn;
@@ -1275,13 +1273,11 @@ void iLinkCGWMidWeekKeyRotation() {
}
}
int AutoCert::execute(std::vector<std::string> const &) {
CGWFailoverFromPrimaryToBackupComponents();
return 0;
AutoCert::AutoCert()
: CLI::App(
"Provide control over JetFIX so that we can go through AutoCert",
"auto-cert") {
callback(CGWFailoverFromPrimaryToBackupComponents);
}
std::string AutoCert::help() {
return "Provide control over JetFIX so that we can go through AutoCert";
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -10
View File
@@ -1,15 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class AutoCert : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
struct AutoCert : public CLI::App {
AutoCert();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+122 -115
View File
@@ -15,7 +15,6 @@
#include "lib/utils/TimeStamp.h"
#include "lib/utils/Uuid.h"
#include "lib/utils/mpmc_bounded_queue.h"
#include "server/sparkd/cmds/CmdRegistry.h"
#include "lib/tracing/PerfLog.h"
#include <benchmark/benchmark.h>
@@ -42,63 +41,63 @@
#include <date/date.h>
#include <tbb/concurrent_queue.h>
#include <tbb/concurrent_vector.h>
namespace bts {
namespace cmd {
namespace bts::cmds {
using namespace orderman;
static void test_date_conversion(benchmark::State &state) {
static void test_date_conversion(::benchmark::State &state) {
auto lldt = datetime::ll_now();
for (auto _ : state) {
benchmark::DoNotOptimize(datetime::utc_ll_to_timepoint(lldt));
::benchmark::DoNotOptimize(datetime::utc_ll_to_timepoint(lldt));
}
}
BENCHMARK(test_date_conversion);
static void gen_id(benchmark::State &state) {
static void gen_id(::benchmark::State &state) {
std::string scope = "benchmark";
for (auto _ : state) {
auto id = utils::reserve_ids(scope, 1);
benchmark::DoNotOptimize(id + 1 > 0);
::benchmark::DoNotOptimize(id + 1 > 0);
}
}
BENCHMARK(gen_id);
static void mutex_lock_unlock(benchmark::State &state) {
static void mutex_lock_unlock(::benchmark::State &state) {
int i = 0;
std::mutex lock;
for (auto _ : state) {
std::lock_guard<std::mutex> l(lock);
benchmark::DoNotOptimize(i++);
::benchmark::DoNotOptimize(i++);
}
}
BENCHMARK(mutex_lock_unlock);
static void spin_lock_unlock(benchmark::State &state) {
static void spin_lock_unlock(::benchmark::State &state) {
int i = 0;
std::atomic_flag _mutex = ATOMIC_FLAG_INIT;
for (auto _ : state) {
while (_mutex.test_and_set(std::memory_order_acquire))
;
benchmark::DoNotOptimize(i++);
::benchmark::DoNotOptimize(i++);
_mutex.clear(std::memory_order_release);
}
}
BENCHMARK(spin_lock_unlock);
static void bts_spinlock_lock_unlock(benchmark::State &state) {
static void bts_spinlock_lock_unlock(::benchmark::State &state) {
int i = 0;
utils::SpinLock l;
for (auto _ : state) {
std::lock_guard<utils::SpinLock> g(l);
benchmark::DoNotOptimize(i++);
::benchmark::DoNotOptimize(i++);
}
}
BENCHMARK(bts_spinlock_lock_unlock);
static void serialize_update(benchmark::State &state) {
static void serialize_update(::benchmark::State &state) {
api::orderman::Update update;
update.set_order_key(1);
update.set_price(1.2);
@@ -121,7 +120,7 @@ struct IntId {
uint32_t value;
};
static void binary_search_of_guid(benchmark::State &state) {
static void binary_search_of_guid(::benchmark::State &state) {
using Id = utils::Uuid;
using Map = boost::container::flat_map<Id, int>;
auto sz = state.range(0);
@@ -136,12 +135,12 @@ static void binary_search_of_guid(benchmark::State &state) {
auto it = m.find(target);
if (it != m.end())
found_count++;
benchmark::DoNotOptimize(found_count);
::benchmark::DoNotOptimize(found_count);
}
}
BENCHMARK(binary_search_of_guid)->Range(8, 8 << 10);
static void binary_search_of_int64(benchmark::State &state) {
static void binary_search_of_int64(::benchmark::State &state) {
using Id = IntId;
using Map = boost::container::flat_map<Id, int>;
auto sz = state.range(0);
@@ -156,12 +155,12 @@ static void binary_search_of_int64(benchmark::State &state) {
auto it = m.find(target);
if (it != m.end())
found_count++;
benchmark::DoNotOptimize(found_count);
::benchmark::DoNotOptimize(found_count);
}
}
BENCHMARK(binary_search_of_int64)->Range(8, 8 << 10);
static void binary_search_of_int32(benchmark::State &state) {
static void binary_search_of_int32(::benchmark::State &state) {
using Id = IntId;
using Map = boost::container::flat_map<Id, int>;
auto sz = state.range(0);
@@ -176,14 +175,14 @@ static void binary_search_of_int32(benchmark::State &state) {
auto it = m.find(target);
if (it != m.end())
found_count++;
benchmark::DoNotOptimize(found_count);
::benchmark::DoNotOptimize(found_count);
}
}
BENCHMARK(binary_search_of_int32)->Range(8, 8 << 10);
static void guid_gen(benchmark::State &state) {
static void guid_gen(::benchmark::State &state) {
for (auto _ : state) {
benchmark::DoNotOptimize(utils::Uuid());
::benchmark::DoNotOptimize(utils::Uuid());
}
}
BENCHMARK(guid_gen);
@@ -192,56 +191,56 @@ struct A {
int x;
};
static void shared_ptr_cpy(benchmark::State &state) {
static void shared_ptr_cpy(::benchmark::State &state) {
auto a = std::make_shared<A>();
for (auto _ : state) {
std::shared_ptr<A> b;
benchmark::DoNotOptimize(b = a);
::benchmark::DoNotOptimize(b = a);
}
}
BENCHMARK(shared_ptr_cpy);
static void alloc_free(benchmark::State &state) {
static void alloc_free(::benchmark::State &state) {
for (auto _ : state) {
auto a = std::make_shared<A>();
benchmark::DoNotOptimize(a.get());
::benchmark::DoNotOptimize(a.get());
}
}
BENCHMARK(alloc_free);
static void new_delete(benchmark::State &state) {
static void new_delete(::benchmark::State &state) {
for (auto _ : state) {
auto a = new A();
delete a;
benchmark::DoNotOptimize(a);
::benchmark::DoNotOptimize(a);
}
}
BENCHMARK(new_delete);
static void boost_pool(benchmark::State &state) {
static void boost_pool(::benchmark::State &state) {
boost::pool<> p(sizeof(A));
for (auto _ : state) {
auto a = p.malloc();
p.free(a);
benchmark::DoNotOptimize(a);
::benchmark::DoNotOptimize(a);
}
}
BENCHMARK(boost_pool);
static void boost_object_pool(benchmark::State &state) {
static void boost_object_pool(::benchmark::State &state) {
boost::object_pool<A> p;
for (auto _ : state) {
auto a = p.malloc();
p.free(a);
benchmark::DoNotOptimize(a);
::benchmark::DoNotOptimize(a);
}
}
BENCHMARK(boost_object_pool);
static void protected_boost_pool(benchmark::State &state) {
static void protected_boost_pool(::benchmark::State &state) {
boost::pool<> p(sizeof(A));
std::atomic_flag _mutex = ATOMIC_FLAG_INIT;
for (auto _ : state) {
@@ -253,31 +252,31 @@ static void protected_boost_pool(benchmark::State &state) {
;
p.free(a);
_mutex.clear(std::memory_order_release);
benchmark::DoNotOptimize(a);
::benchmark::DoNotOptimize(a);
}
}
BENCHMARK(protected_boost_pool);
static void boost_singleton_pool(benchmark::State &state) {
static void boost_singleton_pool(::benchmark::State &state) {
using Pool = boost::singleton_pool<struct Single, sizeof(A)>;
for (auto _ : state) {
auto a = Pool::malloc();
Pool::free(a);
benchmark::DoNotOptimize(a);
::benchmark::DoNotOptimize(a);
}
}
BENCHMARK(boost_singleton_pool);
static void atomic_inc(benchmark::State &state) {
static void atomic_inc(::benchmark::State &state) {
std::atomic<size_t> i(0);
for (auto _ : state) {
benchmark::DoNotOptimize(i++);
::benchmark::DoNotOptimize(i++);
}
}
BENCHMARK(atomic_inc);
static void mem_copy(benchmark::State &state) {
static void mem_copy(::benchmark::State &state) {
char *src = new char[state.range(0)];
char *dst = new char[state.range(0)];
memset(src, 'x', state.range(0));
@@ -291,93 +290,79 @@ static void mem_copy(benchmark::State &state) {
BENCHMARK(mem_copy)->Arg(8)->Arg(64)->Arg(128)->Arg(512)->Arg(1 << 10)->Arg(
8 << 10);
static void alloc_and_copy_from_pool(benchmark::State &state) {
static void alloc_and_copy_from_pool(::benchmark::State &state) {
boost::pool<> p(sizeof(A));
for (auto _ : state) {
auto a = new (p.malloc()) A(); // NOLINT
a->~A(); // NOLINT
p.free(a);
benchmark::DoNotOptimize(a);
::benchmark::DoNotOptimize(a);
}
}
BENCHMARK(alloc_and_copy_from_pool);
static void lambda_copy(benchmark::State &state) {
static void lambda_copy(::benchmark::State &state) {
auto p = std::make_shared<A>();
std::function<int()> f = [p]() { return 1; };
std::array<std::function<int()>, 1> buffer;
for (auto _ : state) {
benchmark::DoNotOptimize(buffer[0] = f);
::benchmark::DoNotOptimize(buffer[0] = f);
}
}
BENCHMARK(lambda_copy);
static void boost_pool_allocator(benchmark::State &state) {
static void boost_pool_allocator(::benchmark::State &state) {
boost::fast_pool_allocator<A> p;
for (auto _ : state) {
auto ptr = std::allocate_shared<A>(p);
benchmark::DoNotOptimize(ptr.get());
::benchmark::DoNotOptimize(ptr.get());
}
}
BENCHMARK(boost_pool_allocator);
int Benchmark::execute(std::vector<std::string> const &args) {
bts_assert(args.size() < 99);
int argc = args.size() + 1;
char *argv[100];
argv[0] = (char *)"spark";
for (size_t i = 0; i < args.size(); i++)
argv[i + 1] = (char *)args[i].c_str();
benchmark::Initialize(&argc, argv);
benchmark::RunSpecifiedBenchmarks();
return 0;
}
static void boost_lockless_queue(benchmark::State &state) {
static void boost_lockless_queue(::benchmark::State &state) {
using FreeList = boost::lockfree::queue<void *>;
FreeList queue(1000000);
for (auto _ : state) {
void *result;
queue.push(nullptr);
benchmark::DoNotOptimize(queue.pop(result));
::benchmark::DoNotOptimize(queue.pop(result));
}
}
BENCHMARK(boost_lockless_queue);
static void tbb_concurrent_queue(benchmark::State &state) {
static void tbb_concurrent_queue(::benchmark::State &state) {
using FreeList = tbb::concurrent_queue<void *>;
FreeList queue;
for (auto _ : state) {
void *result;
queue.push(nullptr);
benchmark::DoNotOptimize(queue.try_pop(result));
::benchmark::DoNotOptimize(queue.try_pop(result));
}
}
BENCHMARK(tbb_concurrent_queue);
static void boost_spsc(benchmark::State &state) {
static void boost_spsc(::benchmark::State &state) {
boost::lockfree::spsc_queue<int> q(10000);
int i = 0;
for (auto _ : state) {
q.push(++i);
benchmark::DoNotOptimize(q.pop(i));
::benchmark::DoNotOptimize(q.pop(i));
}
}
BENCHMARK(boost_spsc);
static void boost_lockless_stack(benchmark::State &state) {
static void boost_lockless_stack(::benchmark::State &state) {
using FreeList = boost::lockfree::stack<void *>;
FreeList stack;
for (auto _ : state) {
void *result;
stack.push(nullptr);
benchmark::DoNotOptimize(stack.pop(result));
::benchmark::DoNotOptimize(stack.pop(result));
}
}
BENCHMARK(boost_lockless_stack);
@@ -407,44 +392,44 @@ struct PPQueue {
}
};
static void get_thread_id(benchmark::State &state) {
static void get_thread_id(::benchmark::State &state) {
for (auto _ : state) {
auto tid = std::this_thread::get_id();
benchmark::DoNotOptimize(tid == std::this_thread::get_id());
::benchmark::DoNotOptimize(tid == std::this_thread::get_id());
}
}
BENCHMARK(get_thread_id);
static void native_order_fix_data_create(benchmark::State &state) {
static void native_order_fix_data_create(::benchmark::State &state) {
auto sz = state.range(0);
for (auto _ : state) {
auto p = new uint8_t[sz];
delete[] p;
benchmark::DoNotOptimize(p);
::benchmark::DoNotOptimize(p);
}
}
BENCHMARK(native_order_fix_data_create)->Range(8, 8 << 14);
static void native_order_fix_data_fragment(benchmark::State &state) {
static void native_order_fix_data_fragment(::benchmark::State &state) {
for (auto _ : state) {
auto p = new jetfix::FIXFragment();
delete p;
benchmark::DoNotOptimize(p);
::benchmark::DoNotOptimize(p);
}
}
BENCHMARK(native_order_fix_data_fragment);
static void small_vec(benchmark::State &state) {
static void small_vec(::benchmark::State &state) {
auto i = 0;
for (auto _ : state) {
boost::container::small_vector<size_t, 1> vec;
vec.emplace_back(i++);
benchmark::DoNotOptimize(vec[0]);
::benchmark::DoNotOptimize(vec[0]);
}
}
BENCHMARK(small_vec);
static void inter_thread_spsc(benchmark::State &state) {
static void inter_thread_spsc(::benchmark::State &state) {
boost::lockfree::spsc_queue<int> q1(10);
boost::lockfree::spsc_queue<int> q2(10);
@@ -479,7 +464,7 @@ static void inter_thread_spsc(benchmark::State &state) {
}
BENCHMARK(inter_thread_spsc);
static void inter_thread_mpmc(benchmark::State &state) {
static void inter_thread_mpmc(::benchmark::State &state) {
using Queue = boost::lockfree::queue<int>;
Queue q1(100), q2(100);
@@ -514,7 +499,7 @@ static void inter_thread_mpmc(benchmark::State &state) {
}
BENCHMARK(inter_thread_mpmc);
static void inter_thread_mpmc_spdlog(benchmark::State &state) {
static void inter_thread_mpmc_spdlog(::benchmark::State &state) {
using Queue = spdlog::details::mpmc_bounded_queue<int>;
Queue q1(8), q2(8);
@@ -549,7 +534,7 @@ static void inter_thread_mpmc_spdlog(benchmark::State &state) {
}
BENCHMARK(inter_thread_mpmc_spdlog);
static void same_thread_mpmc_spdlog(benchmark::State &state) {
static void same_thread_mpmc_spdlog(::benchmark::State &state) {
using Queue = spdlog::details::mpmc_bounded_queue<int>;
Queue q1(8);
int i = 0;
@@ -561,26 +546,26 @@ static void same_thread_mpmc_spdlog(benchmark::State &state) {
}
BENCHMARK(same_thread_mpmc_spdlog);
static void moodycamel_concurrent_queue(benchmark::State &state) {
static void moodycamel_concurrent_queue(::benchmark::State &state) {
using Q = moodycamel::ConcurrentQueue<int>;
Q queue(8);
int i = 0;
for (auto _ : state) {
queue.enqueue(++i); // NOLINT
benchmark::DoNotOptimize(queue.try_dequeue(i));
::benchmark::DoNotOptimize(queue.try_dequeue(i));
}
}
BENCHMARK(moodycamel_concurrent_queue);
static void moodycamel_spsc_queue(benchmark::State &state) {
static void moodycamel_spsc_queue(::benchmark::State &state) {
using Queue = moodycamel::ReaderWriterQueue<int>;
Queue q1(8);
int i = 0;
for (auto _ : state) {
q1.enqueue(++i);
benchmark::DoNotOptimize(q1.try_dequeue(i));
::benchmark::DoNotOptimize(q1.try_dequeue(i));
}
}
BENCHMARK(moodycamel_spsc_queue);
@@ -596,28 +581,28 @@ struct MyDirect {
int inc(int &x) { return x += 1; };
};
static void derived_call(benchmark::State &state) {
static void derived_call(::benchmark::State &state) {
int x = 0;
MyDerived d;
MyBase &b = d;
for (auto _ : state) {
benchmark::DoNotOptimize(b.inc(x));
::benchmark::DoNotOptimize(b.inc(x));
}
}
BENCHMARK(derived_call);
static void direct_call(benchmark::State &state) {
static void direct_call(::benchmark::State &state) {
int x = 0;
MyDerived b;
for (auto _ : state) {
benchmark::DoNotOptimize(b.inc(x));
::benchmark::DoNotOptimize(b.inc(x));
}
}
BENCHMARK(direct_call);
static void get_time_of_day(benchmark::State &state) {
static void get_time_of_day(::benchmark::State &state) {
std::string result(strlen("20121204-14:44:02.999"), ' ');
struct timeval tv;
for (auto _ : state) {
@@ -630,7 +615,7 @@ static void get_time_of_day(benchmark::State &state) {
}
BENCHMARK(get_time_of_day);
static void fix_time_stamp(benchmark::State &state) {
static void fix_time_stamp(::benchmark::State &state) {
std::string r(128, 32);
auto now = datetime::now();
for (auto _ : state) {
@@ -639,7 +624,7 @@ static void fix_time_stamp(benchmark::State &state) {
}
BENCHMARK(fix_time_stamp);
static void perf_log(benchmark::State &state) {
static void perf_log(::benchmark::State &state) {
trace::PerfLog l;
for (auto _ : state) {
l.log(trace::CorID{"sim", 0}, "this", 1, "txt", 3);
@@ -647,7 +632,7 @@ static void perf_log(benchmark::State &state) {
}
BENCHMARK(perf_log);
void int_price_less(benchmark::State &state) {
void int_price_less(::benchmark::State &state) {
for (auto _ : state) {
state.PauseTiming();
int64_t count = state.range(0);
@@ -662,7 +647,7 @@ void int_price_less(benchmark::State &state) {
}
BENCHMARK(int_price_less)->Range(8, 8 << 11);
void set_hashset_insert(benchmark::State &state) {
void set_hashset_insert(::benchmark::State &state) {
std::unordered_set<uintptr_t> s;
size_t x = 0;
for (auto _ : state) {
@@ -670,7 +655,7 @@ void set_hashset_insert(benchmark::State &state) {
s.insert(x);
}
}
void set_stdset_insert(benchmark::State &state) {
void set_stdset_insert(::benchmark::State &state) {
std::set<uintptr_t> s;
size_t x = 0;
for (auto _ : state) {
@@ -678,7 +663,7 @@ void set_stdset_insert(benchmark::State &state) {
s.insert(x);
}
}
void set_btreeset_insert(benchmark::State &state) {
void set_btreeset_insert(::benchmark::State &state) {
btree::btree_set<uintptr_t> s;
size_t x = 0;
for (auto _ : state) {
@@ -687,36 +672,36 @@ void set_btreeset_insert(benchmark::State &state) {
}
}
void set_hashset_iterate(benchmark::State &state) {
void set_hashset_iterate(::benchmark::State &state) {
std::unordered_set<uintptr_t> s;
size_t x = 0;
for (auto _ : state) {
x = std::rand();
s.insert(x);
for (const auto &p : s) {
benchmark::DoNotOptimize(p);
::benchmark::DoNotOptimize(p);
}
}
}
void set_stdset_iterate(benchmark::State &state) {
void set_stdset_iterate(::benchmark::State &state) {
std::set<uintptr_t> s;
size_t x = 0;
for (auto _ : state) {
x = std::rand();
s.insert(x);
for (const auto &p : s) {
benchmark::DoNotOptimize(p);
::benchmark::DoNotOptimize(p);
}
}
}
void set_btreeset_iterate(benchmark::State &state) {
void set_btreeset_iterate(::benchmark::State &state) {
btree::btree_set<uintptr_t> s;
size_t x = 0;
for (auto _ : state) {
x = std::rand();
s.insert(x);
for (const auto &p : s) {
benchmark::DoNotOptimize(p);
::benchmark::DoNotOptimize(p);
}
}
}
@@ -732,7 +717,7 @@ struct Chunk {
size_t a, b, c, d;
};
void tbb_concurrent_vector_insert(benchmark::State &state) {
void tbb_concurrent_vector_insert(::benchmark::State &state) {
tbb::concurrent_vector<Chunk> vec;
size_t x = 0;
for (auto _ : state) {
@@ -740,32 +725,32 @@ void tbb_concurrent_vector_insert(benchmark::State &state) {
}
}
BENCHMARK(tbb_concurrent_vector_insert);
void tbb_concurrent_vector_iterate(benchmark::State &state) {
void tbb_concurrent_vector_iterate(::benchmark::State &state) {
tbb::concurrent_vector<Chunk> vec;
for (size_t x = 0; x < 9999; ++x) {
vec.push_back({++x, ++x, ++x, ++x});
}
for (auto _ : state) {
for (const auto &v : vec) {
benchmark::DoNotOptimize(v);
::benchmark::DoNotOptimize(v);
}
}
}
BENCHMARK(tbb_concurrent_vector_iterate);
void tbb_concurrent_vector_index(benchmark::State &state) {
void tbb_concurrent_vector_index(::benchmark::State &state) {
tbb::concurrent_vector<Chunk> vec;
for (size_t x = 0; x < 9999; ++x) {
vec.push_back({++x, ++x, ++x, ++x});
}
for (auto _ : state) {
for (size_t x = 0; x < vec.size(); ++x) {
benchmark::DoNotOptimize(vec[x]);
::benchmark::DoNotOptimize(vec[x]);
}
}
}
BENCHMARK(tbb_concurrent_vector_index);
void concurrent_append_collection_insert(benchmark::State &state) {
void concurrent_append_collection_insert(::benchmark::State &state) {
ConcurrentAppendCollection<Chunk> vec;
size_t x = 0;
for (auto _ : state) {
@@ -773,34 +758,56 @@ void concurrent_append_collection_insert(benchmark::State &state) {
}
}
BENCHMARK(concurrent_append_collection_insert);
void concurrent_append_collection_iterate(benchmark::State &state) {
void concurrent_append_collection_iterate(::benchmark::State &state) {
ConcurrentAppendCollection<Chunk> vec;
for (size_t x = 0; x < 9999; ++x) {
vec.push_back({++x, ++x, ++x, ++x});
}
for (auto _ : state) {
for (const auto &v : vec) {
benchmark::DoNotOptimize(v);
::benchmark::DoNotOptimize(v);
}
}
}
BENCHMARK(concurrent_append_collection_iterate);
void concurrent_append_collection_index(benchmark::State &state) {
void concurrent_append_collection_index(::benchmark::State &state) {
ConcurrentAppendCollection<Chunk> vec;
for (size_t x = 0; x < 9999; ++x) {
vec.push_back({++x, ++x, ++x, ++x});
}
for (auto _ : state) {
for (size_t x = 0; x < vec.size(); ++x) {
benchmark::DoNotOptimize(vec[x]);
::benchmark::DoNotOptimize(vec[x]);
}
}
}
BENCHMARK(concurrent_append_collection_index);
std::string Benchmark::help() {
return "Run micro-benchmarks. Use --benchmark_filter=regex to run only "
"specific benchmarks.";
struct Benchmark::Args {
std::vector<std::string> args;
};
Benchmark::Benchmark()
: CLI::App(
"Run micro-benchmarks. Use --benchmark_filter=regex to run only "
"specific benchmarks.",
"benchmark"),
_args(new Benchmark::Args()) {
add_option("args", _args->args);
callback([&] {
bts_assert(_args->args.size() < 99);
int argc = _args->args.size() + 1;
char *argv[100];
argv[0] = (char *)"spark";
for (size_t i = 0; i < _args->args.size(); i++)
argv[i + 1] = (char *)_args->args[i].c_str();
::benchmark::Initialize(&argc, argv);
::benchmark::RunSpecifiedBenchmarks();
});
}
} // namespace cmd
} // namespace bts
Benchmark::~Benchmark() {}
} // namespace bts::cmds
+10 -9
View File
@@ -1,15 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
struct Benchmark : public CLI::App {
Benchmark();
~Benchmark();
class Benchmark : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
@@ -1,4 +1,4 @@
#include "CMECertCmd.h"
#include "CMECert.h"
#include "lib/logging/logging.h"
#include "lib/orderman/actors/ilink3/IL3Dispatcher.h"
@@ -17,8 +17,7 @@
#include <string>
#include <thread>
namespace bts {
namespace cmd {
namespace bts::cmds {
using namespace orderman::ilink3;
using namespace api::orderman;
@@ -566,49 +565,47 @@ struct TestRunner {
}
};
int CMECert::execute(const std::vector<std::string> &) {
// Typically a timestamp
SessionSaver ss;
ss.load();
Config config{"NGG", "004", false, [](auto, auto, auto) {}};
IL3Reactor reactor;
IL3MessageFramer framer;
auto &connection = reactor.register_connection(
{// local dev tunnel
TCPEndpoint{"127.0.0.1", 64530}, TCPEndpoint{"127.0.0.1", 64531},
// Direct connection
TCPEndpoint{"69.50.112.199", 64530},
TCPEndpoint{"69.50.12.205", 64530}});
CMECert::CMECert() : CLI::App("Tool to drive CME Cert", "cme-cert") {
callback([] {
// Typically a timestamp
SessionSaver ss;
ss.load();
Config config{"NGG", "004", false, [](auto, auto, auto) {}};
IL3Reactor reactor;
IL3MessageFramer framer;
auto &connection = reactor.register_connection(
{// local dev tunnel
TCPEndpoint{"127.0.0.1", 64530}, TCPEndpoint{"127.0.0.1", 64531},
// Direct connection
TCPEndpoint{"69.50.112.199", 64530},
TCPEndpoint{"69.50.12.205", 64530}});
// NGG primary - 69.50.112.199 64530
// NGG backup - 69.50.112.205 64530
// NGG primary - 69.50.112.199 64530
// NGG backup - 69.50.112.205 64530
IL3GW gw(connection, framer, nullptr, ss.current, config, {});
gw.subscribe_session([&](auto &s) { ss.session_changed(s); });
gw.connect();
IL3GW gw(connection, framer, nullptr, ss.current, config, {});
gw.subscribe_session([&](auto &s) { ss.session_changed(s); });
gw.connect();
TestRunner run(reactor, gw);
TestRunner run(reactor, gw);
///////////////////////////
//
// The current test
//
///////////////////////////
///////////////////////////
//
// The current test
//
///////////////////////////
run.scratch_space();
run.scratch_space();
///////////////////////////
///////////////////////////
for (auto x = 0; x < 5; ++x) {
reactor.do_work();
sleep(1);
}
gw.send_terminate();
gw.flush();
return 0;
for (auto x = 0; x < 5; ++x) {
reactor.do_work();
sleep(1);
}
gw.send_terminate();
gw.flush();
});
}
std::string CMECert::help() { return "Tool to drive CME Cert"; }
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+10
View File
@@ -0,0 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
namespace bts::cmds {
struct CMECert : public CLI::App {
CMECert();
};
} // namespace bts::cmds
-15
View File
@@ -1,15 +0,0 @@
#pragma once
#include "ICmd.h"
namespace bts {
namespace cmd {
class CMECert : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
};
} // namespace cmd
} // namespace bts
-57
View File
@@ -1,57 +0,0 @@
#include "CmdRegistry.h"
#include "ICmd.h"
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>
#include <fstream>
#include <vector>
namespace bts {
namespace cmd {
std::map<std::string, std::unique_ptr<ICmd>> CmdRegistry::_cmds;
int CmdRegistry::dispatch(int argc, char **argv) {
if (argc < 2)
return _cmds["help"]->execute(std::vector<std::string>());
else {
std::vector<std::string> args;
for (int i = 1; i < argc; i++) {
std::string arg(argv[i]);
if (boost::starts_with(boost::to_lower_copy(arg), "-configserver")) {
// skip configserver flag as well as url
i++;
} else
args.push_back(arg);
}
ICmd *cmd = retrieve(std::string(args[0]));
if (cmd == 0) {
return _cmds["help"]->execute(std::vector<std::string>());
} else {
args.erase(args.begin());
return cmd->execute(args);
}
}
}
ICmd *CmdRegistry::retrieve(std::string const &name) {
std::string lname(name);
boost::to_lower(lname);
if (_cmds.find(lname) == _cmds.end())
return 0;
else
return _cmds[lname].get();
}
std::vector<std::string> CmdRegistry::cmds() {
std::vector<std::string> result;
for (auto &p : _cmds)
result.push_back(p.first);
return result;
}
} // namespace cmd
} // namespace bts
-36
View File
@@ -1,36 +0,0 @@
#pragma once
#include "lib/utils/bts_assert.h"
#include <boost/algorithm/string.hpp>
#include <map>
#include <memory>
#include <string>
#include <vector>
#define BEGIN_CMD
#define REGISTER(CMD) bts::cmd::CmdRegistry::register_cmd<bts::cmd::CMD>(#CMD)
#define END_CMD return bts::cmd::CmdRegistry::dispatch(argc, argv)
namespace bts {
namespace cmd {
class ICmd;
class CmdRegistry {
public:
template <class CMD> static void register_cmd(std::string const &name) {
std::string lname = name;
boost::to_lower(lname);
bts_assert(_cmds.find(lname) == _cmds.end());
_cmds[lname].reset(new CMD());
}
static std::vector<std::string> cmds();
static int dispatch(int argc, char **argv);
static ICmd *retrieve(std::string const &name);
private:
static std::map<std::string, std::unique_ptr<ICmd>> _cmds;
};
} // namespace cmd
} // namespace bts
+17 -11
View File
@@ -1,18 +1,24 @@
#include "ConfigCheck.h"
#include "lib/config/ConfigCheck.h"
namespace bts {
namespace cmd {
namespace bts::cmds {
int ConfigCheck::execute(std::vector<std::string> const &args) {
std::vector<const char *> ch_args;
for (auto const &s : args)
ch_args.push_back(s.c_str());
configcheck(ch_args.size(), &ch_args[0]);
return 0;
struct ConfigCheck::Args {
std::vector<std::string> args;
};
ConfigCheck::ConfigCheck()
: CLI::App("Print current configuration", "config-check"),
_args(new ConfigCheck::Args()) {
add_option("args", _args->args);
callback([&] {
std::vector<const char *> ch_args;
for (auto const &s : _args->args)
ch_args.push_back(s.c_str());
configcheck(ch_args.size(), &ch_args[0]);
});
}
std::string ConfigCheck::help() { return "Config check help."; }
ConfigCheck::~ConfigCheck() {}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+10 -9
View File
@@ -1,15 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
struct ConfigCheck : public CLI::App {
ConfigCheck();
~ConfigCheck();
class ConfigCheck : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+4 -11
View File
@@ -1,17 +1,10 @@
#include "server/sparkd/cmds/FIXSim.h"
#include "server/sparkd/exchange_sim/ExchangeSim.h"
namespace bts {
namespace cmd {
namespace bts::cmds {
int FIXSim::execute(std::vector<std::string> const &) {
bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM);
return 0;
FIXSim::FIXSim() : CLI::App("Starts a standalone FIX simulator", "fix-sim") {
callback([] { bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM); });
}
std::string FIXSim::help() {
return "FIXSim - Starts a standalone FIX simulator ";
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -10
View File
@@ -1,15 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class FIXSim : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
struct FIXSim : public CLI::App {
FIXSim();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
-29
View File
@@ -1,29 +0,0 @@
#include "Help.h"
#include "server/sparkd/cmds/CmdRegistry.h"
#include <iostream>
namespace bts {
namespace cmd {
int Help::execute(std::vector<std::string> const &args) {
auto commands = CmdRegistry::cmds();
if (args.size() != 1 || CmdRegistry::retrieve(args[0]) == 0) {
std::cerr << "Syntax: sparkd <cmd> <args> or sparkd help <cmd>, where "
"cmd is one of: "
<< std::endl;
for (auto &cmd : commands)
std::cerr << "\t" << cmd << std::endl;
return 0;
} else {
auto cmd = CmdRegistry::retrieve(args[0]);
std::cerr << "sparkd " << args[0] << ":" << std::endl
<< cmd->help() << std::endl;
return 0;
}
}
std::string Help::help() { return "Display help."; }
} // namespace cmd
} // namespace bts
-15
View File
@@ -1,15 +0,0 @@
#pragma once
#include "ICmd.h"
namespace bts {
namespace cmd {
class Help : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
};
} // namespace cmd
} // namespace bts
-17
View File
@@ -1,17 +0,0 @@
#pragma once
#include <string>
#include <vector>
namespace bts {
namespace cmd {
class ICmd {
public:
virtual int execute(std::vector<std::string> const &args) = 0;
virtual std::string help() = 0;
virtual ~ICmd() {}
};
} // namespace cmd
} // namespace bts
+136 -224
View File
@@ -35,8 +35,7 @@
#include <netinet/in.h>
#include <sys/types.h>
namespace bts {
namespace cmd {
namespace bts::cmds {
using namespace std;
using namespace datetime;
@@ -44,130 +43,52 @@ using namespace bts::api::auth;
using namespace bts::crypto;
using namespace bts::config;
std::string KeyCode::helpstr() {
std::ostringstream oss;
oss << "\n"
<< "ENCODE -keyfile:[keyfile] -pass:password -users:<nusers> "
"-expiration:<2020-05-18> [-ip:<ip_address>] [-port:<spark_port>] "
"[-allow_concurrent]\n "
<< "DECODE -keyfile:[keyfile] -pass:password -source:[file] \n "
<< "KEYGEN -keyfile:[keyfile_prefix]\n "
<< "CHECKIPS \n ";
struct EncryptArgs {
std::string keyfile;
std::string pass;
std::string users;
std::string expiration;
std::string source;
std::string ip;
uint16_t port;
bool allow_concurrent;
};
return oss.str();
}
struct DecryptArgs {
std::string keyfile;
std::string pass;
std::string users;
std::string expiration;
std::string source;
};
int checkips() {
auto ipmap = bts::os::host_ips();
for (const auto &x : ipmap)
std::cout << x.first << " => " << x.second << std::endl;
return 0;
}
struct KeygenArgs {
std::string keyfile;
std::string pass;
std::string users;
std::string expiration;
};
std::string parse_date(std::string const &date) {
auto the_date = bts::datetime::parse_iso_date_time_string(date);
return bts::datetime::iso_date_time_string(the_date);
}
void get_cmd(
const std::vector<std::string> &args,
std::string &keyfile,
std::string &pass,
std::string &users,
std::string &expiration,
std::string *source = nullptr,
std::string *ip = nullptr,
std::string *port = nullptr,
bool *allow_concurrent = nullptr) {
for (size_t i = 1; i < args.size(); i++) {
size_t pass_pos = args[i].find("-pass:");
if (pass_pos != args[i].npos)
pass = args[i].substr(pass_pos + 6);
void eraseAllSubStr(std::string &mainStr, const std::string &toErase) {
size_t pos = std::string::npos;
size_t file_pos = args[i].find("-keyfile:");
if (file_pos != args[i].npos)
keyfile = args[i].substr(file_pos + 9);
size_t u_pos = args[i].find("-users:");
if (u_pos != args[i].npos)
users = args[i].substr(u_pos + 7);
size_t e_pos = args[i].find("-expiration:");
if (e_pos != args[i].npos)
expiration = parse_date(args[i].substr(e_pos + 12));
size_t pay_pos = args[i].find("-source:");
if (source && pay_pos != args[i].npos)
*source = args[i].substr(pay_pos + 8);
size_t i_pos = args[i].find("-ip:");
if (ip && i_pos != args[i].npos)
*ip = args[i].substr(i_pos + 4);
size_t p_pos = args[i].find("-port:");
if (port && p_pos != args[i].npos)
*port = args[i].substr(p_pos + 6);
size_t c_pos = args[i].find("-allow_concurrent");
if (allow_concurrent && c_pos != args[i].npos)
*allow_concurrent = true;
// Search for the substring in string in a loop untill nothing is found
while ((pos = mainStr.find(toErase)) != std::string::npos) {
// If found then erase it from string
mainStr.erase(pos, toErase.length());
}
}
int keygen(const std::vector<std::string> &args) {
std::string keyfile, pass, users, expiration;
get_cmd(args, keyfile, pass, users, expiration);
if (keyfile != "") {
PrivateKeyPtr priv = PrivateKey::create(2048, AlgoType::RSA);
std::string n1 = (keyfile + "_private.pem"),
n2 = (keyfile + "_public.pem");
// I can't get anything but DES to work here.
priv->write_file(n1, CipherType::DES);
priv->public_key()->write_file(n2);
std::cout << "Key pair generated " << n1 << " and " << n2 << std::endl;
return 0;
} else {
std::cout << KeyCode::helpstr();
return -1;
}
}
// int Logon::digest(const std::vector<std::string>& args)
// {
// std::string pass, keyfile, payload, user, expiration, extra_data;
// get_cmd(args, pass, keyfile, payload, user, expiration, extra_data);
// if(extra_data != "")
// {
// unsigned int len;
// //unsigned char* digest;
// //Digest::allocate_array(&digest, &len);
// //Digest::compute_digest(reinterpret_cast<const unsigned
// char*>(payload.c_str()), payload.size(), digest, &len);
// std::string digest = Digest::compute_digest(extra_data);
// len = digest.size();
// UNUSED(len);
// std::cout << std::endl << std::endl;
// //for(size_t i = 0; i < len; i++)
// // printf("%02x", digest[i]);
// std::cout << digest;
// std::cout << std::endl << std::endl;
// return 0;
// }
// else
// {
// std::cout << help();
// return -1;
// }
// }
void write_keycode(
const std::string &users,
const std::string &expiration,
const std::string &ip,
const std::string &port,
uint16_t port,
bool concurrent,
bts::config::KeyCode &kc) {
std::ostringstream out_stream;
@@ -184,102 +105,68 @@ void write_keycode(
ip != "127.0.0.1", "you can't generate a keycode for loopback, just "
"leave the parameter blank");
kc.set_ip(ip);
kc.set_port(port);
kc.set_port(std::to_string(port));
kc.set_allow_concurrent(concurrent);
}
void eraseAllSubStr(std::string &mainStr, const std::string &toErase) {
size_t pos = std::string::npos;
// Search for the substring in string in a loop untill nothing is found
while ((pos = mainStr.find(toErase)) != std::string::npos) {
// If found then erase it from string
mainStr.erase(pos, toErase.length());
}
void check_ips() {
CryptoInit::initialize();
auto ipmap = bts::os::host_ips();
for (const auto &x : ipmap)
std::cout << x.first << " => " << x.second << std::endl;
}
int encrypt(
const std::string &keyfile,
const std::string &pass,
const std::string &users,
const std::string &expiration,
const std::string &ip,
const std::string &port,
bool concurrent) {
void encrypt(const EncryptArgs &args) {
CryptoInit::initialize();
std::string protobuf;
std::string kc_string;
std::stringstream oss, encrypted;
CachedPassword cache(args.pass);
PrivateKeyPtr key = PrivateKey::read_file(args.keyfile, &cache);
if (keyfile == "") {
std::cout << KeyCode::helpstr();
return -1;
} else {
std::string kc_string;
std::stringstream oss, encrypted;
CachedPassword cache(pass);
PrivateKeyPtr key = PrivateKey::read_file(keyfile, &cache);
bts::config::KeyCode kc;
write_keycode(users, expiration, ip, port, concurrent, kc);
if (google::protobuf::TextFormat::PrintToString(kc, &kc_string)) {
oss << kc_string;
key->encrypt(oss, encrypted);
} else
return -1;
// int ret = openssl::RSA_private_encrypt(sz-1, pay, &buffer[0],
// key->pkey.rsa, RSA_PKCS1_PADDING);
EncryptedKeyCode ekc;
auto kcstr = encrypted.str();
eraseAllSubStr(kcstr, "-----BEGIN BTS-----\n");
eraseAllSubStr(kcstr, "\n-----END BTS-----\n");
kcstr.erase(std::remove(kcstr.begin(), kcstr.end(), '\n'), kcstr.end());
// kcstr = boost::replace_all_copy<string>(kcstr, "\n", "\\n");
ekc.set_keycode(kcstr);
std::string pb;
google::protobuf::TextFormat::PrintToString(ekc, &pb);
std::cout << std::endl << std::endl;
std::cout << "#nusers: " << users << std::endl;
std::cout << "#allow concurrent : "
<< (kc.allow_concurrent() ? "true" : "false") << std::endl;
std::cout << "#expires: "
<< datetime::simple_date_time_string(
datetime::ll_to_ptime(kc.expiration()))
<< " " << std::to_string(kc.expiration()) << std::endl;
std::cout << "#generation_time: "
<< datetime::simple_date_time_string(
datetime::ll_to_ptime(kc.generation_time()))
<< " " << std::to_string(kc.generation_time()) << std::endl;
std::cout << "EncryptedKeyCode {" << std::endl;
std::cout << " value {" << std::endl;
std::cout << " " << pb;
std::cout << " }" << std::endl;
std::cout << "}" << std::endl;
std::cout << std::endl << std::endl;
return 0;
}
bts::config::KeyCode kc;
write_keycode(
args.users, args.expiration, args.ip, args.port, args.allow_concurrent,
kc);
if (!google::protobuf::TextFormat::PrintToString(kc, &kc_string))
return;
oss << kc_string;
key->encrypt(oss, encrypted);
EncryptedKeyCode ekc;
auto kcstr = encrypted.str();
eraseAllSubStr(kcstr, "-----BEGIN BTS-----\n");
eraseAllSubStr(kcstr, "\n-----END BTS-----\n");
kcstr.erase(std::remove(kcstr.begin(), kcstr.end(), '\n'), kcstr.end());
ekc.set_keycode(kcstr);
std::string pb;
google::protobuf::TextFormat::PrintToString(ekc, &pb);
std::cout << std::endl << std::endl;
std::cout << "#nusers: " << args.users << std::endl;
std::cout << "#allow concurrent : "
<< (kc.allow_concurrent() ? "true" : "false") << std::endl;
std::cout << "#expires: "
<< datetime::simple_date_time_string(
datetime::ll_to_ptime(kc.expiration()))
<< " " << std::to_string(kc.expiration()) << std::endl;
std::cout << "#generation_time: "
<< datetime::simple_date_time_string(
datetime::ll_to_ptime(kc.generation_time()))
<< " " << std::to_string(kc.generation_time()) << std::endl;
std::cout << "EncryptedKeyCode {" << std::endl;
std::cout << " value {" << std::endl;
std::cout << " " << pb;
std::cout << " }" << std::endl;
std::cout << "}" << std::endl;
std::cout << std::endl << std::endl;
}
int encode(const std::vector<std::string> &args) {
std::string keyfile, pass, users, expiration, ip, port;
bool concurrent = false;
get_cmd(
args, keyfile, pass, users, expiration, nullptr, &ip, &port,
&concurrent);
return encrypt(keyfile, pass, users, expiration, ip, port, concurrent);
}
int decode(const std::vector<std::string> &args) {
std::string keyfile, pass, users, expiration, source;
get_cmd(args, keyfile, pass, users, expiration, &source);
if (keyfile != "") {
PublicKeyPtr pub = bts::crypto::PublicKey::read_file(keyfile);
if (pub && source != "") {
// KeycodePayloadHelper helper;
void decrypt(const DecryptArgs &args) {
CryptoInit::initialize();
if (args.keyfile != "") {
PublicKeyPtr pub = bts::crypto::PublicKey::read_file(args.keyfile);
if (pub && args.source != "") {
bts::config::EncryptedKeyCode ekc;
std::fstream fs(source);
// google::protobuf::io::IstreamInputStream pay_stream(&fs);
// google::protobuf::TextFormat::Parse(&pay_stream, &ekc);
std::fstream fs(args.source);
std::string str(
(std::istreambuf_iterator<char>(fs)),
std::istreambuf_iterator<char>());
@@ -293,39 +180,64 @@ int decode(const std::vector<std::string> &args) {
google::protobuf::TextFormat::PrintToString(kc, &string_out);
std::cout << string_out << std::endl;
}
// int ret = openssl::RSA_public_decrypt(sz, data, &buffer[0],
// key->pkey.rsa, RSA_PKCS1_PADDING);
std::cout << std::endl << std::endl;
return 0;
std::cout << "\n\n";
} else {
std::cout << KeyCode::helpstr();
return -1;
std::cout << "\nFailed\n";
}
}
int KeyCode::execute(const std::vector<std::string> &args) {
void keygen(const KeygenArgs &args) {
CryptoInit::initialize();
if (args.size() > 0) {
if (boost::to_lower_copy(args[0]) == "keycode" ||
boost::to_lower_copy(args[0]) == "encode") {
return encode(args);
}
if (boost::to_lower_copy(args[0]) == "keygen") {
return keygen(args);
}
if (boost::to_lower_copy(args[0]) == "decode") {
return decode(args);
}
if (boost::to_lower_copy(args[0]) == "checkips") {
return checkips();
}
}
std::cout << help() << std::endl;
return -1;
PrivateKeyPtr priv = PrivateKey::create(2048, AlgoType::RSA);
std::string n1 = (args.keyfile + "_private.pem"),
n2 = (args.keyfile + "_public.pem");
// I can't get anything but DES to work here.
priv->write_file(n1, CipherType::DES);
priv->public_key()->write_file(n2);
std::cout << "Key pair generated " << n1 << " and " << n2 << std::endl;
}
std::string KeyCode::help() { return helpstr(); }
struct KeyCode::Args {
EncryptArgs encrypt;
DecryptArgs decrypt;
KeygenArgs keygen;
};
} // namespace cmd
} // namespace bts
KeyCode::KeyCode() : CLI::App("key-code"), _args(new KeyCode::Args()) {
require_subcommand();
auto *cmd = this;
// check ips
cmd->add_subcommand("check-ips")->callback(check_ips);
// decrypt
auto *decrypt_cmd = cmd->add_subcommand("decrypt");
decrypt_cmd->add_option("--expiration", _args->decrypt.expiration);
decrypt_cmd->add_option("--key-file", _args->decrypt.keyfile)->required();
decrypt_cmd->add_option("--pass", _args->decrypt.pass);
decrypt_cmd->add_option("--source", _args->decrypt.source);
decrypt_cmd->add_option("--users", _args->decrypt.users);
decrypt_cmd->callback([&] { decrypt(_args->decrypt); });
// encrypt
auto *encrypt_cmd = cmd->add_subcommand("encrypt");
encrypt_cmd->add_flag("--allow-concurrent", _args->encrypt.allow_concurrent);
encrypt_cmd->add_option("--expiration", _args->encrypt.expiration);
encrypt_cmd->add_option("--ip", _args->encrypt.ip);
encrypt_cmd->add_option("--key-file", _args->encrypt.keyfile)->required();
encrypt_cmd->add_option("--pass", _args->encrypt.pass);
encrypt_cmd->add_option("--port", _args->encrypt.port);
encrypt_cmd->add_option("--source", _args->encrypt.source);
encrypt_cmd->add_option("--users", _args->encrypt.users);
encrypt_cmd->callback([&] { encrypt(_args->encrypt); });
// key-gen
auto *keygen_cmd = cmd->add_subcommand("key-gen");
keygen_cmd->add_option("--expiration", _args->keygen.expiration);
keygen_cmd->add_option("--key-file", _args->keygen.keyfile)->required();
keygen_cmd->add_option("--pass", _args->keygen.pass);
keygen_cmd->add_option("--users", _args->keygen.users);
keygen_cmd->callback([&] { keygen(_args->keygen); });
}
KeyCode::~KeyCode() {}
} // namespace bts::cmds
+19 -19
View File
@@ -1,29 +1,29 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
#include "ICmd.h"
namespace bts::crypto {
namespace bts {
namespace crypto {
class PublicKey;
}
namespace config {
} // namespace bts::crypto
namespace bts::config {
class EncryptedKeyCode;
class KeyCode;
} // namespace config
namespace cmd {
bool decrypt(
const bts::crypto::PublicKey &public_key,
const bts::config::EncryptedKeyCode &kc,
bts::config::KeyCode *dk);
class KeyCode : public ICmd {
public:
enum class PublicPrivate { Public, Private };
int execute(std::vector<std::string> const &args);
std::string help();
} // namespace bts::config
static std::string helpstr();
namespace bts::cmds {
struct KeyCode : public CLI::App {
KeyCode();
~KeyCode();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+10 -9
View File
@@ -1,17 +1,18 @@
#include "Md.h"
#include "server/sparkd/cmds/CmdRegistry.h"
#include "server/sparkd/md/run_md.h"
#include <iostream>
namespace bts {
namespace cmd {
namespace bts::cmds {
int Md::execute(std::vector<std::string> const &args) {
bts::md::md_tool(args);
return 0;
struct Md::Args {
std::vector<std::string> args;
};
Md::Md() : CLI::App("", "md"), _args(new Md::Args()) {
add_option("args", _args->args);
callback([&] { md::md_tool(_args->args); });
}
std::string Md::help() { return "Md command"; }
Md::~Md() {}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+10 -9
View File
@@ -1,15 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
struct Md : public CLI::App {
Md();
~Md();
class Md : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+57 -46
View File
@@ -9,8 +9,7 @@
#include <fstream>
#include <unordered_map>
namespace bts {
namespace cmd {
namespace bts::cmds {
using namespace jetfix;
@@ -98,9 +97,13 @@ typedef FIXMsg<
std::string to_str(std::string const &s) { return s; }
template <class V> std::string to_str(V const &v) { return std::to_string(v); }
template <class V>
std::string to_str(V const &v) {
return std::to_string(v);
}
template <class F, class G> std::string get_if_valid(G const &update) {
template <class F, class G>
std::string get_if_valid(G const &update) {
if (update.template has<F>())
return to_str(update.template get<F>());
else
@@ -393,54 +396,62 @@ void print_orders(
}
}
int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
std::ifstream input(args[0]);
std::ofstream books_stream(args[0] + "_books.csv");
std::ofstream msgs_stream(args[0] + "_msgs.csv");
std::ofstream trades_stream(args[0] + "_trades.csv");
std::ofstream orders_stream(args[0] + "_orders.csv");
struct ParseHistoricalFix::Args {
std::vector<std::string> args;
};
std::unordered_map<std::string, Book> books;
TradeSummaryEvent current_trade_summary;
ParseHistoricalFix::ParseHistoricalFix()
: CLI::App(
"ParseHistoricalFIX <input> will produce 2 files, "
"<input>_mdentries.csv and <input>order_entries.csv",
"parse-historical-fix"),
_args(new ParseHistoricalFix::Args()) {
add_option("args", _args->args);
callback([&] {
auto &args = _args->args;
std::ifstream input(args[0]);
std::ofstream books_stream(args[0] + "_books.csv");
std::ofstream msgs_stream(args[0] + "_msgs.csv");
std::ofstream trades_stream(args[0] + "_trades.csv");
std::ofstream orders_stream(args[0] + "_orders.csv");
MDIncrementalUpdate update;
FIXParser<MDIncrementalUpdate> parser;
std::unordered_map<std::string, Book> books;
TradeSummaryEvent current_trade_summary;
std::string line;
size_t c = 0;
auto ok = 0;
auto start = Timer::now();
MDIncrementalUpdate update;
FIXParser<MDIncrementalUpdate> parser;
print_book_header(books_stream);
print_msg_headers(msgs_stream);
print_trades_header(trades_stream);
print_orders_header(orders_stream);
std::string line;
size_t c = 0;
auto ok = 0;
auto start = Timer::now();
while (std::getline(input, line)) {
c++;
auto parsed_ptr = parser.parse(InMsgBuffer(
line.data(), line.size(), std::shared_ptr<std::string>{}));
if (parsed_ptr) {
auto msg =
std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
update_and_maybe_print_book(*msg, books, books_stream);
print_msgs(c, *msg, msgs_stream);
update_and_maybe_print_trades(
*msg, current_trade_summary, trades_stream);
print_orders(*msg, orders_stream);
print_book_header(books_stream);
print_msg_headers(msgs_stream);
print_trades_header(trades_stream);
print_orders_header(orders_stream);
while (std::getline(input, line)) {
c++;
auto parsed_ptr = parser.parse(InMsgBuffer(
line.data(), line.size(), std::shared_ptr<std::string>{}));
if (parsed_ptr) {
auto msg = std::dynamic_pointer_cast<const MDIncrementalUpdate>(
parsed_ptr);
update_and_maybe_print_book(*msg, books, books_stream);
print_msgs(c, *msg, msgs_stream);
update_and_maybe_print_trades(
*msg, current_trade_summary, trades_stream);
print_orders(*msg, orders_stream);
}
ok += (bool)(parsed_ptr);
}
ok += (bool)(parsed_ptr);
}
auto sec_duration = sec_diff(Timer::now(), start);
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
<< " out of " << c << " time: " << sec_duration << " secs\n";
return 0;
auto sec_duration = sec_diff(Timer::now(), start);
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
<< " out of " << c << " time: " << sec_duration << " secs\n";
});
}
std::string ParseHistoricalFIX::help() {
return "ParseHistoricalFIX <input> will produce 2 files, "
"<input>_mdentries.csv and <input>order_entries.csv";
}
ParseHistoricalFix::~ParseHistoricalFix() {}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+10 -9
View File
@@ -1,15 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
struct ParseHistoricalFix : public CLI::App {
ParseHistoricalFix();
~ParseHistoricalFix();
class ParseHistoricalFIX : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+44 -46
View File
@@ -3,56 +3,54 @@
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
namespace bts {
namespace cmd {
namespace bts::cmds {
int PrintHedgeOrders::execute(std::vector<std::string> const &) {
persistence::SqliteStore order_store("orders");
auto all_orders = order_store.retrieve_all()["orders"];
PrintHedgeOrders::PrintHedgeOrders()
: CLI::App(
"Print out order ids for all hedge orders"
"which were completed at least one second being created",
"print-hedge-orders") {
callback([&] {
persistence::SqliteStore order_store("orders");
auto all_orders = order_store.retrieve_all()["orders"];
std::map<uint32_t, std::pair<std::string, double>> fill_times;
persistence::SqliteStore fill_store("fills");
auto all_fills = fill_store.retrieve_all()["fills"];
for (auto const &kvp : all_fills) {
api::orderman::Fill fill_rec;
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill_rec)) {
auto ts = utils::pb::to_double_time(fill_rec.timestamp());
auto &p = fill_times[fill_rec.order_key()];
auto &min_time = p.second;
if (min_time < ts) {
p.first = kvp.first;
min_time = ts;
}
}
}
std::cout << "order_key,time_to_first_fill,fill_id"
<< "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all_orders)
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() && def.custom_def().has_limit() &&
def.sub_type() ==
api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
auto create_ts = utils::pb::to_double_time(state.created());
auto &first_fill = fill_times[rec.key()];
auto delta = first_fill.second - create_ts;
if (first_fill.first.size())
std::cout << kvp.first << "," << delta << "," << first_fill.first
<< "\n";
std::map<uint32_t, std::pair<std::string, double>> fill_times;
persistence::SqliteStore fill_store("fills");
auto all_fills = fill_store.retrieve_all()["fills"];
for (auto const &kvp : all_fills) {
api::orderman::Fill fill_rec;
if (utils::pb::from_string<api::orderman::Fill>(
kvp.second, &fill_rec)) {
auto ts = utils::pb::to_double_time(fill_rec.timestamp());
auto &p = fill_times[fill_rec.order_key()];
auto &min_time = p.second;
if (min_time < ts) {
p.first = kvp.first;
min_time = ts;
}
}
}
return 0;
std::cout << "order_key,time_to_first_fill,fill_id"
<< "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all_orders)
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() && def.custom_def().has_limit() &&
def.sub_type() ==
api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
auto create_ts = utils::pb::to_double_time(state.created());
auto &first_fill = fill_times[rec.key()];
auto delta = first_fill.second - create_ts;
if (first_fill.first.size())
std::cout << kvp.first << "," << delta << ","
<< first_fill.first << "\n";
}
}
});
}
std::string PrintHedgeOrders::help() {
return "Print out order ids for all hedge orders"
"which were completed at least one second being created";
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -10
View File
@@ -1,15 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class PrintHedgeOrders : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
struct PrintHedgeOrders : public CLI::App {
PrintHedgeOrders();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+13 -16
View File
@@ -2,11 +2,9 @@
#include "server/sparkd/cmds/spartan/market_data/handler/ReactorMDP3FeedHandler.h"
#include "server/sparkd/cmds/spartan/system/reactor/Reactor.h"
namespace bts {
namespace cmd {
int test_md(std::vector<std::string> const &) {
namespace bts::cmds {
void test_md() {
using namespace bts::spartan;
using namespace bts::spartan::mdp3;
@@ -26,8 +24,7 @@ int test_md(std::vector<std::string> const &) {
void on_recovery_complete(InitialSnapshot const &) override {
std::cout << "on_recovery_complete\n";
}
void on_event_end(MDP3Event const &) override {
}
void on_event_end(MDP3Event const &) override {}
void on_packet_loss() override {
std::cout << "on_real_time_channel_loss\n";
}
@@ -38,18 +35,18 @@ int test_md(std::vector<std::string> const &) {
handler.start();
for (;;)
reactor.do_work();
return 0;
}
int QuickTest::execute(std::vector<std::string> const &) {
return 0;
static void execute() {
// put test code here
}
std::string QuickTest::help() {
return "This is a place holder to make it easy to quickly write "
"discardable "
"test code";
QuickTest::QuickTest()
: CLI::App(
"This is a place holder to make it easy to quickly "
"write discardable test code",
"quick-test") {
callback(execute);
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -10
View File
@@ -1,15 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class QuickTest : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
struct QuickTest : public CLI::App {
QuickTest();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+7 -8
View File
@@ -1,5 +1,4 @@
#include "Run.h"
#include "ICmd.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/config/config.h"
@@ -46,10 +45,9 @@
#include <thread>
namespace bts {
namespace cmd {
namespace bts::cmds {
int Run::execute(std::vector<std::string> const &) {
static void execute() {
using namespace bts;
LOG(Gen, Warn,
"------------------------------------ Sparkd launched "
@@ -123,9 +121,10 @@ int Run::execute(std::vector<std::string> const &) {
subprocess::run_all();
quickbus.run();
return 0;
}
std::string Run::help() { return "\t run default sparkd services"; }
} // namespace cmd
} // namespace bts
Run::Run() : CLI::App("Run default sparkd services", "run") {
callback(execute);
}
} // namespace bts::cmds
+5 -10
View File
@@ -1,15 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class Run : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
struct Run : public CLI::App {
Run();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+6 -7
View File
@@ -24,10 +24,9 @@
using namespace bts::position;
using namespace bts::api::position;
namespace bts {
namespace cmd {
namespace bts::cmds {
int Span::execute(const std::vector<std::string> &) {
static void execute() {
srand(time(NULL));
auto &disp = utils::Dispatcher::current();
MarginServiceCore core(disp);
@@ -55,10 +54,10 @@ int Span::execute(const std::vector<std::string> &) {
std::cout << "Waiting for margin results, press any key to exit"
<< std::endl;
disp.run();
return 0;
}
std::string Span::help() { return "\t test CME CORE Span integration \n"; }
Span::Span() : CLI::App("Test CME CORE Span integration", "span") {
callback(execute);
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -10
View File
@@ -1,15 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class Span : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
struct Span : public CLI::App {
Span();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+33 -362
View File
@@ -1,368 +1,39 @@
#include "Spartan.h"
#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h"
#include "server/sparkd/cmds/spartan/market_data/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/market_data/feed_control/IFeedController.h"
#include "server/sparkd/cmds/spartan/market_data/feed_control/PickedFeed.h"
#include "server/sparkd/cmds/spartan/market_data/handler/MDP3FeedHandler.h"
#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/system/PCapReader.h"
#include "server/sparkd/cmds/spartan/cmds/BookStats.h"
#include "server/sparkd/cmds/spartan/cmds/CSV.h"
#include "server/sparkd/cmds/spartan/cmds/Compress.h"
#include "server/sparkd/cmds/spartan/cmds/FindPolys.h"
#include "server/sparkd/cmds/spartan/cmds/Icebergs.h"
#include "server/sparkd/cmds/spartan/cmds/Json.h"
#include "server/sparkd/cmds/spartan/cmds/ListSec.h"
#include "server/sparkd/cmds/spartan/cmds/PolygonBooks.h"
#include "server/sparkd/cmds/spartan/cmds/SQL.h"
#include "server/sparkd/cmds/spartan/cmds/Sim.h"
#include "server/sparkd/cmds/spartan/cmds/SimBooks.h"
#include "server/sparkd/cmds/spartan/cmds/Stats.h"
#include "server/sparkd/cmds/spartan/cmds/ValidateFifo.h"
#include "server/sparkd/cmds/spartan/cmds/ValuePolys.h"
#include "lib/jetfix/codec/f_toa.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "lib/utils/PBUtils.h"
#include "server/sparkd/cmds/spartan/back_testing/run/run_sim.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/FillCounter.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.h"
#include "server/sparkd/cmds/spartan/market_data/parsing/MDP3Parser.h"
#include "server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h"
#include "server/sparkd/cmds/spartan/market_data/utils/queries/BookStatRecorder.h"
namespace bts::cmds {
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.h"
#include "server/sparkd/cmds/spartan/util/args_parser.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include "server/sparkd/cmds/spartan/util/sql/adapters/polygon_defs.h"
#include "server/sparkd/cmds/spartan/util/sql/sql_utils.h"
using namespace spartan::cmds;
#include <Pcap++/PcapFileDevice.h>
#include <boost/algorithm/string/predicate.hpp>
#include <fmt/format.h>
#include <fstream>
#include <iostream>
namespace bts {
namespace cmd {
#define INFO(...) LOG(OrderMan, Info, __VA_ARGS__)
#define WARN(...) LOG(OrderMan, Warn, __VA_ARGS__)
#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__)
using namespace spartan::mdp3;
auto drop_extensions = [](auto const &str) {
size_t lastindex = str.find_last_of(".");
return str.substr(0, lastindex);
};
inline std::string output_path(
const std::string &input_path,
const char *output_ext,
const char *input_ext = "") {
return bts::spartan::file::change_file_ext(
input_path, output_ext, input_ext);
Spartan::Spartan() : CLI::App("Futures trading system for devs", "spartan") {
// subcommands
add_subcommand(CLI::App_p(new BookStats()));
add_subcommand(CLI::App_p(new Compress()));
add_subcommand(CLI::App_p(new CSV()));
add_subcommand(CLI::App_p(new FindPolys()));
add_subcommand(CLI::App_p(new Icebergs()));
add_subcommand(CLI::App_p(new Json()));
add_subcommand(CLI::App_p(new ListSec()));
add_subcommand(CLI::App_p(new PolygonBooks()));
add_subcommand(CLI::App_p(new Sim()));
add_subcommand(CLI::App_p(new SimBooks()));
add_subcommand(CLI::App_p(new SQL()));
add_subcommand(CLI::App_p(new Stats()));
add_subcommand(CLI::App_p(new ValidateFifo()));
add_subcommand(CLI::App_p(new ValuePolys()));
}
void find_polys(std::vector<std::string> const &args) {
PolygonFinder::Options options;
if (args.size() > 2)
options.active_threshold_fills = std::stoi(args[2]);
if (args.size() > 3)
options.max_vertices = std::stoi(args[3]);
if (args.size() > 4)
options.max_securities = std::stoi(args[4]);
DefinitionCollector def_collector;
PolygonFinder poly_finder(options);
observe_channel(args[1], {&poly_finder, &def_collector});
for (size_t id = 1; id <= poly_finder.result.size(); id++)
std::cout << id << " -> "
<< polygon_descriptor(
poly_finder.result[id], def_collector.ptr_result)
<< std::endl;
}
void list_sec(std::string const &input_file) {
DefinitionCollector def;
FillCounter fill_counter;
observe_channel(input_file, {&def, &fill_counter});
auto fill_counts = fill_counter.result_map();
for (auto const &s : def.result) {
std::cout << s.second.asset() << "|" << s.second.symbol() << "|"
<< fill_counts[s.second.security_id()];
for (auto const &l : s.second.legs()) {
std::cout << "|";
if (def.ptr_result[l.security_id])
std::cout << def.ptr_result[l.security_id]->symbol();
else
std::cout << "?" << l.security_id;
}
std::cout << std::endl;
}
}
template <class T>
inline int run(const std::string &pcap, T obs) {
observe_channel(pcap, obs);
return 0;
}
void value_polys(std::vector<std::string> const &args) {
DefinitionCollector def;
PolygonFinder poly_finder;
// first pass, just identify the polygons
observe_channel(args[1], {&poly_finder, &def});
// second pass, value the polygons
PolygonOpportunityFinder op_finder{poly_finder.result, def.ptr_result};
observe_channel(args[1], {&op_finder});
auto poly_v =
std::ofstream(drop_extensions(args[1]) + ".polygon_values.csv");
for (auto &v : op_finder.result)
poly_v << v << std::endl;
auto poly_op =
std::ofstream(drop_extensions(args[1]) + ".polygon_opportunities.csv");
spartan::mdp3::poly_opportunities_to_csv(
aggregate(op_finder.result), poly_finder.result, def.ptr_result,
poly_op);
}
template <class B>
void print_book(std::ostream &books, B const &b) {
char buffer[128];
for (auto level : b) {
d_toa(level.price.price(), buffer, sizeof(buffer), 0);
books << "," << buffer << "," << level.quantity.value() << ","
<< level.n_orders;
}
}
int icebergs_to_csv(
const std::unordered_map<OrderKey, IcebergInput> &ibs,
const std::string &out_csv) {
const char *ok_col = "ok";
const char *eid_col = "eid";
const char *total_col = "total_qty";
const char *show_qty_col = "show_qty";
const char *fills_col = "fill_qty";
// validate args
if (out_csv.empty())
return 1;
// write icebergs to csv
auto ofs = std::ofstream(out_csv);
// write cols
ofs << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col
<< "," << fills_col;
// write rows
for (auto &p : ibs) {
for (auto [eid, show_qty] : p.second.show_quantities()) {
ofs << std::endl
<< p.first << "," << eid << "," << p.second.total_qty() << ","
<< show_qty << "," << p.second.count_fills_by(eid);
}
}
ofs.flush();
return 0;
}
int polygon_books(
const std::string &pcap,
const std::string &db,
const std::string &id_string,
const std::string &out_csv) {
const char *eid_col = "eid";
const char *pid_col = "pid";
const char *log_header = "PolygonBookCSV";
// the nth leg col denotes the last event in which the nth leg's book changed
const int num_leg_cols = 10;
// validate args
if (out_csv.empty())
return 1;
auto ids = bts::cmd::args::parse_set<int>(
id_string, [](const std::string &s) { return std::stoi(s); });
if (ids.empty())
return 1;
INFO(
"generating polygon books for pids: {}",
orderman::Formatter::fmt_values(ids));
SyntheticBookRecorder sbr(
spartan::sql::adapters::polygon_defs::read_sql(db.c_str(), ids));
observe_channel(pcap, sbr);
// don't prepend a comma before calling 'append_book' (it does that
// internally)
auto append_book = [](std::ofstream &out,
const SmallBook &bk) -> std::ofstream & {
uint32_t i = 0;
bts_assert(bk.size() <= 10);
for (; i < bk.size(); i++)
out << "," << bk[i].price << "," << bk[i].quantity.value();
while (i++ < Book::max_depth())
out << ",NaN,0";
return out;
};
auto append_leg_book_eids =
[&](std::ostream &out, const SyntheticMBPUpdate &upd) -> std::ostream & {
int lcount = 0;
for (const auto &p : upd.leg_book_eids) {
out << "," << p.second;
if (++lcount == num_leg_cols) {
WARN(
"{} processing book update for polygon with more than {} legs "
"({} "
"'leg-book-event-id' columns are accounted for in the csv, "
"event "
"ids for susbequent legs will be discarded).",
*log_header, num_leg_cols, num_leg_cols);
break;
}
}
while (lcount++ < num_leg_cols)
out << ",0";
return out;
};
// write polygon books to csv
auto ofs = std::ofstream(out_csv);
// add cols
ofs << eid_col << "," << pid_col;
for (int i = 0; i < num_leg_cols; i++)
ofs << ",leg_" << i << "_eid";
auto add_bk_cols = [&](const char *side_prefix) {
for (int i = 0; i < 10; i++)
ofs << "," << side_prefix << "_p_" << i << "," << side_prefix << "_q_"
<< i;
};
add_bk_cols("bid");
add_bk_cols("ask");
for (const auto &p : sbr.result()) {
for (const auto &upd : p.second) {
ofs << std::endl;
ofs << p.first << "," << upd.synthetic_id.value();
append_leg_book_eids(ofs, upd);
append_book(ofs, upd.bids);
append_book(ofs, upd.asks);
}
}
ofs.flush();
return 0;
}
size_t total_orders = 0;
void stats(std::string const &pcap) {
auto out_path = drop_extensions(drop_extensions(pcap)) + ".order_counts.csv";
auto out_file = std::ofstream{out_path};
out_file << "sid,n_orders" << std::endl;
struct StatPrinter : IMDP3FeedObserver {
StatPrinter(std::ostream &out) : out{out} {}
std::ostream &out;
void on_event_end(MDP3Event const &ev) override {
for (auto s : ev) {
total_orders += s->orders().size();
}
}
} stats{out_file};
observe_channel(pcap, stats);
std::cout << total_orders << std::endl;
}
std::map<std::string, std::function<int(std::vector<std::string> const &)>>
extensions = {
{"stats",
[](auto const &args) {
stats(args[1]);
return 0;
}},
{"value_polys",
[](auto const &args) {
value_polys(args);
return 0;
}},
{"sim",
[](auto const &args) {
run_sim(args[1]);
return 0;
}},
{"list_sec",
[](auto const &args) {
list_sec(args[1]);
return 0;
}},
{"book_stats",
[](const auto &args) {
auto out_path = args.size() > 2
? args[2]
: spartan::file::change_file_ext(
args[1], "book_stats.csv", "pcap.xz");
std::ofstream out_csv(out_path);
int res = run(args[1], BookStatRecorder(out_csv));
return res;
}},
{"find_polys",
[](auto const &args) {
find_polys(args);
return 0;
}},
{"value_polys",
[](auto const &args) {
value_polys(args);
return 0;
}},
{"validate_fifo",
[](auto const &args) {
auto ibs = iceberg_file_cache::find_icebergs(args[1]);
return run(args[1], MatcherValidationObserver(ibs));
}},
};
int Spartan::execute(std::vector<std::string> const &args) {
auto error = 1;
if (args.size() > 0) {
if (args[0] == "json" && args.size() >= 2) {
if (args.size() == 2)
error = spartan::mdp3::to_json(args[1]);
else if (args.size() == 4)
error = spartan::mdp3::to_json(
args[1], std::atoi(args[2].c_str()),
std::atoi(args[3].c_str()));
} else if (args[0] == "compress" && args.size() == 3)
error = spartan::mdp3::compress(args[1], args[2]);
else if (args[0] == "icebergs_csv" && args.size() >= 2) {
auto out_csv = args.size() > 2
? args[2]
: bts::spartan::file::change_file_ext(
args[1], "icebergs.csv", "pcap.xz");
auto ibs = IcebergFinder::find_all(args[1]);
return icebergs_to_csv(ibs, out_csv);
} else if (args[0] == "csv" && args.size() == 2)
error = spartan::mdp3::to_csv(args[1]);
else if (args[0] == "sql" && args.size() == 2)
error = spartan::mdp3::to_sql(args[1]);
else if (args[0] == "polygon_books" && args.size() >= 4) {
auto out_csv = args.size() > 4
? args[4]
: bts::spartan::file::change_file_ext(
args[1], "polygon_books.csv", "pcap.xz");
error = polygon_books(args[1], args[2], args[3], out_csv);
} else if (extensions.count(args[0]) && args.size() >= 2) {
return extensions[args[0]](args);
}
}
if (error)
std::cout << help();
return error;
}
std::string Spartan::help() {
auto str = fmt::format("\tjson <pcap>[.xz] [start_packet packet_count]\n"
"\tcompress <pcap>[.xz] <out-pcap>\n"
"\tpolygon_books <pcap>[.xz] <sqlite> <polygon_ids "
"(delim ',')> [out-csv]\n"
"\tsql <pcap>[.xz]\n"
"\tcsv <pcap>[.xz]\n");
for (auto const &p : extensions)
str += fmt::format("\t{} <pcap>[.xz] ...\n", p.first);
return str;
}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+5 -17
View File
@@ -1,22 +1,10 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
class Spartan : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
private:
int run_from_pcap(std::vector<std::string> const &);
int run_from_mcast(std::vector<std::string> const &);
int parse_pcap(std::vector<std::string> const &);
int analyze_pcap(std::vector<std::string> const &);
void run(std::vector<std::string> const &);
struct Spartan : public CLI::App {
Spartan();
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
-19
View File
@@ -1,19 +0,0 @@
#include "Test.h"
#include "ICmd.h"
#include "lib/testing/catch_test_runner.h"
namespace bts {
namespace cmd {
int Test::execute(std::vector<std::string> const &args) {
return run_tests(args);
}
std::string Test::help() {
return "\t see "
"https://github.com/philsquared/Catch/blob/master/docs/"
"command-line.md";
}
} // namespace cmd
} // namespace bts
-15
View File
@@ -1,15 +0,0 @@
#pragma once
#include "ICmd.h"
namespace bts {
namespace cmd {
class Test : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
};
} // namespace cmd
} // namespace bts
+35 -30
View File
@@ -20,42 +20,47 @@
#include <string>
#include <vector>
namespace bts {
namespace cmd {
namespace bts::cmds {
int Web::execute(const std::vector<std::string> &args) {
config::QuickbusNodeCfg qb_cfg;
auto &ports = config::get<config::Ports>();
qb_cfg.set_my_port(ports.qbus_port());
qbus::Quickbus quickbus(qb_cfg);
api::ServiceContext sc(&quickbus);
struct Web::Args {
bool md;
};
metrics::MetricsContainer stats;
metrics::publish_system_metrics(stats);
trace::PerfLog perf_log;
http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port());
Web::Web()
: CLI::App(
"Run only the services required for the web admin. This cannot be "
"run concurrently with 'sparkd run.'",
"web"),
_args(new Web::Args()) {
add_flag("--md", _args->md, "Start market data");
callback([&] {
config::QuickbusNodeCfg qb_cfg;
auto &ports = config::get<config::Ports>();
qb_cfg.set_my_port(ports.qbus_port());
qbus::Quickbus quickbus(qb_cfg);
api::ServiceContext sc(&quickbus);
user::AuthServiceServer login_server(sc);
if (std::find(args.begin(), args.end(), "-md") != args.end()) {
md::run_market_datapublishers(nullptr, nullptr);
}
metrics::MetricsContainer stats;
metrics::publish_system_metrics(stats);
trace::PerfLog perf_log;
http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port());
api::synthetic::SyntheticSecurityService synthetic_service(sc);
md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service);
md::QueryServerPtr md_query_server(
md::run_query_server(sc, mdclient, stats));
user::AuthServiceServer login_server(sc);
if (_args->md) {
md::run_market_datapublishers(nullptr, nullptr);
}
admin::AdminServiceServer admin_service(sc);
api::synthetic::SyntheticSecurityService synthetic_service(sc);
md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service);
md::QueryServerPtr md_query_server(
md::run_query_server(sc, mdclient, stats));
utils::Dispatcher::current().run();
return 0;
admin::AdminServiceServer admin_service(sc);
utils::Dispatcher::current().run();
});
}
std::string Web::help() {
return "\t run only the services required for the web admin. This cannot "
"be run concurrently with 'sparkd run'\n\tAdd -md to start market "
"data too.";
}
Web::~Web() {}
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
+10 -10
View File
@@ -1,16 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
#include "ICmd.h"
namespace bts::cmds {
namespace bts {
namespace cmd {
struct Web : public CLI::App {
Web();
~Web();
class Web : public ICmd {
public:
int execute(std::vector<std::string> const &args);
std::string help();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace cmd
} // namespace bts
} // namespace bts::cmds
@@ -72,7 +72,7 @@ void PolyObserver::eval() {
t->value = 0;
int qty = std::numeric_limits<int>::max();
for (auto &l : t->polygon.parts) {
auto mkt_side = securities::other_side(side_from_ratio(l.ratio));
auto mkt_side = side_from_ratio(l.ratio);
auto &mkt = me->books[l.sec_id].mbp_side(mkt_side);
auto this_qty = mkt[0].quantity.value();
auto this_ratio = this_qty / std::abs(l.ratio);
@@ -294,7 +294,7 @@ void SniperModel::maybe_force_fills() {
}
if (order.order.quantity.value() > total_fills) {
Price worst_price = order.order.price;
if (order.order.side == api::SIDE_BUY)
if (order.order.side == Side::Buy)
worst_price += me->sim->get_security(order.order.security)
.min_price_increment_int_price() *
100;
@@ -315,7 +315,7 @@ void SniperModel::eval(std::vector<Target *> const &targets) {
int qty = me->max_qty;
size_t p_idx = 0;
for (auto &l : t->polygon.parts) {
auto mkt_side = securities::other_side(side_from_ratio(l.ratio));
auto mkt_side = other_side(side_from_ratio(l.ratio));
auto &leg_sec_info = t->leg_sec_info[p_idx++];
auto &mkt = leg_sec_info->book.mbp_side(mkt_side);
auto this_qty = mkt[0].quantity.value();
@@ -344,7 +344,7 @@ bool SniperModel::within_safety_margin(Target const &t) const {
auto &leg_mkt = leg_mkts.emplace_back();
std::get<0>(leg_mkt) = Quantity{leg.ratio};
std::get<1>(leg_mkt) = {};
auto leg_book_side = securities::other_side(side_from_ratio(leg.ratio));
auto leg_book_side = other_side(side_from_ratio(leg.ratio));
std::get<2>(leg_mkt) = me->books[leg.sec_id].book.mbp_side(leg_book_side);
}
@@ -439,8 +439,8 @@ void SniperModel::fire(Shot const &s) {
if (me->ms_to_work_shot == 0) {
shot.forced_completion = true;
for (auto &p : shot.target.polygon.parts) {
auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL;
auto mkt_side = securities::other_side(part_side);
auto part_side = from_ratio(p.ratio);
auto mkt_side = other_side(part_side);
auto &book = me->books[p.sec_id].book.mbp_side(mkt_side);
Price worst_price;
@@ -453,7 +453,7 @@ void SniperModel::fire(Shot const &s) {
break;
}
}
if (part_side == api::SIDE_BUY)
if (part_side == Side::Buy)
worst_price +=
me->sim->get_security(p.sec_id).min_price_increment_int_price() *
100;
@@ -476,8 +476,8 @@ void SniperModel::fire(Shot const &s) {
}
} else {
for (auto &p : shot.target.polygon.parts) {
auto part_side = p.ratio > 0 ? api::SIDE_BUY : api::SIDE_SELL;
auto mkt_side = securities::other_side(part_side);
auto part_side = from_ratio(p.ratio);
auto mkt_side = other_side(part_side);
auto &book = me->books[p.sec_id].book.mbp_side(mkt_side);
auto limit_price = book[0].price;
@@ -485,7 +485,7 @@ void SniperModel::fire(Shot const &s) {
me->sim->get_security(p.sec_id).min_price_increment_int_price() *
me->pay_up_ticks;
if (part_side == api::SIDE_BUY)
if (part_side == Side::Buy)
limit_price += pay_up_adj;
else
limit_price -= pay_up_adj;
@@ -498,7 +498,7 @@ void SniperModel::fire(Shot const &s) {
order_change.change = OrderChange::added;
order_change.order.key = me->sim->send(order_change);
me->sim->log(
"sent_order", api::Side_Name(part_side),
"sent_order", Side_Name(part_side),
me->sim->get_security(p.sec_id).symbol(), book[0].price.to_double(),
book[0].price.to_double(), book[0].quantity.value());
shot.working[order_change.order.key].order = order_change;
@@ -95,8 +95,8 @@ void FifoImpliedSecurityMatcher::print_legs(std::ostream &out) const {
<< " " << leg_size;
if (append_bbo) {
append_bbo = false;
auto bid = top_of_book(api::Side::SIDE_BUY);
auto ask = top_of_book(api::Side::SIDE_SELL);
auto bid = top_of_book(Side::Buy);
auto ask = top_of_book(Side::Sell);
out << std::setw(8) << FMT("(", bid.quantity, ") ") << std::setw(6)
<< bid.price << std::setw(6) << ask.price
<< FMT("(", ask.quantity, ")");
@@ -108,7 +108,7 @@ void FifoImpliedSecurityMatcher::print_legs(std::ostream &out) const {
///////////////////////
// book related
///////////////////////
ImpliedBookSide FifoImpliedSecurityMatcher::book(api::Side side) const {
ImpliedBookSide FifoImpliedSecurityMatcher::book(Side side) const {
// this isn't the best way to do this (copying),
// but the ImpliedBook only has 2 levels in practice,
// so it should be fine.
@@ -122,7 +122,7 @@ ImpliedBookSide FifoImpliedSecurityMatcher::book(api::Side side) const {
return ib;
}
SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(api::Side side) const {
SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(Side side) const {
SmallBook b;
_book(side, b, 1);
if (b.empty())
@@ -131,18 +131,18 @@ SmallBookLevel FifoImpliedSecurityMatcher::top_of_book(api::Side side) const {
}
void FifoImpliedSecurityMatcher::_book(
api::Side side, SmallBook &dest, uint32_t depth) const {
Side side, SmallBook &dest, uint32_t depth) const {
std::vector<std::tuple<Quantity, Price, SmallBook>> im_vec;
for (const auto &p : _me->implied_mkt) {
SmallBook bk;
auto [sec_matcher, leg_qty] = p.second;
bts_assert(leg_qty.value() != 0);
auto leg_side = leg_qty.value() > 0 ? securities::other_side(side) : side;
auto leg_side = leg_qty.value() > 0 ? side : other_side(side);
for (auto bl : sec_matcher->book(leg_side))
bk.emplace_back(bl.price, bl.quantity);
im_vec.emplace_back(leg_qty, Price{}, bk);
}
return book_utils::compute_synthetic_book(dest, im_vec, depth);
book_utils::compute_synthetic_book(dest, im_vec, depth);
}
void FifoImpliedSecurityMatcher::add_leg(
@@ -156,12 +156,12 @@ void FifoImpliedSecurityMatcher::add_leg(
Quantity FifoImpliedSecurityMatcher::fill(
OrderKey key,
api::Side side,
Side side,
Quantity qty,
std::vector<MatchEffect> *effects,
FillDepot &fills) {
SmallBook b;
auto oside = securities::other_side(side);
auto oside = other_side(side);
_book(oside, b, ImpliedBookSide::max_depth());
auto orig_qty = qty;
// can't fill more than the volume in the book
@@ -174,8 +174,7 @@ Quantity FifoImpliedSecurityMatcher::fill(
auto leg_size = std::get<1>(kv.second);
// todo: support other leg sizes (keeping it simple for now)
bts_assert(leg_size.value() == 1 || leg_size.value() == -1);
api::Side leg_side =
leg_size.value() > 0 ? side : securities::other_side(side);
Side leg_side = leg_size.value() > 0 ? side : other_side(side);
FifoSecurityMatcher &m = *std::get<0>(kv.second);
if (local_fills)
effects = new std::vector<MatchEffect>();
@@ -20,10 +20,10 @@ struct FifoImpliedSecurityMatcher {
FifoImpliedSecurityMatcher(SecurityID security_id, std::string symbol);
~FifoImpliedSecurityMatcher();
ImpliedBookSide book(api::Side) const;
ImpliedBookSide book(Side) const;
Quantity fill(
OrderKey,
api::Side,
Side,
Quantity,
std::vector<MatchEffect> *,
FillDepot &);
@@ -42,7 +42,7 @@ struct FifoImpliedSecurityMatcher {
std::string leg_sym);
SecurityID security_id() const;
std::string symbol() const;
SmallBookLevel top_of_book(api::Side side) const;
SmallBookLevel top_of_book(Side side) const;
void print_legs(std::ostream &out = std::cout) const;
private:
@@ -52,7 +52,7 @@ struct FifoImpliedSecurityMatcher {
FifoImpliedSecurityMatcher(const FifoImpliedSecurityMatcher &);
std::string _log_header() const;
void _book(api::Side side, SmallBook &dest, uint32_t depth) const;
void _book(Side side, SmallBook &dest, uint32_t depth) const;
uint32_t _fill_order(const MDP3Order &order);
};
} // namespace bts::spartan::mdp3
@@ -23,11 +23,11 @@ namespace bts::spartan::mdp3 {
///////////////////////
// static inline std::string _log_header() { return "FifoMatcherContext"; }
static inline Quantity _norm_qty(api::Side side, Quantity qty) {
static inline Quantity _norm_qty(Side side, Quantity qty) {
switch (side) {
case api::Side::SIDE_BUY:
case Side::Buy:
return qty;
case api::Side::SIDE_SELL:
case Side::Sell:
return -qty;
default:
bts_assert_msg(false, FMT("unexpected side '", side, "'"));
@@ -130,12 +130,12 @@ void FifoMatcherContext::implied_matching_status(
///////////////////////
// book
///////////////////////
BookSide FifoMatcherContext::book(SecurityID sec_id, api::Side side) {
BookSide FifoMatcherContext::book(SecurityID sec_id, Side side) {
return _get_matcher(sec_id)->book(side);
}
ImpliedBookSide
FifoMatcherContext::implied_book(SecurityID sec_id, api::Side side) {
FifoMatcherContext::implied_book(SecurityID sec_id, Side side) {
return _get_matcher(sec_id)->implied_book(side);
}
@@ -222,8 +222,8 @@ using BBORow = std::tuple<
void FifoMatcherContext::print_all_bbos(std::ostream &out) {
std::vector<BBORow> rows;
for (auto kv : _me->matchers) {
auto bid = kv.second->matcher()->book_level(api::Side::SIDE_BUY, 0);
auto ask = kv.second->matcher()->book_level(api::Side::SIDE_SELL, 0);
auto bid = kv.second->matcher()->book_level(Side::Buy, 0);
auto ask = kv.second->matcher()->book_level(Side::Sell, 0);
rows.emplace_back(std::make_tuple(
_me->symbols[kv.first], std::get<0>(bid), std::get<1>(bid),
std::get<0>(ask), std::get<1>(ask)));
@@ -31,8 +31,8 @@ struct FifoMatcherContext {
void implied_matching_status(SecurityID sec_id, bool value);
bool implied_matching_status(SecurityID sec_id) const;
void clear();
BookSide book(SecurityID sec_id, api::Side side);
ImpliedBookSide implied_book(SecurityID sec_id, api::Side side);
BookSide book(SecurityID sec_id, Side side);
ImpliedBookSide implied_book(SecurityID sec_id, Side side);
// the Fills param records individual fills (potentially more than one per
// order)
@@ -85,11 +85,10 @@ struct FifoMatcherContext {
struct Comp {
bool operator()(const MDP3Order &a, const MDP3Order &b) {
if (a.side == b.side) {
return bts::orderman::price_comp::is_better(
a.side, a.price, b.price) ||
return a.price.is_better(a.side, b.price) ||
(a.price == b.price && a.priority < b.priority);
}
return a.side == api::Side::SIDE_BUY;
return a.side == Side::Buy;
}
};
std::set<MDP3Order, Comp> sorted_orders;
@@ -102,10 +101,10 @@ struct FifoMatcherContext {
effects->emplace_back(MBPUpdate());
MBPUpdate &mbp = std::get<MBPUpdate>(effects->back());
mbp.security_id = sec_id;
mbp.mbp_buy = matcher->book(api::Side::SIDE_BUY);
mbp.mbp_sell = matcher->book(api::Side::SIDE_SELL);
mbp.implied_mbp_buy = matcher->implied_book(api::Side::SIDE_BUY);
mbp.implied_mbp_sell = matcher->implied_book(api::Side::SIDE_SELL);
mbp.mbp_buy = matcher->book(Side::Buy);
mbp.mbp_sell = matcher->book(Side::Sell);
mbp.implied_mbp_buy = matcher->implied_book(Side::Buy);
mbp.implied_mbp_sell = matcher->implied_book(Side::Sell);
}
}
};
@@ -91,15 +91,14 @@ FifoMergedSecurityMatcher::add_implied_matcher(SecurityID sec) {
return m;
}
FifoImpliedSecurityMatcher *
FifoMergedSecurityMatcher::_best_implied_matcher(api::Side side) {
FifoMergedSecurityMatcher::_best_implied_matcher(Side side) {
if (!_me->implied_matching_status)
return nullptr;
bts::orderman::SidePriceComp cmp(side);
FifoImpliedSecurityMatcher *rv = nullptr;
Price best_price;
for (auto &im : _me->implied_matchers) {
auto itop = im->top_of_book(side);
if (itop.quantity.value() > 0 && cmp(itop.price, best_price)) {
if (itop.quantity.value() > 0 && itop.price.is_better(side, best_price)) {
rv = im;
best_price = itop.price;
}
@@ -137,8 +136,7 @@ void FifoMergedSecurityMatcher::on_order_change(
"unable to process order-update event: failed to find order.");
break;
}
if (orderman::price_comp::is_better(
ch.order.side, ch.order.price, prev_ord->price)) {
if (ch.order.price.is_better(ch.order.side, prev_ord->price)) {
// price improved, try to fill the order
auto new_ch = ch;
_fill(new_ch.order, effects, fills, false);
@@ -174,10 +172,10 @@ void FifoMergedSecurityMatcher::on_order_change(
/////////////////////////
// books
/////////////////////////
BookSide FifoMergedSecurityMatcher::book(api::Side side) {
BookSide FifoMergedSecurityMatcher::book(Side side) {
return matcher()->book(side);
}
ImpliedBookSide FifoMergedSecurityMatcher::implied_book(api::Side side) {
ImpliedBookSide FifoMergedSecurityMatcher::implied_book(Side side) {
if (_me->implied_matchers.empty())
return ImpliedBookSide();
// In theory, computing the quantity of any price level beyond the top of the
@@ -195,8 +193,8 @@ ImpliedBookSide FifoMergedSecurityMatcher::implied_book(api::Side side) {
// the only functionality needed is forward iteration,
// and erase (the latter for optimization).
std::vector<std::tuple<ImpliedBookSide, int>> ibooks;
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp cmp(oside);
auto oside = other_side(side);
bts::orderman::SidePriceComp cmp(map_side(oside));
for (auto it = _me->implied_matchers.begin();
it != _me->implied_matchers.end(); it++) {
auto res = ibooks.emplace_back((*it)->book(oside), 0);
@@ -239,11 +237,11 @@ ImpliedBookSide FifoMergedSecurityMatcher::implied_book(api::Side side) {
}
void FifoMergedSecurityMatcher::merged_book(
api::Side side, std::vector<BookLevel> &dest) {
Side side, std::vector<BookLevel> &dest) {
//
std::vector<std::tuple<ImpliedBookSide, int>> ibooks;
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp cmp(oside);
auto oside = other_side(side);
bts::orderman::SidePriceComp cmp(map_side(oside));
for (auto it = _me->implied_matchers.begin();
it != _me->implied_matchers.end(); it++) {
auto res = ibooks.emplace_back((*it)->book(oside), 0);
@@ -303,11 +301,6 @@ void FifoMergedSecurityMatcher::_fill(
std::vector<MatchEffect> *effects,
FillDepot &fills,
bool reset_show_qty) {
if (order.iceberg_quantity.has_value()) {
// the order's quantity cannot exceed the show qty (but it can be
// smaller).
bts_assert(order.quantity <= order.iceberg_quantity.value().show);
}
auto rem = _fill(
order.key, order.side, order.price, get_total_qty(order), effects,
fills);
@@ -322,14 +315,14 @@ tuple_to_small_book_level(std::tuple<Price, Quantity> t) {
Quantity FifoMergedSecurityMatcher::_fill(
OrderKey key,
api::Side side,
Side side,
Price price,
Quantity quantity,
std::vector<MatchEffect> *effects,
FillDepot &fills) {
//
auto oside = bts::securities::other_side(side);
bts::orderman::SidePriceComp oside_comp(oside);
auto oside = other_side(side);
bts::orderman::SidePriceComp oside_comp(map_side(oside));
while (quantity.value() > 0) {
// Fill one price level at a time.
// While one market may have the best 1st price level,
@@ -29,9 +29,9 @@ struct FifoMergedSecurityMatcher {
const FifoSecurityMatcher *matcheri() const;
const std::vector<FifoImpliedSecurityMatcher *> &implied_matchers();
// books
BookSide book(api::Side);
ImpliedBookSide implied_book(api::Side);
void merged_book(api::Side, std::vector<BookLevel> &);
BookSide book(Side);
ImpliedBookSide implied_book(Side);
void merged_book(Side, std::vector<BookLevel> &);
private:
struct PImpl;
@@ -39,7 +39,7 @@ struct FifoMergedSecurityMatcher {
// hide copy constr
FifoMergedSecurityMatcher(const FifoMergedSecurityMatcher &);
FifoImpliedSecurityMatcher *_best_implied_matcher(api::Side);
FifoImpliedSecurityMatcher *_best_implied_matcher(Side);
// the order's params (quantity, and iceberg_quantity if applicable)
// will be appropriately adjusted according to the number of fills.
void _fill(
@@ -49,7 +49,7 @@ struct FifoMergedSecurityMatcher {
bool reset_show_qty);
Quantity _fill(
OrderKey,
api::Side,
Side,
Price,
Quantity,
std::vector<MatchEffect> *,
@@ -15,10 +15,6 @@ inline bool in_this_level(MatcherOrder const *o_ptr) {
return o_ptr != nullptr && o_ptr->is_active;
}
api::Side _opp_side(api::Side side) {
return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL
: api::Side::SIDE_BUY;
}
// O(n) linear search
MatcherOrder *FifoPriceLevelMatcher::_find(OrderKey key) {
for (MatcherOrder *op : _orders)
@@ -93,7 +89,7 @@ FifoPriceLevelMatcher::_one_pass_cross_order(
? OrderUpdate::Status::PARTIALLY_FILLED
: OrderUpdate::Status::FILLED,
_sec,
_opp_side(_side),
other_side(_side),
Quantity(filled),
_price,
OrderUpdate::FillType::AGGRESSIVE};
@@ -141,7 +137,7 @@ FifoPriceLevelMatcher::FifoPriceLevelMatcher(
std::map<OrderKey, MatcherOrder> &master_order_map,
SecurityID sec,
Price price,
api::Side side,
Side side,
const Trace &trace)
: _master_order_map(&master_order_map), _sec(sec), _price(price),
_side(side), _orders(), _aux_usage(), _priority(0), _trace(trace) {}
@@ -177,8 +173,7 @@ FifoPriceLevelMatcher::operator=(FifoPriceLevelMatcher &&o) {
}
bool FifoPriceLevelMatcher::operator<(const FifoPriceLevelMatcher &o) const {
bts_assert(_side == o._side);
// bid book is sorted high to low (since highest is the top of the book)
return _side == api::Side::SIDE_SELL ? _price < o._price : o._price < _price;
return _price.is_better(_side, o._price);
}
// simple accessors ///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
@@ -296,7 +291,7 @@ void FifoPriceLevelMatcher::cleanup() {
std::string FifoPriceLevelMatcher::print_mbo(size_t n_orders) const {
std::ostringstream buff;
auto [n_ords, qty] = n_orders_qty();
buff << (side() == api::Side::SIDE_BUY ? "B" : "S") << "," << price();
buff << (side() == Side::Buy ? "B" : "S") << "," << price();
buff << "," << qty << "," << n_ords << ":";
size_t i = 0;
for (MatcherOrder *o : _orders) {
@@ -19,7 +19,7 @@ class FifoPriceLevelMatcher {
std::map<OrderKey, MatcherOrder> *_master_order_map;
SecurityID _sec;
Price _price;
api::Side _side;
Side _side;
std::vector<MatcherOrder *> _orders;
std::vector<MatcherOrder *> _aux_usage;
uint32_t _priority;
@@ -52,7 +52,7 @@ class FifoPriceLevelMatcher {
std::map<OrderKey, MatcherOrder> &,
SecurityID sec,
Price price,
api::Side side,
Side side,
const Trace &);
FifoPriceLevelMatcher(const FifoPriceLevelMatcher &);
FifoPriceLevelMatcher(FifoPriceLevelMatcher &&);
@@ -82,7 +82,7 @@ class FifoPriceLevelMatcher {
bool populated() const;
Price price() const { return _price; }
double price_double() const { return _price.to_double(); }
api::Side side() const { return _side; }
Side side() const { return _side; }
MatcherOrder *find_order(OrderKey key) { return _find(key); }
BookLevel book_level() const {
return BookLevel{_price, quantity(), static_cast<int>(n_orders())};
@@ -90,7 +90,7 @@ class FifoPriceLevelMatcher {
// used for testing
void copy_values_to(std::vector<MatcherOrder> &out_v) const;
// mutators (silly reason) /////////////////////////////////////////////////
void set_price_side(Price p, api::Side s) {
void set_price_side(Price p, Side s) {
_price = p;
_side = s;
}
@@ -12,9 +12,9 @@ namespace bts::spartan::mdp3 {
inline bool
_is_cross(const MDP3Order &ord, const FifoPriceLevelMatcher &price_level) {
return ord.side != price_level.side() &&
((ord.side == api::Side::SIDE_BUY &&
((ord.side == Side::Buy &&
ord.price >= price_level.price()) ||
(ord.side == api::Side::SIDE_SELL &&
(ord.side == Side::Sell &&
ord.price <= price_level.price()));
}
@@ -23,14 +23,9 @@ inline Quantity total_quantity(const MDP3Order &ord) {
: ord.quantity;
}
inline api::Side _opp_side(api::Side side) {
return side == api::Side::SIDE_BUY ? api::Side::SIDE_SELL
: api::Side::SIDE_BUY;
}
// O(lg(N)) on the price levels
FifoPriceLevelMatcher *
FifoSecurityMatcher::_find_price_level(api::Side side, Price price) {
FifoSecurityMatcher::_find_price_level(Side side, Price price) {
auto &bk = _book(side);
_search_tok.set_price_side(price, side);
auto it = bk.find(_search_tok);
@@ -51,7 +46,7 @@ FifoSecurityMatcher::_find_order(OrderKey key) {
return std::tuple<FifoPriceLevelMatcher *, MatcherOrder *>(nullptr, nullptr);
}
FifoPriceLevelMatcher *FifoSecurityMatcher::_top(api::Side side) {
FifoPriceLevelMatcher *FifoSecurityMatcher::_top(Side side) {
for (auto &plevel : _book(side))
if (plevel.populated())
return &plevel;
@@ -81,7 +76,7 @@ void FifoSecurityMatcher::_cleanup() {
}
FifoPriceLevelMatcher *FifoSecurityMatcher::_get_or_add_price_level(
api::Side side, Price price) {
Side side, Price price) {
auto *price_level = _find_price_level(side, price);
if (price_level == nullptr) {
auto &same_side_book = _book(side);
@@ -100,7 +95,7 @@ void FifoSecurityMatcher::_add_resting_order(const MDP3Order &ord) {
void FifoSecurityMatcher::_on_new_order(
const MDP3Order &ord, std::vector<MatchEffect> *res) {
_log_trace_order(ord, "FifoSecurityMatcher::_on_new_order begin");
auto &opp_book = _book(_opp_side(ord.side));
auto &opp_book = _book(other_side(ord.side));
uint32_t new_ord_rem = total_quantity(ord).value();
for (FifoPriceLevelMatcher &pl : opp_book) {
if (_is_cross(ord, pl)) {
@@ -220,7 +215,7 @@ void FifoSecurityMatcher::_on_order_not_affecting_books(
case OrderChange::updated_price: {
// since this does not affect the market all we need to do
// is deal with the cross
auto &opp_book = _book(_opp_side(ord.side));
auto &opp_book = _book(other_side(ord.side));
uint32_t new_ord_rem = total_quantity(ord).value();
for (const FifoPriceLevelMatcher &pl : opp_book) {
if (!pl.populated())
@@ -273,7 +268,7 @@ FifoSecurityMatcher::FifoSecurityMatcher(
_order_map,
SecurityID{},
Price(),
api::Side::SIDE_BUY,
Side::Buy,
Trace()),
_sec(args.security_id), _event_id(args.event_id), _trace(),
_ignore_ops_on_deleted_orders(args.ignore_ops_on_deleted_orders),
@@ -325,23 +320,23 @@ const BookSide &FifoSecurityMatcher::BookKeeper::mbp() {
}
return _mbp;
}
BookSide FifoSecurityMatcher::book(api::Side side) {
return (side == api::Side::SIDE_BUY ? _bids : _asks).mbp();
BookSide FifoSecurityMatcher::book(Side side) {
return (side == Side::Buy ? _bids : _asks).mbp();
}
size_t FifoSecurityMatcher::total_n_orders(api::Side side) const {
size_t FifoSecurityMatcher::total_n_orders(Side side) const {
auto rv = 0;
for (const auto &p_level : _book(side))
rv += p_level.n_orders();
return rv;
}
Quantity FifoSecurityMatcher::total_qty(api::Side side) const {
Quantity FifoSecurityMatcher::total_qty(Side side) const {
auto rv = 0;
for (const auto &p_level : _book(side))
rv += p_level.total_quantity().value();
return Quantity(rv);
}
std::tuple<Price, Quantity>
FifoSecurityMatcher::book_level(api::Side side, size_t idx) const {
FifoSecurityMatcher::book_level(Side side, size_t idx) const {
size_t i = 0;
for (const auto &p_level : _book(side))
if (p_level.populated() && i++ == idx)
@@ -384,19 +379,18 @@ void FifoSecurityMatcher::on_order_completed(OrderKey key) {
ord != nullptr, "did not find order that was supposedely 'completed.'");
MDP3Order order;
order.key = key;
order.side = api::SIDE_UNDEFINED;
SmallOrderChange ch(order, OrderChange::ChangeType::removed);
on_order_change(ch);
}
Quantity FifoSecurityMatcher::fill(
OrderKey aggr_key,
api::Side side,
Side side,
Price agg_price,
Quantity qty,
std::vector<MatchEffect> *res,
bool add_aggr_fill) {
FifoPriceLevelMatcher *top = _top(_opp_side(side));
FifoPriceLevelMatcher *top = _top(other_side(side));
if (top != nullptr) {
bts_assert(top->price() == agg_price || agg_price.is_nan());
auto t = top->on_new_opp_order(aggr_key, qty, add_aggr_fill, res);
@@ -465,7 +459,7 @@ std::string FifoSecurityMatcher::print_book(size_t n_levels) {
// auto rem_if = [](const auto& p_lev) { return !p_lev.populated(); };
std::ostringstream buff;
auto add_to_buff = [&](const auto &pl) {
std::string side = pl.side() == api::Side::SIDE_BUY ? "B" : "S";
std::string side = pl.side() == Side::Buy ? "B" : "S";
buff << side << ":" << pl.price_double() << ":" << pl.quantity().value()
<< ":" << pl.n_orders() << std::endl;
};
@@ -59,13 +59,13 @@ class FifoSecurityMatcher : public SecurityMatcher {
// helpers /////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// complexity: O(lg(n))
FifoPriceLevelMatcher *_find_price_level(api::Side, Price price);
boost::container::flat_set<FifoPriceLevelMatcher> &_book(api::Side side) {
return (side == api::Side::SIDE_BUY ? _bids : _asks).book();
FifoPriceLevelMatcher *_find_price_level(Side, Price price);
boost::container::flat_set<FifoPriceLevelMatcher> &_book(Side side) {
return (side == Side::Buy ? _bids : _asks).book();
}
const boost::container::flat_set<FifoPriceLevelMatcher> &
_book(api::Side side) const {
return (side == api::Side::SIDE_BUY ? _bids : _asks).booki();
_book(Side side) const {
return (side == Side::Buy ? _bids : _asks).booki();
}
void _invalidate_mbp() {
_bids.invalidate();
@@ -76,7 +76,7 @@ class FifoSecurityMatcher : public SecurityMatcher {
// for price level in the book (flat_set)
std::tuple<FifoPriceLevelMatcher *, MatcherOrder *>
_find_order(OrderKey key);
FifoPriceLevelMatcher *_top(api::Side side);
FifoPriceLevelMatcher *_top(Side side);
void _cleanup();
// order interaction
void _on_new_order(const MDP3Order &, std::vector<MatchEffect> * = nullptr);
@@ -99,7 +99,7 @@ class FifoSecurityMatcher : public SecurityMatcher {
~FifoSecurityMatcher();
// SecurityMatcher implementation //////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
BookSide book(api::Side) override;
BookSide book(Side) override;
// The qty, and add_aggr_fill params address a couple of issues wrt 'merged
// matching:' Particular quantities of passive fills need to be generated
// when an aggressor order is filled against an implied market, while the
@@ -116,20 +116,20 @@ class FifoSecurityMatcher : public SecurityMatcher {
// necessarily have the 2nd best price as well.
Quantity fill(
OrderKey aggr_key,
api::Side side,
Side side,
Price price,
Quantity qty,
std::vector<MatchEffect> *res,
bool add_aggr_fill);
std::tuple<Price, Quantity>
book_level(api::Side side, size_t idx) const override;
book_level(Side side, size_t idx) const override;
void on_order_change(
SmallOrderChange const &, std::vector<MatchEffect> * = nullptr) override;
void on_order_completed(OrderKey key);
// For testing
std::optional<std::string> has_same_orders(Orders const &);
size_t total_n_orders(api::Side) const;
Quantity total_qty(api::Side) const;
size_t total_n_orders(Side) const;
Quantity total_qty(Side) const;
// Removes existing orders.
void clear();
// Removes existing orders, then populates the book with the
@@ -170,6 +170,6 @@ class FifoSecurityMatcher : public SecurityMatcher {
// into the matcher, as the order's visible quantity is often less than
// the show quantity (e.g., after partial fills, before the next reload).
void _add_resting_order(const MDP3Order &);
FifoPriceLevelMatcher *_get_or_add_price_level(api::Side side, Price);
FifoPriceLevelMatcher *_get_or_add_price_level(Side side, Price);
};
} // namespace bts::spartan::mdp3
@@ -1,50 +0,0 @@
#pragma once
#include "lib/md/IntPrice.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/fill_depot.h"
#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h"
#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h"
#include <functional>
#include <iostream>
#include <map>
#include <string>
#include <tuple>
#include <vector>
namespace bts::spartan::mdp3 {
struct ImpliedFifoSecurityMatcher {
ImpliedFifoSecurityMatcher(SecurityID security_id)
: _security_id(security_id), _implied_mkt() {}
ImpliedBook book(api::Side side) const;
Quantity
fill(OrderKey key, api::Side side, Quantity qty, FillDepot &fills);
// Note: this class is NOT responsible for deleting any FifoSecurityMatcher
// pointers. The FifoMatcherContext creates one FifoMergedSecurityMatcher per
// each security, and each FifoMergedSecurityMatcher creates/owns a
// FifoSecurityMatcher pointer (and is responsible for deleting it). The
// FifoSecurityMatcher pointer passed to this method should be owned by a
// FifoMergedSecurityMatcher.
// Note the absense of a 'side' parameter: unit_qty should be positive for
// Buys, negative for Sells.
void add_leg(
SecurityID sec_id, FifoSecurityMatcher *m, Quantity unit_qty) {
_implied_mkt.insert({sec_id, {m, unit_qty}});
}
SecurityID security_id() const { return _security_id; }
std::tuple<Price, Quantity> top_of_book(api::Side side) const;
private:
SecurityID _security_id;
std::map<SecurityID, std::tuple<FifoSecurityMatcher *, Quantity>>
_implied_mkt;
// hide copy ctr
ImpliedFifoSecurityMatcher(const ImpliedFifoSecurityMatcher &);
void _book(
api::Side side,
std::vector<std::tuple<Price, Quantity>> &dest) const;
uint32_t _fill_order(const MDP3Order &order);
};
} // namespace bts::spartan::mdp3
@@ -50,10 +50,10 @@ template <int L> inline bool mbp_equals(const MBP<L> &a, const MBP<L> &b) {
struct MBPData {
MBPData() {}
MBPData(FifoMergedSecurityMatcher &mat)
: bids(mat.book(api::Side::SIDE_BUY)),
asks(mat.book(api::Side::SIDE_SELL)),
ibids(mat.implied_book(api::Side::SIDE_BUY)),
iasks(mat.implied_book(api::Side::SIDE_SELL)) {}
: bids(mat.book(Side::Buy)),
asks(mat.book(Side::Sell)),
ibids(mat.implied_book(Side::Buy)),
iasks(mat.implied_book(Side::Sell)) {}
BookSide bids, asks;
ImpliedBookSide ibids, iasks;
};
@@ -111,6 +111,11 @@ void MergedMatcher::match(
std::vector<MatchEffect> *res) {
_match_count_this_per++;
_match_count++;
// The caller is not obligated to pass an empty vector. Consider an event
// that involves multiple securities; perhaps the caller wants to put the
// effects for each security in the same vector. The elements that were
// already present in the effects vector are skipped/ignored.
uint32_t effects_origin = res == nullptr ? 0 : res->size();
// This could be better optimized, but the idea is to
// 1) take a snapshot of every security's book/implied book,
// 2) process the order changes,
@@ -152,18 +157,20 @@ void MergedMatcher::match(
_asset_matcher(c.order.security).on_order_change(c, res, fills);
}
// log/validate order updates
for (const auto &u : *res)
_trace.log_outgoing_update(std::get<OrderUpdate>(u));
for (const auto &u : *res)
MatcherUtils::validate_outgoing_update(std::get<OrderUpdate>(u));
// add mbp updates
for (auto &ass_kv : mbps) {
for (auto &sec_kv : ass_kv.second) {
auto new_mbp = sec_kv.second.get_new_mbp();
if (new_mbp.has_value()) {
res->emplace_back(MBPUpdate{
sec_kv.first, new_mbp.value().bids, new_mbp.value().asks,
new_mbp.value().ibids, new_mbp.value().iasks});
if (res != nullptr) {
for (auto it = res->begin() + effects_origin; it != res->end(); it++)
_trace.log_outgoing_update(std::get<OrderUpdate>(*it));
for (auto it = res->begin() + effects_origin; it != res->end(); it++)
MatcherUtils::validate_outgoing_update(std::get<OrderUpdate>(*it));
// add mbp updates
for (auto &ass_kv : mbps) {
for (auto &sec_kv : ass_kv.second) {
auto new_mbp = sec_kv.second.get_new_mbp();
if (new_mbp.has_value()) {
res->emplace_back(MBPUpdate{
sec_kv.first, new_mbp.value().bids, new_mbp.value().asks,
new_mbp.value().ibids, new_mbp.value().iasks});
}
}
}
}
@@ -179,11 +186,11 @@ void MergedMatcher::match(
}
}
BookSide MergedMatcher::book(SecurityID id, api::Side s) {
BookSide MergedMatcher::book(SecurityID id, Side s) {
return _asset_matcher(id).book(id, s);
}
ImpliedBookSide MergedMatcher::implied_book(SecurityID id, api::Side s) {
ImpliedBookSide MergedMatcher::implied_book(SecurityID id, Side s) {
return _asset_matcher(id).implied_book(id, s);
}
@@ -235,8 +242,8 @@ void DirectMatcher::match(
// books
for (auto cm : dirty)
res->emplace_back(MBPUpdate{
cm.first, cm.second->book(api::Side::SIDE_BUY),
cm.second->book(api::Side::SIDE_SELL), ImpliedBookSide(), ImpliedBookSide()});
cm.first, cm.second->book(Side::Buy),
cm.second->book(Side::Sell), ImpliedBookSide(), ImpliedBookSide()});
auto now = TimeUtil::GetCurrentTime();
auto usecs = TimeUtil::TimestampToMicroseconds(now) -
TimeUtil::TimestampToMicroseconds(_per_start_time);
@@ -249,7 +256,7 @@ void DirectMatcher::match(
}
}
BookSide DirectMatcher::book(SecurityID id, api::Side s) {
BookSide DirectMatcher::book(SecurityID id, Side s) {
auto it = _matcher_map.find(id);
bts_assert(it != _matcher_map.end());
return it->second.book(s);
@@ -40,8 +40,8 @@ class MergedMatcher {
void
match(ChangeSrc, SmallOrderChangeVec const &, std::vector<MatchEffect> *);
BookSide book(SecurityID, api::Side);
ImpliedBookSide implied_book(SecurityID, api::Side);
BookSide book(SecurityID, Side);
ImpliedBookSide implied_book(SecurityID, Side);
template <class TIter> void init(TIter first, TIter last) {
while (first != last) {
@@ -84,8 +84,8 @@ class DirectMatcher {
void
match(ChangeSrc, SmallOrderChangeVec const &, std::vector<MatchEffect> *);
BookSide book(SecurityID, api::Side);
ImpliedBookSide implied_book(SecurityID, api::Side);
BookSide book(SecurityID, Side);
ImpliedBookSide implied_book(SecurityID, Side);
template <class Iter> void reset(SecurityID sec, Iter first, Iter last) {
_matcher_map.try_emplace(sec).first->second.reset(first, last);
@@ -24,21 +24,18 @@ struct Trace {
bool is_ignored_event(size_t eid) const {
return ignored_events.find(eid) != ignored_events.end();
}
std::string _side(api::Side s) {
return s == api::Side::SIDE_BUY ? "BUY" : "SELL";
}
void log_incoming_change(const SmallOrderChange &c) {
if (traced_orders.find(c.order.key) != traced_orders.end())
LOG(OrderMan, Diag,
"Matcher incoming change:{} ok:{} side:{} price:{} qty:{} ib:{}",
c.change, c.order.key, _side(c.order.side), c.order.price,
c.change, c.order.key, c.order.side, c.order.price,
c.order.quantity, c.order.iceberg_quantity);
}
void log_outgoing_update(const OrderUpdate &u) {
if (traced_orders.find(u.key) != traced_orders.end())
LOG(OrderMan, Diag,
"Matcher outgoing update:{} ok:{} side:{} last_fp:{} last_fq:{}",
u.status, u.key, _side(u.side), u.last_price, u.last_fills);
u.status, u.key, u.side, u.last_price, u.last_fills);
}
};
@@ -86,7 +83,7 @@ struct MatcherOrder {
return std::make_tuple(resting_qty, cross_qty);
}
OrderKey key;
api::Side side;
Side side;
Price price;
Quantity quantity;
uint32_t priority = 0;
@@ -94,7 +91,7 @@ struct MatcherOrder {
bool is_active = true;
MatcherOrder(
OrderKey key,
api::Side side,
Side side,
Price price,
Quantity qty, // working qty out there
uint32_t priority,
@@ -13,9 +13,9 @@ class SecurityMatcher {
// for performance reasons. Its internal state will change
// when this method is called if its book is dirty,
// and has to be recomputed.
virtual BookSide book(api::Side) = 0;
virtual BookSide book(Side) = 0;
virtual std::tuple<Price, Quantity>
book_level(api::Side, size_t level = 0) const = 0;
book_level(Side, size_t level = 0) const = 0;
// mutators
virtual void on_order_change(
SmallOrderChange const &, std::vector<MatchEffect> * = nullptr) = 0;
@@ -2,11 +2,11 @@
#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h"
namespace bts::spartan::mdp3 {
inline MDP3Fill::Aggressor _to_aggressor(api::Side s) {
inline MDP3Fill::Aggressor _to_aggressor(Side s) {
switch (s) {
case api::Side::SIDE_BUY:
case Side::Buy:
return MDP3Fill::Aggressor::THIS_BUY;
case api::Side::SIDE_SELL:
case Side::Sell:
return MDP3Fill::Aggressor::THIS_SELL;
default:
return MDP3Fill::Aggressor::OTHER;
@@ -25,7 +25,7 @@ _to_passive_fill(const OrderUpdate &upd, bool implied_aggressor) {
}
inline MDP3Fill
_to_aggressor_fill(OrderKey k, api::Side side, const OrderUpdate &upd) {
_to_aggressor_fill(OrderKey k, Side side, const OrderUpdate &upd) {
MDP3Fill aggr;
aggr.key = k;
aggr.price = upd.last_price;
@@ -216,8 +216,8 @@ inline void print_detailed_order_changes(
template <class TIter>
inline void print_instrument_snapshots(
TIter first, TIter last, std::ostream &out = std::cout) {
auto book_volume = [](const SecuritySnapshot *snap, api::Side side) {
const auto &b = snap->book_side(map_side(side));
auto book_volume = [](const SecuritySnapshot *snap, Side side) {
const auto &b = snap->book_side(side);
int vol = ranges::accumulate(
b | ranges::views::transform(
[](const BookLevel &l) { return l.quantity.value(); }),
@@ -226,8 +226,8 @@ inline void print_instrument_snapshots(
};
auto total_book_volume = [&](const SecuritySnapshot *snap) {
return Quantity(
book_volume(snap, api::Side::SIDE_BUY).value() +
book_volume(snap, api::Side::SIDE_SELL).value());
book_volume(snap, Side::Buy).value() +
book_volume(snap, Side::Sell).value());
};
bts::spartan::table_display::print_table(
out, first, last,
@@ -253,11 +253,11 @@ inline void print_instrument_snapshots(
[](const auto *s) { return FMT(s->cleared_volume()); }},
{" Bid Vol ", 10,
[&](const auto *s) {
return FMT(book_volume(s, api::Side::SIDE_BUY));
return FMT(book_volume(s, Side::Buy));
}},
{" Ask Vol ", 10,
[&](const auto *s) {
return FMT(book_volume(s, api::Side::SIDE_SELL));
return FMT(book_volume(s, Side::Sell));
}},
{" Tot Vol ", 10,
[&](const auto *s) { return FMT(total_book_volume(s)); }},
@@ -269,19 +269,19 @@ inline void print_instrument_snapshots(
[](const auto *s) { return FMT(s->fills().size()); }},
{" Qty ", 8,
[](const auto *s) {
return FMT(s->book(api::Side::SIDE_BUY)[0].quantity);
return FMT(s->book(Side::Buy)[0].quantity);
}},
{" Bid ", 8,
[](const auto *s) {
return FMT(s->book(api::Side::SIDE_BUY)[0].price);
return FMT(s->book(Side::Buy)[0].price);
}},
{" Ask ", 8,
[](const auto *s) {
return FMT(s->book(api::Side::SIDE_SELL)[0].price);
return FMT(s->book(Side::Sell)[0].price);
}},
{" Qty ", 8,
[](const auto *s) {
return FMT(s->book(api::Side::SIDE_SELL)[0].quantity);
return FMT(s->book(Side::Sell)[0].quantity);
}},
{"DImp", 4,
[](const auto *s) {
@@ -295,18 +295,18 @@ inline void print_instrument_snapshots(
}},
{" I Qty ", 8,
[](const auto *s) {
return FMT(s->implied_book(api::Side::SIDE_BUY)[0].quantity);
return FMT(s->implied_book(Side::Buy)[0].quantity);
}},
{" I Bid ", 8,
[](const auto *s) {
return FMT(s->implied_book(api::Side::SIDE_BUY)[0].price);
return FMT(s->implied_book(Side::Buy)[0].price);
}},
{" I Ask ", 8,
[](const auto *s) {
return FMT(s->implied_book(api::Side::SIDE_SELL)[0].price);
return FMT(s->implied_book(Side::Sell)[0].price);
}},
{" I Qty ", 8, [](const auto *s) {
return FMT(s->implied_book(api::Side::SIDE_SELL)[0].quantity);
return FMT(s->implied_book(Side::Sell)[0].quantity);
}}});
out << std::endl;
}
@@ -25,16 +25,16 @@ class RandomGenerator {
return static_cast<int>(random_double(min, max));
}
api::Side random_side() {
return random_int(1, 100) % 2 == 0 ? api::Side::SIDE_BUY
: api::Side::SIDE_SELL;
Side random_side() {
return random_int(1, 100) % 2 == 0 ? Side::Buy
: Side::Sell;
}
};
MDP3Order create_order(
uint64_t key,
int32_t sec,
api::Side side,
Side side,
uint64_t priority,
double price,
uint32_t qty,
@@ -51,13 +51,13 @@ MDP3Order create_order(
}
MDP3Order create_empty_order(uint64_t key) {
return create_order(key, 0, api::Side::SIDE_BUY, 0, 0.0, 0);
return create_order(key, 0, Side::Buy, 0, 0.0, 0);
}
TEST_CASE("fifo_matching_engine_single_level", "[matcher] [spartan]") {
auto sec_key = 10;
double price = 100.0;
api::Side side = api::Side::SIDE_BUY;
Side side = Side::Buy;
Trace trace;
std::map<OrderKey, MatcherOrder> master_ord_map;
FifoPriceLevelMatcher price_level(
@@ -124,7 +124,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher] [spartan]") {
auto &ord_59 = out_ords.at(59);
REQUIRE(ord_59.quantity.value() == 60);
auto ord_59_mod = create_order(
ord_59.key.value(), 60, api::Side::SIDE_BUY, 0, price, 20);
ord_59.key.value(), 60, Side::Buy, 0, price, 20);
price_level.on_order_update(ord_59_mod);
// stays at the same level
price_level.copy_values_to(out_ords);
@@ -140,7 +140,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher] [spartan]") {
auto &ord_59 = out_ords.at(59);
REQUIRE(ord_59.quantity.value() == 60);
auto ord_59_mod = create_order(
ord_59.key.value(), 60, api::Side::SIDE_BUY, 0, price, 61);
ord_59.key.value(), 60, Side::Buy, 0, price, 61);
price_level.on_order_update(ord_59_mod);
// goes to the end
price_level.copy_values_to(out_ords);
@@ -151,19 +151,19 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher] [spartan]") {
SECTION("iceberg_simple") {
auto k = 0;
auto price = 100.0;
auto o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, 100.0, 1);
auto o = create_order(k++, sec_key, Side::Buy, 0, 100.0, 1);
price_level.on_new_same_side_order(o);
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 2);
o = create_order(k++, sec_key, Side::Buy, 0, price, 2);
price_level.on_new_same_side_order(o);
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 3);
o = create_order(k++, sec_key, Side::Buy, 0, price, 3);
price_level.on_new_same_side_order(o);
// iceberg, show qty=2, total_qty=100
o = create_order(
k++, sec_key, api::Side::SIDE_BUY, 0, price, 2,
k++, sec_key, Side::Buy, 0, price, 2,
IcebergQuantity{Quantity(2), Quantity(100)});
price_level.on_new_same_side_order(o);
// non ib, to get the visible qty to 15
o = create_order(k++, sec_key, api::Side::SIDE_BUY, 0, price, 7);
o = create_order(k++, sec_key, Side::Buy, 0, price, 7);
price_level.on_new_same_side_order(o);
REQUIRE(price_level.quantity().value() == 15);
REQUIRE(price_level.total_quantity().value() == 113);
@@ -198,7 +198,7 @@ TEST_CASE("fifo_matching_engine_single_level", "[matcher] [spartan]") {
}
}
TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
TEST_CASE("fifo_matching_engine_book", "[matcher]") {
auto verify_book_level = [](const BookLevel &bl, double price, uint32_t qty,
uint32_t n_orders) {
REQUIRE(bl.price == Price::from_double<9>(price));
@@ -210,7 +210,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
FifoSecurityMatcher sec_me(SecurityID(100));
auto order_key = 0;
auto priority = 0;
auto add_to_book = [&](uint32_t n_orders, api::Side side, double price,
auto add_to_book = [&](uint32_t n_orders, Side side, double price,
bool affect_book = true) -> void {
for (uint32_t i = 1; i <= n_orders; i++) {
auto o =
@@ -232,7 +232,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
add_to_book((lp.i + 1) * 10, Side::Buy, lp.p);
// asks
for (struct {
int i = 0;
@@ -243,10 +243,10 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
add_to_book((lp.i + 1) * 10, Side::Sell, lp.p);
// verify bids
{
BookSide bid_bk = sec_me.book(api::Side::SIDE_BUY);
BookSide bid_bk = sec_me.book(Side::Buy);
double exp_price = 15.09;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = 100 - i * 10;
@@ -258,7 +258,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
// verify asks
{
BookSide ask_bk = sec_me.book(api::Side::SIDE_SELL);
BookSide ask_bk = sec_me.book(Side::Sell);
double exp_price = 15.10;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = (i + 1) * 10;
@@ -282,7 +282,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
add_to_book((lp.i + 1) * 10, Side::Buy, lp.p);
// asks
for (struct {
int i = 0;
@@ -293,9 +293,9 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
add_to_book((lp.i + 1) * 10, Side::Sell, lp.p);
auto verify_bids = [&]() {
BookSide bid_bk = sec_me.book(api::Side::SIDE_BUY);
BookSide bid_bk = sec_me.book(Side::Buy);
double exp_price = 15.09;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = 100 - i * 10;
@@ -306,7 +306,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
};
auto verify_asks = [&]() {
BookSide ask_bk = sec_me.book(api::Side::SIDE_SELL);
BookSide ask_bk = sec_me.book(Side::Sell);
double exp_price = 15.10;
for (int i = 0; i < n_levels; i++) {
auto exp_n_orders = (i + 1) * 10;
@@ -321,11 +321,11 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
// bids: 15.00-15.09
// asks: 15.10-15.19
// add a passive order, should have no effect on the markets
add_to_book(100, api::Side::SIDE_BUY, 15.09, false);
add_to_book(100, Side::Buy, 15.09, false);
verify_bids();
verify_asks();
// add an aggressive order
add_to_book(100, api::Side::SIDE_BUY, 15.19, false);
add_to_book(100, Side::Buy, 15.19, false);
verify_bids();
verify_asks();
}
@@ -341,7 +341,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
add_to_book((lp.i + 1) * 10, Side::Buy, lp.p);
// asks
for (struct {
int i = 0;
@@ -352,7 +352,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
add_to_book((lp.i + 1) * 10, Side::Sell, lp.p);
// remove one order at a time from the second level
// 90 orders starting with id 360
auto b = 360;
@@ -368,7 +368,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
remvd += qty;
auto now_qty = init_qty - remvd;
auto now_n_orders = init_n_orders - (i + 1);
BookSide bk = sec_me.book(api::Side::SIDE_BUY);
BookSide bk = sec_me.book(Side::Buy);
if (now_qty == 0)
// whole level got removed
// now level 1 should be 15.07
@@ -376,7 +376,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
else
verify_book_level(bk[1], 15.08, now_qty, now_n_orders);
}
BookSide bk = sec_me.book(api::Side::SIDE_BUY);
BookSide bk = sec_me.book(Side::Buy);
verify_book_level(bk[1], 15.07, 3240, 80);
}
SECTION("modify") {
@@ -391,7 +391,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
add_to_book((lp.i + 1) * 10, Side::Buy, lp.p);
// asks
for (struct {
int i = 0;
@@ -402,28 +402,28 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
add_to_book((lp.i + 1) * 10, Side::Sell, lp.p);
// modify qty of all orders to 1
auto b = 360;
for (auto i = 0; i < 90; i++) {
auto id = i + b;
auto ord =
create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.08, 1);
create_order(id, sec_key, Side::Buy, id, 15.08, 1);
sec_me.on_order_change(OrderChange{
create_empty_order(ord.key.value()), ord, OrderChange::updated});
}
BookSide bk = sec_me.book(api::Side::SIDE_BUY);
BookSide bk = sec_me.book(Side::Buy);
verify_book_level(bk[1], 15.08, 90, 90);
// now change all these prices to move to a diff level
for (auto i = 0; i < 90; i++) {
auto id = i + b;
auto ord =
create_order(id, sec_key, api::Side::SIDE_BUY, id, 15.09, 1);
create_order(id, sec_key, Side::Buy, id, 15.09, 1);
sec_me.on_order_change(OrderChange{
create_empty_order(ord.key.value()), ord, OrderChange::updated});
}
// 15.08 is gone
bk = sec_me.book(api::Side::SIDE_BUY);
bk = sec_me.book(Side::Buy);
verify_book_level(bk[1], 15.07, 3240, 80);
verify_book_level(bk[0], 15.09, 5140, 190);
}
@@ -439,7 +439,7 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
add_to_book((lp.i + 1) * 10, Side::Buy, lp.p);
// asks
for (struct {
int i = 0;
@@ -450,73 +450,73 @@ TEST_CASE("fifo_matching_engine_book", "[matcher] [spartan]") {
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
add_to_book((lp.i + 1) * 10, Side::Sell, lp.p);
// send a new sell at 15.09 to take the first 10 orders
auto ord =
create_order(10000, sec_key, api::Side::SIDE_SELL, 10000, 15.09, 55);
create_order(10000, sec_key, Side::Sell, 10000, 15.09, 55);
sec_me.on_order_change(OrderChange{MDP3Order{}, ord, OrderChange::added});
auto bk = sec_me.book(api::Side::SIDE_BUY);
auto bk = sec_me.book(Side::Buy);
verify_book_level(bk[0], 15.09, 4995, 90);
verify_book_level(bk[1], 15.08, 4095, 90);
verify_book_level(bk[2], 15.07, 3240, 80);
// now completely sweep 2 complete levels and make the rest a working
// order
ord = create_order(
10001, sec_key, api::Side::SIDE_SELL, 10001, 15.08, 10000);
10001, sec_key, Side::Sell, 10001, 15.08, 10000);
sec_me.on_order_change(OrderChange{MDP3Order{}, ord, OrderChange::added});
bk = sec_me.book(api::Side::SIDE_BUY);
bk = sec_me.book(Side::Buy);
verify_book_level(bk[0], 15.07, 3240, 80);
// failing (11/16)
auto bk_ask = sec_me.book(api::Side::SIDE_SELL);
auto bk_ask = sec_me.book(Side::Sell);
verify_book_level(bk_ask[0], 15.08, 910, 1);
}
// SECTION("IB_cross") {
// // create 10 level book and cross the first 3 levels completely.
// // verify the resting IB is correct
// auto n_levels = 10;
// // bids
// for (struct {
// int i = 0;
// double p = 15;
// void inc() {
// i++;
// p += 0.01;
// }
// } lp;
// lp.i < n_levels; lp.inc())
// add_to_book((lp.i + 1) * 10, api::Side::SIDE_BUY, lp.p);
// // asks
// for (struct {
// int i = 0;
// double p = 15.10;
// void inc() {
// i++;
// p += 0.01;
// }
// } lp;
// lp.i < n_levels; lp.inc())
// add_to_book((lp.i + 1) * 10, api::Side::SIDE_SELL, lp.p);
// auto bid_bk = sec_me.book(api::Side::SIDE_BUY);
// verify_book_level(bid_bk[0], 15.09, 5050, 100);
// // the bid book is
// // 15.09 5050 100
// // 15.08 4095 90
// // 15.07 3240 80
// // 15.06 2485 70
// // send a new sell at 15.07 to take the first 3 levels
// auto cross_ib = create_order(
// 10000, sec_key, api::Side::SIDE_SELL, 10000, 15.07, 13,
// IcebergQuantity{Quantity(13), Quantity(12500)});
// sec_me.on_order_change(
// OrderChange{MDP3Order{}, cross_ib, OrderChange::added});
// bid_bk = sec_me.book(api::Side::SIDE_BUY);
// verify_book_level(bid_bk[0], 15.06, 2485, 70);
// auto ask_bk = sec_me.book(api::Side::SIDE_SELL);
// verify_book_level(ask_bk[0], 15.07, 13, 1);
// }
SECTION("IB_cross") {
// create 10 level book and cross the first 3 levels completely.
// verify the resting IB is correct
auto n_levels = 10;
// bids
for (struct {
int i = 0;
double p = 15;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, Side::Buy, lp.p);
// asks
for (struct {
int i = 0;
double p = 15.10;
void inc() {
i++;
p += 0.01;
}
} lp;
lp.i < n_levels; lp.inc())
add_to_book((lp.i + 1) * 10, Side::Sell, lp.p);
auto bid_bk = sec_me.book(Side::Buy);
verify_book_level(bid_bk[0], 15.09, 5050, 100);
// the bid book is
// 15.09 5050 100
// 15.08 4095 90
// 15.07 3240 80
// 15.06 2485 70
// send a new sell at 15.07 to take the first 3 levels
auto cross_ib = create_order(
10000, sec_key, Side::Sell, 10000, 15.07, 13,
IcebergQuantity{Quantity(13), Quantity(12500)});
sec_me.on_order_change(
OrderChange{MDP3Order{}, cross_ib, OrderChange::added});
bid_bk = sec_me.book(Side::Buy);
verify_book_level(bid_bk[0], 15.06, 2485, 70);
auto ask_bk = sec_me.book(Side::Sell);
verify_book_level(ask_bk[0], 15.07, 13, 1);
}
}
TEST_CASE("perf", "[matcher] [spartan]") {
TEST_CASE("perf", "[matcher]") {
return;
RandomGenerator rnd_gen;
const size_t n_levels = 500;
@@ -532,7 +532,7 @@ TEST_CASE("perf", "[matcher] [spartan]") {
int p_lev = rnd_gen.random_int(0, n_levels - 1);
bts_assert(p_lev >= 0);
double price;
if (ord.side == api::Side::SIDE_BUY)
if (ord.side == Side::Buy)
price = top_bid - (p_lev * tick_size);
else
price = (top_bid + tick_size) + (p_lev * tick_size);
@@ -567,8 +567,8 @@ TEST_CASE("perf", "[matcher] [spartan]") {
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Adding " << aux.size() << " took "
<< 1.0 * nsecs / 1000000 << " msecs" << std::endl;
size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) +
sec_m.total_n_orders(Side::Sell);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
@@ -593,8 +593,8 @@ TEST_CASE("perf", "[matcher] [spartan]") {
std::cout << "Removing " << n_orders << " took "
<< 1.0 * nsecs / 1000000 << " msecs" << std::endl;
n_orders += inc;
size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) +
sec_m.total_n_orders(Side::Sell);
REQUIRE(comp_n_orders == 0);
}
}
@@ -618,8 +618,8 @@ TEST_CASE("perf", "[matcher] [spartan]") {
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Updaing (qty dec) " << n_orders << " took "
<< 1.0 * nsecs / 1000000 << " msecs" << std::endl;
size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) +
sec_m.total_n_orders(Side::Sell);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
@@ -644,8 +644,8 @@ TEST_CASE("perf", "[matcher] [spartan]") {
TimeUtil::TimestampToNanoseconds(t0);
std::cout << "Updaing (qty inc) " << n_orders << " took "
<< 1.0 * nsecs / 1000000 << " msecs" << std::endl;
size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) +
sec_m.total_n_orders(Side::Sell);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
@@ -663,7 +663,7 @@ TEST_CASE("perf", "[matcher] [spartan]") {
for (auto &oc : aux) {
oc.change = OrderChange::ChangeType::updated;
double inc = tick_size;
if (oc.order.side == api::Side::SIDE_BUY)
if (oc.order.side == Side::Buy)
inc = -1 * inc;
oc.order.price =
Price::from_double<9>(oc.order.price.to_double() + inc),
@@ -675,8 +675,8 @@ TEST_CASE("perf", "[matcher] [spartan]") {
std::cout << "Updating (price change no cross) " << n_orders
<< " took " << 1.0 * nsecs / 1000000 << " msecs"
<< std::endl;
size_t comp_n_orders = sec_m.total_n_orders(api::Side::SIDE_BUY) +
sec_m.total_n_orders(api::Side::SIDE_SELL);
size_t comp_n_orders = sec_m.total_n_orders(Side::Buy) +
sec_m.total_n_orders(Side::Sell);
REQUIRE(comp_n_orders == n_orders);
n_orders += inc;
}
@@ -693,13 +693,13 @@ TEST_CASE("perf", "[matcher] [spartan]") {
// send one lot buy orders to completely cross
// the offer market
size_t buy_orders_to_send =
sec_m.total_qty(api::Side::SIDE_SELL).value();
sec_m.total_qty(Side::Sell).value();
OrderChange ochange;
ochange.change = OrderChange::ChangeType::added;
ochange.order = MDP3Order{
OrderKey(0),
SecurityID(100),
api::Side::SIDE_BUY,
Side::Buy,
MDP3OrderPriority(0),
Price::from_double<9>(10000),
Quantity(1),
@@ -721,7 +721,7 @@ TEST_CASE("perf", "[matcher] [spartan]") {
std::cout << "Crossing (with new orders) " << n_orders << " took "
<< 1.0 * nsecs / 1000000 << " msecs" << std::endl;
// whole sell side should be gone
REQUIRE(sec_m.total_n_orders(api::Side::SIDE_SELL) == 0);
REQUIRE(sec_m.total_n_orders(Side::Sell) == 0);
n_orders += inc;
}
}
@@ -611,8 +611,8 @@ double Sim::price_at_market(SecurityID id, Quantity qty, Side s) const {
}
PandLSummary Sim::pnl() const {
auto mkt_pricer = [this](SecurityID id, Quantity qty, api::Side s) {
return price_at_market(id, qty, map_side(s));
auto mkt_pricer = [this](SecurityID id, Quantity qty, Side s) {
return price_at_market(id, qty, s);
};
PandL pl{me->instruments, mkt_pricer};
@@ -710,8 +710,7 @@ void Sim::validate_matcher_books(Side side) {
for (auto &[sec_id, snap] : me->snapshots) {
if (snap.ready_to_trade()) {
if (snap.book_side(side) !=
simulation.matcher.book(sec_id, map_side(side))) {
if (snap.book_side(side) != simulation.matcher.book(sec_id, side)) {
LOG(Spartan, Error,
"Mismatch between matcher and pcap books on event: {}. "
@@ -722,7 +721,7 @@ void Sim::validate_matcher_books(Side side) {
LOG(Spartan, Error, "{}", snap.book_side(side));
LOG(Spartan, Error, "Matcher Book:");
LOG(Spartan, Error, "{}",
simulation.matcher.book(sec_id, map_side(side)));
simulation.matcher.book(sec_id, side));
bts_assert(false);
}
}
@@ -78,11 +78,7 @@ double PandL::position_liquidation_value(
OutrightPosition const &outright_position) const {
double liquidation_proceeds = 0;
for (auto &[id, this_pos] : outright_position) {
auto mkt_action = api::SIDE_UNDEFINED;
if (this_pos > 0)
mkt_action = api::SIDE_SELL;
else
mkt_action = api::SIDE_BUY;
auto mkt_action = this_pos > 0 ? Side::Sell : Side::Buy;
auto cash_flow = me->pricer(id, Quantity{(int)this_pos}, mkt_action);
liquidation_proceeds += cash_flow;
}
@@ -97,14 +93,14 @@ double PandL::price_to_cash(SecurityID id, double p) const {
return get_sec(id).to_cash(p);
}
int PandL::mkt_to_cash_flow(api::Side s) const {
if (s == api::SIDE_BUY)
int PandL::mkt_to_cash_flow(Side s) const {
if (s == Side::Buy)
return -1;
else
return 1;
}
int PandL::mkt_to_contract_flow(api::Side s) const {
int PandL::mkt_to_contract_flow(Side s) const {
return -1 * mkt_to_cash_flow(s);
}
@@ -35,8 +35,8 @@ class PandL {
struct PImpl;
SecurityDefinition const &get_sec(SecurityID id) const;
double price_to_cash(SecurityID id, double p) const;
int mkt_to_cash_flow(api::Side s) const;
int mkt_to_contract_flow(api::Side s) const;
int mkt_to_cash_flow(Side s) const;
int mkt_to_contract_flow(Side s) const;
std::unique_ptr<PImpl> me;
};
@@ -0,0 +1,112 @@
#include "PCapOrderChangeBootstrapper.h"
#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h"
#include "server/sparkd/cmds/spartan/market_data/gen/src/SecurityTradingStatus.h"
#include <unordered_map>
namespace bts::spartan::mdp3 {
using TradingStatus = ::mdp3::SecurityTradingStatus;
const size_t NO_EVENT = 0;
struct SecStatusInfo {
size_t current_event = NO_EVENT;
size_t last_status_change_event = NO_EVENT;
size_t consecutive_events_with_status = 0;
TradingStatus::Value status = TradingStatus::PreOpen;
};
template <class GetChanges>
inline OrderChanges _order_changes(
std::unordered_map<SecurityID, SecStatusInfo> &status_info_map,
const SecuritySnapshot *s,
size_t event_id,
GetChanges incremental_changes) {
auto res = status_info_map.emplace(
s->security_id(), SecStatusInfo{event_id, event_id, 1, s->trading_status()});
if (res.second)
return {};
SecStatusInfo &status_info = res.first->second;
// This class is designed to be used in conjunction with pcap observation.
// Its purpose is to bootstrap incremental order changes, which occur as time
// moves in one direction: forward. Assuming that event ids increase over
// time, it is reasonable to expect the sequence of event id parameters to be
// passed to this class in (weak) ascending order.
bts_assert(event_id >= status_info.current_event);
if (status_info.current_event == event_id) {
// If event id == NO_EVENT, then this is the first time this security
// has been observed.
if (event_id != NO_EVENT) {
// The security's trading status has already been recorded for this
// event. Assert that it hasn't mysteriously changed.
bts_assert(status_info.status == s->trading_status());
}
} else {
status_info.current_event = event_id;
if (s->trading_status() == status_info.status)
status_info.consecutive_events_with_status++;
else {
status_info.last_status_change_event = event_id;
status_info.consecutive_events_with_status = 1;
status_info.status = s->trading_status();
}
}
if (status_info.status != TradingStatus::ReadyToTrade)
return {};
// If the security's status just changed (on this event), the
// 'consecutive_events_with_status' field would be at its logical minimum: 1.
// Assert that this value is logical.
bts_assert(status_info.consecutive_events_with_status > 0);
switch (status_info.consecutive_events_with_status) {
case 1:
// The security just transitioned to 'ReadyToTrade'. Presumably, all
// preopen trades are flushed in this event, so it is ignored.
return {};
case 2:
// This is the first event AFTER the security transitioned to
// 'ReadyToTrade'. Presumably, the preopen trades were flushed in the
// last event, so this is the first 'normal' event for the security.
// Bootstrap the incremental change sequence by adding all resting orders.
OrderChanges resting_orders;
for (const auto &p : s->orders()) {
auto &ch = resting_orders.emplace_back();
ch.order = p.second;
ch.change = OrderChange::ChangeType::added;
}
return resting_orders;
}
// This is a 'normal' update, and the incremental change sequence has already
// been bootstrapped. Simply return the incremental changes.
return incremental_changes(s);
}
struct PCapOrderChangeBootstrapper::PImpl {
std::unordered_map<SecurityID, SecStatusInfo> status_info_map;
};
PCapOrderChangeBootstrapper::PCapOrderChangeBootstrapper()
: _me(new PCapOrderChangeBootstrapper::PImpl()) {}
PCapOrderChangeBootstrapper::~PCapOrderChangeBootstrapper() {}
OrderChanges PCapOrderChangeBootstrapper::order_changes(
const SecuritySnapshot *s, size_t event_id) {
struct {
inline OrderChanges operator()(const SecuritySnapshot *s) {
return s->order_changes();
}
} get_changes;
return _order_changes(_me->status_info_map, s, event_id, get_changes);
}
OrderChanges PCapOrderChangeBootstrapper::deduced_order_changes(
const SecuritySnapshot *s, size_t event_id) {
struct {
inline OrderChanges operator()(const SecuritySnapshot *s) {
return s->deduced_order_changes();
}
} get_changes;
return _order_changes(_me->status_info_map, s, event_id, get_changes);
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,30 @@
#pragma once
#include "server/sparkd/cmds/spartan/market_data/common/mdp3_common.h"
#include <memory>
namespace bts::spartan::mdp3 {
/**
* IFeedObservers may use this class to ignore (deduced)order changes until all
* preopen trades have been processed. The (deduced_)order_changes methods
* return empty vectors until the second consecutive event in which a security's
* trading status is 'ReadyToTrade' (pending preopen trades are flushed in the
* first event), at which point the returned vectors consist of 'added' changes
* for all resting orders to bootstrap the incremental change sequence. After
* that, 'deduced_order_changes', and 'order_changes' simply return
* SecuritySnapshot::deduced_order_changes and SecuritySnapshot::order_changes,
* respectively.
*/
struct PCapOrderChangeBootstrapper {
PCapOrderChangeBootstrapper();
~PCapOrderChangeBootstrapper();
OrderChanges order_changes(const SecuritySnapshot *, size_t event_id);
OrderChanges deduced_order_changes(const SecuritySnapshot *, size_t event_id);
private:
struct PImpl;
std::unique_ptr<PImpl> _me;
};
} // namespace bts::spartan::mdp3
@@ -25,17 +25,11 @@ struct MBPUpdate {
BookSide mbp_sell;
ImpliedBookSide implied_mbp_buy;
ImpliedBookSide implied_mbp_sell;
BookSide const &mbp_side(api::Side side) const {
if (side == api::SIDE_BUY)
return mbp_buy;
else
return mbp_sell;
BookSide const &mbp_side(Side side) const {
return side == Side::Buy ? mbp_buy : mbp_sell;
}
const ImpliedBookSide &implied_mbp_side(api::Side side) const {
if (side == api::SIDE_BUY)
return implied_mbp_buy;
else
return implied_mbp_sell;
const ImpliedBookSide &implied_mbp_side(Side side) const {
return side == Side::Buy ? implied_mbp_buy : implied_mbp_sell;
}
};
@@ -44,7 +38,7 @@ struct OrderUpdate {
enum Status { FILLED, PARTIALLY_FILLED, CANCELED };
Status status;
SecurityID security_id;
api::Side side;
Side side;
Quantity last_fills;
Price last_price;
enum FillType { PASSIVE, AGGRESSIVE };
@@ -0,0 +1,34 @@
#include "BookStats.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/BookStatRecorder.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <fstream>
#include <string>
namespace bts::spartan::cmds {
struct BookStats::Args {
std::string pcap;
std::string out;
};
BookStats::BookStats()
: CLI::App(
"Generate a csv file of various statistics about book updates",
"book-stats"),
_args(new BookStats::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] {
if (_args->out.empty())
_args->out =
file::change_file_ext(_args->pcap, "book_stats.csv", "pcap.xz");
std::ofstream out(_args->out);
mdp3::BookStatRecorder bsr(out);
mdp3::observe_channel(_args->pcap, bsr);
});
}
BookStats::~BookStats() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct BookStats : public CLI::App {
BookStats();
~BookStats();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,19 @@
#include "CSV.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
namespace bts::spartan::cmds {
struct CSV::Args {
std::string pcap;
};
CSV::CSV()
: CLI::App("Generate csv files for various pcap-derived tables", "csv"),
_args(new CSV::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] { mdp3::to_csv(_args->pcap); });
}
CSV::~CSV() {}
} // namespace bts::spartan::cmds
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct CSV : public CLI::App {
CSV();
~CSV();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,19 @@
#include "Compress.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
namespace bts::spartan::cmds {
struct Compress::Args {
std::string pcap;
std::string out;
};
Compress::Compress() : CLI::App("", "compress"), _args(new Compress::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] { mdp3::compress(_args->pcap, _args->out); });
}
Compress::~Compress() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Compress : public CLI::App {
Compress();
~Compress();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,51 @@
#include "FindPolys.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/market_data/utils/polygon_utils.h"
#include <iostream>
namespace bts::spartan::cmds {
const int UNSET = -1;
struct FindPolys::Args {
std::string pcap;
int active_threshold_fills = UNSET;
int max_vertices = UNSET;
int max_securities = UNSET;
};
FindPolys::FindPolys()
: CLI::App("Find polygons in a pcap file", "find-polys"),
_args(new FindPolys::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option(
"active-threshold-fills,--active-threshold-fills",
_args->active_threshold_fills);
add_option("max-vertices,--max-vertices", _args->max_vertices);
add_option("max-securities,--max-securities", _args->max_securities);
callback([&] {
// init options (from _args->
mdp3::PolygonFinder::Options options;
if (_args->active_threshold_fills != UNSET)
options.active_threshold_fills = _args->active_threshold_fills;
if (_args->max_vertices != UNSET)
options.max_vertices = _args->max_vertices;
if (_args->max_securities != UNSET)
options.max_securities = _args->max_securities;
mdp3::DefinitionCollector def_collector;
mdp3::PolygonFinder poly_finder(options);
mdp3::observe_channel(_args->pcap, {&poly_finder, &def_collector});
for (size_t id = 1; id <= poly_finder.result.size(); id++)
std::cout << id << " -> "
<< mdp3::polygon_descriptor(
poly_finder.result[id], def_collector.ptr_result)
<< std::endl;
});
}
FindPolys::~FindPolys() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct FindPolys : public CLI::App {
FindPolys();
~FindPolys();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,51 @@
#include "Icebergs.h"
#include "server/sparkd/cmds/spartan/domain/IcebergQuantity.h"
#include "server/sparkd/cmds/spartan/domain/OrderKey.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h"
#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <fstream>
#include <string>
namespace bts::spartan::cmds {
struct Icebergs::Args {
std::string pcap;
std::string out;
};
Icebergs::Icebergs()
: CLI::App("Generate a csv file of iceberg orders in a pcap", "icebergs"),
_args(new Icebergs::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] {
const char *ok_col = "ok";
const char *eid_col = "eid";
const char *total_col = "total_qty";
const char *show_qty_col = "show_qty";
const char *fills_col = "fill_qty";
if (_args->out.empty())
_args->out =
file::change_file_ext(_args->pcap, "icebergs.csv", "pcap.xz");
std::ofstream out(_args->out);
auto ibs = mdp3::IcebergFinder::find_all(_args->pcap);
// write cols
out << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col
<< "," << fills_col;
// write rows
for (auto &i : ibs) {
auto relevent_eids = i.second.all_eids();
for (auto eid : relevent_eids)
out << i.first.value() << "," << eid << ","
<< i.second.total_qty().value() << ","
<< std::get<1>(i.second.show_qty_at(eid)) << ","
<< i.second.count_fills_by(eid) << "\n";
}
});
}
Icebergs::~Icebergs() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Icebergs : public CLI::App {
Icebergs();
~Icebergs();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,25 @@
#include "Json.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
namespace bts::spartan::cmds {
struct Json::Args {
uint32_t start_packet = 0;
uint32_t packet_count = 0;
std::string pcap{};
};
Json::Json()
: CLI::App("Generate a json file for data packets in a pcap", "json"),
_args(new Json::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("start-packet,--start-packet", _args->start_packet);
add_option("packet-count,--packet-count", _args->packet_count);
callback([&] {
mdp3::to_json(_args->pcap, _args->start_packet, _args->packet_count);
});
}
Json::~Json() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Json : public CLI::App {
Json();
~Json();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,37 @@
#include "ListSec.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/FillCounter.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include <iostream>
namespace bts::spartan::cmds {
struct ListSec::Args {
std::string pcap;
};
ListSec::ListSec() : CLI::App("List securities in a pcap file", "list-sec"), _args(new ListSec::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] {
mdp3::DefinitionCollector def;
mdp3::FillCounter fill_counter;
mdp3::observe_channel(_args->pcap, {&def, &fill_counter});
auto fill_counts = fill_counter.result_map();
for (auto const &s : def.result) {
std::cout << s.second.asset() << "|" << s.second.symbol() << "|"
<< fill_counts[s.second.security_id()];
for (auto const &l : s.second.legs()) {
std::cout << "|";
if (def.ptr_result[l.security_id])
std::cout << def.ptr_result[l.security_id]->symbol();
else
std::cout << "?" << l.security_id;
}
std::cout << std::endl;
}
});
}
ListSec::~ListSec() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct ListSec : public CLI::App {
ListSec();
~ListSec();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,122 @@
#include "PolygonBooks.h"
#include "lib/orderman/orderman_utils/Formatter.h"
#include "server/sparkd/cmds/spartan/domain/book/Book.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/SyntheticBookRecorder.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include "server/sparkd/cmds/spartan/util/sql/adapters/polygon_defs.h"
#include "server/sparkd/cmds/spartan/util/sql/sql_utils.h"
#include <iostream>
#include <unordered_set>
namespace bts::spartan::cmds {
#define INFO(...) LOG(OrderMan, Info, __VA_ARGS__)
#define INFO_FN(...) \
LOG(OrderMan, Info, "[{}] {}", std::string(__FUNCTION__), \
fmt::format(__VA_ARGS__))
#define WARN(...) LOG(OrderMan, Warn, __VA_ARGS__)
#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__)
struct PolygonBooks::Args {
std::string pcap;
std::string db;
std::vector<int> security_ids;
std::string out_csv;
};
PolygonBooks::PolygonBooks()
: CLI::App(
"Generate a csv file of book updates for polygons", "polygon-books"),
_args(new PolygonBooks::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("data-base,--db,--data-base", _args->db)->required();
add_option("security-ids,--security-ids", _args->security_ids)->required();
add_option("out,--out", _args->out_csv);
callback([&] {
const char *eid_col = "eid";
const char *pid_col = "pid";
const char *log_header = "PolygonBookCSV";
// the nth leg col denotes the last event in which the nth leg's book
// changed
const int num_leg_cols = 10;
// validate args
if (_args->out_csv.empty())
_args->out_csv =
file::change_file_ext(_args->pcap, "polygon_books.csv", "pcap.xz");
if (_args->out_csv.empty() || _args->security_ids.empty())
return;
std::unordered_set<int> ids;
for (auto sid : _args->security_ids)
ids.insert(sid);
INFO(
"generating polygon books for pids: {}",
orderman::Formatter::fmt_values(ids));
mdp3::SyntheticBookRecorder sbr(
spartan::sql::adapters::polygon_defs::read_sql(
_args->db.c_str(), ids));
mdp3::observe_channel(_args->pcap, sbr);
// don't prepend a comma before calling 'append_book' (it does that
// internally)
auto append_book = [](std::ofstream &out,
const mdp3::SmallBook &bk) -> std::ofstream & {
uint32_t i = 0;
bts_assert(bk.size() <= 10);
for (; i < bk.size(); i++)
out << "," << bk[i].price << "," << bk[i].quantity.value();
while (i++ < mdp3::BookSide::max_depth())
out << ",NaN,0";
return out;
};
auto append_leg_book_eids =
[&](std::ostream &out,
const mdp3::SyntheticMBPUpdate &upd) -> std::ostream & {
int lcount = 0;
for (const auto &p : upd.leg_book_eids) {
out << "," << p.second;
if (++lcount == num_leg_cols) {
WARN(
"{} processing book update for polygon with more than {} "
"legs "
"({} "
"'leg-book-event-id' columns are accounted for in the csv, "
"event "
"ids for susbequent legs will be discarded).",
*log_header, num_leg_cols, num_leg_cols);
break;
}
}
while (lcount++ < num_leg_cols)
out << ",0";
return out;
};
// write polygon books to csv
auto ofs = std::ofstream(_args->out_csv);
// add cols
ofs << eid_col << "," << pid_col;
for (int i = 0; i < num_leg_cols; i++)
ofs << ",leg_" << i << "_eid";
auto add_bk_cols = [&](const char *side_prefix) {
for (int i = 0; i < 10; i++)
ofs << "," << side_prefix << "_p_" << i << "," << side_prefix
<< "_q_" << i;
};
add_bk_cols("bid");
add_bk_cols("ask");
for (const auto &p : sbr.result()) {
for (const auto &upd : p.second) {
ofs << std::endl;
ofs << p.first << "," << upd.synthetic_id.value();
append_leg_book_eids(ofs, upd);
append_book(ofs, upd.bids);
append_book(ofs, upd.asks);
}
}
ofs.flush();
});
}
PolygonBooks::~PolygonBooks() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct PolygonBooks : public CLI::App {
PolygonBooks();
~PolygonBooks();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,17 @@
#include "SQL.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
namespace bts::spartan::cmds {
struct SQL::Args {
std::string pcap;
};
SQL::SQL() : CLI::App("Generate a sqlite database for a pcap file", "sql"), _args(new SQL::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] { mdp3::to_sql(_args->pcap); });
}
SQL::~SQL() {}
} // namespace bts::spartan::cmds
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct SQL : public CLI::App {
SQL();
~SQL();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,20 @@
#include "Sim.h"
#include "server/sparkd/cmds/spartan/back_testing/run/run_sim.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include <functional>
namespace bts::spartan::cmds {
struct Sim::Args {
std::string config_path;
};
Sim::Sim()
: CLI::App("Run trading simulation(s)", "sim"), _args(new Sim::Args()) {
add_option("config-file,--config-file", _args->config_path)->required();
callback([&] { mdp3::run_sim(_args->config_path); });
}
Sim::~Sim() {}
} // namespace bts::spartan::cmds
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Sim : public CLI::App {
Sim();
~Sim();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,91 @@
#include "SimBooks.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h"
#include "server/sparkd/cmds/spartan/domain/SecurityCriteria.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/recorders/MatcherBookRecorder.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <boost/container/small_vector.hpp>
namespace bts::spartan::cmds {
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
#define INFO(...) _LOG(Info, __VA_ARGS__)
struct SimBooks::Args {
std::string pcap;
std::string out;
std::vector<std::string> assets;
std::vector<std::string> security_groups;
mdp3::SecurityCriteria security_criteria;
};
SimBooks::SimBooks()
: CLI::App(
"Generate a csv file of book updates by feeding deduced orders from "
"a pcap file to the simulation matcher",
"sim-books"),
_args(new SimBooks::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
add_option(
"--assets", _args->assets,
"Specify asset(s) to filter the set of securities for which books are "
"generated");
add_option(
"--security-groups", _args->security_groups,
"Specify security group(s) to filter the set of securities for which "
"books are generated");
callback([&] {
// Validate args
bts_assert(_args->pcap.size());
if (_args->out.empty())
_args->out =
file::change_file_ext(_args->pcap, "sim_books.csv", "pcap.xz");
bts_assert(_args->out.size());
// Sweep pcap for securityies and icebergs
mdp3::DefinitionCollector dc;
// mdp3::IcebergFinder ice_finder;
INFO(
"Begin observe '{}' (1/2): detect all securityies and iceberg orders",
_args->pcap);
mdp3::observe_channel(_args->pcap, dc);
auto ibs = mdp3::iceberg_file_cache::find_icebergs(_args->pcap);
INFO(
"End observe '{}' (1/2): detected {} securities, and {} iceberg "
"orders",
_args->pcap, dc.result.size(), ibs.size());
// Init MergedMatcher
mdp3::MergedMatcher matcher;
for (auto p : dc.result)
matcher.add_def(p.second);
// Init MatcherBookRecorder
mdp3::SecurityCriteria security_criteria;
if (_args->assets.size() || _args->security_groups.size()) {
// assets
auto &assets = security_criteria.clauses.emplace_back().assets;
assets.insert(
assets.end(), _args->assets.begin(), _args->assets.end());
// security groups
auto &security_groups =
security_criteria.clauses.emplace_back().assets;
security_groups.insert(
security_groups.end(), _args->security_groups.begin(),
_args->security_groups.end());
}
mdp3::MatcherBookRecorder rec({matcher, ibs, security_criteria, true});
// Init FeedRecorderWrapper and observe pcap to generate the csv
std::ofstream out(_args->out);
mdp3::FeedRecorderWrapper rec_wrap(out, &rec);
INFO("Begin observe '{}' (2/2): record matcher books", _args->pcap);
mdp3::observe_channel(_args->pcap, rec_wrap);
INFO("End observe '{}' (2/2): recorded matcher book", _args->pcap);
out.flush();
});
}
SimBooks::~SimBooks() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct SimBooks : public CLI::App {
SimBooks();
~SimBooks();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,44 @@
#include "Stats.h"
#include "server/sparkd/cmds/spartan/domain/SecuritySnapshot.h"
#include "server/sparkd/cmds/spartan/market_data/handler/IMDP3FeedObserver.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
namespace bts::spartan::cmds {
size_t total_orders = 0;
static void execute(const std::string &pcap, std::ostream &out_file) {
out_file << "sid,n_orders" << std::endl;
struct StatPrinter : spartan::mdp3::IMDP3FeedObserver {
StatPrinter(std::ostream &out) : out{out} {}
std::ostream &out;
void on_event_end(spartan::mdp3::MDP3Event const &ev) override {
for (const auto *s : ev)
total_orders += s->orders().size();
}
} stats{out_file};
observe_channel(pcap, stats);
std::cout << total_orders << std::endl;
}
struct Stats::Args {
std::string pcap;
std::string out;
};
Stats::Stats() : CLI::App("", "stats"), _args(new Stats::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] {
if (_args->out.empty())
_args->out =
spartan::file::change_file_ext(_args->pcap, "order_counts.csv");
std::ofstream out(_args->out);
execute(_args->pcap, out);
});
}
Stats::~Stats() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Stats : public CLI::App {
Stats();
~Stats();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,26 @@
#include "ValidateFifo.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/matcher_validation_observer.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <iostream>
namespace bts::spartan::cmds {
struct ValidateFifo::Args {
std::string pcap;
};
ValidateFifo::ValidateFifo()
: CLI::App("", "validate-fifo"), _args(new ValidateFifo::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] {
auto ibs = mdp3::IcebergFinder::find_all(_args->pcap);
mdp3::MatcherValidationObserver obs(ibs);
mdp3::observe_channel(_args->pcap, obs);
});
}
ValidateFifo::~ValidateFifo() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct ValidateFifo : public CLI::App {
ValidateFifo();
~ValidateFifo();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,50 @@
#include "ValuePolys.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonFinder.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/PolygonOpportunityFinder.h"
#include "server/sparkd/cmds/spartan/market_data/utils/mdp3_utils.h"
#include "server/sparkd/cmds/spartan/util/file_stream_utils.h"
#include <iostream>
namespace bts::spartan::cmds {
struct ValuePolys::Args {
std::string pcap;
};
ValuePolys::ValuePolys()
: CLI::App(
"Generate csv files of polygon values and opportunities. Each of "
"which are calculated from the polygon book updates in a pcap file",
"value-polys"),
_args(new ValuePolys::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] {
mdp3::DefinitionCollector def;
mdp3::PolygonFinder poly_finder;
// first pass, just identify the polygons
mdp3::observe_channel(_args->pcap, {&poly_finder, &def});
// second pass, value the polygons
mdp3::PolygonOpportunityFinder op_finder{
poly_finder.result, def.ptr_result};
mdp3::observe_channel(_args->pcap, {&op_finder});
auto poly_v = std::ofstream(
file::change_file_ext(_args->pcap, "polygon_values.csv"));
for (auto &v : op_finder.result)
poly_v << v << std::endl;
auto poly_op = std::ofstream(
file::change_file_ext(_args->pcap, "polygon_opportunities.csv"));
spartan::mdp3::poly_opportunities_to_csv(
aggregate(op_finder.result), poly_finder.result, def.ptr_result,
poly_op);
});
}
ValuePolys::~ValuePolys() {}
} // namespace bts::spartan::cmds
@@ -0,0 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct ValuePolys : public CLI::App {
ValuePolys();
~ValuePolys();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -0,0 +1,41 @@
#include "SecurityCriteria.h"
#include <algorithm>
namespace bts::spartan::mdp3 {
#define CONTAINS(container, value) \
(std::find(container.begin(), container.end(), value) != container.end())
bool SecurityCriteriaClause::is_defined() const {
return exchanges.size() || security_types.size() || assets.size() ||
security_ids.size();
}
bool SecurityCriteriaClause::includes(const SecurityDefinition &def) const {
if (!is_defined())
return false;
if (security_types.size() && !CONTAINS(security_types, def.security_type()))
return false;
if (assets.size() && !CONTAINS(assets, def.asset()))
return false;
if (security_ids.size() && !security_ids.count(def.security_id()))
return false;
return true;
}
bool SecurityCriteria::is_defined() const {
for (const auto &c : clauses)
if (c.is_defined())
return true;
return false;
}
bool SecurityCriteria::includes(const SecurityDefinition &def) const {
for (const auto &c : clauses) {
if (c.includes(def))
return true;
}
return false;
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,37 @@
#pragma once
#include "SecurityDefinition.h"
#include "SecurityID.h"
#include "lib/domain_model/Security.pb.h"
#include <boost/container/small_vector.hpp>
#include <string>
#include <unordered_set>
namespace bts::spartan::mdp3 {
/**
* A group of properties that define a subset of securities. Each field
* (exchanges, security_types, etc.) is a discrete set of potential values for a
* particular property of the security. The subset of securities defined by the
* SecurityCriteriaClause consists of all those with properties that are
* contained in the [non-empty] discrete set(s).
*/
struct SecurityCriteriaClause {
boost::container::small_vector<api::Venue, 0> exchanges;
boost::container::small_vector<api::SecurityType, 0> security_types;
boost::container::small_vector<std::string, 0> assets;
std::unordered_set<SecurityID> security_ids;
bool is_defined() const;
bool includes(const SecurityDefinition &instrument) const;
};
/**
* Represents a subset of securities as the union of [SecurityCriteriaClause] subsets.
*/
struct SecurityCriteria {
bool is_defined() const;
bool includes(const SecurityDefinition &instrument) const;
boost::container::small_vector<SecurityCriteriaClause, 1> clauses;
};
} // namespace bts::spartan::mdp3
@@ -89,8 +89,7 @@ SecurityDefinition::SecurityDefinition(
spread.noLegs().forEach([this](auto const &l) {
LegDef ll{
SecurityID{l.legSecurityID()},
l.legSide() == ::mdp3::LegSide::BuySide ? api::SIDE_BUY
: api::SIDE_SELL,
l.legSide() == ::mdp3::LegSide::BuySide ? Side::Buy : Side::Sell,
Quantity{l.legRatioQty()}};
legs_.push_back(ll);
});
@@ -12,7 +12,7 @@ namespace bts::spartan::mdp3 {
struct LegDef {
SecurityID security_id;
api::Side side;
Side side;
Quantity ratio;
};
Loaded 100 of 122 files, more files were not shown because too many files have changed in this diff. Show more