From 16bf52e193dd923bac7a9ba77cfd73e050b44af1 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 6 Dec 2021 16:29:56 -0500 Subject: [PATCH 1/5] refactor: Clean up MarketDataSubscription * Define as class (not struct) to agree w/ fwd decls * Make ctr inline since its defined in header * Pass params to ctr by value (and std::move) rather than by const ref (and copy) --- src/lib/md/MarketDataSubscription.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/md/MarketDataSubscription.h b/src/lib/md/MarketDataSubscription.h index abd64f306..c42ca9c90 100644 --- a/src/lib/md/MarketDataSubscription.h +++ b/src/lib/md/MarketDataSubscription.h @@ -7,12 +7,13 @@ namespace bts::md { -struct MarketDataSubscription { +class MarketDataSubscription { + public: api::domain::Venue venue() const; MarketDataSubscription() = default; explicit MarketDataSubscription( - const api::md::MarketDataSubscriptionParams ¶ms); + api::md::MarketDataSubscriptionParams params); MarketDataSubscription(const MarketDataSubscription &) noexcept = default; MarketDataSubscription(MarketDataSubscription &&) noexcept = default; @@ -31,9 +32,9 @@ struct MarketDataSubscription { // Definition ////////////////////////////////////////////////// -MarketDataSubscription::MarketDataSubscription( - const api::md::MarketDataSubscriptionParams ¶ms) - : params_(params) {} +inline MarketDataSubscription::MarketDataSubscription( + api::md::MarketDataSubscriptionParams params) + : params_(std::move(params)) {} inline const api::md::MarketDataSubscriptionParams & MarketDataSubscription::params() const { From cd6fcacc4fbe749e29e7bd8363e3ec841a5802aa Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 6 Dec 2021 16:52:34 -0500 Subject: [PATCH 2/5] refactor: Clean up CryptoCommand * Replace BinanceDemo w/ general purpose LiveLadderDemo * Get rid of CryptoConfig.proto, as it inappropriately depended on non-config protos --- src/lib/config/protos/CryptoConfig.proto | 7 -- src/lib/core/ProtoReader.h | 64 +++++++++++++++++++ src/server/sparkd/cmds/common/Validators.cpp | 18 ++++++ src/server/sparkd/cmds/common/Validators.h | 11 ++++ src/server/sparkd/cmds/crypto/BinanceDemo.cpp | 44 ------------- src/server/sparkd/cmds/crypto/BinanceDemo.h | 17 ----- .../sparkd/cmds/crypto/BinanceDemoCmd.cpp | 20 ------ src/server/sparkd/cmds/crypto/CryptoCmd.cpp | 4 +- .../sparkd/cmds/crypto/LiveLadderDemo.cpp | 25 ++++++++ .../sparkd/cmds/crypto/LiveLadderDemo.h | 15 +++++ .../sparkd/cmds/crypto/LiveLadderDemoCmd.cpp | 44 +++++++++++++ .../{BinanceDemoCmd.h => LiveLadderDemoCmd.h} | 6 +- 12 files changed, 182 insertions(+), 93 deletions(-) delete mode 100644 src/lib/config/protos/CryptoConfig.proto create mode 100644 src/lib/core/ProtoReader.h create mode 100644 src/server/sparkd/cmds/common/Validators.cpp create mode 100644 src/server/sparkd/cmds/common/Validators.h delete mode 100644 src/server/sparkd/cmds/crypto/BinanceDemo.cpp delete mode 100644 src/server/sparkd/cmds/crypto/BinanceDemo.h delete mode 100644 src/server/sparkd/cmds/crypto/BinanceDemoCmd.cpp create mode 100644 src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp create mode 100644 src/server/sparkd/cmds/crypto/LiveLadderDemo.h create mode 100644 src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.cpp rename src/server/sparkd/cmds/crypto/{BinanceDemoCmd.h => LiveLadderDemoCmd.h} (65%) diff --git a/src/lib/config/protos/CryptoConfig.proto b/src/lib/config/protos/CryptoConfig.proto deleted file mode 100644 index 9c1cdc49f..000000000 --- a/src/lib/config/protos/CryptoConfig.proto +++ /dev/null @@ -1,7 +0,0 @@ -syntax='proto3'; -package bts.config; -import "lib/api/md/MarketDataSubscriptionParams.proto"; - -message CryptoConfig { - repeated bts.api.md.MarketDataSubscriptionParams crypto_md = 1; -} diff --git a/src/lib/core/ProtoReader.h b/src/lib/core/ProtoReader.h new file mode 100644 index 000000000..d597b09a7 --- /dev/null +++ b/src/lib/core/ProtoReader.h @@ -0,0 +1,64 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace bts::core { + +class ProtoReader { + public: + template + static bool read_proto(const std::string &path, T &result); + + template + static bool read_proto( + const std::string &path, T &result, std::ostream &err); + + template + static std::optional read_proto(const std::string &path); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +bool ProtoReader::read_proto( + const std::string &path, T &result, std::ostream &err) { + std::ifstream f(path); + if (f.good()) { + std::string str( + (std::istreambuf_iterator(f)), + (std::istreambuf_iterator())); + if (google::protobuf::TextFormat::ParseFromString(str, &result)) { + return true; + } + err << "Invalid config in " << path << "\n"; + } else { + err << "File not found in " << path << "\n"; + } + return false; +} + +template +bool ProtoReader::read_proto(const std::string &path, T &result) { + struct NullBuffer : std::streambuf { + int overflow(int c) override { return c; } + }; + NullBuffer null_buff; + std::ostream no_op(&null_buff); + return read_proto(path, result, no_op); +} + +template +std::optional ProtoReader::read_proto(const std::string &path) { + T result; + if (read_proto(path, result)) { + return result; + } + return std::nullopt; +} + +} // namespace spartan::core diff --git a/src/server/sparkd/cmds/common/Validators.cpp b/src/server/sparkd/cmds/common/Validators.cpp new file mode 100644 index 000000000..63afed1ac --- /dev/null +++ b/src/server/sparkd/cmds/common/Validators.cpp @@ -0,0 +1,18 @@ +#include "Validators.h" +#include "server/sparkd/cmds/spartan/utils/FileUtils.h" +#include + +namespace bts::cmds { + +CLI::Validator Validators::file_exists() { + return CLI::Validator( + [](const std::string &path) -> std::string { + if (spartan::FileUtils::file_exists(path)) { + return ""; + } + return fmt::format("The file '{}' does not exist.", path); + }, + "file_exists_validator", "Validates that the file exists"); +} + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/common/Validators.h b/src/server/sparkd/cmds/common/Validators.h new file mode 100644 index 000000000..d8b3abeda --- /dev/null +++ b/src/server/sparkd/cmds/common/Validators.h @@ -0,0 +1,11 @@ +#pragma once +#include + +namespace bts::cmds { + +class Validators { + public: + static CLI::Validator file_exists(); +}; + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/BinanceDemo.cpp b/src/server/sparkd/cmds/crypto/BinanceDemo.cpp deleted file mode 100644 index b6f4e0eab..000000000 --- a/src/server/sparkd/cmds/crypto/BinanceDemo.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "BinanceDemo.h" -#include "lib/config/config.h" -#include "lib/config/protos/CryptoConfig.pb.h" -#include "lib/md/MarketDataGateway.h" -#include "lib/md/MarketDataRegistration.h" -#include "lib/md/MarketDataSubscription.h" -#include "lib/md/listeners/LiveLadderPrinter.h" -#include "server/sparkd/cmds/spartan/utils/FileUtils.h" -#include - -namespace bts::cmds { - -struct BinanceDemo::PImpl { - std::vector subs; -}; - -BinanceDemo::BinanceDemo() : me_(std::make_unique()) { - char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; - putenv(ssl_cert_file); - md::MarketDataGateway gw; - md::listeners::LiveLadderPrinter ladder_printer; - auto crypto = config::get(); - for (const auto &sub_params : crypto.crypto_md()) { - me_->subs.emplace_back(sub_params); - md::MarketDataSubscription sub(sub_params); - switch (sub_params.VenueParams_case()) { - case api::md::MarketDataSubscriptionParams::kBinance: { - gw.register_listener(sub, &ladder_printer); - break; - } - default: - break; - } - } - const std::size_t n_loops = 16; - for (std::size_t n = 0; n < n_loops; n++) { - std::cout << "Main loop iter " << n + 1; - std::this_thread::sleep_for(std::chrono::seconds(5)); - } -} - -BinanceDemo::~BinanceDemo() = default; - -} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/BinanceDemo.h b/src/server/sparkd/cmds/crypto/BinanceDemo.h deleted file mode 100644 index b7de78026..000000000 --- a/src/server/sparkd/cmds/crypto/BinanceDemo.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once -#include -#include - -namespace bts::cmds { - -class BinanceDemo { - public: - BinanceDemo(); - ~BinanceDemo(); - - private: - struct PImpl; - std::unique_ptr me_; -}; - -} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/BinanceDemoCmd.cpp b/src/server/sparkd/cmds/crypto/BinanceDemoCmd.cpp deleted file mode 100644 index e1777faf4..000000000 --- a/src/server/sparkd/cmds/crypto/BinanceDemoCmd.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "BinanceDemoCmd.h" -#include "BinanceDemo.h" -#include -#include - -namespace bts::cmds { - -struct BinanceDemoCmd::PImpl { - std::optional demo; -}; - -BinanceDemoCmd::BinanceDemoCmd() - : CLI::App("Display binance live depth", "binance-demo"), - me_(std::make_unique()) { - callback([this]() { me_->demo.emplace(); }); -} - -BinanceDemoCmd::~BinanceDemoCmd() = default; - -} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/CryptoCmd.cpp b/src/server/sparkd/cmds/crypto/CryptoCmd.cpp index 6827374b2..2d1514e5a 100644 --- a/src/server/sparkd/cmds/crypto/CryptoCmd.cpp +++ b/src/server/sparkd/cmds/crypto/CryptoCmd.cpp @@ -1,6 +1,6 @@ #include "CryptoCmd.h" -#include "BinanceDemoCmd.h" #include "DeribitDemoCmd.h" +#include "LiveLadderDemoCmd.h" #include namespace bts::cmds { @@ -12,7 +12,7 @@ struct CryptoCmd::Args { CryptoCmd::CryptoCmd() : CLI::App("Execute commands for the new crypto system", "crypto"), args_(std::make_unique()) { - add_subcommand(std::make_shared()); + add_subcommand(std::make_shared()); add_subcommand(std::make_shared()); } diff --git a/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp b/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp new file mode 100644 index 000000000..d471dd2e8 --- /dev/null +++ b/src/server/sparkd/cmds/crypto/LiveLadderDemo.cpp @@ -0,0 +1,25 @@ +#include "LiveLadderDemo.h" +#include "lib/md/MarketDataGateway.h" +#include "lib/md/MarketDataRegistration.h" +#include "lib/md/MarketDataSubscription.h" +#include "lib/md/listeners/LiveLadderPrinter.h" +#include + +namespace bts::cmds { + +LiveLadderDemo::LiveLadderDemo(md::MarketDataSubscription subscription) { + char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; + putenv(ssl_cert_file); + + md::MarketDataGateway gw; + md::listeners::LiveLadderPrinter ladder_printer; + gw.register_listener(subscription, &ladder_printer); + + const std::size_t n_loops = 16; + for (std::size_t n = 0; n < n_loops; n++) { + std::cout << "Main loop iter " << n + 1; + std::this_thread::sleep_for(std::chrono::seconds(5)); + } +} + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/LiveLadderDemo.h b/src/server/sparkd/cmds/crypto/LiveLadderDemo.h new file mode 100644 index 000000000..3a3153184 --- /dev/null +++ b/src/server/sparkd/cmds/crypto/LiveLadderDemo.h @@ -0,0 +1,15 @@ +#pragma once +#include + +namespace bts::md { +class MarketDataSubscription; +} + +namespace bts::cmds { + +class LiveLadderDemo { + public: + explicit LiveLadderDemo(md::MarketDataSubscription subscription); +}; + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.cpp b/src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.cpp new file mode 100644 index 000000000..a791b61b8 --- /dev/null +++ b/src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.cpp @@ -0,0 +1,44 @@ +#include "LiveLadderDemoCmd.h" +#include "LiveLadderDemo.h" +#include "lib/api/md/MarketDataSubscriptionParams.pb.h" +#include "lib/core/ProtoReader.h" +#include "lib/logging/logging.h" +#include "lib/md/MarketDataSubscription.h" +#include "server/sparkd/cmds/common/Validators.h" +#include "server/sparkd/cmds/crypto/LiveLadderDemo.h" +#include "server/sparkd/cmds/spartan/utils/FileUtils.h" +#include +#include + +namespace bts::cmds { + +struct LiveLadderDemoCmd::PImpl { + std::string subscription_in; + std::optional demo; +}; + +LiveLadderDemoCmd::LiveLadderDemoCmd() + : CLI::App("Display live depth", "ladder-demo"), + me_(std::make_unique()) { + add_option( + "subscription,--subscription", me_->subscription_in, + "Path to a json file that defines a MarketDataSubscriptionParams " + "object") + ->required() + ->check(Validators::file_exists()); + callback([this]() { + auto subscriptions = + core::ProtoReader::read_proto( + me_->subscription_in); + if (!subscriptions.has_value()) { + LOG(md, bts::Error, + "Unable to parse MarketDataSubscriptionParams from file '{}'", + me_->subscription_in); + } + me_->demo.emplace(md::MarketDataSubscription{std::move(*subscriptions)}); + }); +} + +LiveLadderDemoCmd::~LiveLadderDemoCmd() = default; + +} // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/BinanceDemoCmd.h b/src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.h similarity index 65% rename from src/server/sparkd/cmds/crypto/BinanceDemoCmd.h rename to src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.h index c67be80c0..5c5c929a9 100644 --- a/src/server/sparkd/cmds/crypto/BinanceDemoCmd.h +++ b/src/server/sparkd/cmds/crypto/LiveLadderDemoCmd.h @@ -4,10 +4,10 @@ namespace bts::cmds { -class BinanceDemoCmd : public CLI::App { +class LiveLadderDemoCmd : public CLI::App { public: - BinanceDemoCmd(); - ~BinanceDemoCmd(); + LiveLadderDemoCmd(); + ~LiveLadderDemoCmd(); private: struct PImpl; From ed0924df02b1fca859a78c2f0c3cc4d0a4021b76 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 6 Dec 2021 17:08:29 -0500 Subject: [PATCH 3/5] fix build: Replace now-deprecated option in web build Replace '-p' w/ '--mode production' --- src/web/api/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/api/build.sh b/src/web/api/build.sh index 42287ff38..fabe8adde 100755 --- a/src/web/api/build.sh +++ b/src/web/api/build.sh @@ -85,7 +85,7 @@ if [ "$FULL" == "1" ]; then cp -r ../public/* $OUT/ui/ dotnet restore -s ../../../../externals/nuget/packages if [ "$MODE" == "release" ]; then - dotnet fable webpack -- --env BUILD_PATH=$OUT -p + dotnet fable webpack -- --env BUILD_PATH=$OUT --mode production else dotnet fable webpack -- --env BUILD_PATH=$OUT fi From e09d4d24bab303f3bb13ba2eca007dbfa4ed4bdc Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 7 Dec 2021 16:15:14 -0500 Subject: [PATCH 4/5] build: Add option to build sdk (or not) The sdk isn't needed right now, and it adds several seconds to the build --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 71fa85c1d..99844c2e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ set (SPARK_PYTHON "${EXTERNALS_ROOT}/python") set (SPARK_BIN "${EXTERNALS_ROOT}/bin") set (SPARK_INCLUDE "${EXTERNALS_ROOT}/include") set (SPARK_LIB "${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}") +set (BUILD_SDK OFF CACHE BOOL "Build the sdk") include_directories (BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib) link_directories (${SPARK_LIB}) @@ -47,7 +48,9 @@ else () add_subdirectory(src/lib) add_subdirectory(src/web) add_subdirectory(doc/user) - add_subdirectory(src/sdk) + if (BUILD_SDK) + add_subdirectory(src/sdk) + endif() add_subdirectory(src/data_files) add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/server From e769926429cffd8949c33a0c0c4764b507be495f Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 7 Dec 2021 17:49:05 -0500 Subject: [PATCH 5/5] Update externals --- externals | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals b/externals index deb80e672..7bcc559c1 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit deb80e67290e1f430dddff50a929b7c6ae130184 +Subproject commit 7bcc559c1a5262f924d65e4cacec190ffc1fe8e0