diff --git a/CMakeLists.txt b/CMakeLists.txt index b00d05f22..442bfd067 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ if(WIN32) else() add_compile_definitions(SPARK_ENABLE_CCAPI) add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA) + add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT) add_compile_definitions(CCAPI_ENABLE_EXCHANGE_FTX) add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE) add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US) diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 6a67abf2d..76e7dca43 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -9,7 +9,8 @@ file( GLOB_RECURSE Config_Protos RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} config/protos/*.proto - api/domain/SecurityMetadataCollection.proto) + api/domain/SecurityMetadataCollection.proto + api/md/MarketDataConfig.proto) file( GLOB_RECURSE Service_Protos RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/src/lib/api/md/BinanceMarketDataConfig.proto b/src/lib/api/md/BinanceMarketDataConfig.proto new file mode 100644 index 000000000..e323518aa --- /dev/null +++ b/src/lib/api/md/BinanceMarketDataConfig.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package bts.api.md; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; + +message BinanceMarketDataConfig { + // TODO +} diff --git a/src/lib/api/md/DepthStreamConfig.proto b/src/lib/api/md/DepthStreamConfig.proto deleted file mode 100644 index 31f296d8f..000000000 --- a/src/lib/api/md/DepthStreamConfig.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -import "lib/api/domain/Venue.proto"; - -package bts.api.md; - -message DepthStreamConfig { - bts.api.domain.Venue venue = 1; - repeated string symbols = 2; - uint32 num_levels = 3; -} diff --git a/src/lib/api/md/FTXMarketDataConfig.proto b/src/lib/api/md/FTXMarketDataConfig.proto new file mode 100644 index 000000000..32536f1d2 --- /dev/null +++ b/src/lib/api/md/FTXMarketDataConfig.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +import "lib/api/domain/SecurityCriteria.proto"; + +package bts.api.md; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; + +message FTXMarketDataConfig { + uint32 num_book_levels = 1; + bts.api.domain.SecurityCriteria security_criteria = 2; +} diff --git a/src/lib/api/md/MarketDataConfig.proto b/src/lib/api/md/MarketDataConfig.proto index 6fd886490..59a82e1ff 100644 --- a/src/lib/api/md/MarketDataConfig.proto +++ b/src/lib/api/md/MarketDataConfig.proto @@ -1,10 +1,16 @@ syntax = "proto3"; -import "lib/api/md/DepthStreamConfig.proto"; +import "lib/api/md/FTXMarketDataConfig.proto"; +import "lib/api/md/BinanceMarketDataConfig.proto"; + +option csharp_namespace = "bts.api.md"; +option cc_enable_arenas = true; package bts.api.md; message MarketDataConfig { - repeated bts.api.md.DepthStreamConfig depth_streams = 1; - uint32 num_dispatcher_threads = 2; + oneof venue { + bts.api.md.FTXMarketDataConfig ftx = 1; + bts.api.md.BinanceMarketDataConfig binance = 2; + } } diff --git a/src/lib/collections/DiscreteValuesFilter.h b/src/lib/collections/DiscreteValuesFilter.h new file mode 100644 index 000000000..a4294c040 --- /dev/null +++ b/src/lib/collections/DiscreteValuesFilter.h @@ -0,0 +1,98 @@ +#pragma once +#include +#include + +namespace bts::collections { + +template +class DiscreteValuesFilter { + public: + using Values = std::vector; + + bool is_active() const; + bool is_included(Value value) const; + void include(Value value); + bool include(const Values &values); + void exclude(Value value); + bool exclude(const Values &values); + + void reset(Values values, bool is_included); + + private: + bool are_values_included_ = false; + Values values_; + + // Adds a value to the list if it isn't there already + void add_value_(Value value); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +bool DiscreteValuesFilter::is_active() const { + return are_values_included_ || !values_.empty(); +} + +template +bool DiscreteValuesFilter::is_included(Value value) const { + return are_values_included_ == + (std ::find(values_.begin(), values_.end(), value) != values_.end()); +} + +template +void DiscreteValuesFilter::include(Value value) { + if (are_values_included_) { + add_value_(value); + return; + } + values_.erase(std::find(values_.begin(), values_.end(), value)); +} + +template +bool DiscreteValuesFilter::include(const Values &values) { + auto orig_sz = this->values_.size(); + auto orig_is_included = are_values_included_; + for (auto t : values) { + include(t); + } + return this->values_.size() != orig_sz || + orig_is_included != are_values_included_; +} + +template +void DiscreteValuesFilter::exclude(Value value) { + if (are_values_included_) { + values_.erase(std::find(values_.begin(), values_.end(), value)); + return; + } + add_value_(value); +} + +template +bool DiscreteValuesFilter::exclude(const Values &values) { + auto orig_sz = this->values_.size(); + auto orig_is_included = are_values_included_; + for (auto t : values) { + exclude(t); + } + return this->values_.size() != orig_sz || + orig_is_included != are_values_included_; +} + +template +void DiscreteValuesFilter::reset(Values values, bool is_included) { + this->values_ = std::move(values); + are_values_included_ = is_included; +} + +template +void DiscreteValuesFilter::add_value_(Value value) { + auto it = std::find(values_.begin(), values_.end(), value); + if (it == values_.end()) { + values_.push_back(value); + } +} + +} // namespace bts::collections diff --git a/src/lib/collections/Find.h b/src/lib/collections/Find.h new file mode 100644 index 000000000..c91e4c1b7 --- /dev/null +++ b/src/lib/collections/Find.h @@ -0,0 +1,6 @@ +#pragma once + +#define FIND(range, value) std::find(range.begin(), range.end(), value) + +#define FIND_IF(range, predicate) \ + std::find_if(range.begin(), range.end(), predicate) diff --git a/src/lib/collections/Remove.h b/src/lib/collections/Remove.h new file mode 100644 index 000000000..bfa7ad587 --- /dev/null +++ b/src/lib/collections/Remove.h @@ -0,0 +1,58 @@ +#pragma once +#include +#include +#include + +// Removes all occurrences of value +#define REMOVE(container, value) \ + container.erase( \ + std::remove(container.begin(), container.end(), value), \ + container.end()) + +// Removes all values that satisfy the predicate +#define REMOVE_IF(container, predicate) \ + container.erase( \ + std::remove_if(container.begin(), container.end(), predicate), \ + container.end()) + +// Removes first occurrence of value +#define REMOVE_FIRST(range, value) \ + range.erase(std::find(range.begin(), range.end(), value)) + +// Removes first value that satisfies the predicate +#define REMOVE_FIRST_IF(range, predicate) \ + range.erase(std::find_if(range.begin(), range.end(), predicate)) + +namespace bts::collections { + +class Remove { + public: + template + using TCount = typename std::result_of::type; + + template + static TCount remove(Range &range, Value value); + + template + static TCount remove_if(Range &range, Predicate predicate); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +Remove::TCount Remove::remove(Range &range, Value value) { + auto sz_before = range.size(); + REMOVE_RANGE(range, value); + return sz_before - range.size(); +} + +template +Remove::TCount Remove::remove_if(Range &range, Predicate predicate) { + auto sz_before = range.size(); + REMOVE_RANGE_IF(range, predicate); + return sz_before - range.size(); +} + +} // namespace bts::collections diff --git a/src/lib/config/ConfigurationState.cpp b/src/lib/config/ConfigurationState.cpp index f30d882a8..7fa2a2140 100644 --- a/src/lib/config/ConfigurationState.cpp +++ b/src/lib/config/ConfigurationState.cpp @@ -103,8 +103,7 @@ string ConfigurationState::settings_path() { void ConfigurationState::initialize_defaults(IConfigLog &log) { initialize_defaults( - log, - {std::string("lib/config/protos/"), std::string("lib/api/domain/")}); + log, {"lib/config/protos/", "lib/api/domain/", "lib/api/md/"}); } void ConfigurationState::initialize_defaults( @@ -545,10 +544,12 @@ void ConfigurationState::generate_all_config_proto( // add a meta type and field for each generated message proto::DescriptorPool::generated_pool(); - auto find_proto_file = [&proto_dirs](const std::string &name) -> const proto::FileDescriptor * { + auto find_proto_file = + [&proto_dirs](const std::string &name) -> const proto::FileDescriptor * { for (const auto &proto_dir : proto_dirs) { - const auto * res = proto::DescriptorPool::generated_pool()->FindFileByName( - proto_dir + name); + const auto *res = + proto::DescriptorPool::generated_pool()->FindFileByName( + proto_dir + name); if (res != nullptr) { return res; } @@ -615,8 +616,8 @@ bool is_valid_key_descriptor(const proto::Descriptor *property_desc) { return false; } -const proto::Message * -ConfigurationState::try_get_message(const std::string &property_name) { +const proto::Message *ConfigurationState::try_get_message( + const std::string &property_name) { initialize_settings(); const proto::Message *message; @@ -655,8 +656,8 @@ const proto::Message *ConfigurationState::try_get_message_or_default( return nullptr; } -const proto::Message & -ConfigurationState::get_message(const proto::Descriptor *property_desc) { +const proto::Message &ConfigurationState::get_message( + const proto::Descriptor *property_desc) { if (!is_valid_key_descriptor(property_desc)) { throw InvalidKeyException(property_desc->name().c_str()); } @@ -711,8 +712,8 @@ const proto::Message *ConfigurationState::try_get_value( return nullptr; } -const google::protobuf::Descriptor * -ConfigurationState::get_descriptor_by_name(const std::string &name) { +const google::protobuf::Descriptor *ConfigurationState::get_descriptor_by_name( + const std::string &name) { const proto::Message *pmsg = _defaults.get(); if (pmsg) { // Get the metadata descriptor (Meta = rule+value fields) diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 5ff4b5df8..c79576013 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -2,7 +2,7 @@ # Common Market data options # -MarketDataConfig { +HistoricMarketDataConfig { rule: "DeploymentType.type == 'PROD'" value { use_sim: false @@ -45,12 +45,12 @@ CMEOptions { } } -ICEPublisherOptions { - value { - network_interface: "eth1" - } -} - +ICEPublisherOptions { + value { + network_interface: "eth1" + } +} + #to make this work in dev clone the operations repo #go here: c:\\code\\bts\\operations\\ice\\ICE_iMpactMulticastFeed_GettingStarted-1_1_36\\bin\\easyTunnel> #modify the easyTunnelProxy.properties file to replace diff --git a/src/lib/config/defaults/MarketDataConfig.conf b/src/lib/config/defaults/MarketDataConfig.conf new file mode 100644 index 000000000..4d6ebe0bb --- /dev/null +++ b/src/lib/config/defaults/MarketDataConfig.conf @@ -0,0 +1,14 @@ +MarketDataConfig { + value { + ftx { + security_criteria { + clauses { + security_types: FUTURE + security_types: SPOT + symbols: "BTC" + symbols: "ETH" + } + } + } + } +} diff --git a/src/lib/config/protos/market_data.proto b/src/lib/config/protos/market_data.proto index e84c36e93..b3f6c663f 100644 --- a/src/lib/config/protos/market_data.proto +++ b/src/lib/config/protos/market_data.proto @@ -1,7 +1,7 @@ syntax = 'proto2'; package bts.md; -message MarketDataConfig { +message HistoricMarketDataConfig { // Allows you to turn off in-proc market data entirely optional bool use_md = 1 [ default = true ]; optional bool use_onixs = 2 [ default = true ]; @@ -251,4 +251,4 @@ message FeedHandlerConfig { repeated string channel = 1; optional string network_interface_a = 3 [ default = "0.0.0.0" ]; optional string network_interface_b = 4 [ default = "" ]; -} \ No newline at end of file +} diff --git a/src/lib/core/BoolName.h b/src/lib/core/BoolName.h new file mode 100644 index 000000000..c21c6e5d0 --- /dev/null +++ b/src/lib/core/BoolName.h @@ -0,0 +1,39 @@ +#pragma once + +namespace bts::core { + +class BoolName { + public: + explicit constexpr BoolName(bool value); + + [[nodiscard]] static constexpr const char *true_false(bool value); + [[nodiscard]] static constexpr const char *yes_no(bool value); + [[nodiscard]] static constexpr const char *y_n(bool value); + [[nodiscard]] static constexpr const char *on_off(bool value); + + private: + bool value_; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr const char *BoolName::true_false(bool value) { + return value ? "true" : "false"; +} + +constexpr const char *BoolName::yes_no(bool value) { + return value ? "yes" : "no"; +} + +constexpr const char *BoolName::y_n(bool value) { + // NOLINTNEXTLINE + return value ? "y" : "n"; +} + +constexpr const char *BoolName::on_off(bool value) { + return value ? "on" : "off"; +} + +} // namespace bts::core diff --git a/src/lib/core/NaN.h b/src/lib/core/NaN.h new file mode 100644 index 000000000..e9a46102e --- /dev/null +++ b/src/lib/core/NaN.h @@ -0,0 +1,24 @@ +#pragma once +#include + +namespace bts::core { + +class NaN { + public: + static constexpr double value(); + static constexpr float valuef(); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr double NaN::value() { + return std::numeric_limits::quiet_NaN(); +} + +constexpr float NaN::valuef() { + return std::numeric_limits::quiet_NaN(); +} + +} // namespace bts::core diff --git a/src/lib/core/StringContains.h b/src/lib/core/StringContains.h new file mode 100644 index 000000000..30fae9f7e --- /dev/null +++ b/src/lib/core/StringContains.h @@ -0,0 +1,70 @@ +#pragma once +#include +#include +#include + +namespace bts::core { + +class StringContains { + public: + template + using AllSame = std::enable_if_t...>>; + + template + using AllString = AllSame; + + constexpr bool contains(const std::string &str, const std::string &substr); + constexpr bool contains(const std::string &str, const char *substr); + constexpr bool contains(const char *str, const char *substr); + + template + constexpr bool contains( + const std::string &str, const std::string &substr, Ts... substrs); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +constexpr bool StringContains::contains( + const std::string &str, const std::string &substr) { + return str.find(substr) != std::string::npos; +} + +constexpr bool StringContains::contains( + const std::string &str, const char *substr) { + return str.find(substr) != std::string::npos; +} + +constexpr bool StringContains::contains(const char *str, const char *substr) { + if (str[0] == '\0') { + return false; + } + if (substr[0] == '\0') { + return false; + } + + for (std::size_t i = 0; str[i] != '\0'; i++) { + bool is_match = true; + for (std::size_t j = 0; substr[j] < '\0'; j++) { + if (str[i + j] != substr[j]) { + is_match = false; + break; + } + } + if (is_match) { + return true; + } + } + return false; +} + +template +constexpr bool StringContains::contains( + const std::string &str, const std::string &substr, Ts... substrs) { + return contains(str, substr) || contains(str, substrs...); +} + +#define STRING_CONTAINS(x, ...) (bts::StringContains(x, __VA_ARGS__)) + +} // namespace bts::core diff --git a/src/lib/core/StringFinder.h b/src/lib/core/StringFinder.h new file mode 100644 index 000000000..7a709dc11 --- /dev/null +++ b/src/lib/core/StringFinder.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + +namespace bts::core { + +class StringFinder { + public: + static std::size_t find(std::string str, char ch, int occurrence = 1); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::size_t StringFinder::find( + std::string str, char ch, int occurrence) { + int occur = 0; + + // Loop to find the Nth + // occurrence of the character + for (std::size_t i = 0; i < str.length(); i++) { + if (str[i] == ch) { + occur += 1; + } + if (occur == occurrence) + return i; + } + return std::string::npos; +} + +} // namespace bts::core diff --git a/src/lib/core/StringJoiner.h b/src/lib/core/StringJoiner.h new file mode 100644 index 000000000..76a8a1d85 --- /dev/null +++ b/src/lib/core/StringJoiner.h @@ -0,0 +1,178 @@ +#pragma once +#include +#include +#include +#include + +namespace bts::core { + +class StringJoiner { + public: + template + static std::ostream &join( + const std::string &delimiter, std::ostream &out, Arg arg); + + template + static std::string join(const std::string &delimiter, Arg arg); + + template + static std::ostream &join( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args); + + template + static std::string join(const std::string &delimiter, Arg arg, Args... args); + + template + static std::ostream &join( + Iter first, Iter last, const std::string &delimiter, std::ostream &out); + + template + static std::string join(Iter first, Iter last, const std::string &delimiter); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::string join_non_empties(const std::string &delimiter, Arg arg); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::string join_non_empties( + const std::string &delimiter, Arg arg, Args... args); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::ostream &join_non_empties( + const std::string &delimiter, std::ostream &out, Arg arg); + + /** + * Converts the args to strings (using the standard output operator), then + * joins the non-empty ones with the specified delimiter. + */ + template + static std::ostream &join_non_empties( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args); + + private: + template + static std::ostream &join_non_empties_( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +std::ostream &StringJoiner::join( + const std::string & /*delimiter*/, std::ostream &out, Arg arg) { + // special case for single item (don't use delimiter) + out << arg; + return out; +} + +template +std::string StringJoiner::join(const std::string & /*delimiter*/, Arg arg) { + // special case for single item (don't use delimiter) + std::stringstream out; + out << arg; + return out.str(); +} + +template +std::ostream &StringJoiner::join( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args) { + out << arg << delimiter; + return join(delimiter, out, args...); +} + +template +std::string StringJoiner::join( + const std::string &delimiter, Arg arg, Args... args) { + std::stringstream out; + out << arg << delimiter; + join(delimiter, out, args...); + return out.str(); +} + +template +std::ostream &StringJoiner::join( + Iter it, Iter last, const std::string &delimiter, std::ostream &out) { + if (it == last) { + return out; + } + out << *it; + it++; + for (; it != last; it++) { + out << delimiter << *it; + } + return out; +} + +template +std::string StringJoiner::join( + Iter it, Iter last, const std::string &delimiter) { + std::stringstream out; + join(it, last, delimiter, out); + return out.str(); +} + +template +std::string StringJoiner::join_non_empties( + const std::string & /*delimiter*/, Arg arg) { + std::stringstream ss; + ss << arg; + return ss.str(); +} + +template +std::string StringJoiner::join_non_empties( + const std::string &delimiter, Arg arg, Args... args) { + std::stringstream ss; + std::ostream &out = ss; + join_non_empties(delimiter, out, arg, args...); + return ss.str(); +} + +template +std::ostream &StringJoiner::join_non_empties( + const std::string & /*delimiter*/, std::ostream &out, Arg arg) { + out << arg; + return out; +} + +template +std::ostream &StringJoiner::join_non_empties( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args) { + std::stringstream ss; + ss << arg; + auto arg_str = ss.str(); + if (arg_str.empty()) { + return join_non_empties(delimiter, out, args...); + } + out << arg; + return join_non_empties_(delimiter, out, args...); +} + +template +std::ostream &StringJoiner::join_non_empties_( + const std::string &delimiter, std::ostream &out, Arg arg, Args... args) { + std::stringstream ss; + ss << arg; + auto arg_str = ss.str(); + if (arg_str.empty()) { + return join_non_empties_(delimiter, out, args...); + } + out << delimiter << arg_str; + join(delimiter, out, args...); + return out; +} + +} // namespace bts::core diff --git a/src/lib/io/JsonReader.cpp b/src/lib/io/JsonReader.cpp new file mode 100644 index 000000000..d2ce6ad85 --- /dev/null +++ b/src/lib/io/JsonReader.cpp @@ -0,0 +1,420 @@ +#include "JsonReader.h" +#include "lib/core/TimeStamp.h" +#include "lib/logging/logging.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/datetime.h" +#include +#include +#include + +namespace bts::io { + +using core::TimeStamp; +using std::int32_t; +using std::int64_t; +using std::numeric_limits; +using std::stoull; +using std::string; +using std::uint32_t; +using std::uint64_t; + +// 'read' fn bodies should be wrapped between BEGIN/END_READ_JSON tags. +// These macros abstract out repetative exception handling code (and a nullptr check). + +#define BEGIN_READ_JSON \ + if (!json_) { \ + return false; \ + } \ + try + +#define END_READ_JSON(type_name) \ + catch (const std::exception &e) { \ + LOG(Gen, Diag, "Failed to parse json to '{}'. Exception: {}", \ + std::string{type_name}, std::string{e.what()}); \ + } \ + catch (...) { \ + bts_assert_msgf( \ + false, "Failed to parse json to '{}'. Exception: unknown", \ + std::string{type_name}); \ + } \ + return false; + +// The idea here is we parse numbers to wide types (either int64, uint64, or double), +// then cast that value to the (potentially narrower) output type. This macro evaluates +// to true if 'wide_val' is within the bounds of the result type (otherwise false). +#define NUM_WITHIN_BOUNDS(wide_val, result_t) \ + (wide_val >= \ + static_cast(numeric_limits::min()) && \ + wide_val <= \ + static_cast(numeric_limits::max())) + +static const nlohmann::json *get_field_json( + const nlohmann::json *json, const std::string &name) { + if (!json || !json->is_object()) { + return nullptr; + } + auto it = json->find(name); + if (it == json->end()) { + return nullptr; + } + return &it.value(); +} + +JsonReader::JsonReader(const nlohmann::json *json) : json_(json) {} + +JsonReader::JsonReader(const nlohmann::json *json, const std::string &name) + : json_(get_field_json(json, name)) {} + +bool JsonReader::read(const string &name, string &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, int32_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, uint32_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, int64_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, uint64_t &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, double &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(const string &name, TimeStamp &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read_epoch_nanos(const string &name, TimeStamp &value) const { + return JsonReader{json_, name}.read_epoch_nanos(value); +} + +bool JsonReader::read_epoch_micros(const string &name, TimeStamp &value) const { + return JsonReader{json_, name}.read_epoch_micros(value); +} + +bool JsonReader::read_epoch_millis(const string &name, TimeStamp &value) const { + return JsonReader{json_, name}.read_epoch_millis(value); +} + +bool JsonReader::read(const string &name, nlohmann::json &result) const { + return JsonReader{json_, name}.read(result); +} + +bool JsonReader::read(int32_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + result = static_cast(d); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to int32 ({})", + d, result); + } + return true; + } + case nlohmann::json::value_t::number_integer: { + const auto il = json_->get(); + if (!NUM_WITHIN_BOUNDS(il, int32_t)) { + LOG(Gen, Error, + "Failed to convert field of type 'number_integer' ({}) to " + "uint32: value is negative.", + il); + } + result = static_cast(il); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + const auto ul = json_->get(); + if (ul > static_cast(numeric_limits::max())) { + LOG(Gen, Error, + "Failed to convert field of type 'number_unsigned' ({}) to " + "int32: value is too large.", + ul); + return false; + } + result = static_cast(ul); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoi(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("int32") +} + +bool JsonReader::read(uint32_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + result = static_cast(d); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to uint32 ({})", + d, result); + } + return true; + } + case nlohmann::json::value_t::number_integer: { + const auto i = json_->get(); + if (!NUM_WITHIN_BOUNDS(i, uint32_t)) { + LOG(Gen, Error, + "Failed to convert field of type 'number_integer' ({}) to " + "uint32: value is negative.", + i); + return false; + } + result = static_cast(i); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + const auto ul = json_->get(); + if (ul > static_cast(numeric_limits::max())) { + LOG(Gen, Error, + "Failed to convert field of type 'number_unsigned' ({}) to " + "int32: value is too large.", + ul); + return false; + } + result = static_cast(ul); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoul(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("uint32") +} + +bool JsonReader::read(int64_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to int64 ({})", + d, result); + } + result = static_cast(d); + return true; + } + case nlohmann::json::value_t::number_integer: { + result = json_->get(); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + const auto ul = json_->get(); + if (NUM_WITHIN_BOUNDS(ul, int64_t)) { + LOG(Gen, Error, + "Potential loss of data converting field of type " + "'number_unsigned' ({}) to int64 ({})", + ul, result); + return false; + } + result = static_cast(ul); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoul(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("int64") +} + +bool JsonReader::read(uint64_t &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: { + const auto d = json_->get(); + if (result != d) { + LOG(Gen, Warn, + "Potential loss of data converting field of type " + "'number_float' ({}) to uint64 ({})", + d, result); + } + result = static_cast(d); + return true; + } + case nlohmann::json::value_t::number_integer: { + const auto il = json_->get(); + if (NUM_WITHIN_BOUNDS(il, uint64_t)) { + LOG(Gen, Error, + "Potential loss of data converting field of type " + "'number_integer' ({}) to uint64 ({})", + il, result); + return false; + } + result = static_cast(il); + return true; + } + case nlohmann::json::value_t::number_unsigned: { + result = json_->get(); + return true; + } + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stoul(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("uint64") +} + +bool JsonReader::read(double &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::number_float: + case nlohmann::json::value_t::number_integer: + case nlohmann::json::value_t::number_unsigned: + json_->get_to(result); + return true; + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + result = std::stod(str); + return true; + } + default: + return false; + } + } + END_READ_JSON("double") +} + +bool JsonReader::read(string &result) const { + BEGIN_READ_JSON { + json_->get_to(result); + return true; + } + END_READ_JSON("string") +} + +bool JsonReader::read(nlohmann::json &result) const { + BEGIN_READ_JSON { + json_->get_to(result); + return true; + } + END_READ_JSON("json-object") +} + +bool JsonReader::parse_time_stamp_(const string &str, TimeStamp &value) const { + return parse_iso_time_stamp_(str, value) || + parse_epoch_nanos_time_stamp_(str, value); +} + +bool JsonReader::parse_iso_time_stamp_( + const string &str, TimeStamp &value) const { + value = TimeStamp::from_iso_string(str); + return true; +} + +bool JsonReader::parse_epoch_nanos_time_stamp_( + const string &str, TimeStamp &value) const { + uint64_t nanos = stoull(str); + value = TimeStamp{nanos}; + return true; +} + +bool JsonReader::read(TimeStamp &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::null: + case nlohmann::json::value_t::object: + case nlohmann::json::value_t::array: + case nlohmann::json::value_t::boolean: + case nlohmann::json::value_t::binary: + case nlohmann::json::value_t::discarded: + return false; + case nlohmann::json::value_t::string: { + const auto str = json_->get(); + return parse_iso_time_stamp_(str, result) || + parse_epoch_nanos_time_stamp_(str, result); + } + case nlohmann::json::value_t::number_integer: + case nlohmann::json::value_t::number_unsigned: + case nlohmann::json::value_t::number_float: + // Maybe we could assume seconds here, idk though...when in doubt, prob + // better to be cautious than cavalier + return false; + } + } + END_READ_JSON("TimeStamp") +} + +bool JsonReader::read_iso( + const std::string &name, core::TimeStamp &result) const { + return JsonReader{json_, name}.read_iso(result); +} + +bool JsonReader::read_iso(core::TimeStamp &result) const { + BEGIN_READ_JSON { + switch (json_->type()) { + case nlohmann::json::value_t::string: { + return parse_iso_time_stamp_(json_->get(), result); + } + default: + return false; + } + } + END_READ_JSON("TimeStamp") +} + +bool JsonReader::read_epoch_nanos(TimeStamp &value) const { + int64_t nanos; + if (!read(nanos)) { + return false; + } + value = TimeStamp::from_nanos(nanos); + return true; +} + +bool JsonReader::read_epoch_micros(TimeStamp &value) const { + int64_t micros; + if (!read(micros)) { + return false; + } + value = TimeStamp::from_micros(micros); + return true; +} + +bool JsonReader::read_epoch_millis(TimeStamp &value) const { + int64_t millis; + if (!read(millis)) { + return false; + } + value = TimeStamp::from_millis(millis); + return true; +} + +} // namespace bts::io diff --git a/src/lib/io/JsonReader.h b/src/lib/io/JsonReader.h new file mode 100644 index 000000000..022fb906b --- /dev/null +++ b/src/lib/io/JsonReader.h @@ -0,0 +1,123 @@ +#pragma once +#include "lib/core/TimeStamp.h" +#include +#include +#include +#include + +namespace bts::io { + +class JsonReader { + public: + explicit JsonReader(const nlohmann::json *json); + + JsonReader(const nlohmann::json *json, const std::string &name); + + bool read(const std::string &name, std::string &result) const; + bool read(const std::string &name, std::int32_t &result) const; + bool read(const std::string &name, std::uint32_t &result) const; + bool read(const std::string &name, std::int64_t &result) const; + bool read(const std::string &name, std::uint64_t &result) const; + bool read(const std::string &name, double &result) const; + bool read(const std::string &name, nlohmann::json &result) const; + bool read(const std::string &name, core::TimeStamp &result) const; + bool read_iso(const std::string &name, core::TimeStamp &result) const; + bool read_epoch_nanos( + const std::string &name, core::TimeStamp &result) const; + bool read_epoch_micros( + const std::string &name, core::TimeStamp &result) const; + bool read_epoch_millis( + const std::string &name, core::TimeStamp &result) const; + + template + bool read(const std::string &name, T &result) const; + + template + T read(const std::string &name) const; + + bool read(std::string &result) const; + bool read(std::int32_t &result) const; + bool read(std::uint32_t &result) const; + bool read(std::int64_t &result) const; + bool read(std::uint64_t &result) const; + bool read(double &result) const; + bool read(nlohmann::json &result) const; + bool read(core::TimeStamp &result) const; + bool read_iso(core::TimeStamp &result) const; + bool read_epoch_nanos(core::TimeStamp &result) const; + bool read_epoch_micros(core::TimeStamp &result) const; + bool read_epoch_millis(core::TimeStamp &result) const; + + template + bool read(T &result) const; + + /** + * @brief Reads a json field into \p result + * + * @details If the read fails, assigns \p default_value to \p result + * @return True if the read succeeds, otherwise false + */ + template + bool read_or(const std::string &name, T &result, T default_value) const; + + private: + const nlohmann::json *json_; + + bool parse_time_stamp_( + const std::string &str, core::TimeStamp &result) const; + bool parse_iso_time_stamp_( + const std::string &str, core::TimeStamp &result) const; + bool parse_epoch_nanos_time_stamp_( + const std::string &str, core::TimeStamp &result) const; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +template +inline bool JsonReader::read(const std::string &name, T &result) const { + if (!json_ || !json_->is_object()) { + return false; + } + auto it = json_->find(name); + if (it == json_->end()) { + return false; + } + it->get_to(result); + return true; +} + +template +inline bool JsonReader::read(T &result) const { + try { + json_->get_to(result); + return true; + } catch (...) { + const auto j_str = json_->dump(2); + LOG(md, Error, "Exception parsing json:\n{}", j_str); + return false; + } +} + +template +inline T JsonReader::read(const std::string &name) const { + bts_assert(json_ && json_->is_object()); + auto it = json_->find(name); + bts_assert_msgf(it != json_->end(), "json doesn't contain field '{}'", name); + T result; + it->get_to(result); + return result; +} + +template +inline bool JsonReader::read_or( + const std::string &name, T &result, T default_value) const { + if (read(name, result)) { + return true; + } + result = default_value; + return false; +} + +} // namespace bts::io diff --git a/src/lib/io/JsonType.h b/src/lib/io/JsonType.h new file mode 100644 index 000000000..41964334a --- /dev/null +++ b/src/lib/io/JsonType.h @@ -0,0 +1,47 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/EnumMacros.h" +#include +#include +#include + +namespace bts::io { + +// clang-format off +#define FOR_ALL_VALUES_T(per_value, t) \ + per_value(t, null) \ + per_value(t, object) \ + per_value(t, array) \ + per_value(t, string) \ + per_value(t, boolean) \ + per_value(t, number_integer) \ + per_value(t, number_unsigned) \ + per_value(t, number_float) \ + per_value(t, binary) \ + per_value(t, discarded) + +#define FOR_ALL_VALUES(per_value) FOR_ALL_VALUES_T(per_value, nlohmann::detail::value_t) +// clang-format on + +/** + * @brief A wrapper class around nlohmann::detail::value_t that provides + * basic functionality, such as name/parse fns, output operator, etc. + */ +class JsonType { + public: + using Value = nlohmann::detail::value_t; + ENUM_BASIC_MEMBERS(JsonType) +}; + +ENUM_OUTPUT_OPS(JsonType) +ENUM_NAME_DEF(JsonType, FOR_ALL_VALUES) +ENUM_PARSE_DEF(JsonType, FOR_ALL_VALUES) +ENUM_FROM_JSON(JsonType) +ENUM_EQUALITY_OPS(JsonType) + +// undefine macros +#undef FOR_ALL_VALUES +#undef FOR_ALL_VALUES_T + +} // namespace bts::io +#endif diff --git a/src/lib/md/CCApiCorrelationID.h b/src/lib/md/CCApiCorrelationID.h index 42b7a87aa..22b1b537b 100644 --- a/src/lib/md/CCApiCorrelationID.h +++ b/src/lib/md/CCApiCorrelationID.h @@ -1,57 +1,275 @@ #pragma once #ifdef SPARK_ENABLE_CCAPI -#include "lib/api/domain/Venue.pb.h" -#include "lib/md/MarketDataListenerID.h" - +#include "lib/core/FMT_IMPL.h" +#include "lib/core/StringFinder.h" +#include "lib/core/StringJoiner.h" +#include "lib/utils/bts_assert.h" #include -#include -#include +#include #include -#include +#include namespace bts::md { class CCApiCorrelationID { public: - static std::string fmt_market_depth( - api::domain::Venue venue, - std::string desc, - MarketDataListenerID listener_id); + static constexpr const char *delimiter(); - static std::optional< - std::tuple> - parse_market_depth(const std::string &corr_id_str); + static CCApiCorrelationID from_string(const std::string &str); + + template + static CCApiCorrelationID from_parts(Parts... parts); + + /** + * @brief Gets the number of parts + * + * @return constexpr std::size_t The number of parts + */ + constexpr std::size_t size() const; + + constexpr bool empty() const; + /** + * Gets a vector of the next \p len parts (or however many are available, if + * fewer than \p len). The cursor remains unchanged. + * + * @param len The number of parts to get + * @param require_len Whether or not to assert at least \p len parts are + * available + * + * @return A vector of pointers to the next \p len parts (or fewer if less + * than \p len are available) + */ + std::vector peek_parts( + std::size_t len = 1, bool require_len = true) const; + + std::vector peek_parts_at( + std::size_t index, std::size_t len = 1, bool require_len = true) const; + + /** + * Gets a vector of the next \p len parts (or however many are available, if + * fewer than \p len). The cursor is incremented by the size of the returned + * vector. + * + * @param len The number of parts to get + * @param require_len Whether or not to assert at least \p len parts are + * available to consume + * + * @return A vector of pointers to the next \p len parts (or fewer if less + * than \p len are available to consume) + */ + std::vector consume_parts( + std::size_t len, bool require_len = true); + + template + bool consume_part(T &value, TParse parse = {}); + + template + bool consume_parts(std::size_t len, T &value, TParse parse = {}); + + constexpr const std::string &to_string() const; + std::string peek_string(std::size_t len = 1, bool require_len = true) const; + std::string peek_string_at( + std::size_t index, std::size_t len = 1, bool require_len = true) const; + std::string consume_string(std::size_t len, bool require_len = true); + CCApiCorrelationID trim_front() const; + + private: + std::vector parts_; + std::size_t cursor_ = 0; + std::string str_; + + /** + * @brief Advances 'cursor_' by the specified amount. If 'str_' is non-null, + * it is trimmed the same number of parts (i.e., so 'cursor_' and 'str_' + * remain in sync). + * + * @details Don't mutate 'cursor_' outside of this method (or risk + * desynchronizing it with 'str_'). + * + * @param len + */ + void advance_(std::size_t len); }; +constexpr bool operator==( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs); + +constexpr bool operator!=( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs); + +std::ostream &operator<<( + std::ostream &out, const CCApiCorrelationID &corr_id) noexcept; + ////////////////////////////////////////////////// // Definition ////////////////////////////////////////////////// -inline std::string CCApiCorrelationID::fmt_market_depth( - api::domain::Venue venue, - std::string desc, - MarketDataListenerID listener_id) { - return fmt::format( - "{}.{}.{}", api::domain::Venue_Name(venue), desc, listener_id); +constexpr const char *CCApiCorrelationID::delimiter() { return "."; } + +inline CCApiCorrelationID CCApiCorrelationID::from_string( + const std::string &str) { + std::vector parts; + boost::algorithm::split(parts, str, [](auto c) { return c == '.'; }); + CCApiCorrelationID rv; + rv.parts_ = std::move(parts); + rv.cursor_ = 0; + rv.str_ = str; + return rv; } -inline std::optional< - std::tuple> -CCApiCorrelationID::parse_market_depth(const std::string &corr_id_str) { - std::vector parts; - boost::algorithm::split(parts, corr_id_str, [](auto c) { return c == '.'; }); - if (parts.size() != 3) { - return std::nullopt; +template +inline CCApiCorrelationID CCApiCorrelationID::from_parts(Parts... parts) { + // TODO: create the parts vector/string simultaneously (more efficient) + auto parts_str = core::StringJoiner::join(delimiter(), parts...); + return from_string(parts_str); +} + +inline std::vector CCApiCorrelationID::peek_parts( + std::size_t len, bool require_len) const { + return peek_parts_at(0, len, require_len); +} + +inline std::vector CCApiCorrelationID::peek_parts_at( + std::size_t index, std::size_t len, bool require_len) const { + std::vector rv; + auto adj_cursor = cursor_ + index; + std::size_t rem_len = + parts_.size() <= adj_cursor ? 0 : parts_.size() - adj_cursor; + if (len > rem_len) { + bts_assert_msgf( + !require_len, + "Expected at least {} more parts in correlation id ({} remain). " + "cursor: {}; total_parts_len: {}", + len, rem_len, cursor_, parts_.size()); + len = rem_len; } - api::domain::Venue venue; - if (!api::domain::Venue_Parse(parts.at(0), &venue)) { - return std::nullopt; + for (std::size_t i = 0; i < len; i++) { + rv.push_back(&parts_.at(adj_cursor + i)); } - auto sec_desc = parts.at(1); - auto listener_id = MarketDataListenerID{ - static_cast(std::stoul(parts.at(2)))}; - return std::make_tuple(venue, sec_desc, listener_id); + return rv; +} + +inline std::vector CCApiCorrelationID::consume_parts( + std::size_t len, bool require_len) { + auto rv = peek_parts(len, require_len); + advance_(rv.size()); + return rv; +} + +constexpr const std::string &CCApiCorrelationID::to_string() const { + return str_; +} + +inline std::string CCApiCorrelationID::peek_string( + std::size_t len, bool require_len) const { + return peek_string_at(0, len, require_len); +} + +inline std::string CCApiCorrelationID::peek_string_at( + std::size_t index, std::size_t len, bool require_len) const { + auto sub_parts = peek_parts_at(index, len, require_len); + std::string rv; + bool first = true; + for (const auto *p : sub_parts) { + rv.append(*p); + if (first) { + first = false; + } else { + rv.append(delimiter()); + } + } + return rv; +} + +inline std::string CCApiCorrelationID::consume_string( + std::size_t len, bool require_len) { + auto sub_parts = consume_parts(len, require_len); + std::string rv; + bool first = true; + for (const auto *p : sub_parts) { + rv.append(*p); + if (first) { + first = false; + } else { + rv.append(delimiter()); + } + } + return rv; +} + +template +inline bool CCApiCorrelationID::consume_part(T &value, TParse parse) { + return consume_parts(value, parse, 1); +} + +template +inline bool CCApiCorrelationID::consume_parts( + std::size_t len, T &value, TParse parse) { + bts_assert(len > 0); + if (size() < len) { + return false; + } + auto str = consume_string(len); + return parse(str, &value); +} + +inline CCApiCorrelationID CCApiCorrelationID::trim_front() const { + if (cursor_ >= parts_.size()) { + return CCApiCorrelationID{}; + } + std::vector sub_parts{parts_.begin() + cursor_, parts_.end()}; + auto parts_str = fmt::to_string( + fmt::join(parts_.begin() + cursor_, parts_.end(), delimiter())); + CCApiCorrelationID rv; + rv.parts_ = std::move(sub_parts); + rv.cursor_ = 0; + rv.str_ = std::move(parts_str); + return rv; +} + +inline void CCApiCorrelationID::advance_(std::size_t len) { + if (len == 0) { + return; + } + cursor_ += len; + auto idx = core::StringFinder::find(str_, '.', len); + bts_assert(idx >= 0); + str_ = str_.substr(idx); +} + +constexpr std::size_t CCApiCorrelationID::size() const { + if (cursor_ > parts_.size()) { + return 0; + } + return parts_.size() - cursor_; +} + +constexpr bool CCApiCorrelationID::empty() const { return size() == 0; } + +constexpr bool operator==( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs) { + return lhs.to_string() == rhs.to_string(); +} + +constexpr bool operator!=( + const CCApiCorrelationID &lhs, const CCApiCorrelationID &rhs) { + return !(lhs.to_string() == rhs.to_string()); +} + +inline std::ostream &operator<<( + std::ostream &out, const CCApiCorrelationID &corr_id) noexcept { + out << corr_id.to_string(); + return out; } } // namespace bts::md + +template <> +struct std::hash { + std::size_t operator()(const bts::md::CCApiCorrelationID &corr_id) const { + return std::hash()(corr_id.to_string()); + } +}; + +FMT_IMPL(bts::md::CCApiCorrelationID, cid, "{}", cid.to_string()); #endif diff --git a/src/lib/md/CCApiEventDemuxer.cpp b/src/lib/md/CCApiEventDemuxer.cpp new file mode 100644 index 000000000..e7cc1a34a --- /dev/null +++ b/src/lib/md/CCApiEventDemuxer.cpp @@ -0,0 +1,166 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiEventDemuxer.h" +#include "lib/collections/DiscreteValuesFilter.h" +#include "lib/collections/Find.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/ICCApiMessageHandler.h" + +#include +#include +#include +#include +#include + +namespace bts::md { + +using bts::collections::DiscreteValuesFilter; +using std::string; +using std::unordered_map; +using std::vector; + +struct CCApiEventDemuxer::HandlerData { + ICCApiMessageHandler *handler; + DiscreteValuesFilter msg_type_filter; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const vector &msgs, + const CCApiCorrelationID &corr_id) { + if (!msg_type_filter.is_active()) { + // Handler is listening to msgs of all types + handler->on_messages(event, session, msgs, corr_id); + return; + } + + std::vector filtered_msgs; + for (const auto *msg : msgs) { + if (msg_type_filter.is_included(msg->getType())) { + filtered_msgs.push_back(msg); + } + } + if (!filtered_msgs.empty()) { + handler->on_messages(event, session, filtered_msgs, corr_id); + } + } +}; + +struct CCApiEventDemuxer::PImpl { + std::size_t corr_id_len; + unordered_map> correlated_handlers; +}; + +CCApiEventDemuxer::CCApiEventDemuxer(std::size_t corr_id_len) + : me_(std::make_unique()) { + me_->corr_id_len = corr_id_len; +} + +CCApiEventDemuxer::~CCApiEventDemuxer() = default; + +bool CCApiEventDemuxer::processEvent( + const ccapi::Event &event, ccapi::Session *session) { + unordered_map> + demuxed_msgs; + for (const auto &msg : event.getMessageList()) { + switch (msg.getType()) { + case ccapi::Message::Type::SESSION_CONNECTION_UP: + LOG(md, Info, "Session connection up"); + continue; + case ccapi::Message::Type::SESSION_CONNECTION_DOWN: + LOG(md, Warn, "Session connection down"); + continue; + default: + break; + } + for (const auto &corr_id_str : msg.getCorrelationIdList()) { + auto child_corr_id = CCApiCorrelationID::from_string(corr_id_str); + // trim the front parts off (they're all the same) + child_corr_id.consume_parts(me_->corr_id_len); + auto [it, _] = demuxed_msgs.try_emplace(child_corr_id); + it->second.push_back(&msg); + } + } + + for (const auto &[hnd_corr_id, msgs] : demuxed_msgs) { + auto hnds_it = me_->correlated_handlers.find(hnd_corr_id); + if (hnds_it == me_->correlated_handlers.end()) { + continue; + } + for (auto &hnd : hnds_it->second) { + hnd.on_messages(event, session, msgs, hnd_corr_id); + } + } + return true; +} + +void CCApiEventDemuxer::on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + unordered_map> + demuxed_msgs; + for (const auto *msg : msgs) { + for (const auto &corr_id_str : msg->getCorrelationIdList()) { + auto handler_corr_id = CCApiCorrelationID::from_string(corr_id_str); + // trim the front parts off (they're all the same) + handler_corr_id.consume_parts(me_->corr_id_len); + auto [it, _] = demuxed_msgs.try_emplace(handler_corr_id); + it->second.push_back(msg); + } + } + + for (const auto &[handler_corr_id, dmsgs] : demuxed_msgs) { + auto hnds_it = me_->correlated_handlers.find(handler_corr_id); + if (hnds_it == me_->correlated_handlers.end()) { + LOG(md, Error, + "No handler registered with correlation id '{}' (msgs will be " + "ignored)", + handler_corr_id); + continue; + } + for (auto &hnd : hnds_it->second) { + hnd.on_messages(event, session, dmsgs, handler_corr_id); + } + } +} + +bool CCApiEventDemuxer::has_handler(const CCApiCorrelationID &corr_id) const { + return me_->correlated_handlers.find(corr_id) != + me_->correlated_handlers.end(); +} + +bool CCApiEventDemuxer::add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector msg_types) { + auto &hnds = me_->correlated_handlers.try_emplace(corr_id).first->second; + auto it = FIND_IF( + hnds, [handler](const auto &hnd) { return hnd.handler == handler; }); + if (it != hnds.end()) { + return it->msg_type_filter.include(msg_types); + } + auto &hnd = hnds.emplace_back(); + hnd.handler = handler; + if (!msg_types.empty()) { + hnd.msg_type_filter.reset(std::move(msg_types), true); + } + return true; +} + +bool CCApiEventDemuxer::remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector types) { + auto corr_hnd_it = me_->correlated_handlers.find(corr_id); + if (corr_hnd_it == me_->correlated_handlers.end()) { + return false; + } + auto &hnds = corr_hnd_it->second; + auto it = FIND_IF( + hnds, [handler](const auto &hnd) { return hnd.handler == handler; }); + return it != hnds.end() && it->msg_type_filter.exclude(types); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiEventDemuxer.h b/src/lib/md/CCApiEventDemuxer.h new file mode 100644 index 000000000..02e208eda --- /dev/null +++ b/src/lib/md/CCApiEventDemuxer.h @@ -0,0 +1,82 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/ICCApiMessageHandler.h" +#include +#include +#include +#include +#include + +namespace bts::md { + +/** + * Demuxes ccapi::Events by CorrelationID to ICCApiMessageHandlers + */ +class CCApiEventDemuxer : public ccapi::EventHandler, + public ICCApiMessageHandler { + public: + explicit CCApiEventDemuxer(std::size_t corr_id_len = 0); + ~CCApiEventDemuxer() override; + + bool processEvent( + const ccapi::Event &event, ccapi::Session *session) override; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + /** + * @brief Determines whether or not a handler has been added to a particular + * correlation id + * + * @param corr_id The correlation id to check + * + * @return True if a handler has been added for the specified correlation id, + * otherwise false + */ + bool has_handler(const CCApiCorrelationID &corr_id) const; + + /** + * @brief Connects a handler to ccapi callbacks with the specified + * correlation id + * + * @param corr_id The correlation id of the callbacks the handler will + * receive + * @param handler The handler to connect + * + * @return True if the handler is added, otherwise False (i.e., the handler + * had already been added to the specified correlation id) + */ + bool add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector types = {}); + + /** + * @brief Disconnects a handler from ccapi callbacks with the specified + * correlation id + * + * @param corr_id The correlation id of the callbacks the handler will no + * longer receive + * @param handler The handler to disconnect + * + * @return True if the handler is removed, otherwise False (either handler + * was already disconnected, or it wasn't added for the specified correlation + * id in the first place) + */ + bool remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector types = {}); + + private: + struct PImpl; + struct HandlerData; + + std::unique_ptr me_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiListenerAdaptor.cpp b/src/lib/md/CCApiListenerAdaptor.cpp deleted file mode 100644 index 58142748f..000000000 --- a/src/lib/md/CCApiListenerAdaptor.cpp +++ /dev/null @@ -1,165 +0,0 @@ -#ifdef SPARK_ENABLE_CCAPI -#include "CCApiListenerAdaptor.h" -#include "lib/api/md/MarketDataConfig.pb.h" -#include "lib/collections/Contains.h" -#include "lib/domain/BUY_SELL.h" -#include "lib/domain/SecurityID.h" -#include "lib/logging/logging.h" -#include "lib/md/CCApiCorrelationID.h" -#include "lib/md/CCApiDepthWalker.h" -#include "lib/md/CCApiSecurityParser.h" -#include "lib/md/IMarketDataListener.h" -#include "lib/md/MarketDataListenerData.h" -#include "lib/persistence/IDGenerator.h" - -#include -#include -#include -#include - -namespace bts::md { - -using namespace bts::domain; -using std::string; -using std::unique_ptr; -using std::unordered_map; -using std::vector; - -struct CCApiListenerAdaptor::PImpl { - PImpl( - IMarketDataListener *const listener, - persistence::IDGenerator *const sec_id_store) - : listener(listener), sec_id_store(sec_id_store) {} - - IMarketDataListener *const listener; - - persistence::IDGenerator *const sec_id_store; - unordered_map> securities; - - void on_instruments(const ccapi::Message &msg, api::domain::Venue venue) { - vector all_secs; - auto new_secs = CCApiSecurityParser::parse(msg.getElementList()); - for (const auto &sec : new_secs) { - const auto &desc = sec.security_common.desc; - bts_assert(!desc.empty()); - auto sec_id = SecurityID{ - sec_id_store->get_or_add(api::domain::Venue_Name(venue), desc)}; - if (CONTAINS_KEY(securities, sec_id)) { - // don't replace existing secs (in case code downstream relies on - // ptr equality) - continue; - } - auto sec_ptr = std::make_unique(sec); - sec_ptr->security_common.id.value() = sec_id_store->get_or_add( - api::domain::Venue_Name(venue), sec_ptr->security_common.desc); - all_secs.push_back(sec_ptr.get()); - securities[sec_id] = std::move(sec_ptr); - } - listener->on_securities(all_secs); - } - - void on_instruments(const ccapi::Message &msg, const string &corr_id) { - // Correlation ID is the Venue - api::domain::Venue venue; - if (!api::domain::Venue_Parse(corr_id, &venue)) { - LOG(md, Error, "Failed to parse venue from ccapi correlation id '{}'", - corr_id); - return; - } - on_instruments(msg, venue); - } - - void on_book(const ccapi::Message &msg, SecurityID sec_id) { - if (msg.getType() != - ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH) { - return; - } - - Book book; - book.set_security_id(sec_id); - // The market data arrives in the following order: - // * bid levels (by price descending) - // * ask levels (by price ascending) - CCApiDepthWalker bid_walker{"BID_PRICE", "BID_SIZE", &book[BUY]}; - CCApiDepthWalker ask_walker{"ASK_PRICE", "ASK_SIZE", &book[SELL]}; - const auto &levels = msg.getElementList(); - for (const auto &level : levels) { - // Each level has 2 entries, either: - // ["BID_PRICE", "BID_SIZE"], or - // ["ASK_PRICE", "ASK_SIZE"] - const auto &props = level.getNameValueMap(); - for (const auto &[name, value] : props) { - bid_walker.parse_field(name, value) || - ask_walker.parse_field(name, value); - } - } - - // validate the book - bts_assert(bid_walker.price_idx == bid_walker.qty_idx); - bts_assert(ask_walker.price_idx == ask_walker.qty_idx); - bts_assert(book.is_sorted()); - bts_assert(book[BUY].count_populated() == bid_walker.price_idx); - bts_assert(book[SELL].count_populated() == ask_walker.price_idx); - listener->on_book(book); - } - - void on_book(const ccapi::Message &msg, const string &corr_id) { - // Correlation ID is a formatted string: - // ".." - const auto corr_id_res = CCApiCorrelationID::parse_market_depth(corr_id); - bts_assert(corr_id_res.has_value()); - const auto [venue, sec_desc, listener_id] = *corr_id_res; - LOG(md, Diag, "listener_id: {}", listener_id); // to build gcc - const auto sec_id = - sec_id_store->get_or_add(api::domain::Venue_Name(venue), sec_desc); - on_book(msg, SecurityID{sec_id}); - } - - SecurityID get_sec_id( - const api::domain::Venue venue, const string &sec_desc) { - const auto sec_id = - sec_id_store->get_or_add(api::domain::Venue_Name(venue), sec_desc); - return SecurityID{sec_id}; - } -}; - -CCApiListenerAdaptor::CCApiListenerAdaptor( - IMarketDataListener *const listener, - persistence::IDGenerator *const sec_id_store) - : me_(std::make_unique(listener, sec_id_store)) {} - -CCApiListenerAdaptor::~CCApiListenerAdaptor() = default; - -bool CCApiListenerAdaptor::processEvent( - const ccapi::Event &event, ccapi::Session *) { - LOG(md, Diag, "Received ccapi event: {}", event.toString()); - const auto &msgs = event.getMessageList(); - for (const auto &msg : msgs) { - const auto &corr_ids = msg.getCorrelationIdList(); - for (const auto &corr_id : corr_ids) { - switch (msg.getType()) { - case ccapi::Message::Type::GET_INSTRUMENTS: { - // This message should only arrive with a RESPONSE event - bts_assert(event.getType() == ccapi::Event::Type::RESPONSE); - me_->on_instruments(msg, corr_id); - break; - } - case ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH: { - // This message should only arrive with a SUBSCRIPTION_DATA event - bts_assert( - event.getType() == ccapi::Event::Type::SUBSCRIPTION_DATA); - me_->on_book(msg, corr_id); - break; - } - default: - LOG(md, Warn, "Unexpected ccapi message type '{}'", - ccapi::Message::typeToString(msg.getType())); - break; - } - } - } - return true; -} - -} // namespace bts::md -#endif diff --git a/src/lib/md/CCApiListenerAdaptor.h b/src/lib/md/CCApiListenerAdaptor.h deleted file mode 100644 index 3f67d04fd..000000000 --- a/src/lib/md/CCApiListenerAdaptor.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once -#ifdef SPARK_ENABLE_CCAPI -#include "lib/api/domain/Venue.pb.h" - -#include -#include - -namespace bts::api::md { -class MarketDataConfig; -} - -namespace bts::persistence { -class IDGenerator; -} - -namespace bts::md { - -class IMarketDataListener; - -class CCApiListenerAdaptor : public ccapi::EventHandler { - public: - CCApiListenerAdaptor( - IMarketDataListener *const listener, - persistence::IDGenerator *const sec_id_store); - - ~CCApiListenerAdaptor(); - - bool processEvent( - const ccapi::Event &event, ccapi::Session *session) override; - - private: - struct PImpl; - std::unique_ptr me_; -}; - -} // namespace bts::md -#endif diff --git a/src/lib/md/CCApiMarketDepthAdaptor.cpp b/src/lib/md/CCApiMarketDepthAdaptor.cpp new file mode 100644 index 000000000..5ee549d06 --- /dev/null +++ b/src/lib/md/CCApiMarketDepthAdaptor.cpp @@ -0,0 +1,78 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiMarketDepthAdaptor.h" +#include "lib/domain/BUY_SELL.h" +#include "lib/domain/Book.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiDepthWalker.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/MarketDataListenerData.h" + +#include +#include +#include +#include +#include + +namespace bts::md { + +static bool parse(const ccapi::Message &msg, domain::Book &book) { + if (msg.getType() != ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH) { + return false; + } + + // The market data arrives in the following order: + // * bid levels (by price descending) + // * ask levels (by price ascending) + CCApiDepthWalker bid_walker{ + CCAPI_BEST_BID_N_PRICE, CCAPI_BEST_BID_N_SIZE, &book[BUY]}; + CCApiDepthWalker ask_walker{ + CCAPI_BEST_ASK_N_PRICE, CCAPI_BEST_ASK_N_SIZE, &book[SELL]}; + const auto &levels = msg.getElementList(); + for (const auto &level : levels) { + // Each level has 2 entries, either: + // ["BID_PRICE", "BID_SIZE"], or + // ["ASK_PRICE", "ASK_SIZE"] + const auto &props = level.getNameValueMap(); + for (const auto &[name, value] : props) { + bid_walker.parse_field(name, value) || + ask_walker.parse_field(name, value); + } + } + + // validate the book + bts_assert(bid_walker.price_idx == bid_walker.qty_idx); + bts_assert(ask_walker.price_idx == ask_walker.qty_idx); + bts_assert(book.is_sorted()); + bts_assert(book[BUY].count_populated() == bid_walker.price_idx); + bts_assert(book[SELL].count_populated() == ask_walker.price_idx); + return true; +} + +static domain::Book parse(const ccapi::Message &msg) { + domain::Book book; + parse(msg, book); + return book; +} + +CCApiMarketDepthAdaptor::CCApiMarketDepthAdaptor( + domain::SecurityID security_id, IMarketDataListener *listener) + : security_id_(security_id), listener_(listener) {} + +const domain::Book &CCApiMarketDepthAdaptor::last_book() const { + return last_book_; +} + +void CCApiMarketDepthAdaptor::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const std::vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + for (const auto *msg : msgs) { + auto book = parse(*msg); + book.set_security_id(security_id_); + listener_->on_book(book); + } +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMarketDepthAdaptor.h b/src/lib/md/CCApiMarketDepthAdaptor.h new file mode 100644 index 000000000..c25bb9269 --- /dev/null +++ b/src/lib/md/CCApiMarketDepthAdaptor.h @@ -0,0 +1,44 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/Book.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include + +namespace ccapi { +class Event; +class Message; +class Session; +} // namespace ccapi + +namespace bts::md { + +class CCApiCorrelationID; + +/** + * @brief A bts::md::ICCApiMessageHandler that translates + * market depth messages into books for an bts::md::IMarketDataListener + */ +class CCApiMarketDepthAdaptor : public ICCApiMessageHandler { + public: + CCApiMarketDepthAdaptor() = default; + + CCApiMarketDepthAdaptor( + domain::SecurityID security_id, IMarketDataListener *listener); + + const domain::Book &last_book() const; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + domain::SecurityID security_id_; + IMarketDataListener *listener_; + domain::Book last_book_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMessageHandler.h b/src/lib/md/CCApiMessageHandler.h new file mode 100644 index 000000000..6f3cd0962 --- /dev/null +++ b/src/lib/md/CCApiMessageHandler.h @@ -0,0 +1,46 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/utils/Delegate.h" +#include +#include +#include + +namespace bts::md { + +/** + * @brief A proxy handler that emits messages to a bindable callback delegate. + * It is designed to be used with a CCApiSession. + * + * @details This class is useful for creating handlers without the overhead of + * defining a new class. It's especially convenient for cases where a 'master' + * class handles callbacks for multiple message types. If a proxy handler is + * used for each message type, rather than a single handler for all message + * types, one avoids the need to parse the message-type from the correlation id + * and demux the callback with switch cases. + */ +class CCApiMessageHandler : public ICCApiMessageHandler { + public: + using Callback = core::Delegate &, + const CCApiCorrelationID &)>; + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override { + callback_(event, session, msgs, corr_id); + } + + Callback &callback() { return callback_; } + + private: + Callback callback_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMessageLogger.cpp b/src/lib/md/CCApiMessageLogger.cpp new file mode 100644 index 000000000..b2146ca57 --- /dev/null +++ b/src/lib/md/CCApiMessageLogger.cpp @@ -0,0 +1,33 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiMessageLogger.h" +#include "lib/core/StringJoiner.h" +#include "lib/core/TimeStamp.h" +#include +#include +#include +#include + +namespace bts::md { + +using bts::core::StringJoiner; +using std::string; + +CCApiMessageLogger::CCApiMessageLogger(std::string name) + : name_(std::move(name)) {} + +void CCApiMessageLogger::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) { + + auto msg_strs = msgs | ranges::views::transform([](const ccapi::Message *m) { + return m->toStringPretty(); + }); + auto msgs_str = StringJoiner::join(msg_strs.begin(), msg_strs.end(), "\n"); + LOG(md, Info, "Received CCApi messages. name: {}; corr_id: {}; msgs:\n{}", + name_, corr_id, msgs_str); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiMessageLogger.h b/src/lib/md/CCApiMessageLogger.h new file mode 100644 index 000000000..6d16d3576 --- /dev/null +++ b/src/lib/md/CCApiMessageLogger.h @@ -0,0 +1,34 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/Book.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include + +namespace ccapi { +class Event; +class Message; +class Session; +} // namespace ccapi + +namespace bts::md { + +class CCApiCorrelationID; + +class CCApiMessageLogger : public ICCApiMessageHandler { + public: + CCApiMessageLogger() = default; + explicit CCApiMessageLogger(std::string name); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + std::string name_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiReader.h b/src/lib/md/CCApiReader.h new file mode 100644 index 000000000..4874aa66f --- /dev/null +++ b/src/lib/md/CCApiReader.h @@ -0,0 +1,155 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/utils/bts_assert.h" +#include +#include +#include +#include + +namespace bts::md { + +class CCApiReader { + public: + using NameValues = std::map; + + explicit CCApiReader(const NameValues *name_values); + + bool read(const std::string &key, std::string &value) const; + bool read(const std::string &key, nlohmann::json &value) const; + + template + bool read(const std::string &key, Value &value) const; + + template + bool read(const std::string &key, Value &value, Parse parse = {}) const; + + private: + const NameValues *name_values_ptr_; + + const std::string *get_(const std::string &key) const; + + bool parse_(const std::string &str, double &value) const noexcept; + bool parse_(const std::string &str, std::int32_t &value) const noexcept; + bool parse_(const std::string &str, std::int64_t &value) const noexcept; + bool parse_(const std::string &str, std::uint32_t &value) const noexcept; + bool parse_(const std::string &str, std::uint64_t &value) const noexcept; + + template + bool parse_(const std::string &str, Value &value) const; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline CCApiReader::CCApiReader(const NameValues *name_values) + : name_values_ptr_(name_values) {} + +inline bool CCApiReader::read( + const std::string &key, std::string &value) const { + const auto *value_str = get_(key); + if (value_str == nullptr) { + return false; + } + value = *value_str; + return true; +} + +inline bool CCApiReader::read( + const std::string &key, nlohmann::json &value) const { + const auto *value_str = get_(key); + if (value_str == nullptr) { + return false; + } + auto js = nlohmann::json::parse(*value_str); + value = std::move(js); + return true; +} + +template +inline bool CCApiReader::read(const std::string &key, Value &value) const { + const auto *value_str = get_(key); + if (value_str == nullptr) { + return false; + } + return parse_(*value_str, value); +} + +template +inline bool CCApiReader::read( + const std::string &key, Value &value, Parse parse) const { + const auto *value_str = get_(key); + return value_str != nullptr && parse(*value_str, value); +} + +inline const std::string *CCApiReader::get_(const std::string &key) const { + bts_assert(name_values_ptr_ != nullptr); + auto it = name_values_ptr_->find(key); + return it == name_values_ptr_->end() ? nullptr : &it->second; +} + +template +inline bool CCApiReader::parse_(const std::string &str, Value &value) const { + std::stringstream ss; + ss << str; + ss >> value; + return true; +} + +inline bool CCApiReader::parse_( + const std::string &str, double &value) const noexcept { + try { + value = std::stod(str); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::int32_t &value) const noexcept { + try { + value = std::stoi(str); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::int64_t &value) const noexcept { + try { + value = std::stol(str); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::uint32_t &value) const noexcept { + try { + const auto ul = std::stoul(str); + // the only error stoul won't throw + if (ul > std::numeric_limits::max()) { + return false; + } + value = static_cast(value); + return true; + } catch (...) { + return false; + } +} + +inline bool CCApiReader::parse_( + const std::string &str, std::uint64_t &value) const noexcept { + try { + value = std::stoul(str); + return true; + } catch (...) { + return false; + } +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiSecurityParser.h b/src/lib/md/CCApiSecurityParser.h deleted file mode 100644 index 1110f5f45..000000000 --- a/src/lib/md/CCApiSecurityParser.h +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once -#ifdef SPARK_ENABLE_CCAPI -#include "lib/core/StringFiller.h" -#include "lib/domain/Security.h" - -#include -#include -#include - -namespace bts::md { - -class CCApiSecurityParser { - public: - template > - static domain::Security parse(const TMap &name_values); - - template > - static void parse(domain::Security &security, const TMap &name_values); - - static std::vector parse( - const std::vector &securities_in); - - static void parse( - const std::vector &securities_in, - std::vector &securities_out); -}; - -////////////////////////////////////////////////// -// Definition -////////////////////////////////////////////////// - -template -inline domain::Security CCApiSecurityParser::parse(const TMap &name_values) { - domain::Security sec; - parse(sec, name_values); - return sec; -} - -template -inline void CCApiSecurityParser::parse( - domain::Security &security, const TMap &props) { - auto &scomm = security.security_common; - for (const auto &[name, value] : props) { - if (name == CCAPI_INSTRUMENT) { - scomm.desc = value; - continue; - } - if (name == CCAPI_ORDER_PRICE_INCREMENT) { - double min_price_inc = std::stod(value); - scomm.min_price_increment = domain::Price::from_double(min_price_inc); - continue; - } - if (name == CCAPI_ORDER_QUANTITY_INCREMENT) { - scomm.min_quantity_increment = std::stod(value); - continue; - } - if (name == CCAPI_UNDERLYING_SYMBOL) { - scomm.symbol = value; - // TODO: make sure these name-mappings are appropriate - core::StringFiller::fill_if_empty(scomm.asset, value); - core::StringFiller::fill_if_empty(scomm.group, value); - continue; - } - if (name == CCAPI_BASE_ASSET) { - scomm.asset = value; - // TODO: make sure these name-mappings are appropriate - core::StringFiller::fill_if_empty(scomm.symbol, value); - core::StringFiller::fill_if_empty(scomm.group, value); - continue; - } - if (name == CCAPI_QUOTE_ASSET) { - if (!api::domain::Currency_Parse(value, &scomm.currency)) { - const auto default_currency = api::domain::Currency::USD; - scomm.currency = default_currency; - LOG(md, Error, - "Failed to parse currency type from string '{}'. Falling back " - "to default value '{}'", - value, default_currency); - } - } - if (name == CCAPI_ORDER_QUANTITY_MIN) { - scomm.min_quantity = std::stod(value); - continue; - } - } -} - -inline void CCApiSecurityParser::parse( - const std::vector &securities_in, - std::vector &securities_out) { - for (const auto &el : securities_in) { - const std::map &name_vals = - el.getNameValueMap(); - auto sec = CCApiSecurityParser::parse(name_vals); - securities_out.push_back(sec); - } -} - -inline std::vector CCApiSecurityParser::parse( - const std::vector &securities_in) { - std::vector secs; - parse(securities_in, secs); - return secs; -} - -} // namespace bts::md -#endif diff --git a/src/lib/md/CCApiSession.cpp b/src/lib/md/CCApiSession.cpp index 249b7effc..e70659789 100644 --- a/src/lib/md/CCApiSession.cpp +++ b/src/lib/md/CCApiSession.cpp @@ -1,91 +1,145 @@ #ifdef SPARK_ENABLE_CCAPI #include "CCApiSession.h" -#include "lib/api/md/DepthStreamConfig.pb.h" +#include "lib/logging/logging.h" #include "lib/md/CCApiCorrelationID.h" -#include "lib/md/CCApiListenerAdaptor.h" -#include "lib/md/MarketDataListenerData.h" -#include "lib/md/MarketDataRegistrationFactory.h" -#include "lib/persistence/IDGenerator.h" +#include "lib/md/CCApiEventDemuxer.h" +#include "lib/md/ICCApiMessageHandler.h" -#include +#include #include -#include +#include #include #include #include -#include +#include namespace bts::md { +using ccapi::EventDispatcher; +using ccapi::Request; +using ccapi::Session; +using ccapi::SessionConfigs; +using ccapi::SessionOptions; +using ccapi::Subscription; +using std::vector; + struct CCApiSession::PImpl { - PImpl( - const MarketDataListenerData listener_data, - bts::persistence::IDGenerator *const sec_id_store) - : listener_data(listener_data), sec_id_store(sec_id_store), - ccapi_dispatcher( - static_cast(listener_data.cfg.num_dispatcher_threads())), - ccapi_listener(listener_data.listener, sec_id_store), - ccapi_session( - ccapi_opts, ccapi_cfgs, &ccapi_listener, &ccapi_dispatcher) {} + PImpl(EventDispatcher *dispatcher, SessionConfigs cfgs, SessionOptions opts) + : dispatcher(dispatcher), demuxer(), cfgs(std::move(cfgs)), + opts(std::move(opts)), + session(this->opts, this->cfgs, &this->demuxer, dispatcher) {} - MarketDataListenerData listener_data; - bts::persistence::IDGenerator *sec_id_store; - - ccapi::SessionOptions ccapi_opts; - ccapi::SessionConfigs ccapi_cfgs; - - // Dispatches ccapi callbacks - ccapi::EventDispatcher ccapi_dispatcher; - - // Handles ccapi sec/md callbacks - CCApiListenerAdaptor ccapi_listener; + EventDispatcher *dispatcher; + CCApiEventDemuxer demuxer; + SessionConfigs cfgs; + SessionOptions opts; // Manages ccapi connection to exchanges - ccapi::Session ccapi_session; + Session session; - void request_securities(api::domain::Venue venue) { - auto venue_name = api::domain::Venue_Name(venue); - auto venue_name_lower = boost::algorithm::to_lower_copy(venue_name); - // The 'venue' in the request must be lower-case (ccapi expects this). The - // correlation id, however, should remain unchanged (in the callback, we - // parse the venue with 'api::domain::Venue_Parse') - ccapi::Request req( - ccapi::Request::Operation::GET_INSTRUMENTS, venue_name_lower); - req.setCorrelationId(venue_name); - ccapi_session.sendRequest(req); - } - - void subscribe_md(const api::md::DepthStreamConfig &ds) { - const auto opts = - fmt::format("{}={}", CCAPI_MARKET_DEPTH_MAX, ds.num_levels()); - auto venue_name = bts::api::domain::Venue_Name(ds.venue()); - boost::algorithm::to_lower(venue_name); - for (const auto &symbol : ds.symbols()) { - const auto corr_id = CCApiCorrelationID::fmt_market_depth( - ds.venue(), symbol, listener_data.id); - ccapi::Subscription sub( - venue_name, symbol, CCAPI_MARKET_DEPTH, opts, corr_id); - ccapi_session.subscribe(sub); - } + void check_handler(const CCApiCorrelationID &corr_id) { + bts_assert_msg(!corr_id.empty(), "Must specify non-empty correlation id"); + bts_assert_msgf( + demuxer.has_handler(corr_id), + "No handler is associated with the request's correlation id '{}'", + corr_id); } }; -CCApiSession::CCApiSession( - MarketDataListenerData listener_data, - bts::persistence::IDGenerator *sec_id_store) - : me_(std::make_unique(std::move(listener_data), sec_id_store)) { +CCApiSession::CCApiSession(EventDispatcher *dispatcher) + : CCApiSession(dispatcher, SessionConfigs{}, SessionOptions{}) {} - std::unordered_set venues; - for (auto ds : me_->listener_data.cfg.depth_streams()) { - // Limit one sec request per Venue - if (venues.insert(ds.venue()).second) { - me_->request_securities(ds.venue()); - } - me_->subscribe_md(ds); - } +CCApiSession::CCApiSession( + EventDispatcher *dispatcher, SessionConfigs cfgs, SessionOptions opts) + : me_(std::make_unique( + dispatcher, std::move(cfgs), std::move(opts))) {} + +CCApiSession::~CCApiSession() { me_->session.stop(); } + +EventDispatcher *CCApiSession::dispatcher() { return me_->dispatcher; } + +bool CCApiSession::add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector msg_types) { + return me_->demuxer.add_handler(corr_id, handler, msg_types); } -CCApiSession::~CCApiSession() { me_->ccapi_session.stop(); } +bool CCApiSession::remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + vector msg_types) { + return me_->demuxer.remove_handler(corr_id, handler, msg_types); +} + +void CCApiSession::request(Request req) { + LOG(md, Info, "Requesting {}", req.toString()); + auto corr_id_str = req.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + me_->check_handler(corr_id); + me_->session.sendRequest(req); +} + +void CCApiSession::request(Request req, ICCApiMessageHandler *handler) { + auto corr_id_str = req.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler); + request(req); +} + +void CCApiSession::request( + BatchRequest batch_req, vector msg_types) { + if (batch_req.empty()) { + return; + } + vector reqs; + for (auto &[tmp_sub, handler] : batch_req) { + auto &sub = reqs.emplace_back(std::move(tmp_sub)); + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler, msg_types); + } + auto req_strs = + ranges::transform_view(reqs, [](const auto &r) { return r.toString(); }); + LOG(md, Info, "Requesting batch:\n{}", + fmt::to_string(fmt::join(req_strs, "\n"))); + me_->session.sendRequest(reqs); +} + +void CCApiSession::subscribe(Subscription sub) { + LOG(md, Info, "Subscribing {}", sub.toString()); + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + me_->check_handler(corr_id); + me_->session.subscribe(sub); +} + +void CCApiSession::subscribe(Subscription sub, ICCApiMessageHandler *handler) { + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler); + subscribe(sub); +} + +void CCApiSession::subscribe( + CCApiSession::BatchSubscription batch_sub, + vector msg_types) { + if (batch_sub.empty()) { + return; + } + vector subs; + for (auto &[tmp_sub, handler] : batch_sub) { + auto &sub = subs.emplace_back(std::move(tmp_sub)); + auto corr_id_str = sub.getCorrelationId(); + auto corr_id = CCApiCorrelationID::from_string(corr_id_str); + add_handler(corr_id, handler, msg_types); + } + auto sub_strs = + ranges::transform_view(subs, [](const auto &s) { return s.toString(); }); + LOG(md, Info, "Subscribing batch:\n{}", + fmt::to_string(fmt::join(sub_strs, "\n"))); + me_->session.subscribe(subs); +} } // namespace bts::md #endif diff --git a/src/lib/md/CCApiSession.h b/src/lib/md/CCApiSession.h index 3edc9fd53..b77fb8bc0 100644 --- a/src/lib/md/CCApiSession.h +++ b/src/lib/md/CCApiSession.h @@ -1,33 +1,84 @@ #pragma once #ifdef SPARK_ENABLE_CCAPI -#include "lib/api/domain/Venue.pb.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/ICCApiMessageHandler.h" +#include +#include #include +#include +#include namespace ccapi { -class Session; -} - -namespace bts::persistence { -class IDGenerator; -} +class EventDispatcher; +class EventHandler; +class SessionConfigs; +class SessionOptions; +class Request; +class Subscription; +} // namespace ccapi namespace bts::md { -struct MarketDataListenerData; +class CCApiSubscription; +class ICCApiMessageHandler; +/** + * @brief Provides facilities to request/subscribe data from exchanges + * + * @details The basic workflow to request/subscribe data is as follows: + * First, create a CCApiCorrelationID to associate with the req/sub. Next, add a + * handler with that particular CCApiCorrelationID. Then, create the req/sub + * object with that same CCApiCorrelationID and pass it to the req/sub method. + * The handler should receive all callbacks associated with that + * CCApiCorrelationId. For convenience, there are request/subscribe overloads + * with an ICCApiMessageHandler* param, so you may set the handler and + * and make the request/subscription in one call. + */ class CCApiSession { public: - explicit CCApiSession( - MarketDataListenerData listener_data, - bts::persistence::IDGenerator *sec_id_store); + using BatchRequest = + std::vector>; - CCApiSession(const CCApiSession &) = delete; - CCApiSession(CCApiSession &&) = default; - CCApiSession &operator=(const CCApiSession &) = delete; - CCApiSession &operator=(CCApiSession &&) = default; + using BatchSubscription = + std::vector>; + + explicit CCApiSession(ccapi::EventDispatcher *dispatcher); + + CCApiSession( + ccapi::EventDispatcher *dispatcher, + ccapi::SessionConfigs cfgs, + ccapi::SessionOptions opts); ~CCApiSession(); + ccapi::EventDispatcher *dispatcher(); + + bool add_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector msg_types = {}); + + bool remove_handler( + const CCApiCorrelationID &corr_id, + ICCApiMessageHandler *handler, + std::vector msg_types = {}); + + void request(ccapi::Request req); + + void request(ccapi::Request req, ICCApiMessageHandler *handler); + + void request( + BatchRequest batch_req, + std::vector msg_types = {}); + + void subscribe(ccapi::Subscription sub); + + void subscribe(ccapi::Subscription sub, ICCApiMessageHandler *handler); + + void subscribe( + BatchSubscription batch_sub, + std::vector msg_types = {}); + private: struct PImpl; std::unique_ptr me_; diff --git a/src/lib/md/CCApiTradeAdaptor.cpp b/src/lib/md/CCApiTradeAdaptor.cpp new file mode 100644 index 000000000..e65bc8790 --- /dev/null +++ b/src/lib/md/CCApiTradeAdaptor.cpp @@ -0,0 +1,70 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiTradeAdaptor.h" +#include "lib/core/TimeStamp.h" +#include "lib/domain/SecurityID.h" +#include "lib/domain/TradeRecord.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiTradeRecord.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/MarketDataListenerData.h" + +#include +#include +#include +#include + +namespace bts::md { + +using bts::core::TimeStamp; +using bts::domain::SecurityID; +using std::vector; + +CCApiTradeAdaptor::CCApiTradeAdaptor( + SecurityID sec_id, IMarketDataListener *listener) + : sec_id_(sec_id), listener_(listener) {} + +void CCApiTradeAdaptor::on_messages( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) { + std::vector trades; + for (const auto *msg : msgs) { + switch (msg->getType()) { + case ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE: + case ccapi::Message::Type::MARKET_DATA_EVENTS_AGG_TRADE: + break; + default: + bts_assert_msgf( + false, "Received CCApi msg of unexpected type '{}'", + ccapi::Message::typeToString(msg->getType())); + return; + } + LOG(md, Diag, "Received CCApi trade msg:\n{}", msg->toStringPretty()); + auto ts = TimeStamp::from_int64( + msg->getTimeReceived().time_since_epoch().count()); + auto sec_desc = corr_id.peek_string(); + auto ccapi_trades = CCApiTradeRecord::parse(*msg); + trades.reserve(trades.size() + ccapi_trades.size()); + for (const auto &ccapi_trade : ccapi_trades) { + auto &trade = trades.emplace_back(); + trade.sec_id = sec_id_; + trade.time_stamp = ts; + ccapi_trade.to_trade_record(trade); + } + } + if (trades.empty()) { + LOG(md, Warn, "Received ccapi trade msg with no trades:\n{}", + event.toStringPretty()); + return; + } + std::vector trade_ptrs; + trade_ptrs.reserve(trades.size()); + for (const auto &trade : trades) { + trade_ptrs.push_back(&trade); + } + listener_->on_trades(trade_ptrs); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiTradeAdaptor.h b/src/lib/md/CCApiTradeAdaptor.h new file mode 100644 index 000000000..5c6fe63d2 --- /dev/null +++ b/src/lib/md/CCApiTradeAdaptor.h @@ -0,0 +1,39 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/md/CCApiTradeRecord.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include + +namespace ccapi { +class Event; +class Message; +class Session; +} // namespace ccapi + +namespace bts::md { + +class CCApiCorrelationID; + +class CCApiTradeAdaptor : public ICCApiMessageHandler { + public: + CCApiTradeAdaptor() = default; + + CCApiTradeAdaptor(domain::SecurityID sec_id, IMarketDataListener *listener); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + const std::vector &last_trades() const; + + private: + domain::SecurityID sec_id_; + IMarketDataListener *listener_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiTradeRecord.cpp b/src/lib/md/CCApiTradeRecord.cpp new file mode 100644 index 000000000..5a871976b --- /dev/null +++ b/src/lib/md/CCApiTradeRecord.cpp @@ -0,0 +1,93 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "CCApiTradeRecord.h" +#include "lib/core/BoolName.h" +#include "lib/domain/AggressorSide.h" +#include "lib/domain/Price.h" +#include "lib/domain/Quantity.h" +#include "lib/domain/TradeRecord.h" +#include "lib/io/JsonReader.h" +#include "lib/md/CCApiReader.h" +#include "lib/utils/bts_chrono.h" +#include "lib/utils/datetime.h" +#include +#include +#include +#include +#include +#include + +namespace bts::md { + +using bts::core::BoolName; +using bts::domain::TradeRecord; +using std::vector; + +void CCApiTradeRecord::to_trade_record(TradeRecord &value) const { + // This alias allows the underlying type in 'domain::Quantity' to change + // without breaking this code (will change from uint to double soon) + using QuantityValue = std::remove_reference< + decltype(std::declval().value())>::type; + + value.trade_price = domain::Price::from_double(last_price); + // Temp code to show non-zero values until we have double quantities + value.trade_quantity.value() = + static_cast(std::max(last_size * 1000, 1.0)); + + value.agg_side = is_buyer_maker ? domain::AggressorSide::Buy + : domain::AggressorSide::Sell; +} + +bool CCApiTradeRecord::parse( + const ccapi::Element &source, + CCApiTradeRecord &result, + bool is_aggregated) { + CCApiReader reader{&source.getNameValueMap()}; + return reader.read(CCAPI_LAST_PRICE, result.last_price) && + reader.read(CCAPI_LAST_SIZE, result.last_size) && + reader.read( + (is_aggregated ? CCAPI_AGG_TRADE_ID : CCAPI_TRADE_ID), + result.trade_id) && + reader.read(CCAPI_IS_BUYER_MAKER, result.is_buyer_maker); +} + +std::vector CCApiTradeRecord::parse( + const ccapi::Message &msg) { + std::vector result; + bts_assert(parse(msg, result)); + return result; +} + +bool CCApiTradeRecord::parse( + const ccapi::Message &msg, vector &result) { + bool is_aggregated = false; + switch (msg.getType()) { + case ccapi::Message::Type::MARKET_DATA_EVENTS_AGG_TRADE: + is_aggregated = true; + break; + case ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE: + break; + default: + return false; + } + + const auto &trade_els = msg.getElementList(); + result.reserve(result.size() + trade_els.size()); + for (const auto &trade_el : trade_els) { + auto &trade_rec = result.emplace_back(); + if (!CCApiTradeRecord::parse(trade_el, trade_rec, is_aggregated)) { + return false; + } + } + return true; +} + +std::ostream &operator<<( + std::ostream &out, const CCApiTradeRecord &tr) noexcept { + out << "trade_id: " << tr.trade_id << "; price: " << tr.last_price << "; " + << tr.last_size + << "; is_buyer_maker: " << BoolName::yes_no(tr.is_buyer_maker); + return out; +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/CCApiTradeRecord.h b/src/lib/md/CCApiTradeRecord.h new file mode 100644 index 000000000..e5af3e9a4 --- /dev/null +++ b/src/lib/md/CCApiTradeRecord.h @@ -0,0 +1,50 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/NaN.h" +#include "lib/core/TimeStamp.h" +#include +#include +#include + +namespace bts::domain { +class TradeRecord; +} + +namespace ccapi { +class Element; +class Message; +} // namespace ccapi + +namespace bts::md { + +/** + * @brief Represents a 'trade' ccapi::Element. Each field has a corresponding + * entry in 'ccapi::Element#getNameValueMap()' + */ +class CCApiTradeRecord { + public: + bool is_buyer_maker = false; + double last_price = core::NaN::value(); + double last_size = core::NaN::value(); + std::uint64_t trade_id = 0; + + static std::vector parse(const ccapi::Message &source); + + static bool parse( + const ccapi::Element &source, + CCApiTradeRecord &result, + bool is_aggregated); + + static bool parse( + const ccapi::Message &msg, std::vector &result); + + void to_trade_record(domain::TradeRecord &value) const; +}; + +void from_json(const nlohmann::json &j, CCApiTradeRecord &r); +void to_json(nlohmann::json &j, const CCApiTradeRecord &r); + +std::ostream &operator<<(std::ostream &out, const CCApiTradeRecord &r) noexcept; + +} // namespace bts::md +#endif diff --git a/src/lib/md/ICCApiMessageHandler.cpp b/src/lib/md/ICCApiMessageHandler.cpp new file mode 100644 index 000000000..d4da329af --- /dev/null +++ b/src/lib/md/ICCApiMessageHandler.cpp @@ -0,0 +1,10 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "ICCApiMessageHandler.h" +#include "lib/md/ICCApiMessageHandler.h" + +namespace bts::md { + +ICCApiMessageHandler::~ICCApiMessageHandler() = default; + +} // namespace bts::md +#endif diff --git a/src/lib/md/ICCApiMessageHandler.h b/src/lib/md/ICCApiMessageHandler.h new file mode 100644 index 000000000..c002dd057 --- /dev/null +++ b/src/lib/md/ICCApiMessageHandler.h @@ -0,0 +1,32 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/CCApiCorrelationID.h" + +#include +#include + +namespace ccapi { +class Event; +class Session; +} // namespace ccapi + +namespace bts::md { + +/** + * Provides a callback similar to ccapi::EventHandler::processEvent. This + * class is designed to work with the CCApiEventDemuxer, which provides + * facilities to subscribe only to messages with a particular correlation id. + */ +class ICCApiMessageHandler { + public: + virtual ~ICCApiMessageHandler(); + + virtual void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) = 0; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/MarketDataGateway.cpp b/src/lib/md/MarketDataGateway.cpp index 783036f68..66090cd9b 100644 --- a/src/lib/md/MarketDataGateway.cpp +++ b/src/lib/md/MarketDataGateway.cpp @@ -1,10 +1,11 @@ #ifdef SPARK_ENABLE_CCAPI #include "MarketDataGateway.h" #include "lib/api/domain/Venue.pb.h" -#include "lib/api/md/DepthStreamConfig.pb.h" #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/collections/Contains.h" #include "lib/core/ProtoPrinter.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityID.h" #include "lib/logging/logging.h" #include "lib/md/CCApiSession.h" #include "lib/md/IMarketDataListener.h" @@ -13,89 +14,104 @@ #include "lib/md/MarketDataRegistration.h" #include "lib/md/MarketDataRegistrationError.h" #include "lib/md/MarketDataRegistrationFactory.h" +#include "lib/md/MarketDataSubscription.h" +#include "lib/md/ftx/FTXSubscription.h" #include "lib/persistence/IDGenerator.h" #include "lib/utils/bts_assert.h" +#include #include #include +#include #include namespace bts::md { +using bts::api::domain::Venue; +using bts::api::md::MarketDataConfig; +using bts::domain::Security; +using bts::domain::SecurityID; +using bts::persistence::IDGenerator; using std::string; +using std::unique_ptr; +using std::unordered_map; using std::vector; -static void validate( - const api::md::DepthStreamConfig &ds, - std::uint32_t index, - vector &errors) { - if (!api::domain::Venue_IsValid(ds.venue())) { - errors.push_back(fmt::format( - "DepthStreamConfig at index {} 'venue' is invalid", index)); - } - if (ds.num_levels() == 0) { - errors.push_back(fmt::format( - "DepthStreamConfig at index {} 'num_levels' is 0", index)); - } - if (ds.symbols().empty()) { - errors.push_back(fmt::format( - "DepthStreamConfig at index {} 'symbols' is empty", index)); - } -} - -static void validate( - const api::md::MarketDataConfig &cfg, vector errs) { - if (cfg.depth_streams().empty()) { - errs.push_back("MarketDataConfig 'depth_streams' is empty"); - } - std::uint32_t ds_idx = 0; - for (const auto &ds : cfg.depth_streams()) { - validate(ds, ds_idx++, errs); - } +static bool validate(const MarketDataConfig & /*cfg*/, vector &errs) { + auto orig_n_errs = errs.size(); + // TODO + return errs.size() == orig_n_errs; } struct MarketDataGateway::PImpl { + ccapi::EventDispatcher dispatcher; + unordered_map sessions; + MarketDataRegistrationFactory registration_factory; - bts::persistence::IDGenerator &sec_id_store = - bts::persistence::IDGenerator::instance(); - std::unordered_map sessions; + IDGenerator &sec_id_store = IDGenerator::instance(); + unordered_map listeners; + unordered_map> securities; + + CCApiSession &get_session(Venue venue) { + auto it = sessions.find(venue); + if (it == sessions.end()) { + LOG(md, Info, "Creating CCApiSession for venue {}", Venue_Name(venue)); + it = sessions.try_emplace(venue, &dispatcher).first; + } + return it->second; + } }; +#define VENUE_CASE_NAME(x) \ + (MarketDataConfig::descriptor() \ + ->FindFieldByNumber(static_cast(x)) \ + ->name()) + MarketDataGateway::MarketDataGateway() : me_(std::make_unique()) {} MarketDataGateway::~MarketDataGateway() = default; MarketDataRegistration MarketDataGateway::subscribe( - const api::md::MarketDataConfig &cfg, - IMarketDataListener *listener) noexcept { + const MarketDataConfig &cfg, IMarketDataListener *listener) noexcept { auto reg = me_->registration_factory.create(); bts_assert(reg.listener_id.has_value()); - bts_assert(me_->sessions.find(*reg.listener_id) == me_->sessions.end()); - LOG(md, bts::Info, "MarketDataGateway subscribing to data.\n{}\n", + bts_assert(!CONTAINS_KEY(me_->listeners, *reg.listener_id)); + LOG(md, Info, "MarketDataGateway subscribing to data:\n{}", core::ProtoPrinterFactory::create(&cfg)); vector cfg_errs; - validate(cfg, cfg_errs); - if (!cfg_errs.empty()) { + if (!validate(cfg, cfg_errs)) { return MarketDataRegistration{ {MarketDataRegistrationError::Type::CONFIG_ERROR, fmt::to_string(fmt::join(cfg_errs, "\n"))}}; } - MarketDataListenerData listener_data; + MarketDataListenerData &listener_data = me_->listeners[*reg.listener_id]; listener_data.id = *reg.listener_id; listener_data.listener = listener; listener_data.cfg = cfg; - me_->sessions.try_emplace( - listener_data.id, listener_data, &me_->sec_id_store); + switch (cfg.venue_case()) { + case MarketDataConfig::VenueCase::kFtx: { + listener_data.subscription.add(ftx::FTXSubscription{ + &me_->dispatcher, cfg.ftx(), &me_->sec_id_store, + listener_data.listener}); + break; + } + default: + LOG(md, Error, + "Unable to create md subscription because the venue '{}' isn't " + "supported.", + VENUE_CASE_NAME(cfg.venue_case())); + break; + } + // TODO: add more exchanges here return reg; } void MarketDataGateway::unsubscribe(MarketDataListenerID id) { - LOG(md, bts::Info, "Unsubscribing listener {}", id); - bts_assert(CONTAINS_KEY(me_->sessions, id)); - me_->sessions.erase(id); + LOG(md, Info, "Unsubscribing listener {}", id); + me_->listeners.erase(id); } } // namespace bts::md diff --git a/src/lib/md/MarketDataListenerData.h b/src/lib/md/MarketDataListenerData.h index 9f5e64d69..c34421e49 100644 --- a/src/lib/md/MarketDataListenerData.h +++ b/src/lib/md/MarketDataListenerData.h @@ -2,13 +2,25 @@ #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/md/IMarketDataListener.h" #include "lib/md/MarketDataListenerID.h" +#include "lib/md/MarketDataSubscription.h" +#include namespace bts::md { struct MarketDataListenerData { + MarketDataListenerData() = default; + MarketDataListenerData(const MarketDataListenerData &) noexcept = delete; + MarketDataListenerData(MarketDataListenerData &&) noexcept = default; + MarketDataListenerData &operator=(const MarketDataListenerData &) noexcept = + delete; + MarketDataListenerData &operator=(MarketDataListenerData &&) noexcept = + default; + ~MarketDataListenerData() = default; + MarketDataListenerID id; IMarketDataListener *listener; api::md::MarketDataConfig cfg; + MarketDataSubscription subscription; }; } // namespace bts::md diff --git a/src/lib/md/MarketDataSubscription.cpp b/src/lib/md/MarketDataSubscription.cpp new file mode 100644 index 000000000..4bca67083 --- /dev/null +++ b/src/lib/md/MarketDataSubscription.cpp @@ -0,0 +1,27 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "MarketDataSubscription.h" +#include "lib/api/md/MarketDataConfig.pb.h" +#include "lib/md/ftx/FTXSubscription.h" +#include +#include + +namespace bts::md { + +using std::vector; + +struct MarketDataSubscription::PImpl { + vector ftx_subs; + // TODO: add more exchanges +}; + +MarketDataSubscription::MarketDataSubscription() + : me_(std::make_unique()) {} + +MarketDataSubscription::~MarketDataSubscription() = default; + +void MarketDataSubscription::add(ftx::FTXSubscription sub) { + me_->ftx_subs.push_back(std::move(sub)); +} + +} // namespace bts::md +#endif diff --git a/src/lib/md/MarketDataSubscription.h b/src/lib/md/MarketDataSubscription.h new file mode 100644 index 000000000..346b793af --- /dev/null +++ b/src/lib/md/MarketDataSubscription.h @@ -0,0 +1,32 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include + +namespace bts::md::ftx { +class FTXSubscription; +} + +namespace bts::md { + +class MarketDataSubscription { + public: + MarketDataSubscription(); + + MarketDataSubscription(const MarketDataSubscription &) noexcept = delete; + MarketDataSubscription(MarketDataSubscription &&) noexcept = default; + MarketDataSubscription &operator=(const MarketDataSubscription &) noexcept = + delete; + MarketDataSubscription &operator=(MarketDataSubscription &&) noexcept = + default; + + ~MarketDataSubscription(); + + void add(ftx::FTXSubscription); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md +#endif diff --git a/src/lib/md/ccApi/CCMain.cpp b/src/lib/md/ccApi/CCMain.cpp index 28c9e375d..163ed63d9 100644 --- a/src/lib/md/ccApi/CCMain.cpp +++ b/src/lib/md/ccApi/CCMain.cpp @@ -1,11 +1,9 @@ #ifdef SPARK_ENABLE_CCAPI #include "CCMain.h" -#include "lib/api/md/DepthStreamConfig.pb.h" #include "lib/api/md/MarketDataConfig.pb.h" #include "lib/config/config.h" #include "lib/config/protos/market_data.pb.h" -#include "lib/core/ProtoReader.h" #include "lib/logging/logging.h" #include "lib/md/MDProducer.h" #include "lib/md/MarketDataGateway.h" @@ -69,17 +67,7 @@ CCMain::~CCMain() {} void CCMain::run() { LOG(md, Info, "Run::run_ccapi t_id:{} t_name:{}", os::get_thread_id(), os::get_thread_name()); - bts::api::md::MarketDataConfig md_config; - md_config.set_num_dispatcher_threads(1); - auto it = md_config.mutable_depth_streams()->Add(); - it->set_venue(::bts::api::domain::Venue::FTX); - it->set_num_levels(5); - it->mutable_symbols()->Add("BTC-PERP"); - it->mutable_symbols()->Add("BTC/USD"); - - char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt"; - putenv(ssl_cert_file); - + const auto &md_config = config::get(); me->gw.subscribe(md_config, &me->cc_listener); size_t n = 0; diff --git a/src/lib/md/ccApi/CCMarketDataListener.cpp b/src/lib/md/ccApi/CCMarketDataListener.cpp index 172b428c3..95e2e377b 100644 --- a/src/lib/md/ccApi/CCMarketDataListener.cpp +++ b/src/lib/md/ccApi/CCMarketDataListener.cpp @@ -159,22 +159,6 @@ void CCMarketDataListener::on_securities( bool pursue = false; domain::Security sec = *s_ptr; const auto &s_comm = sec.security_common; - // temp - { - if (sec.security_common.desc == "BTC-PERP") { - domain::FutureSecurity fsec = {}; - fsec.expiration = get_ts_from_date(2022, 12, 25); - sec.security = fsec; - sec.security_common.min_price_increment_ammount = 1.0; - pursue = true; - } - if (sec.security_common.desc == "BTC/USD") { - domain::SpotSecurity ssec = {}; - sec.security = ssec; - sec.security_common.min_price_increment_ammount = 1.0; - pursue = true; - } - } auto search_result = _find_sec(sec, true); LOG(md, Info, "CCMarketDataListener::on_securities added new security, exchange: " diff --git a/src/lib/md/ftx/FTXFuture.cpp b/src/lib/md/ftx/FTXFuture.cpp new file mode 100644 index 000000000..ceeacdf74 --- /dev/null +++ b/src/lib/md/ftx/FTXFuture.cpp @@ -0,0 +1,181 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXFuture.h" +#include "lib/api/domain/Currency.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/core/TableFormatter.h" +#include "lib/domain/FutureSecurity.h" +#include "lib/domain/Security.h" +#include "lib/io/JsonReader.h" +#include "lib/logging/logging.h" + +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::api::domain::Currency; +using bts::api::domain::Venue; +using bts::core::TableFormatter; +using bts::domain::FutureSecurity; +using bts::domain::Security; +using bts::io::JsonReader; +using std::string; +using std::stringstream; +using std::vector; + +static std::string deduce_sym_from_name(const std::string &name) { + std::regex word_regex("(\\w+)"); + auto words_begin = + std::sregex_iterator(name.begin(), name.end(), word_regex); + auto words_end = std::sregex_iterator(); + if (words_begin != words_end) { + return words_begin->str(); + } + return ""; +} + +static std::string deduce_sym(const FTXFuture &ffut) { + if (!ffut.underlying.empty()) { + return ffut.underlying; + } + return deduce_sym_from_name(ffut.name); +} + +void FTXFuture::parse(const ccapi::Message &msg, vector &securities) { + const auto &sec_elements = msg.getElementList(); + for (const auto &sec_el : sec_elements) { + const auto &sec_props = sec_el.getNameValueMap(); + auto it = sec_props.find(CCAPI_HTTP_BODY); + if (it == sec_props.end()) { + continue; + } + const auto &body = it->second; + const auto js = nlohmann::json::parse(body); + vector ftx_futs; + if (!js["success"].get()) { + LOG(md, Error, + "FTXFutureParser received json element w/ 'success' false"); + continue; + } + js["result"].get_to(ftx_futs); + for (const auto &ftx_fut : ftx_futs) { + switch (ftx_fut.type.value) { + case FTXSecurityType::FUTURE: + case FTXSecurityType::PERPETUAL: + break; + case FTXSecurityType::MOVE: + case FTXSecurityType::PREDICTION: + // TODO: Add support for 'MOVE' and 'PREDICTION' futs. They may + // complicate things because they have the same exp dates as + // 'FUTURE' futs. This translates to duplicate sym/exp pairs since + // all fall under the umbrella of a uniform 'Future' type in our + // system. + default: + continue; + } + auto &sec = securities.emplace_back(); + auto &scomm = sec.security_common; + scomm.type = ftx_fut.type.to_security_type(); + scomm.venue = Venue::FTX; + + scomm.exchange_sec_id = ftx_fut.name; + scomm.desc = ftx_fut.name; + scomm.symbol = deduce_sym(ftx_fut); + scomm.asset = scomm.symbol; + scomm.group = scomm.symbol; + + scomm.min_price_increment = ftx_fut.price_increment; + scomm.min_quantity = ftx_fut.size_increment; + scomm.min_quantity_increment = ftx_fut.size_increment; + scomm.currency = Currency::USD; + + auto &fut = sec.security.emplace(); + fut.expiration = ftx_fut.expiry; + LOG(md, Diag, "Converted FTXFuture:\n{}to domain::Security:\n{}", + ftx_fut, sec.to_string(true)); + } + } +} + +void from_json(const nlohmann::json &j, FTXFuture &fut) { + const string js_str = j.dump(4); + LOG(md, Diag, "Parsing FTXFuture from json:\n{}\n", js_str); + + JsonReader jr{&j}; + jr.read("ask", fut.ask); + jr.read("bid", fut.bid); + jr.read("change1h", fut.change_1h); + jr.read("change24h", fut.change_24h); + jr.read("changeBod", fut.change_bod); + jr.read("volumeUsd24h", fut.volume_usd_24h); + jr.read("volume", fut.volume); + jr.read("description", fut.description); + jr.read("enabled", fut.enabled); + jr.read("expired", fut.expired); + jr.read("expiry", fut.expiry); + jr.read("index", fut.index); + jr.read("imfFactor", fut.imf_factor); + jr.read("last", fut.last); + jr.read("lowerBound", fut.lower_bound); + jr.read("mark", fut.mark); + jr.read("name", fut.name); + jr.read("openInterest", fut.open_interest); + jr.read("openInterestUsd", fut.open_interest_usd); + jr.read("perpetual", fut.perpetual); + jr.read("positionLimitWeight", fut.position_limit_weight); + jr.read("postOnly", fut.post_only); + jr.read("priceIncrement", fut.price_increment); + jr.read("sizeIncrement", fut.size_increment); + jr.read("underlying", fut.underlying); + jr.read("upperBound", fut.upper_bound); + jr.read("type", fut.type); +} + +string FTXFuture::to_string_pretty() const { + stringstream out; + TableFormatter::SchemaBuilder schema; + schema.add("ask", [](const auto &f) { return f.ask; }); + schema.add("bid", [](const auto &f) { return f.bid; }); + schema.add("change_1h", [](const auto &f) { return f.change_1h; }); + schema.add("change_24h", [](const auto &f) { return f.change_24h; }); + schema.add("change_bod", [](const auto &f) { return f.change_bod; }); + schema.add("volume_usd_24h", [](const auto &f) { return f.volume_usd_24h; }); + schema.add("volume", [](const auto &f) { return f.volume; }); + schema.add("description", [](const auto &f) { return f.description; }); + schema.add("enabled", [](const auto &f) { return f.enabled; }); + schema.add("expired", [](const auto &f) { return f.expired; }); + schema.add("expiry", [](const auto &f) { return f.expiry; }); + schema.add("index", [](const auto &f) { return f.index; }); + schema.add("imf_factor", [](const auto &f) { return f.imf_factor; }); + schema.add("last", [](const auto &f) { return f.last; }); + schema.add("lower_bound", [](const auto &f) { return f.lower_bound; }); + schema.add("mark", [](const auto &f) { return f.mark; }); + schema.add("name", [](const auto &f) { return f.name; }); + schema.add("open_interest", [](const auto &f) { return f.open_interest; }); + schema.add( + "open_interest_usd", [](const auto &f) { return f.open_interest_usd; }); + schema.add("perpetual", [](const auto &f) { return f.perpetual; }); + schema.add("position_limit_weight", [](const auto &f) { + return f.position_limit_weight; + }); + schema.add("post_only", [](const auto &f) { return f.post_only; }); + schema.add( + "price_increment", [](const auto &f) { return f.price_increment; }); + schema.add("size_increment", [](const auto &f) { return f.size_increment; }); + schema.add("underlying", [](const auto &f) { return f.underlying; }); + schema.add("upper_bound", [](const auto &f) { return f.upper_bound; }); + schema.add("type", [](const auto &f) { return f.type; }); + TableFormatter::format_record(out, *this, schema.result); + return out.str(); +} + +std::ostream &operator<<(std::ostream &out, const FTXFuture &fut) noexcept { + out << fut.name; + return out; +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXFuture.h b/src/lib/md/ftx/FTXFuture.h new file mode 100644 index 000000000..10ab39d32 --- /dev/null +++ b/src/lib/md/ftx/FTXFuture.h @@ -0,0 +1,73 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/TimeStamp.h" +#include "lib/md/ftx/FTXSecurityType.h" + +#include +#include +#include +#include +#include + +namespace bts::domain { +class Security; +} + +namespace ccapi { +class Message; +} + +namespace bts::md::ftx { + +/*! + * @brief Represents a 'future' at FTX + * + * @details The fields mirror those in the response json for 'futures' + * requests. https://docs.ftx.com/reference/list-all-futures + */ +class FTXFuture { + public: + double ask; + double bid; + double change_1h; + double change_24h; + double change_bod; + double volume_usd_24h; + double volume; + std::string description; + bool enabled; + bool expired; + core::TimeStamp expiry; + double index; + double imf_factor; + double last; + double lower_bound; + double mark; + std::string name; + double open_interest; + double open_interest_usd; + bool perpetual; + double position_limit_weight; + bool post_only; + double price_increment; + double size_increment; + std::string underlying; + double upper_bound; + FTXSecurityType type; + + static void parse( + const ccapi::Message &msg, std::vector &securities); + + /** + * @brief Provides a detailed string representation of the object in + * easy-to-read table format, with a row for each field. + */ + std::string to_string_pretty() const; +}; + +void from_json(const nlohmann::json &j, FTXFuture &fut); + +std::ostream &operator<<(std::ostream &out, const FTXFuture &fut) noexcept; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp new file mode 100644 index 000000000..f6fa569db --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.cpp @@ -0,0 +1,54 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXHistoricalPriceAdaptor.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/ftx/FTXHistoricalPriceRecord.h" + +#include +#include +#include + +namespace bts::md::ftx { + +using bts::domain::SecurityID; +using std::vector; + +struct FTXHistoricalPriceAdaptor::PImpl { + SecurityID sec_id; + IMarketDataListener *listener; +}; + +FTXHistoricalPriceAdaptor::FTXHistoricalPriceAdaptor( + SecurityID sec_id, IMarketDataListener *listener) + : me_(std::make_unique()) { + me_->sec_id = sec_id; + me_->listener = listener; +} + +FTXHistoricalPriceAdaptor::~FTXHistoricalPriceAdaptor() = default; + +void FTXHistoricalPriceAdaptor::on_messages( + const ccapi::Event & /*event*/, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + vector price_recs; + for (const auto *msg : msgs) { + if (!FTXHistoricalPriceRecord::parse(*msg, price_recs)) { + continue; + } + bts_assert(!price_recs.empty()); + auto latest_it = std::max_element( + price_recs.begin(), price_recs.end(), + [](const auto &lhs, const auto &rhs) { + return lhs.start_time < rhs.start_time; + }); + auto sec_stats = latest_it->to_security_statistics(me_->sec_id); + me_->listener->on_security_statistics(sec_stats); + break; + } +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceAdaptor.h b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.h new file mode 100644 index 000000000..4c3f81fba --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceAdaptor.h @@ -0,0 +1,31 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/md/ICCApiMessageHandler.h" + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::md::ftx { + +class FTXHistoricalPriceAdaptor : public ICCApiMessageHandler { + public: + FTXHistoricalPriceAdaptor( + domain::SecurityID sec_id, IMarketDataListener *listener); + + ~FTXHistoricalPriceAdaptor(); + + void on_messages( + const ccapi::Event &event, + ccapi::Session *session, + const std::vector &msgs, + const CCApiCorrelationID &corr_id) override; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceRecord.cpp b/src/lib/md/ftx/FTXHistoricalPriceRecord.cpp new file mode 100644 index 000000000..c678c4045 --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceRecord.cpp @@ -0,0 +1,90 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXHistoricalPriceRecord.h" +#include "lib/domain/Price.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/io/JsonReader.h" +#include "lib/md/CCApiReader.h" +#include +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::domain::Price; +using bts::domain::SecurityID; +using bts::domain::SecurityMarketDataStatistics; +using bts::io::JsonReader; +using std::map; +using std::string; +using std::vector; + +static bool parse( + const map &name_values, + std::vector &price_recs) { + CCApiReader ccapi_reader{&name_values}; + nlohmann::json js; + if (!ccapi_reader.read(CCAPI_HTTP_BODY, js)) { + return false; + } + JsonReader jreader{&js}; + if (!jreader.read("success")) { + LOG(md, Error, + "FTXHistoricalPriceRecord json field 'success' is false:\n{}", js); + return false; + } + if (!jreader.read("result", price_recs)) { + LOG(md, Error, + "Failed to read FTXHistoricalPriceRecord json from field " + "'result':\n{}", + js); + } + return !price_recs.empty(); +} + +bool FTXHistoricalPriceRecord::parse( + const ccapi::Message &msg, + std::vector &price_recs) { + auto orig_sz = price_recs.size(); + for (const auto &sec_el : msg.getElementList()) { + const auto &hp_props = sec_el.getNameValueMap(); + auto it = hp_props.find(CCAPI_HTTP_BODY); + if (it == hp_props.end()) { + continue; + } + bts::md::ftx::parse(hp_props, price_recs); + } + return price_recs.size() > orig_sz; +} + +void FTXHistoricalPriceRecord::to_security_statistics( + SecurityMarketDataStatistics &sec_stats) const { + sec_stats.high = Price::from_double(high); + sec_stats.low = Price::from_double(low); + sec_stats.open = Price::from_double(open); + sec_stats.settle = Price::from_double(close); + sec_stats.volume.value() = volume; +} + +SecurityMarketDataStatistics FTXHistoricalPriceRecord::to_security_statistics( + SecurityID &sec_id) const { + SecurityMarketDataStatistics sec_stats; + to_security_statistics(sec_stats); + sec_stats.security_id = sec_id; + return sec_stats; +} + +void from_json(const nlohmann::json &j, FTXHistoricalPriceRecord &p) { + JsonReader jr{&j}; + jr.read("startTime", p.start_time); + jr.read("open", p.open); + jr.read("close", p.close); + jr.read("high", p.high); + jr.read("low", p.low); + jr.read("volume", p.volume); +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXHistoricalPriceRecord.h b/src/lib/md/ftx/FTXHistoricalPriceRecord.h new file mode 100644 index 000000000..353a6c95a --- /dev/null +++ b/src/lib/md/ftx/FTXHistoricalPriceRecord.h @@ -0,0 +1,41 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/TimeStamp.h" +#include "lib/domain/SecurityID.h" +#include + +namespace bts::domain { +class SecurityMarketDataStatistics; +} + +namespace ccapi { +class Event; +class Message; +} // namespace ccapi + +namespace bts::md::ftx { + +class FTXHistoricalPriceRecord { + public: + core::TimeStamp start_time; + double open; + double close; + double high; + double low; + double volume; + + static bool parse( + const ccapi::Message &msg, + std::vector &price_recs); + + void to_security_statistics( + domain::SecurityMarketDataStatistics &sec_stats) const; + + domain::SecurityMarketDataStatistics to_security_statistics( + domain::SecurityID &sec_id) const; +}; + +void from_json(const nlohmann::json &j, FTXHistoricalPriceRecord &p); + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXMarket.cpp b/src/lib/md/ftx/FTXMarket.cpp new file mode 100644 index 000000000..07c222f91 --- /dev/null +++ b/src/lib/md/ftx/FTXMarket.cpp @@ -0,0 +1,217 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXMarket.h" +#include "lib/api/domain/Currency.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/core/NaN.h" +#include "lib/core/TableFormatter.h" +#include "lib/domain/FutureSecurity.h" +#include "lib/domain/Security.h" +#include "lib/domain/SpotSecurity.h" +#include "lib/io/JsonReader.h" +#include "lib/logging/logging.h" +#include "lib/md/ftx/FTXSecurityType.h" + +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::api::domain::Currency; +using bts::api::domain::Currency_Name; +using bts::api::domain::Currency_Parse; +using bts::api::domain::SecurityType; +using bts::api::domain::Venue; +using bts::core::NaN; +using bts::core::TableFormatter; +using bts::domain::Security; +using bts::io::JsonReader; +using std::ostream; +using std::string; +using std::vector; + +static std::string deduce_symbol(const std::string &name) { + std::regex word_regex("(\\w+)"); + auto words_begin = + std::sregex_iterator(name.begin(), name.end(), word_regex); + auto words_end = std::sregex_iterator(); + if (words_begin != words_end) { + return words_begin->str(); + } + return ""; +} + +static std::string deduce_symbol(const FTXMarket &fsec) { + if (!fsec.underlying.empty()) { + return fsec.underlying; + } + switch (fsec.type.value) { + case FTXSecurityType::SPOT: + return fsec.base_currency; + default: + return deduce_symbol(fsec.name); + } +} + +void FTXMarket::parse(const ccapi::Message &msg, vector &securities) { + const auto &sec_elements = msg.getElementList(); + for (const auto &sec_el : sec_elements) { + const auto &sec_props = sec_el.getNameValueMap(); + auto it = sec_props.find(CCAPI_HTTP_BODY); + if (it == sec_props.end()) { + continue; + } + const auto &body = it->second; + const auto js = nlohmann::json::parse(body); + vector ftx_mkts; + if (!js["success"].get()) { + LOG(md, Error, + "FTXSecurityParser received json element w/ 'success' false"); + continue; + } + js["result"].get_to(ftx_mkts); + for (const auto &ftx_mkt : ftx_mkts) { + auto &sec = securities.emplace_back(); + auto &scomm = sec.security_common; + scomm.desc = ftx_mkt.name; + scomm.venue = Venue::FTX; + + scomm.symbol = deduce_symbol(ftx_mkt); + scomm.asset = scomm.symbol; + scomm.group = scomm.symbol; + + scomm.min_price_increment = ftx_mkt.price_increment; + scomm.min_quantity = ftx_mkt.min_provide_size; + scomm.min_quantity_increment = ftx_mkt.size_increment; + scomm.type = ftx_mkt.type.to_security_type(); + + switch (scomm.type) { + case SecurityType::UNSET: + bts_assert_msgf( + false, "Failed to parse ftx security type from '{}'", + ftx_mkt.type); + break; + case api::domain::SPOT: + sec.security.emplace(); + break; + case api::domain::FUTURE: + sec.security.emplace(); + break; + case api::domain::SPREAD: + bts_assert_msg(false, "Spreads not supported"); + break; + case api::domain::OPTION: + bts_assert_msg(false, "Options not supported"); + break; + case api::domain::SYNTHETIC: + bts_assert_msg(false, "Synthetics not supported"); + break; + case api::domain::SecurityType_INT_MIN_SENTINEL_DO_NOT_USE_: + case api::domain::SecurityType_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert_msgf(false, "Bad security type value '{}'", scomm.type); + break; + } + + // currency + constexpr const auto default_currency = Currency::USD; + if (ftx_mkt.quote_currency.empty()) { + scomm.currency = default_currency; + } else if (!Currency_Parse(ftx_mkt.quote_currency, &scomm.currency)) { + scomm.currency = default_currency; + LOG(md, Error, + "Failed to parse currency type from string '{}'. Falling back" + "to default value '{}'", + ftx_mkt.quote_currency, Currency_Name(default_currency)); + } + LOG(md, Diag, "domain::Security (from FTXMarket):\n{}", + sec.to_string(true)); + } + } +} + +void from_json(const nlohmann::json &j, FTXMarket &mkt) { + const string js_str = j.dump(4); + LOG(md, Diag, "Parsing FTXMarket from json:\n{}\n", js_str); + + JsonReader jr{&j}; + // type-independent props + jr.read("name", mkt.name); + jr.read("type", mkt.type); + jr.read("quoteVolume24h", mkt.quote_volume_24h); + jr.read("change1h", mkt.change_1h); + jr.read("change24h", mkt.change_24h); + jr.read("changeBod", mkt.change_bod); + jr.read("highLeverageFeeExempt", mkt.high_leverage_fee_exempt); + jr.read("minProvideSize", mkt.min_provide_size); + + jr.read("enabled", mkt.enabled); + jr.read_or("ask", mkt.ask, NaN::value()); + jr.read_or("bid", mkt.bid, NaN::value()); + jr.read_or("last", mkt.last, NaN::value()); + jr.read("postOnly", mkt.post_only); + jr.read_or("price", mkt.price, NaN::value()); + jr.read("priceIncrement", mkt.price_increment); + jr.read("sizeIncrement", mkt.size_increment); + jr.read("restricted", mkt.restricted); + jr.read("volumeUsd24h", mkt.volume_usd_24h); + jr.read("largeOrderThreshold", mkt.large_order_threshold); + jr.read("isEtfMarket", mkt.is_etf_market); + + // type-dependent props + switch (mkt.type.value) { + case FTXSecurityType::FUTURE: + // these only apply to futures + jr.read("underlying", mkt.underlying); + break; + case FTXSecurityType::SPOT: + // these only apply to spots + jr.read("baseCurrency", mkt.base_currency); + jr.read("quoteCurrency", mkt.quote_currency); + break; + default: + LOG(md, Error, "Unexpected FTXSecurityType '{}'", + j["type"].get()); + break; + } +} + +ostream &operator<<(ostream &out, const FTXMarket &mkt) noexcept { + core::TableFormatter::SchemaBuilder schema; + schema.add("name", [](const auto &s) { return s.name; }); + schema.add("base_currency", [](const auto &s) { return s.base_currency; }); + schema.add("quote_currency", [](const auto &s) { return s.quote_currency; }); + schema.add( + "quote_volume_24h", [](const auto &s) { return s.quote_volume_24h; }); + schema.add("change_1h", [](const auto &s) { return s.change_1h; }); + schema.add("change_24h", [](const auto &s) { return s.change_24h; }); + schema.add("change_bod", [](const auto &s) { return s.change_bod; }); + schema.add("high_leverage_fee_exempt", [](const auto &s) { + return s.high_leverage_fee_exempt; + }); + schema.add( + "min_provide_size", [](const auto &s) { return s.min_provide_size; }); + schema.add("type", [](const auto &s) { return s.type; }); + schema.add("underlying", [](const auto &s) { return s.underlying; }); + schema.add("enabled", [](const auto &s) { return s.enabled; }); + schema.add("ask", [](const auto &s) { return s.ask; }); + schema.add("bid", [](const auto &s) { return s.bid; }); + schema.add("last", [](const auto &s) { return s.last; }); + schema.add("post_only", [](const auto &s) { return s.post_only; }); + schema.add("price", [](const auto &s) { return s.price; }); + schema.add( + "price_increment", [](const auto &s) { return s.price_increment; }); + schema.add("size_increment", [](const auto &s) { return s.size_increment; }); + schema.add("restricted", [](const auto &s) { return s.restricted; }); + schema.add("volume_usd_24h", [](const auto &s) { return s.volume_usd_24h; }); + schema.add("large_order_threshold", [](const auto &s) { + return s.large_order_threshold; + }); + schema.add("is_etf_market", [](const auto &s) { return s.is_etf_market; }); + TableFormatter::format_record(out, mkt, schema.result); + return out; +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXMarket.h b/src/lib/md/ftx/FTXMarket.h new file mode 100644 index 000000000..bc0a0e2a3 --- /dev/null +++ b/src/lib/md/ftx/FTXMarket.h @@ -0,0 +1,61 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/md/ftx/FTXSecurityType.h" +#include +#include +#include +#include +#include + +namespace bts::domain { +class Security; +} + +namespace ccapi { +class Message; +} + +namespace bts::md::ftx { + +/** + * Represents a 'market' at FTX + * + * @details The fields mirror those in the response json for 'markets' + * requests. https://docs.ftx.com/reference/get-markets + */ +class FTXMarket { + public: + std::string name; + std::string base_currency; + std::string quote_currency; + double quote_volume_24h; + double change_1h; + double change_24h; + double change_bod; + bool high_leverage_fee_exempt; + double min_provide_size; + FTXSecurityType type; + std::string underlying; + bool enabled; + double ask; + double bid; + double last; + bool post_only; + double price; + double price_increment; + double size_increment; + bool restricted; + double volume_usd_24h; + double large_order_threshold; + bool is_etf_market; + + static void parse( + const ccapi::Message &msg, std::vector &securities); +}; + +void from_json(const nlohmann::json &j, FTXMarket &sec); + +std::ostream &operator<<(std::ostream &out, const FTXMarket &sec) noexcept; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXMessageType.h b/src/lib/md/ftx/FTXMessageType.h new file mode 100644 index 000000000..0a13aa2c0 --- /dev/null +++ b/src/lib/md/ftx/FTXMessageType.h @@ -0,0 +1,35 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/core/EnumMacros.h" +#include "lib/core/FMT_IMPL.h" +#include +#include +#include + +namespace bts::md::ftx { + +// clang-format off +#define FOR_ALL_VALUES(per_value) \ + per_value(FTXMessageType, NONE) \ + per_value(FTXMessageType, FUTURES) \ + per_value(FTXMessageType, SECURITIES) \ + per_value(FTXMessageType, DEPTH) \ + per_value(FTXMessageType, TICKER) \ + per_value(FTXMessageType, STATISTICS) +// clang-format on + +/** + * @brief Represents the different message types at FTX + * + * @details Each enum value is associated with a particular type of request (or + * subscription) that is sent to the exchange (FTX), and a particular handler + * type designed to adapt the messages for an IMarketDataListener. + */ +ENUM_CLASS_DEF(FTXMessageType, FOR_ALL_VALUES) + +#undef FOR_ALL_VALUES +} // namespace bts::md::ftx + +FMT_IMPL(bts::md::ftx::FTXMessageType, t, "{}", t.name()); + +#endif diff --git a/src/lib/md/ftx/FTXSecurityRequestor.cpp b/src/lib/md/ftx/FTXSecurityRequestor.cpp new file mode 100644 index 000000000..261efefff --- /dev/null +++ b/src/lib/md/ftx/FTXSecurityRequestor.cpp @@ -0,0 +1,275 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXSecurityRequestor.h" +#include "lib/api/domain/SecurityType.pb.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/FTXMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/collections/Remove.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityCriteria.h" +#include "lib/logging/logging.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiMessageHandler.h" +#include "lib/md/CCApiSession.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/ftx/FTXFuture.h" +#include "lib/md/ftx/FTXMarket.h" +#include "lib/md/ftx/FTXMessageType.h" +#include "lib/persistence/IDGenerator.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using api::domain::SecurityType; +using api::domain::Venue; +using api::domain::Venue_Name; +using bts::domain::Security; +using bts::domain::SecurityID; +using std::string; +using std::unordered_set; +using std::vector; + +#define VENUE_NAME (Venue_Name(Venue::FTX)) + +#define FTX_MSG_TYPE_NAME(x) \ + std::string { FTXMessageType{x}.name() } + +static ccapi::Request markets_req() { + auto secs_corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTX_MSG_TYPE_NAME(FTXMessageType::SECURITIES)); + ccapi::Request secs_req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, "ftx"}; + secs_req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_HOST, "ftx.com"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, "/api/markets"}, + {CCAPI_HTTP_METHOD, "GET"}}); + secs_req.setCorrelationId(secs_corr_id.to_string()); + return secs_req; +} + +static ccapi::Request futures_req() { + auto futs_corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTX_MSG_TYPE_NAME(FTXMessageType::FUTURES)); + ccapi::Request futs_req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, "ftx"}; + futs_req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_HOST, "ftx.com"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, "/api/futures"}, + {CCAPI_HTTP_METHOD, "GET"}}); + futs_req.setCorrelationId(futs_corr_id.to_string()); + return futs_req; +} + +static vector get_sec_descs(const vector &sec_ptrs) { + vector descs{sec_ptrs.size()}; + std::transform( + sec_ptrs.begin(), sec_ptrs.end(), descs.begin(), + [](const auto &s) { return s.desc(); }); + return descs; +} + +static void validate_response(const ccapi::Event &event, bool is_expected) { + bts_assert_msgf( + event.getType() == ccapi::Event::Type::RESPONSE, + "Received event w/ unexpected type:\n{}", event.toStringPretty()); + bts_assert_msgf( + is_expected, "Received unexpected response from FTX:\n{}", + event.toStringPretty()); +} + +struct FTXSecurityRequestor::PImpl { + // General Notes: + // FTX securities must be obtained from 2 different urls: + // 1) https://ftx.com/api/markets + // 2) https://ftx.com/api/futures + // Although all securities are listed in the response from (1), not all + // information is provided for futures (notably, the expiration date is + // missing!). This missing information is obtained from (2). Therefore, + // futures are published when (2) is received, while all other securities + // are published when (1) is received. + + domain::SecurityCriteria criteria; + unordered_set securities; + persistence::IDGenerator *sec_id_store; + // True when a 'markets' request is in flight + bool mkts_req_pending = false; + // True when a 'futures' request is in flight + bool futs_req_pending = false; + // True when someone called FTXSecurityRequestor::request while another + // request was already in flight. In this case, the 'external_req_pending' + // flag is used to signal that another request should be sent immediately + // after the current one finishes. + bool external_req_pending = false; + + // These proxy handlers don't actually process new securities. Member methods + // in this class process the securities; we simply bind them to proxy handler + // callbacks. This is easier than defining separate ICCApiMessageHandler + // impls for 'markets', and 'futures', esp since they would need access to + // state contained in this class. + CCApiMessageHandler futs_handler_proxy; + CCApiMessageHandler mkts_handler_proxy; + Callback callback; + + // The session automatically starts/stops in its ctr/dtr + CCApiSession session; + + explicit PImpl( + ccapi::EventDispatcher *dispatcher, + domain::SecurityCriteria criteria, + persistence::IDGenerator *sec_id_store) + : criteria(std::move(criteria)), sec_id_store(sec_id_store), + session(dispatcher) { + mkts_handler_proxy.callback().bind<&PImpl::on_markets>(*this); + futs_handler_proxy.callback().bind<&PImpl::on_futures>(*this); + } + + bool is_busy() const { return mkts_req_pending || futs_req_pending; } + + void request() { + external_req_pending = false; + mkts_req_pending = true; + futs_req_pending = true; + session.request(markets_req(), &mkts_handler_proxy); + session.request(futures_req(), &futs_handler_proxy); + } + + /** + * @brief Parses securities from a ccapi::Message + * + * @details Only new securities that satisfy the 'SecurityCriteria' are added + * to \p added_secs + * + * @tparam TParser + * @param msg The ccapi::Message to parse + * @param types The types of securities to parse + * @param parser Ther parser to use + */ + template + void parse_secs( + const ccapi::Message &msg, + const vector &types, + vector &added_secs) { + T::parse(msg, added_secs); + // Exclude secs that don't satisfy the SecurityCriteria + auto is_sec_excluded = [this, &types](const Security &sec) { + return !CONTAINS(types, sec.type()) || + !criteria.includes_or_undefined(sec) || + securities.count(sec.desc()) > 0; + }; + REMOVE_IF(added_secs, is_sec_excluded); + // Assign security ids + for (auto &sec : added_secs) { + // Since the ids haven't been set yet, we should expect them to be 0 + bts_assert(sec.id().value() == 0); + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())}; + sec.security_common.id = sec_id; + } + } + + /** + * @brief Parses securities from ccapi messages, updates the securities set, + * and subscribes depth for newly arrived securities + * + * @details Securities that don't pass the criteria filter are ignored + * + * @tparam TParser The type of parser + * @param msgs Vector of ccapi messages containing securities + * @param types The types of securities to update (types must be explicitly + * specified to be included) + */ + template + void update_securities( + const vector &msgs, + const vector &types) { + vector added_secs; + for (const auto *msg : msgs) { + bts_assert_msgf( + msg->getType() == ccapi::Message::Type::GENERIC_PUBLIC_REQUEST, + "Received msg w/ unexpected type:\n{}", msg->toStringPretty()); + parse_secs(*msg, types, added_secs); + } + auto added_sec_descs = get_sec_descs(added_secs); + LOG(md, Info, "Received FTX securities:\n{}", + fmt::to_string(fmt::join(added_sec_descs, "\n"))); + callback(added_secs); + } + + /** + * Handles 'markets' update from CCApiSession + */ + bool on_markets( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + validate_response(event, mkts_req_pending); + LOG(md, Diag, "Received FTX markets:\n{}", event.toStringPretty()); + update_securities(msgs, {SecurityType::SPOT}); + mkts_req_pending = false; + if (external_req_pending && !is_busy()) { + request(); + } + return true; + } + + /** + * Handles 'futures' update from CCApiSession + */ + bool on_futures( + const ccapi::Event &event, + ccapi::Session * /*session*/, + const vector &msgs, + const CCApiCorrelationID & /*corr_id*/) { + validate_response(event, futs_req_pending); + LOG(md, Diag, "Received FTX futures:\n{}", event.toStringPretty()); + update_securities(msgs, {SecurityType::FUTURE}); + futs_req_pending = false; + if (external_req_pending && !is_busy()) { + request(); + } + return true; + } +}; + +FTXSecurityRequestor::FTXSecurityRequestor( + ccapi::EventDispatcher *dispatcher, + domain::SecurityCriteria criteria, + persistence::IDGenerator *sec_id_store) + : me_(std::make_unique( + dispatcher, std::move(criteria), sec_id_store)) {} + +FTXSecurityRequestor::FTXSecurityRequestor(FTXSecurityRequestor &&) noexcept = + default; +FTXSecurityRequestor &FTXSecurityRequestor::operator=( + FTXSecurityRequestor &&) noexcept = default; + +FTXSecurityRequestor::~FTXSecurityRequestor() = default; + +void FTXSecurityRequestor::request() { + if (is_busy()) { + me_->external_req_pending = true; + return; + } + me_->request(); +} + +bool FTXSecurityRequestor::is_busy() const { return me_->is_busy(); } + +FTXSecurityRequestor::Callback &FTXSecurityRequestor::callback() { + return me_->callback; +} + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSecurityRequestor.h b/src/lib/md/ftx/FTXSecurityRequestor.h new file mode 100644 index 000000000..a9312f5c4 --- /dev/null +++ b/src/lib/md/ftx/FTXSecurityRequestor.h @@ -0,0 +1,79 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/domain/SecurityID.h" +#include "lib/utils/Delegate.h" +#include +#include +#include +#include + +namespace ccapi { +class EventDispatcher; +} + +namespace bts::api::md { +class FTXMarketDataConfig; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::domain { +class Security; +class SecurityCriteria; +} // namespace bts::domain + +namespace bts::md::ftx { + +/** + * @brief Requests securities from FTX + * + * @details Uses ccapi infrastructure to send the request + */ +class FTXSecurityRequestor { + public: + FTXSecurityRequestor( + ccapi::EventDispatcher *dispatcher, + domain::SecurityCriteria criteria, + persistence::IDGenerator *sec_id_store); + + FTXSecurityRequestor(const FTXSecurityRequestor &) noexcept = delete; + FTXSecurityRequestor(FTXSecurityRequestor &&) noexcept; + FTXSecurityRequestor &operator=(const FTXSecurityRequestor &) noexcept = + delete; + FTXSecurityRequestor &operator=(FTXSecurityRequestor &&) noexcept; + + ~FTXSecurityRequestor(); + + using Callback = core::Delegate)>; + + Callback &callback(); + + /** + * @brief Request securities from the exchange + * + * @details Does NOT generate a live-updating stream of securities. Callers + * must call 'request' periodically to keep their securities in sync with the + * active set at the exchange. + */ + void request(); + + /** + * @brief Indicates whether or not a request is in-progress + * + * @details Concurrent requests are not supported + */ + bool is_busy() const; + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSecurityType.h b/src/lib/md/ftx/FTXSecurityType.h new file mode 100644 index 000000000..a93da0de1 --- /dev/null +++ b/src/lib/md/ftx/FTXSecurityType.h @@ -0,0 +1,65 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include "lib/api/domain/SecurityType.pb.h" +#include "lib/core/EnumMacros.h" +#include +#include + +namespace bts::md::ftx { + +// clang-format off +#define FOR_ALL_VALUES(per_value) \ + per_value(FTXSecurityType, NONE) \ + per_value(FTXSecurityType, SPOT) \ + per_value(FTXSecurityType, FUTURE) \ + per_value(FTXSecurityType, PERPETUAL) \ + per_value(FTXSecurityType, PREDICTION) \ + per_value(FTXSecurityType, MOVE) +// clang-format on + +/** + * @brief Represents the different security types at FTX + * + * @details The names come directly from FTX's 'markets' and 'futures' response + * payload (json). For more details see the docs: + * https://docs.ftx.com, + * https://docs.ftx.com/#markets, + * https://docs.ftx.com/#futures + */ +class FTXSecurityType { + public: + enum Value { FOR_ALL_VALUES(ENUM_DECLARE_VALUE) }; + + ENUM_BASIC_MEMBERS(FTXSecurityType) + + constexpr api::domain::SecurityType to_security_type() const; +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +ENUM_OUTPUT_OPS(FTXSecurityType) +ENUM_NAME_DEF(FTXSecurityType, FOR_ALL_VALUES) +ENUM_PARSE_DEF(FTXSecurityType, FOR_ALL_VALUES) +ENUM_FROM_JSON(FTXSecurityType) +ENUM_EQUALITY_OPS(FTXSecurityType) + +constexpr api::domain::SecurityType FTXSecurityType::to_security_type() const { + switch (value) { + case SPOT: + return api::domain::SecurityType::SPOT; + case FUTURE: + case PERPETUAL: + case PREDICTION: + case MOVE: + return api::domain::SecurityType::FUTURE; + case NONE: + default: + return api::domain::SecurityType::UNSET; + } +} + +#undef FOR_ALL_VALUES +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSubscription.cpp b/src/lib/md/ftx/FTXSubscription.cpp new file mode 100644 index 000000000..31f55fef1 --- /dev/null +++ b/src/lib/md/ftx/FTXSubscription.cpp @@ -0,0 +1,243 @@ +#ifdef SPARK_ENABLE_CCAPI +#include "FTXSubscription.h" +#include "lib/api/domain/Venue.pb.h" +#include "lib/api/md/FTXMarketDataConfig.pb.h" +#include "lib/collections/Contains.h" +#include "lib/core/ProtoPrinter.h" +#include "lib/domain/Security.h" +#include "lib/domain/SecurityCriteria.h" +#include "lib/domain/SecurityMarketDataStatistics.h" +#include "lib/logging/logging.h" +#include "lib/md/CCApiCorrelationID.h" +#include "lib/md/CCApiMarketDepthAdaptor.h" +#include "lib/md/CCApiMessageHandler.h" +#include "lib/md/CCApiMessageLogger.h" +#include "lib/md/CCApiSession.h" +#include "lib/md/CCApiTradeAdaptor.h" +#include "lib/md/ICCApiMessageHandler.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/md/MarketDataListenerData.h" +#include "lib/md/ftx/FTXHistoricalPriceAdaptor.h" +#include "lib/md/ftx/FTXHistoricalPriceRecord.h" +#include "lib/md/ftx/FTXMessageType.h" +#include "lib/md/ftx/FTXSecurityRequestor.h" +#include "lib/persistence/IDGenerator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bts::md::ftx { + +using bts::api::domain::Venue; +using bts::api::domain::Venue_Name; +using bts::api::md::FTXMarketDataConfig; +using bts::core::ProtoPrinterFactory; +using bts::domain::Security; +using bts::domain::SecurityCriteria; +using bts::domain::SecurityID; +using bts::persistence::IDGenerator; +using std::string; +using std::unique_ptr; +using std::unordered_map; +using std::vector; + +#define VENUE_NAME (Venue_Name(Venue::FTX)) + +#define FTX_MSG_TYPE_NAME(x) \ + std::string { FTXMessageType{x}.name() } + +static ccapi::Request create_stats_req(const string &sec_desc) { + const int SECS_PER_DAY = 86400; + auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTX_MSG_TYPE_NAME(FTXMessageType::STATISTICS), sec_desc); + ccapi::Request req{ + ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, + CCAPI_EXCHANGE_NAME_FTX}; + req.appendParam( + {{CCAPI_HTTP_PROTOCOL, "https"}, + {CCAPI_HTTP_HOST, "ftx.com"}, + {CCAPI_HTTP_PORT, "443"}, + {CCAPI_HTTP_PATH, fmt::format("/api/markets/{}/candles", sec_desc)}, + {CCAPI_HTTP_QUERY_STRING, fmt::format("resolution={}", SECS_PER_DAY)}, + {CCAPI_HTTP_METHOD, "GET"}}); + req.setCorrelationId(corr_id.to_string()); + return req; +} + +static ccapi::Subscription create_depth_sub( + const string &sec_desc, const string &opts) { + auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTXMessageType::DEPTH, sec_desc); + return ccapi::Subscription{ + CCAPI_EXCHANGE_NAME_FTX, sec_desc, CCAPI_MARKET_DEPTH, opts, + corr_id.to_string()}; +} + +static ccapi::Subscription create_ticker_sub( + const string &sec_desc, const string &opts) { + const auto corr_id = CCApiCorrelationID::from_parts( + VENUE_NAME, FTXMessageType::TICKER, sec_desc); + return ccapi::Subscription{ + CCAPI_EXCHANGE_NAME_FTX, sec_desc, CCAPI_TRADE, opts, + corr_id.to_string()}; +} + +struct FTXSubscription::PImpl { + // General Notes: + // FTX securities must be obtained from 2 different urls: + // 1) https://ftx.com/api/markets + // 2) https://ftx.com/api/futures + // Although all securities are listed in the response from (1), not all + // information is provided for futures (notably, the expiration date is + // missing!). This missing information is obtained from (2). Therefore, + // futures are published when (2) is received, while all other securities + // are published when (1) is received. + + unordered_map> securities; + IDGenerator *sec_id_store; + // Don't actually need this, but keeping it around for diagnostic purposes + // (i.e., inspection in debugger, temp logs, etc.) + FTXMarketDataConfig cfg; + SecurityCriteria criteria; + FTXSecurityRequestor sec_requestor; + // The session automatically starts/stops in its ctr/dtr + CCApiSession session; + IMarketDataListener *listener; + + // These aliases are useful for quickly switching the handler types (e.g., + // a special handler that logs messages for testing/diagnostic purposes) + using DepthHandler = CCApiMarketDepthAdaptor; + using TickerHandler = CCApiTradeAdaptor; + using StatsHandler = FTXHistoricalPriceAdaptor; + + unordered_map depth_handlers; + unordered_map ticker_handlers; + unordered_map stats_handlers; + + explicit PImpl( + ccapi::EventDispatcher *dispatcher, + FTXMarketDataConfig cfg, + persistence::IDGenerator *sec_id_store, + IMarketDataListener *listener) + : sec_id_store(sec_id_store), cfg(std::move(cfg)), + criteria(domain::SecurityCriteria::from_proto( + this->cfg.security_criteria())), + sec_requestor(dispatcher, criteria, sec_id_store), session(dispatcher), + listener(listener) { + LOG(md, Info, "Initializing FTXSubscription:\n{}", + ProtoPrinterFactory::create(&this->cfg)); + sec_requestor.callback().bind<&PImpl::on_securities>(*this); + sec_requestor.request(); + } + + DepthHandler &create_depth_handler(string sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + depth_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert(is_new); + return it->second; + } + + TickerHandler &create_ticker_handler(string sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + ticker_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert_msgf( + is_new, "Ticker handler already created for '{}'", sec_desc); + return it->second; + } + + StatsHandler &create_stats_handler(string sec_desc) { + SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)}; + auto [it, is_new] = + stats_handlers.try_emplace(sec_desc, sec_id, listener); + bts_assert_msgf( + is_new, "Statistics handler already created for '{}'", sec_desc); + return it->second; + } + + /** + * @brief Subscribes to depth, ticker, and price data for a list of + * securities + */ + template + void subscribe_market_data(SecDescs securities) { + const string depth_opts = + fmt::format(CCAPI_MARKET_DEPTH_MAX "={}", cfg.num_book_levels()); + const string ticker_opts = ""; + + // Batch subscribe/request ticker, depth, and stats + CCApiSession::BatchSubscription depth_subs; + CCApiSession::BatchSubscription ticker_subs; + CCApiSession::BatchRequest stats_reqs; + + for (const auto &sec_desc : securities) { + depth_subs.push_back( + {create_depth_sub(sec_desc, depth_opts), + &create_depth_handler(sec_desc)}); + ticker_subs.push_back( + {create_ticker_sub(sec_desc, ticker_opts), + &create_ticker_handler(sec_desc)}); + stats_reqs.push_back( + {create_stats_req(sec_desc), &create_stats_handler(sec_desc)}); + } + + session.subscribe( + depth_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_MARKET_DEPTH}); + session.subscribe( + ticker_subs, {ccapi::Message::Type::MARKET_DATA_EVENTS_TRADE}); + session.request( + stats_reqs, {ccapi::Message::Type::GENERIC_PUBLIC_REQUEST}); + } + + /** + * Updates the securities map and subscribes market data for newly arrived + * securities + */ + void on_securities(vector secs) { + // 'sec_descs' is used to make depth subscriptions + vector sec_descs; + sec_descs.reserve(secs.size()); + // 'sec_ptrs' is passed to IMarketDataListeners + vector sec_ptrs; + sec_ptrs.reserve(secs.size()); + + for (auto &s : secs) { + auto id = s.id(); + auto desc = s.desc(); + auto [it, is_new] = securities.try_emplace( + id, std::make_unique(std::move(s))); + bts_assert_msgf(is_new, "Received duplicate FTX security '{}'", desc); + sec_descs.push_back(std::move(desc)); + // add ptr to entry in 'securities' (not the loop var, we moved that) + sec_ptrs.push_back(it->second.get()); + } + + listener->on_securities(sec_ptrs); + subscribe_market_data(sec_descs); + } +}; + +FTXSubscription::FTXSubscription( + ccapi::EventDispatcher *dispatcher, + FTXMarketDataConfig cfg, + IDGenerator *sec_id_store, + IMarketDataListener *listener) + : me_(std::make_unique( + dispatcher, std::move(cfg), sec_id_store, listener)) {} + +FTXSubscription::FTXSubscription(FTXSubscription &&) noexcept = default; +FTXSubscription &FTXSubscription::operator=(FTXSubscription &&) noexcept = + default; + +FTXSubscription::~FTXSubscription() = default; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/md/ftx/FTXSubscription.h b/src/lib/md/ftx/FTXSubscription.h new file mode 100644 index 000000000..b49aca0a0 --- /dev/null +++ b/src/lib/md/ftx/FTXSubscription.h @@ -0,0 +1,50 @@ +#pragma once +#ifdef SPARK_ENABLE_CCAPI +#include +#include + +namespace ccapi { +class EventDispatcher; +} + +namespace bts::api::md { +class FTXMarketDataConfig; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md::ftx { + +/** + * @brief Subscribes to FTX market data + * + * @details Uses ccapi infrastructure to request/subscribe data + */ +class FTXSubscription { + public: + FTXSubscription( + ccapi::EventDispatcher *dispatcher, + api::md::FTXMarketDataConfig cfg, + persistence::IDGenerator *sec_id_store, + IMarketDataListener *listener); + + FTXSubscription(const FTXSubscription &) noexcept = delete; + FTXSubscription(FTXSubscription &&) noexcept; + FTXSubscription &operator=(const FTXSubscription &) noexcept = delete; + FTXSubscription &operator=(FTXSubscription &&) noexcept; + + ~FTXSubscription(); + + private: + struct PImpl; + std::unique_ptr me_; +}; + +} // namespace bts::md::ftx +#endif diff --git a/src/lib/utils/Delegate.h b/src/lib/utils/Delegate.h index cae44c889..455ea11cd 100644 --- a/src/lib/utils/Delegate.h +++ b/src/lib/utils/Delegate.h @@ -36,9 +36,9 @@ class Delegate { static R invoke_static_handler(void *, Args... arg) { return handler(std::forward(arg)...); } - + static R null_target(Arg...) { - if constexpr (std::is_same::value) + if constexpr (std::is_same::value) return; else return {}; @@ -81,3 +81,10 @@ class Delegate { }; } // namespace bts::spartan + +namespace bts::core { + +template +using Delegate = bts::spartan::Delegate; + +} // namespace bts::core diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index f192a64e7..779b7c35a 100644 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -190,7 +190,13 @@ std::string iso_date_time_string(PTime const &t) { PTime parse_iso_date_time_tz(const std::string &date) { // example 1) 2022-10-27T09:30-5:00 // example 2) 2022-10-27T09:30+5:00 + +#ifdef _WIN32 + struct tm tt = {0, 0, 0, 0, 0, 0, 0, 0, 0}; +#else struct tm tt = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +#endif + int tz_hours; int tz_minutes; int num_scanned = sscanf( @@ -213,7 +219,8 @@ PTime parse_iso_date_time_string(std::string const &t) { constexpr const std::size_t time_idx = sizeof("YYYY-MM-DD") - 1; auto actual_time_idx = t.find('T', time_idx); if (actual_time_idx != std::string::npos) { - constexpr const std::size_t time_zone_idx = time_idx + sizeof("HH:MM") - 1; + constexpr const std::size_t time_zone_idx = + time_idx + sizeof("HH:MM") - 1; // time zone must be after time bts_assert(actual_time_idx < time_zone_idx); if (t.find('+', time_zone_idx) != std::string::npos || diff --git a/src/server/sparkd/cmds/md/cmds/Run.cpp b/src/server/sparkd/cmds/md/cmds/Run.cpp index 1564531fb..29711f4e4 100644 --- a/src/server/sparkd/cmds/md/cmds/Run.cpp +++ b/src/server/sparkd/cmds/md/cmds/Run.cpp @@ -115,7 +115,7 @@ void Run::run( LOG(md, Info, "Run::run cleanup:{} run_on_md_thread:{} t_id:{} t_name:{}", cleanup, run_on_md_thread, os::get_thread_id(), get_thread_name()); if (type == Run::MDType::Configured && - !config::get().use_md()) { + !config::get().use_md()) { LOG(md, Info, "Ignoring request to run md: 'use_md' is configured false"); return; } @@ -126,7 +126,7 @@ void Run::run( if (cleanup) md::cmds::Cleanup::execute(); if (type == Run::MDType::Configured) { - const auto &conf = config::get(); + const auto &conf = config::get(); if (conf.use_replay()) type = Run::MDType::Replay; else if (conf.use_onixs()) @@ -155,7 +155,7 @@ void Run::run( if (is_run_emumba) { threading::make_thread("Emumba", []() { run_emumba(); }).detach(); } - if (bts::config::get().use_sim()) { + if (bts::config::get().use_sim()) { threading::make_thread("ExchangeSimGLBX", [&mbo_writer]() { sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo_writer); }).detach();