diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index 07c3e2ca6..033ec0ed5 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OrderService2.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OrderService2.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp) \ No newline at end of file diff --git a/src/lib/api/service_compiler/templates/cpp/server.pb.h b/src/lib/api/service_compiler/templates/cpp/server.pb.h index e582225e0..7fb2c6e67 100644 --- a/src/lib/api/service_compiler/templates/cpp/server.pb.h +++ b/src/lib/api/service_compiler/templates/cpp/server.pb.h @@ -1,60 +1,58 @@ -#pragma once -#include "{{filename name}}.pb.h" -#include "lib/api/public/ApiServer.h" -#include "lib/utils/bts_export.h" -#include -#include -#include -#include -#include - -// #include -// #include -namespace bts { namespace api { - class ServiceContext; -}} - -{{ns-open package}} - -// Forward declaractions -{{#each message_type}} -class {{name}};{{/each}} - -{{#each service}} -// {{comment}} -class {{name}}Base : public bts::api::ApiServer { -protected: - {{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier=""); - - const char* name() const override; - - {{#each method}} - {{#if server_streaming}} - typedef std::function*, const std::vector<{{output_type_cpp}}>*, std::set<{{key_type.cpptype}}>*)> {{cap name}}Publisher; - {{cap name}}Publisher make_{{name}}_publisher() const; - {{cap name}}Publisher make_{{name}}_publisher(const std::string &) const; - {{cap name}}Publisher make_{{name}}_publisher(const {{input_type}}& qualifier) const; - {{else}} - // {{comment}} - virtual void {{name}}( - const {{input_type}}& request, - const api::Response<{{output_type_cpp}}>& reply) = 0; - {{/if}} - {{/each}} - - enum ServiceMethods { - SYS_PING = -1, - {{#each method}} - {{upper name}} = {{@index}}, - {{/each}} - }; - -private: - void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override; - {{#each method}}{{#if server_streaming}} - {{cap name}}Publisher make_{{name}}_publisher(TableHandle) const; - {{/if}}{{/each}} -}; - -{{/each}} -{{ns-close package}} +#pragma once +#include "{{filename name}}.pb.h" +#include "lib/api/public/ApiServer.h" +#include "lib/utils/bts_export.h" +#include +#include +#include +#include +#include + +namespace bts { namespace api { + class ServiceContext; +}} + +{{ns-open package}} + +// Forward declaractions +{{#each message_type}} +class {{name}};{{/each}} + +{{#each service}} +// {{comment}} +class {{name}}Base : public bts::api::ApiServer { +protected: + {{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier=""); + + const char* name() const override; + + {{#each method}} + {{#if server_streaming}} + typedef std::function*, const std::vector<{{output_type_cpp}}>*, std::set<{{key_type.cpptype}}>*)> {{cap name}}Publisher; + {{cap name}}Publisher make_{{name}}_publisher() const; + {{cap name}}Publisher make_{{name}}_publisher(const std::string &) const; + {{cap name}}Publisher make_{{name}}_publisher(const {{input_type}}& qualifier) const; + {{else}} + // {{comment}} + virtual void {{name}}( + const {{input_type}}& request, + const api::Response<{{output_type_cpp}}>& reply) = 0; + {{/if}} + {{/each}} + + enum ServiceMethods { + SYS_PING = -1, + {{#each method}} + {{upper name}} = {{@index}}, + {{/each}} + }; + +private: + void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override; + {{#each method}}{{#if server_streaming}} + {{cap name}}Publisher make_{{name}}_publisher(TableHandle) const; + {{/if}}{{/each}} +}; + +{{/each}} +{{ns-close package}} diff --git a/src/lib/api/services/SyntheticSecurityService.proto b/src/lib/api/services/SyntheticSecurityService.proto new file mode 100644 index 000000000..89d096bc2 --- /dev/null +++ b/src/lib/api/services/SyntheticSecurityService.proto @@ -0,0 +1,53 @@ +syntax="proto3"; +package bts.api.synthetic; +option csharp_namespace="bts.api.synthetic"; + +import "lib/api/apioptions.proto"; +import "lib/domain_model/Security.proto"; +//import "google/protobuf/timestamp.proto"; +import "lib/sparkproto/SparkProtos.proto"; + +message CreateResult { + enum Error { + ERROR_OK = 0; + ERROR_NOT_SYNTHETIC = 1; + ERROR_ID_ALREADY_ASSIGNED = 2; + } + uint32 key = 1; + Error error = 2; +} +message UpdateResult { + enum Error { + ERROR_OK = 0; + ERROR_MISSING_KEY = 1; + ERROR_UNKNOWN_SECURITY = 2; + ERROR_DATABASE = 3; + } + Error error = 1; +} + +message Metadata { + uint32 security_key = 1; + spark.ExecutionStrategy strategy = 2; +} +message MetadataQuery { + repeated uint32 security_key = 1; +} +message MetadataCollection { + repeated Metadata data = 1; +} + +service SyntheticSecurityService { + option(bts.api.service_description)="Create or query synthetic securities."; + + // Create synthetic + rpc create_synthetic(api.Security) returns (CreateResult); + // Subscribe synthetics -- code only + + // set/update metadata // spark.ExecutionStrategy + rpc update_security_metadata(Metadata) returns (UpdateResult); + // query metadata + rpc query_security_metadata(MetadataQuery) returns (MetadataCollection) + { option (bts.api.method_description)="Returns the metadata for the given set of securities if it exists. If no data for a particular key exists it will not be present in the output collection."; } + // subscribe metadata -- code only +} diff --git a/src/lib/domain_model/SecurityKey.h b/src/lib/domain_model/SecurityKey.h new file mode 100644 index 000000000..d9e3021b2 --- /dev/null +++ b/src/lib/domain_model/SecurityKey.h @@ -0,0 +1,5 @@ +#pragma once +#include "lib/utils/ValueWrapper.h" +namespace bts { +using SecurityKey = utils::ValueWrapper; +} \ No newline at end of file diff --git a/src/lib/domain_model/common.h b/src/lib/domain_model/common.h index 2f474c0c1..af6f2a590 100644 --- a/src/lib/domain_model/common.h +++ b/src/lib/domain_model/common.h @@ -1,17 +1,14 @@ -#pragma once - -#include -#include - -#include "lib/utils/ValueWrapper.h" - -namespace bts { -using SecurityKey = utils::ValueWrapper; - - - -using Price = utils::ValueWrapper; -using Quantity = utils::ValueWrapper; -using UserLogin = utils::ValueWrapper; -} - +#pragma once + +#include +#include + +#include "lib/utils/ValueWrapper.h" +#include "lib/domain_model/SecurityKey.h" + +namespace bts { +using Price = utils::ValueWrapper; +using Quantity = utils::ValueWrapper; +using UserLogin = utils::ValueWrapper; +} + diff --git a/src/lib/md/ISyntheticSecurityService.h b/src/lib/md/ISyntheticSecurityService.h new file mode 100644 index 000000000..af6e63a53 --- /dev/null +++ b/src/lib/md/ISyntheticSecurityService.h @@ -0,0 +1,19 @@ +#pragma once + +#include "lib/api/services/SyntheticSecurityService.pb.h" +#include "lib/domain_model/SecurityKey.h" +#include "lib/utils/Event.h" +#include + +namespace bts { +namespace api { +namespace synthetic { + +struct ISyntheticSecurityService { + virtual const std::unordered_map securities() = 0; + virtual utils::EventSubscriber& on_securities() = 0; + virtual const Metadata* get_metadata(SecurityKey) = 0; + virtual utils::EventSubscriber& on_metadata() = 0; +}; + +}}} \ No newline at end of file diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 8a659d931..11742b376 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -4,6 +4,7 @@ #include "lib/utils/Sqlite_utils.h" #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" +#include "lib/md/Shm/FixedSecurity.h" #include #include @@ -14,14 +15,7 @@ using namespace bts::utils; namespace bts { namespace md { - -// schema: -// id, security (binary), expiration time - -struct SecurityDB { - // Store the symbols for debugging purposes - boost::container::flat_set known_keys; -}; +namespace db { /* * Schema: @@ -44,16 +38,19 @@ sqlite3_ptr open_db() { sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(db, - "CREATE TABLE IF NOT EXIST securities (key INTEGER NOT NULL, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL"); + "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); + + sqlite3_exec_nofail(db, + "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); return dbptr; } -std::unique_ptr load_security_db( +size_t load_securities( const std::set& symbols, const std::function& callback) { - auto security_db = std::make_unique(); + size_t count = 0; auto dbptr = open_db(); // Load all existing securities and call the callback @@ -64,7 +61,7 @@ std::unique_ptr load_security_db( auto result = sqlite3_step(select); Security security; while (result == SQLITE_ROW) { - uint32_t security_key = sqlite3_column_int(select, 0); + security_key security_key = sqlite3_column_int(select, 0); auto symbol = std::string( (char*)sqlite3_column_text(select, 1), sqlite3_column_bytes(select, 1)); @@ -76,50 +73,97 @@ std::unique_ptr load_security_db( auto len = sqlite3_column_bytes(select, 4); if (security.ParseFromArray(data, len)) { bts_assert(security_key == security.id()); - security_db->known_keys.insert(SecurityKey(security.id())); + count++; callback(security); } + else { + LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); + } } result = sqlite3_step(select); } - return security_db; + return count; } -bool store_if_new(SecurityDB& db, const api::Security& security) { - auto key = SecurityKey(security.id()); - if (db.known_keys.find(key) == db.known_keys.end()) { - db.known_keys.insert(key); - auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?)"); - auto buffer = security.SerializeAsString(); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, security.id()), dbptr); - SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, security.symbol().c_str(), security.symbol().size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, security.exchange()), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(security)), dbptr); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, datetime::today().to_short()), dbptr); - auto result = sqlite3_step(insert.get()); - if (result != SQLITE_DONE) { - LOG(md, Warn, "Failed to insert security into security db. key={}", security.id()); +void store_if_new(const std::vector& securities) { + auto dbptr = open_db(); + auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION"); + std::string buffer; + for (const api::Security* psecurity : securities) { + if (psecurity && psecurity->SerializeToString(&buffer)) { + SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, psecurity->id()), dbptr); + SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, psecurity->symbol().c_str(), psecurity->symbol().size(), SQLITE_STATIC), dbptr); + SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, psecurity->exchange()), dbptr); + SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr); + SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); + SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, datetime::today().to_short()), dbptr); + auto result = sqlite3_step(insert.get()); + if (result != SQLITE_DONE) { + LOG(md, Warn, "Failed to insert security into security db. key={}", psecurity->id()); + } } - return true; } - return false; + sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION"); +} + + +std::vector load_metadata() { + std::vector values; + auto dbptr = open_db(); + auto db = dbptr.get(); + auto select_stmt = prepare_query(db, "select key,data from metadata"); + auto select = select_stmt.get(); + auto result = sqlite3_step(select); + while(result == SQLITE_ROW) { + uint32_t key = sqlite3_column_int(select, 0); + auto blob = sqlite3_column_blob(select, 1); + auto len = sqlite3_column_bytes(select, 1); + api::synthetic::Metadata data; + if (data.ParseFromArray(blob, len)) { + bts_assert(key == data.security_key()); + values.push_back(data); + } + else { + LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); + } + result = sqlite3_step(select); + } + return values; +} +bool store_metadata(const api::synthetic::Metadata& data) { + auto dbptr = open_db(); + auto insert = utils::prepare_query(dbptr, "insert into metadata (key, data) values (?,?)"); + + SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); + std::string buffer = data.SerializeAsString(); + SQLITE_CHK(sqlite3_bind_blob(insert.get(), 2, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); + auto result = sqlite3_step(insert.get()); + if (result != SQLITE_DONE) { + LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); + return false; + } + return true; } #else namespace bts { namespace md { -struct SecurityDB {}; -std::unique_ptr load_security_db( - const std::set&, - const std::function&) { - return std::make_unique(); -} -bool store_if_new(SecurityDB&, const api::Security&) { - return false; +namespace db { + +size_t load_securities( + const std::set& symbols, + const std::function& callback) { + return 0; } +void store_if_new(const std::vector& handles) {} +// std::vector load_metadata() { +// std::vector result; +// return result; +// } +// void store_metadata(const api::synthetic::Metadata&) { +// } #endif @@ -152,4 +196,4 @@ datetime::Date expiration(const api::Security& security) { } } -}}//namespace \ No newline at end of file +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 19099bfd5..3e7b9e2ec 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -3,26 +3,34 @@ #include "lib/domain_model/common.h" #include "lib/domain_model/Security.pb.h" #include "lib/utils/datetime.h" +#include "lib/md/MDProducerHandle.h" +#include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" #include "UserSymbol.h" #include +#include #include -#include namespace bts { namespace api {class Security;} namespace md { +namespace db { struct SecurityDB; -std::unique_ptr load_security_db( +// Loads securities matching the given filter for producers to pre-populate security lists +// note: possible to use this to optimize initial memory layout? sort by expiration? +size_t load_securities( const std::set& symbols, const std::function& callback); -bool store_if_new(SecurityDB& db, const api::Security& security); +void store_if_new(const std::vector& handles); + +// metadata +std::vector load_metadata(); +bool store_metadata(const api::synthetic::Metadata&); datetime::Date expiration(const api::Security& security); - -}}//namespace \ No newline at end of file +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 69567de2c..b67da9774 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -6,6 +6,7 @@ #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" +#include "lib/md/SecurityDB.h" #include #include // for reservation @@ -17,18 +18,16 @@ using namespace std; using namespace bts::utils; +using std::size_t; namespace bts { namespace md { -//GCC_DIAG_OFF("-Weffc++") // Plain Struct -struct SecurityIDTask { - std::string descriptor; - SecurityType type; - SecurityIDRecorder::callback cb; - void* arg; +struct SecurityIDEntry { + api::Security security; + SecurityIDRecorder::Callback callback; }; -//GCC_DIAG_RESTORE() + struct SecurityIDRecorderState { SecurityIDRecorderState() : work(), @@ -37,7 +36,7 @@ struct SecurityIDRecorderState { select_stmt(nullptr, sqlite3_finalize), insert_stmt(nullptr, sqlite3_finalize) {} - tbb::concurrent_queue work; + tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; sqlite3_stmt_ptr select_stmt; @@ -50,9 +49,7 @@ SecurityIDRecorder::SecurityIDRecorder(): init(); } -SecurityIDRecorder::~SecurityIDRecorder() -{ -} +SecurityIDRecorder::~SecurityIDRecorder() { } void SecurityIDRecorder::init() { string db_dir = config::get().data_path() + "/md/"; @@ -75,100 +72,89 @@ void SecurityIDRecorder::init() { _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); } -void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, callback cb, void* arg) { - SecurityIDTask task; - task.descriptor = descriptor; - task.type = type; - task.cb = cb; - task.arg = arg; - _self->work.push(task); -} - -void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, Callback cb) { - Callback* pcb = new Callback(cb); - get_key(descriptor, type, [](void* data, security_key key) { - Callback* pcb = static_cast(data); - DEFER(if (pcb) delete pcb); // exception safe - (*pcb)(key); - }, pcb); -} - void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } +void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { + if (security.id() != invalid_security_key) { + auto work = std::make_shared(); + work->security = security; + work->callback = callback; + _self->work.push(work); + } +} -bool SecurityIDRecorder::do_work() { - // Check if any handles need work first - work_pending_handles(); - +void SecurityIDRecorder::do_work() { + work_handles(); + work_db(); +} + +void SecurityIDRecorder::work_handles() { + Security security; + MDProducerHandle handle; + while (_self->pending_handle.try_pop(handle)) { + security.Clear(); + convert_security(handle.security(), &security); + assign_key(security, [handle](const api::Security& security) mutable { + handle.set_key(security.id()); + }); + } +} + +void SecurityIDRecorder::work_db() { // Drain the work queue - // Query the database for each security. If it has an ID return it + // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list // Then persist that list - SecurityIDTask temp; - vector new_securities; + // Once a security has a key it is public so record it in the security database + std::shared_ptr temp; + vector> new_entries; + vector new_descriptors; size_t count = 0; - bool incomplete = false; + string descriptor; while (_self->work.try_pop(temp)) { - security_key key = search(temp.descriptor); - if (key != invalid_security_key) { - temp.cb(temp.arg, key); - } - else { - new_securities.push_back(temp); - ++count; - // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat - if (count > 500) { - incomplete = true; - break; + descriptor = security_descriptor(temp->security); + security_key key = search(descriptor); + if (key != invalid_security_key) { + temp->security.set_id(key); + temp->callback(temp->security); + } + else { + // always push in parallel + new_entries.push_back(move(temp)); + new_descriptors.push_back(descriptor); + ++count; + // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat + if (count > 500) { + break; + } } - } } - if (new_securities.size() > 0) { + if (new_entries.size() > 0) { sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); - for (SecurityIDTask& security : new_securities) { - security_key key = insert(security.descriptor, security.type); + for (size_t x = 0; x < new_entries.size(); ++x) { + Security& security = new_entries[x]->security; + security_key key = insert(new_descriptors[x], securitytype_of(security)); bts_assert_d(key); - security.cb(security.arg, key); + security.set_id(key); + new_entries[x]->callback(security); } sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); - } - return incomplete; -} -void SecurityIDRecorder::work_pending_handles() { - MDProducerHandle handle; - vector defer; - while(_self->pending_handle.try_pop(handle)) { - // A spread will not have a descriptor until all of its leg keys are available - string descriptor = security_descriptor(handle.security()); - if (descriptor.empty()) { - defer.push_back(handle); - continue; + // record the securities + std::vector save_these; + save_these.reserve(new_entries.size()); + for (const auto& e : new_entries) { + save_these.push_back(&e->security); } - - MDProducerHandle* phandle = new MDProducerHandle(handle); - DEFER(if (phandle) delete phandle); - get_key(descriptor, handle.security().type, [](void* arg, security_key key) { - auto* phandle = static_cast(arg); - if (phandle) { - phandle->set_key(key); - delete phandle; - } - }, phandle); - phandle = nullptr; - } - - for (auto& handle : defer) { - _self->pending_handle.push(handle); + db::store_if_new(save_these); } } - security_key SecurityIDRecorder::search(const std::string& descriptor) { sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); DEFER(sqlite3_reset(_self->select_stmt.get())); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 649770fee..67b7b5c29 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -2,7 +2,6 @@ #ifndef WIN32 #include "common.h" #include "MDProducerHandle.h" -#include "lib/utils/bts_export.h" #include #include @@ -19,26 +18,21 @@ namespace md { */ struct SecurityIDRecorderState; -class BTS_EXPORT SecurityIDRecorder +class SecurityIDRecorder { public: SecurityIDRecorder(); ~SecurityIDRecorder(); - typedef void(*callback)(void*, security_key); - typedef std::function Callback; + typedef std::function Callback; - // Asyncronously retrieve the security_key for the given descriptor - void get_key(const std::string& descriptor, SecurityType, callback cb, void* arg); - - // A less efficient but higher level get_key which performs heap allocation in the call path - void get_key(const std::string& descriptor, SecurityType, Callback cb); - - // A higher level interface which allows you to just pass in a handle and have it generate a descriptor and set the key + // Fast - for use by market data producers void ensure_key(MDProducerHandle handle); + // Slow - for use by services + void assign_key(const api::Security& security, const Callback& key_assigned); // Someone must call work periodically to do lookups and flush the reserved ids to disk. // returne true if there is more work to be done - bool do_work(); + void do_work(); private: std::unique_ptr _self; @@ -47,7 +41,8 @@ private: security_key search(const std::string& descriptor); security_key insert(const std::string& descriptor, SecurityType); - void work_pending_handles(); + void work_handles(); + void work_db(); BTS_NO_COPY(SecurityIDRecorder); }; diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index 037f09ef1..d609a5efc 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -76,7 +76,7 @@ void write_date_string(uint32_t year, uint32_t month, uint32_t day, std::ostring << year - 2000; } -const std::string security_descriptor(const FixedSecurity& security) { +std::string security_descriptor(const FixedSecurity& security) { const char separator = ' '; std::ostringstream result; // If the same symbol is listed at 2 different exchanges it should have a different ID @@ -118,6 +118,14 @@ const std::string security_descriptor(const FixedSecurity& security) { return result.str(); } +std::string security_descriptor(const api::Security& security) { + std::string result; + with_security(security.Contract_case(), num_legs(security), [&](FixedSecurity fs) { + convert_security(security, fs, true); + result = security_descriptor(fs); + }); + return result; +} void get_contract_spec(const FixedSecurity& source, api::ContractSpecification& spec) { spec.set_cash_per_point(source.cme_unit_of_measure_qty); @@ -218,6 +226,12 @@ SecurityType securitytype_of_contractcase(Security::ContractCase cc) { SecurityType securitytype_of(const Security& s) { return securitytype_of_contractcase(s.Contract_case()); } +size_t num_legs(const Security& sec) { + if (sec.Contract_case() == api::Security::ContractCase::kSpread) { + return sec.spread().legs_size(); + } + return 0; +} void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index be8fb7cc2..35d9588c1 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -128,7 +128,8 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS; typedef FixedSpreadSecurity<128> FixedSpreadSecurityL; typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL; -const std::string security_descriptor(const FixedSecurity&); +std::string security_descriptor(const api::Security&); +// std::string security_descriptor(const FixedSecurity&); // Convert a fixedsecurity to a normal Security - caveat: spreads will be normalized Security convert_security(const FixedSecurity&); @@ -138,6 +139,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit SecurityType securitytype_of_contractcase(Security::ContractCase cc); SecurityType securitytype_of(const Security&); +size_t num_legs(const Security&); // Stack allocates a security of the given type template diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp new file mode 100644 index 000000000..98e553fee --- /dev/null +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -0,0 +1,160 @@ +#ifndef WIN32 +#include "SyntheticSecurityService.h" + +#include "lib/config/config.h" +#include "lib/api/public/ServiceContext.h" +#include "lib/api/services/PersistenceService.pb.h" +#include "lib/md/common.h" +#include "lib/md/SecurityIDRecorder.h" +#include "lib/md/SecurityDB.h" +#include "lib/md/get_symbols.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/Event.h" +#include + +namespace bts { +namespace api { +namespace synthetic { + +struct SecurityLess { + bool operator()(const api::Security& left, const api::Security& right) { + return left.id() < right.id(); + } +}; + +struct SyntheticSecurityService::Pimpl { + Pimpl() : + recorder(), + timer(utils::Dispatcher::background(), 200, [&]() { recorder.do_work(); }), + synthetics(), on_security(), + meta(), on_metadata() + {} + md::SecurityIDRecorder recorder; + utils::DispatcherTimer timer; + std::unordered_map synthetics; + utils::Event on_security; + std::unordered_map meta; + utils::Event on_metadata; +}; + +SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context) + : SyntheticSecurityServiceBase(context), + _me(new Pimpl()) { + + // load securities + std::set symbols; + md::get_system_md_symbols(symbols); + md::db::load_securities(symbols, [=](const api::Security& security) { + if (security.Contract_case() == api::Security::kSynthetic) { + auto key = SecurityKey(security.id()); + _me->synthetics.insert(std::make_pair(key, security)); + } + }); + + // Load metadata + auto metadata = md::db::load_metadata(); + for (const auto& entry : metadata) { + _me->meta.insert(std::make_pair(entry.security_key(), entry)); + } + +} +SyntheticSecurityService::~SyntheticSecurityService() { } + + +const std::unordered_map SyntheticSecurityService::securities() { + return _me->synthetics; +} +utils::EventSubscriber& SyntheticSecurityService::on_securities() { + return _me->on_security; +} +const Metadata* SyntheticSecurityService::get_metadata(SecurityKey key) { + auto iter = _me->meta.find(key); + if (iter != _me->meta.end()) { + return &iter->second; + } + return nullptr; +} +utils::EventSubscriber& SyntheticSecurityService::on_metadata() { + return _me->on_metadata; +} + +void send_reply(const api::Response& fn, CreateResult::Error err, uint32_t key=md::invalid_security_key) { + CreateResult r; + r.set_key(key); + r.set_error(err); + fn(r); +} + +void SyntheticSecurityService::create_synthetic( + const Security& security, + const api::Response& reply) { + + CreateResult result; + // Validate security is synthetic + if (security.Contract_case() != Security::kSynthetic) { + send_reply(reply, CreateResult::ERROR_NOT_SYNTHETIC); + return; + } + if (security.id() != md::invalid_security_key) { + send_reply(reply, CreateResult::ERROR_ID_ALREADY_ASSIGNED); + return; + } + + // Assign security key + _me->recorder.assign_key(security, [=](const api::Security& security) { + // Notify listeners that a new security is added + _me->synthetics.insert(std::make_pair(security.id(), security)); + _me->on_security.publish(security); + // signal success + send_reply(reply, CreateResult::ERROR_OK, security.id()); + }); +} + +void SyntheticSecurityService::update_security_metadata( + const Metadata& request, + const api::Response& reply) { + UpdateResult result; + if (request.security_key() == md::invalid_security_key) { + result.set_error(UpdateResult::ERROR_MISSING_KEY); + reply(result); + return; + } + auto key = SecurityKey(request.security_key()); + if (_me->synthetics.find(key) == _me->synthetics.end()) { + result.set_error(UpdateResult::ERROR_UNKNOWN_SECURITY); + reply(result); + return; + } + // update the on disk database + if (!md::db::store_metadata(request)) { + result.set_error(UpdateResult::ERROR_DATABASE); + reply(result); + return; + } + // update the local database + _me->meta[key] = request; + // notify subscribers + _me->on_metadata.publish(request); + + reply(result); +} + +void SyntheticSecurityService::query_security_metadata( + const MetadataQuery& request, + const api::Response& reply) { + + MetadataCollection result; + for (const auto k : request.security_key()) { + SecurityKey key(k); + auto iter = _me->meta.find(key); + if (iter != _me->meta.end()) { + result.mutable_data()->Add()->CopyFrom(iter->second); + } + } + reply(result); +} + + +}}}//namespace + +#endif \ No newline at end of file diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h new file mode 100644 index 000000000..0f07c0080 --- /dev/null +++ b/src/lib/md/SyntheticSecurityService.h @@ -0,0 +1,49 @@ +#pragma once +#include "lib/md/common.h" +#include "lib/api/services/SyntheticSecurityService.pb.h" +#include "lib/api/services/SyntheticSecurityService.server.pb.h" +#include "lib/domain_model/SecurityKey.h" +#include "ISyntheticSecurityService.h" +#include +#include + + +namespace bts { +namespace api { +namespace synthetic { + + +class SyntheticSecurityService : + private api::synthetic::SyntheticSecurityServiceBase, + public ISyntheticSecurityService +{ + struct Pimpl; + std::unique_ptr _me; + + public: + SyntheticSecurityService(api::ServiceContext& context); + ~SyntheticSecurityService(); + + const std::unordered_map securities() override; + utils::EventSubscriber& on_securities() override; + const Metadata* get_metadata(SecurityKey) override; + utils::EventSubscriber& on_metadata() override; + + private: + void create_synthetic( + const Security& request, + const api::Response& reply) override; + + void update_security_metadata( + const Metadata& request, + const api::Response& reply) override; + + void query_security_metadata( + const MetadataQuery& request, + const api::Response& reply) override; + +}; + +}}}//namespace + + diff --git a/src/lib/md/common.cpp b/src/lib/md/common.cpp index 465eef209..4071f33ec 100644 --- a/src/lib/md/common.cpp +++ b/src/lib/md/common.cpp @@ -19,6 +19,7 @@ namespace bts { os << sym.symbol() << "-" << api::SecurityType_Name(sym.type()); return os; } + } namespace co2 { diff --git a/src/lib/md/common.h b/src/lib/md/common.h index 45ff264c3..cb3ebb646 100644 --- a/src/lib/md/common.h +++ b/src/lib/md/common.h @@ -3,6 +3,7 @@ #include "lib/utils/bts_export.h" #include "lib/utils/datetime.h" +#include "lib/domain_model/SecurityKey.h" #include "lib/md/protos/co2.pb.h" #include "lib/domain_model/Security.pb.h" diff --git a/src/lib/sparkproto/SparkProtos.proto b/src/lib/sparkproto/SparkProtos.proto index 80d19554e..ad36d121d 100644 --- a/src/lib/sparkproto/SparkProtos.proto +++ b/src/lib/sparkproto/SparkProtos.proto @@ -791,6 +791,7 @@ message CrossingHedgeOrder message ExecutionStrategy { + bool deleted = 1; map leg_params_map = 20; } diff --git a/src/lib/utils/Event.h b/src/lib/utils/Event.h new file mode 100644 index 000000000..03d2fe9c0 --- /dev/null +++ b/src/lib/utils/Event.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace bts { +namespace utils { + +template +struct EventSubscriber { + typedef int Handle; + typedef std::function Callback; + + EventSubscriber() : _id(0), _subscribers(), _lock() {} + + Handle subscribe(const Callback& cb) { + std::lock_guard l(_lock); + auto id = ++_id; + _subscribers.push_back(std::make_tuple(id, cb)); + return id; + } + void unsubscribe(Handle h) { + std::lock_guard l(_lock); + _subscribers.erase( + std::remove_if(_subscribers.begin(), + _subscribers.end(), + [h](std::tuple& t) { return std::get(t) == h; })); + } +protected: + void broadcast(const T& item) { + std::lock_guard l(_lock); + for (const auto& t : _subscribers) { + std::get(t)(item); + } + } +private: + Handle _id; + std::list> _subscribers; + std::mutex _lock; + +}; + +template +struct EventPublisher { + typedef int Handle; + virtual void publish(const T&) = 0; +}; + + +template +class Event : + public EventSubscriber, + public EventPublisher { + +public: + void publish(const T& item) { + this->broadcast(item); + } +}; + + +// template +// class ReplayEvent { +// typedef std::function Callback; +// typedef int Handle; +// Handle _id; +// std::vector _items; +// std::list> _subscribers; + +// void broadcast(const T& item) { +// for(const auto& t : _subscribers) { +// std::get(t)(item); +// } +// } + +// public: +// ReplayEvent() : +// _id(0), +// _items(), +// _subscribers() {} +// ~ReplayEvent() {} + +// void publish(T&& item) { +// _items.push_back(move(item)); +// broadcast(_items.back()); +// } +// void publish(const T& item) { +// _items.push_back(item); +// broadcast(item); +// } + +// Handle subscribe(Callback& cb) { +// auto id = ++_id; +// _subscribers.push_back(std::make_tuple(id, cb)); +// for (const auto& item : _items) { +// cb(item); +// } +// return id; +// } +// void unsubscribe(Handle h) { +// _subscribers.erase( +// std::remove_if(_subscribers.begin(), +// _subscribers.end(), +// [h](std::tuple& t) { return std::get(t) == h; })); +// } +// }; + + +}}//namespace \ No newline at end of file diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 89884d146..79040893d 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -15,6 +15,7 @@ #include "server/sparkd/md/run_md.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" #include "server/sparkd/orderman2/server/OrderServiceServer2.h" +#include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" @@ -37,6 +38,7 @@ int Run::execute(std::vector const &) user::UserServiceServer login_server(sc); account::AccountServiceServer account_server(sc); md::run_market_datapublishers(); + api::synthetic::SyntheticSecurityService synthetic_service(sc); md::QueryServerPtr md_query_server(md::run_query_server(sc)); orderman2::OrderServiceServer2 order_service_server(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 7e2814191..c38942a52 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -82,8 +82,7 @@ MDP3Listener::MDP3Listener( MDProducer& producer, SecurityIDRecorder& recorder, co2::SettlementRecorder& settles, - std::vector channels, - utils::Dispatcher& disp) : + std::vector channels) : _index(index), _producer(producer), _packet_arrived(), @@ -93,17 +92,13 @@ MDP3Listener::MDP3Listener( _symbol_filter_storage(), _symbol_filter(), _channels(channels), - _security_recorder_input_timer(disp, 900, [&]() { dequeue_securities(); }), - _security_recorder_input(), _concurrency_monitor(0), _book_updates() { _book_updates.reserve(32); - _security_recorder_input_timer.start(); } MDP3Listener::~MDP3Listener() { - _security_recorder_input_timer.stop(); } // @@ -568,35 +563,9 @@ void MDP3Listener::add_security(const FixedSecurity& security) { // Add it to the index so it can receive messages SecurityData* data = _index.add_security(security.numeric_id, handle); - // Computing the descriptor may be expensive so defer that processing to the background worker thread (see dequeue_securities) - _security_recorder_input.push(data); -} - -void MDP3Listener::dequeue_securities() { - // already running on '_background' thread - vector defer; - - SecurityData* data = nullptr; - while (_security_recorder_input.try_pop(data)) { - bts_assert(data->handle.security().key == invalid_security_key); - - string descriptor = security_descriptor(data->handle.security()); - if (descriptor.empty()) { - defer.push_back(data); - } - else { - _security_recorder.get_key(descriptor, data->handle.security().type, [](void* pv_handle, security_key key) { - SecurityData* pdata = (SecurityData*)pv_handle; - pdata->handle.set_key(key); - }, data); - // Restore saved settlements - _settlement_recorder.restore_settlement(data->handle); - } - } - - // try again later (when spread legs have resolved) - for (auto* retry : defer) { - _security_recorder_input.push(retry); + // Assign a security key + if (data->handle.key() == invalid_security_key) { + _security_recorder.ensure_key(data->handle); } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 221a89b74..979091a6b 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -18,7 +18,7 @@ class MDP3Listener : public onixs::SecurityListener { public: - MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector, utils::Dispatcher&); + MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector); virtual ~MDP3Listener(); // @@ -91,9 +91,10 @@ public: void add_filter_symbol(const std::string& symbol); void heartbeat(); -private: + // Add a new security to the index void add_security(const FixedSecurity& security); - void dequeue_securities(); + +private: enum SecurityFilterAction { ADD, SKIP, DELETE }; template SecurityFilterAction filter_security(int exchange_id, const Sec& sec) { @@ -122,8 +123,6 @@ private: std::vector _symbol_filter_storage; std::set _symbol_filter; std::vector _channels; - utils::DispatcherTimer _security_recorder_input_timer; - tbb::concurrent_queue _security_recorder_input; std::atomic _concurrency_monitor; struct BookData { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index f3548c8a7..1ac976386 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -3,6 +3,7 @@ #include "MDP3Utils.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" +#include "lib/md/SecurityDB.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/proto_helper.h" @@ -16,7 +17,7 @@ using namespace std; using namespace bts::co2; -// using namespace onixs; +using std::size_t; namespace bts { namespace md { @@ -317,11 +318,19 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, utils::Dispatcher::current())); + unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } + // Load the list of known securities and add them to the index for fast startup + db::load_securities(channel.symbols, [&](const api::Security& security) { + with_security(securitytype_of(security), num_legs(security), [&](FixedSecurity& fs) { + convert_security(security, fs, true); + listener->add_security(fs); + }); + }); + handler.registerListener(*listener); handler.registerMarketDataListener(*listener); handler.registerSecurityListener(*listener);