Merged MDTempBranch into master
This commit is contained in:
commit
7bb3758bae
22 files changed
+686
-262
No files matched your search
@@ -1 +1 @@
|
||||
set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OrderService2.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
|
||||
set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OrderService2.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
|
||||
@@ -1,60 +1,58 @@
|
||||
#pragma once
|
||||
#include "{{filename name}}.pb.h"
|
||||
#include "lib/api/public/ApiServer.h"
|
||||
#include "lib/utils/bts_export.h"
|
||||
#include <string>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <memory>
|
||||
|
||||
// #include <iostream>
|
||||
// #include <atomic>
|
||||
namespace bts { namespace api {
|
||||
class ServiceContext;
|
||||
}}
|
||||
|
||||
{{ns-open package}}
|
||||
|
||||
// Forward declaractions
|
||||
{{#each message_type}}
|
||||
class {{name}};{{/each}}
|
||||
|
||||
{{#each service}}
|
||||
// {{comment}}
|
||||
class {{name}}Base : public bts::api::ApiServer {
|
||||
protected:
|
||||
{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier="");
|
||||
|
||||
const char* name() const override;
|
||||
|
||||
{{#each method}}
|
||||
{{#if server_streaming}}
|
||||
typedef std::function<void(const std::vector<{{output_type_cpp}}>*, const std::vector<{{output_type_cpp}}>*, std::set<{{key_type.cpptype}}>*)> {{cap name}}Publisher;
|
||||
{{cap name}}Publisher make_{{name}}_publisher() const;
|
||||
{{cap name}}Publisher make_{{name}}_publisher(const std::string &) const;
|
||||
{{cap name}}Publisher make_{{name}}_publisher(const {{input_type}}& qualifier) const;
|
||||
{{else}}
|
||||
// {{comment}}
|
||||
virtual void {{name}}(
|
||||
const {{input_type}}& request,
|
||||
const api::Response<{{output_type_cpp}}>& reply) = 0;
|
||||
{{/if}}
|
||||
{{/each}}
|
||||
|
||||
enum ServiceMethods {
|
||||
SYS_PING = -1,
|
||||
{{#each method}}
|
||||
{{upper name}} = {{@index}},
|
||||
{{/each}}
|
||||
};
|
||||
|
||||
private:
|
||||
void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override;
|
||||
{{#each method}}{{#if server_streaming}}
|
||||
{{cap name}}Publisher make_{{name}}_publisher(TableHandle) const;
|
||||
{{/if}}{{/each}}
|
||||
};
|
||||
|
||||
{{/each}}
|
||||
{{ns-close package}}
|
||||
#pragma once
|
||||
#include "{{filename name}}.pb.h"
|
||||
#include "lib/api/public/ApiServer.h"
|
||||
#include "lib/utils/bts_export.h"
|
||||
#include <string>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <memory>
|
||||
|
||||
namespace bts { namespace api {
|
||||
class ServiceContext;
|
||||
}}
|
||||
|
||||
{{ns-open package}}
|
||||
|
||||
// Forward declaractions
|
||||
{{#each message_type}}
|
||||
class {{name}};{{/each}}
|
||||
|
||||
{{#each service}}
|
||||
// {{comment}}
|
||||
class {{name}}Base : public bts::api::ApiServer {
|
||||
protected:
|
||||
{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier="");
|
||||
|
||||
const char* name() const override;
|
||||
|
||||
{{#each method}}
|
||||
{{#if server_streaming}}
|
||||
typedef std::function<void(const std::vector<{{output_type_cpp}}>*, const std::vector<{{output_type_cpp}}>*, std::set<{{key_type.cpptype}}>*)> {{cap name}}Publisher;
|
||||
{{cap name}}Publisher make_{{name}}_publisher() const;
|
||||
{{cap name}}Publisher make_{{name}}_publisher(const std::string &) const;
|
||||
{{cap name}}Publisher make_{{name}}_publisher(const {{input_type}}& qualifier) const;
|
||||
{{else}}
|
||||
// {{comment}}
|
||||
virtual void {{name}}(
|
||||
const {{input_type}}& request,
|
||||
const api::Response<{{output_type_cpp}}>& reply) = 0;
|
||||
{{/if}}
|
||||
{{/each}}
|
||||
|
||||
enum ServiceMethods {
|
||||
SYS_PING = -1,
|
||||
{{#each method}}
|
||||
{{upper name}} = {{@index}},
|
||||
{{/each}}
|
||||
};
|
||||
|
||||
private:
|
||||
void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override;
|
||||
{{#each method}}{{#if server_streaming}}
|
||||
{{cap name}}Publisher make_{{name}}_publisher(TableHandle) const;
|
||||
{{/if}}{{/each}}
|
||||
};
|
||||
|
||||
{{/each}}
|
||||
{{ns-close package}}
|
||||
@@ -0,0 +1,53 @@
|
||||
syntax="proto3";
|
||||
package bts.api.synthetic;
|
||||
option csharp_namespace="bts.api.synthetic";
|
||||
|
||||
import "lib/api/apioptions.proto";
|
||||
import "lib/domain_model/Security.proto";
|
||||
//import "google/protobuf/timestamp.proto";
|
||||
import "lib/sparkproto/SparkProtos.proto";
|
||||
|
||||
message CreateResult {
|
||||
enum Error {
|
||||
ERROR_OK = 0;
|
||||
ERROR_NOT_SYNTHETIC = 1;
|
||||
ERROR_ID_ALREADY_ASSIGNED = 2;
|
||||
}
|
||||
uint32 key = 1;
|
||||
Error error = 2;
|
||||
}
|
||||
message UpdateResult {
|
||||
enum Error {
|
||||
ERROR_OK = 0;
|
||||
ERROR_MISSING_KEY = 1;
|
||||
ERROR_UNKNOWN_SECURITY = 2;
|
||||
ERROR_DATABASE = 3;
|
||||
}
|
||||
Error error = 1;
|
||||
}
|
||||
|
||||
message Metadata {
|
||||
uint32 security_key = 1;
|
||||
spark.ExecutionStrategy strategy = 2;
|
||||
}
|
||||
message MetadataQuery {
|
||||
repeated uint32 security_key = 1;
|
||||
}
|
||||
message MetadataCollection {
|
||||
repeated Metadata data = 1;
|
||||
}
|
||||
|
||||
service SyntheticSecurityService {
|
||||
option(bts.api.service_description)="Create or query synthetic securities.";
|
||||
|
||||
// Create synthetic
|
||||
rpc create_synthetic(api.Security) returns (CreateResult);
|
||||
// Subscribe synthetics -- code only
|
||||
|
||||
// set/update metadata // spark.ExecutionStrategy
|
||||
rpc update_security_metadata(Metadata) returns (UpdateResult);
|
||||
// query metadata
|
||||
rpc query_security_metadata(MetadataQuery) returns (MetadataCollection)
|
||||
{ option (bts.api.method_description)="Returns the metadata for the given set of securities if it exists. If no data for a particular key exists it will not be present in the output collection."; }
|
||||
// subscribe metadata -- code only
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
#pragma once
|
||||
#include "lib/utils/ValueWrapper.h"
|
||||
namespace bts {
|
||||
using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag,uint32_t>;
|
||||
}
|
||||
@@ -1,17 +1,14 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "lib/utils/ValueWrapper.h"
|
||||
|
||||
namespace bts {
|
||||
using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag,uint32_t>;
|
||||
|
||||
|
||||
|
||||
using Price = utils::ValueWrapper<struct OrderPriceTag,double>;
|
||||
using Quantity = utils::ValueWrapper<struct QuantityTag,uint32_t>;
|
||||
using UserLogin = utils::ValueWrapper<struct UserLoginTag,std::string>;
|
||||
}
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "lib/utils/ValueWrapper.h"
|
||||
#include "lib/domain_model/SecurityKey.h"
|
||||
|
||||
namespace bts {
|
||||
using Price = utils::ValueWrapper<struct OrderPriceTag,double>;
|
||||
using Quantity = utils::ValueWrapper<struct QuantityTag,uint32_t>;
|
||||
using UserLogin = utils::ValueWrapper<struct UserLoginTag,std::string>;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/api/services/SyntheticSecurityService.pb.h"
|
||||
#include "lib/domain_model/SecurityKey.h"
|
||||
#include "lib/utils/Event.h"
|
||||
#include <unordered_map>
|
||||
|
||||
namespace bts {
|
||||
namespace api {
|
||||
namespace synthetic {
|
||||
|
||||
struct ISyntheticSecurityService {
|
||||
virtual const std::unordered_map<SecurityKey, Security> securities() = 0;
|
||||
virtual utils::EventSubscriber<Security>& on_securities() = 0;
|
||||
virtual const Metadata* get_metadata(SecurityKey) = 0;
|
||||
virtual utils::EventSubscriber<Metadata>& on_metadata() = 0;
|
||||
};
|
||||
|
||||
}}}
|
||||
+85
-41
@@ -4,6 +4,7 @@
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/system.pb.h"
|
||||
#include "lib/md/Shm/FixedSecurity.h"
|
||||
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <boost/filesystem.hpp>
|
||||
@@ -14,14 +15,7 @@ using namespace bts::utils;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
// schema:
|
||||
// id, security (binary), expiration time
|
||||
|
||||
struct SecurityDB {
|
||||
// Store the symbols for debugging purposes
|
||||
boost::container::flat_set<SecurityKey> known_keys;
|
||||
};
|
||||
namespace db {
|
||||
|
||||
/*
|
||||
* Schema:
|
||||
@@ -44,16 +38,19 @@ sqlite3_ptr open_db() {
|
||||
sqlite3_exec_nofail(db, "PRAGMA syncronous=0");
|
||||
|
||||
sqlite3_exec_nofail(db,
|
||||
"CREATE TABLE IF NOT EXIST securities (key INTEGER NOT NULL, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL");
|
||||
"CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);");
|
||||
|
||||
sqlite3_exec_nofail(db,
|
||||
"CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);");
|
||||
|
||||
return dbptr;
|
||||
}
|
||||
|
||||
std::unique_ptr<SecurityDB> load_security_db(
|
||||
size_t load_securities(
|
||||
const std::set<UserSymbol>& symbols,
|
||||
const std::function<void(const api::Security&)>& callback) {
|
||||
|
||||
auto security_db = std::make_unique<SecurityDB>();
|
||||
size_t count = 0;
|
||||
auto dbptr = open_db();
|
||||
|
||||
// Load all existing securities and call the callback
|
||||
@@ -64,7 +61,7 @@ std::unique_ptr<SecurityDB> load_security_db(
|
||||
auto result = sqlite3_step(select);
|
||||
Security security;
|
||||
while (result == SQLITE_ROW) {
|
||||
uint32_t security_key = sqlite3_column_int(select, 0);
|
||||
security_key security_key = sqlite3_column_int(select, 0);
|
||||
auto symbol = std::string(
|
||||
(char*)sqlite3_column_text(select, 1),
|
||||
sqlite3_column_bytes(select, 1));
|
||||
@@ -76,50 +73,97 @@ std::unique_ptr<SecurityDB> load_security_db(
|
||||
auto len = sqlite3_column_bytes(select, 4);
|
||||
if (security.ParseFromArray(data, len)) {
|
||||
bts_assert(security_key == security.id());
|
||||
security_db->known_keys.insert(SecurityKey(security.id()));
|
||||
count++;
|
||||
callback(security);
|
||||
}
|
||||
else {
|
||||
LOG(md, Warn, "Failed to parse security from database. id = {}", security_key);
|
||||
}
|
||||
}
|
||||
result = sqlite3_step(select);
|
||||
}
|
||||
|
||||
return security_db;
|
||||
return count;
|
||||
}
|
||||
|
||||
bool store_if_new(SecurityDB& db, const api::Security& security) {
|
||||
auto key = SecurityKey(security.id());
|
||||
if (db.known_keys.find(key) == db.known_keys.end()) {
|
||||
db.known_keys.insert(key);
|
||||
auto dbptr = open_db();
|
||||
auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?)");
|
||||
auto buffer = security.SerializeAsString();
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, security.id()), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, security.symbol().c_str(), security.symbol().size(), SQLITE_STATIC), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, security.exchange()), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(security)), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, datetime::today().to_short()), dbptr);
|
||||
auto result = sqlite3_step(insert.get());
|
||||
if (result != SQLITE_DONE) {
|
||||
LOG(md, Warn, "Failed to insert security into security db. key={}", security.id());
|
||||
void store_if_new(const std::vector<Security*>& securities) {
|
||||
auto dbptr = open_db();
|
||||
auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
|
||||
sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION");
|
||||
std::string buffer;
|
||||
for (const api::Security* psecurity : securities) {
|
||||
if (psecurity && psecurity->SerializeToString(&buffer)) {
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, psecurity->id()), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, psecurity->symbol().c_str(), psecurity->symbol().size(), SQLITE_STATIC), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, psecurity->exchange()), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr);
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, datetime::today().to_short()), dbptr);
|
||||
auto result = sqlite3_step(insert.get());
|
||||
if (result != SQLITE_DONE) {
|
||||
LOG(md, Warn, "Failed to insert security into security db. key={}", psecurity->id());
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION");
|
||||
}
|
||||
|
||||
|
||||
std::vector<api::synthetic::Metadata> load_metadata() {
|
||||
std::vector<api::synthetic::Metadata> values;
|
||||
auto dbptr = open_db();
|
||||
auto db = dbptr.get();
|
||||
auto select_stmt = prepare_query(db, "select key,data from metadata");
|
||||
auto select = select_stmt.get();
|
||||
auto result = sqlite3_step(select);
|
||||
while(result == SQLITE_ROW) {
|
||||
uint32_t key = sqlite3_column_int(select, 0);
|
||||
auto blob = sqlite3_column_blob(select, 1);
|
||||
auto len = sqlite3_column_bytes(select, 1);
|
||||
api::synthetic::Metadata data;
|
||||
if (data.ParseFromArray(blob, len)) {
|
||||
bts_assert(key == data.security_key());
|
||||
values.push_back(data);
|
||||
}
|
||||
else {
|
||||
LOG(md, Warn, "Failed to parse security metadata for security_key={}", key);
|
||||
}
|
||||
result = sqlite3_step(select);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
bool store_metadata(const api::synthetic::Metadata& data) {
|
||||
auto dbptr = open_db();
|
||||
auto insert = utils::prepare_query(dbptr, "insert into metadata (key, data) values (?,?)");
|
||||
|
||||
SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr);
|
||||
std::string buffer = data.SerializeAsString();
|
||||
SQLITE_CHK(sqlite3_bind_blob(insert.get(), 2, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr);
|
||||
auto result = sqlite3_step(insert.get());
|
||||
if (result != SQLITE_DONE) {
|
||||
LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#else
|
||||
namespace bts {
|
||||
namespace md {
|
||||
struct SecurityDB {};
|
||||
std::unique_ptr<SecurityDB> load_security_db(
|
||||
const std::set<UserSymbol>&,
|
||||
const std::function<void(const api::Security&)>&) {
|
||||
return std::make_unique<SecurityDB>();
|
||||
}
|
||||
bool store_if_new(SecurityDB&, const api::Security&) {
|
||||
return false;
|
||||
namespace db {
|
||||
|
||||
size_t load_securities(
|
||||
const std::set<UserSymbol>& symbols,
|
||||
const std::function<void(const api::Security&)>& callback) {
|
||||
return 0;
|
||||
}
|
||||
void store_if_new(const std::vector<Security*>& handles) {}
|
||||
// std::vector<api::synthetic::Metadata> load_metadata() {
|
||||
// std::vector<api::synthetic::Metadata> result;
|
||||
// return result;
|
||||
// }
|
||||
// void store_metadata(const api::synthetic::Metadata&) {
|
||||
// }
|
||||
|
||||
#endif
|
||||
|
||||
@@ -152,4 +196,4 @@ datetime::Date expiration(const api::Security& security) {
|
||||
}
|
||||
}
|
||||
|
||||
}}//namespace
|
||||
}}}//namespace
|
||||
+13
-5
@@ -3,26 +3,34 @@
|
||||
#include "lib/domain_model/common.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include "lib/md/MDProducerHandle.h"
|
||||
#include "lib/api/services/SyntheticSecurityService.pb.h"
|
||||
#include "common.h"
|
||||
#include "UserSymbol.h"
|
||||
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
|
||||
namespace bts {
|
||||
namespace api {class Security;}
|
||||
namespace md {
|
||||
namespace db {
|
||||
|
||||
struct SecurityDB;
|
||||
|
||||
std::unique_ptr<SecurityDB> load_security_db(
|
||||
// Loads securities matching the given filter for producers to pre-populate security lists
|
||||
// note: possible to use this to optimize initial memory layout? sort by expiration?
|
||||
size_t load_securities(
|
||||
const std::set<UserSymbol>& symbols,
|
||||
const std::function<void(const api::Security&)>& callback);
|
||||
|
||||
bool store_if_new(SecurityDB& db, const api::Security& security);
|
||||
void store_if_new(const std::vector<Security*>& handles);
|
||||
|
||||
// metadata
|
||||
std::vector<api::synthetic::Metadata> load_metadata();
|
||||
bool store_metadata(const api::synthetic::Metadata&);
|
||||
|
||||
datetime::Date expiration(const api::Security& security);
|
||||
|
||||
|
||||
}}//namespace
|
||||
}}}//namespace
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/md/SecurityDB.h"
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <tbb/queuing_mutex.h> // for reservation
|
||||
|
||||
@@ -17,18 +18,16 @@
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::utils;
|
||||
using std::size_t;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
//GCC_DIAG_OFF("-Weffc++") // Plain Struct
|
||||
struct SecurityIDTask {
|
||||
std::string descriptor;
|
||||
SecurityType type;
|
||||
SecurityIDRecorder::callback cb;
|
||||
void* arg;
|
||||
struct SecurityIDEntry {
|
||||
api::Security security;
|
||||
SecurityIDRecorder::Callback callback;
|
||||
};
|
||||
//GCC_DIAG_RESTORE()
|
||||
|
||||
struct SecurityIDRecorderState {
|
||||
SecurityIDRecorderState() :
|
||||
work(),
|
||||
@@ -37,7 +36,7 @@ struct SecurityIDRecorderState {
|
||||
select_stmt(nullptr, sqlite3_finalize),
|
||||
insert_stmt(nullptr, sqlite3_finalize)
|
||||
{}
|
||||
tbb::concurrent_queue<SecurityIDTask> work;
|
||||
tbb::concurrent_queue<std::shared_ptr<SecurityIDEntry>> work;
|
||||
tbb::concurrent_queue<MDProducerHandle> pending_handle;
|
||||
sqlite3_ptr db;
|
||||
sqlite3_stmt_ptr select_stmt;
|
||||
@@ -50,9 +49,7 @@ SecurityIDRecorder::SecurityIDRecorder():
|
||||
init();
|
||||
}
|
||||
|
||||
SecurityIDRecorder::~SecurityIDRecorder()
|
||||
{
|
||||
}
|
||||
SecurityIDRecorder::~SecurityIDRecorder() { }
|
||||
|
||||
void SecurityIDRecorder::init() {
|
||||
string db_dir = config::get<config::System>().data_path() + "/md/";
|
||||
@@ -75,100 +72,89 @@ void SecurityIDRecorder::init() {
|
||||
_self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, callback cb, void* arg) {
|
||||
SecurityIDTask task;
|
||||
task.descriptor = descriptor;
|
||||
task.type = type;
|
||||
task.cb = cb;
|
||||
task.arg = arg;
|
||||
_self->work.push(task);
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, Callback cb) {
|
||||
Callback* pcb = new Callback(cb);
|
||||
get_key(descriptor, type, [](void* data, security_key key) {
|
||||
Callback* pcb = static_cast<Callback*>(data);
|
||||
DEFER(if (pcb) delete pcb); // exception safe
|
||||
(*pcb)(key);
|
||||
}, pcb);
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
|
||||
_self->pending_handle.push(handle);
|
||||
}
|
||||
void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) {
|
||||
if (security.id() != invalid_security_key) {
|
||||
auto work = std::make_shared<SecurityIDEntry>();
|
||||
work->security = security;
|
||||
work->callback = callback;
|
||||
_self->work.push(work);
|
||||
}
|
||||
}
|
||||
|
||||
bool SecurityIDRecorder::do_work() {
|
||||
// Check if any handles need work first
|
||||
work_pending_handles();
|
||||
|
||||
void SecurityIDRecorder::do_work() {
|
||||
work_handles();
|
||||
work_db();
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::work_handles() {
|
||||
Security security;
|
||||
MDProducerHandle handle;
|
||||
while (_self->pending_handle.try_pop(handle)) {
|
||||
security.Clear();
|
||||
convert_security(handle.security(), &security);
|
||||
assign_key(security, [handle](const api::Security& security) mutable {
|
||||
handle.set_key(security.id());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::work_db() {
|
||||
// Drain the work queue
|
||||
// Query the database for each security. If it has an ID return it
|
||||
// Query the database for each security. If it has an ID set it
|
||||
// If it does not have an ID add it to a list
|
||||
// Then persist that list
|
||||
SecurityIDTask temp;
|
||||
vector<SecurityIDTask> new_securities;
|
||||
// Once a security has a key it is public so record it in the security database
|
||||
std::shared_ptr<SecurityIDEntry> temp;
|
||||
vector<std::shared_ptr<SecurityIDEntry>> new_entries;
|
||||
vector<string> new_descriptors;
|
||||
size_t count = 0;
|
||||
bool incomplete = false;
|
||||
string descriptor;
|
||||
while (_self->work.try_pop(temp)) {
|
||||
security_key key = search(temp.descriptor);
|
||||
if (key != invalid_security_key) {
|
||||
temp.cb(temp.arg, key);
|
||||
}
|
||||
else {
|
||||
new_securities.push_back(temp);
|
||||
++count;
|
||||
// Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat
|
||||
if (count > 500) {
|
||||
incomplete = true;
|
||||
break;
|
||||
descriptor = security_descriptor(temp->security);
|
||||
security_key key = search(descriptor);
|
||||
if (key != invalid_security_key) {
|
||||
temp->security.set_id(key);
|
||||
temp->callback(temp->security);
|
||||
}
|
||||
else {
|
||||
// always push in parallel
|
||||
new_entries.push_back(move(temp));
|
||||
new_descriptors.push_back(descriptor);
|
||||
++count;
|
||||
// Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat
|
||||
if (count > 500) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (new_securities.size() > 0) {
|
||||
if (new_entries.size() > 0) {
|
||||
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
||||
|
||||
for (SecurityIDTask& security : new_securities) {
|
||||
security_key key = insert(security.descriptor, security.type);
|
||||
for (size_t x = 0; x < new_entries.size(); ++x) {
|
||||
Security& security = new_entries[x]->security;
|
||||
security_key key = insert(new_descriptors[x], securitytype_of(security));
|
||||
bts_assert_d(key);
|
||||
security.cb(security.arg, key);
|
||||
security.set_id(key);
|
||||
new_entries[x]->callback(security);
|
||||
}
|
||||
|
||||
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
||||
}
|
||||
return incomplete;
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::work_pending_handles() {
|
||||
MDProducerHandle handle;
|
||||
vector<MDProducerHandle> defer;
|
||||
while(_self->pending_handle.try_pop(handle)) {
|
||||
// A spread will not have a descriptor until all of its leg keys are available
|
||||
string descriptor = security_descriptor(handle.security());
|
||||
if (descriptor.empty()) {
|
||||
defer.push_back(handle);
|
||||
continue;
|
||||
// record the securities
|
||||
std::vector<Security*> save_these;
|
||||
save_these.reserve(new_entries.size());
|
||||
for (const auto& e : new_entries) {
|
||||
save_these.push_back(&e->security);
|
||||
}
|
||||
|
||||
MDProducerHandle* phandle = new MDProducerHandle(handle);
|
||||
DEFER(if (phandle) delete phandle);
|
||||
get_key(descriptor, handle.security().type, [](void* arg, security_key key) {
|
||||
auto* phandle = static_cast<MDProducerHandle*>(arg);
|
||||
if (phandle) {
|
||||
phandle->set_key(key);
|
||||
delete phandle;
|
||||
}
|
||||
}, phandle);
|
||||
phandle = nullptr;
|
||||
}
|
||||
|
||||
for (auto& handle : defer) {
|
||||
_self->pending_handle.push(handle);
|
||||
db::store_if_new(save_these);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
security_key SecurityIDRecorder::search(const std::string& descriptor) {
|
||||
sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
|
||||
DEFER(sqlite3_reset(_self->select_stmt.get()));
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
#ifndef WIN32
|
||||
#include "common.h"
|
||||
#include "MDProducerHandle.h"
|
||||
#include "lib/utils/bts_export.h"
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@@ -19,26 +18,21 @@ namespace md {
|
||||
*/
|
||||
|
||||
struct SecurityIDRecorderState;
|
||||
class BTS_EXPORT SecurityIDRecorder
|
||||
class SecurityIDRecorder
|
||||
{
|
||||
public:
|
||||
SecurityIDRecorder();
|
||||
~SecurityIDRecorder();
|
||||
typedef void(*callback)(void*, security_key);
|
||||
typedef std::function<void(security_key)> Callback;
|
||||
typedef std::function<void(const api::Security&)> Callback;
|
||||
|
||||
// Asyncronously retrieve the security_key for the given descriptor
|
||||
void get_key(const std::string& descriptor, SecurityType, callback cb, void* arg);
|
||||
|
||||
// A less efficient but higher level get_key which performs heap allocation in the call path
|
||||
void get_key(const std::string& descriptor, SecurityType, Callback cb);
|
||||
|
||||
// A higher level interface which allows you to just pass in a handle and have it generate a descriptor and set the key
|
||||
// Fast - for use by market data producers
|
||||
void ensure_key(MDProducerHandle handle);
|
||||
// Slow - for use by services
|
||||
void assign_key(const api::Security& security, const Callback& key_assigned);
|
||||
|
||||
// Someone must call work periodically to do lookups and flush the reserved ids to disk.
|
||||
// returne true if there is more work to be done
|
||||
bool do_work();
|
||||
void do_work();
|
||||
|
||||
private:
|
||||
std::unique_ptr<SecurityIDRecorderState> _self;
|
||||
@@ -47,7 +41,8 @@ private:
|
||||
security_key search(const std::string& descriptor);
|
||||
security_key insert(const std::string& descriptor, SecurityType);
|
||||
|
||||
void work_pending_handles();
|
||||
void work_handles();
|
||||
void work_db();
|
||||
|
||||
BTS_NO_COPY(SecurityIDRecorder);
|
||||
};
|
||||
|
||||
@@ -76,7 +76,7 @@ void write_date_string(uint32_t year, uint32_t month, uint32_t day, std::ostring
|
||||
<< year - 2000;
|
||||
}
|
||||
|
||||
const std::string security_descriptor(const FixedSecurity& security) {
|
||||
std::string security_descriptor(const FixedSecurity& security) {
|
||||
const char separator = ' ';
|
||||
std::ostringstream result;
|
||||
// If the same symbol is listed at 2 different exchanges it should have a different ID
|
||||
@@ -118,6 +118,14 @@ const std::string security_descriptor(const FixedSecurity& security) {
|
||||
|
||||
return result.str();
|
||||
}
|
||||
std::string security_descriptor(const api::Security& security) {
|
||||
std::string result;
|
||||
with_security(security.Contract_case(), num_legs(security), [&](FixedSecurity fs) {
|
||||
convert_security(security, fs, true);
|
||||
result = security_descriptor(fs);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
void get_contract_spec(const FixedSecurity& source, api::ContractSpecification& spec) {
|
||||
spec.set_cash_per_point(source.cme_unit_of_measure_qty);
|
||||
@@ -218,6 +226,12 @@ SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
|
||||
SecurityType securitytype_of(const Security& s) {
|
||||
return securitytype_of_contractcase(s.Contract_case());
|
||||
}
|
||||
size_t num_legs(const Security& sec) {
|
||||
if (sec.Contract_case() == api::Security::ContractCase::kSpread) {
|
||||
return sec.spread().legs_size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) {
|
||||
dest.key = set_security_key ? src.id() : invalid_security_key;
|
||||
|
||||
@@ -128,7 +128,8 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS;
|
||||
typedef FixedSpreadSecurity<128> FixedSpreadSecurityL;
|
||||
typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL;
|
||||
|
||||
const std::string security_descriptor(const FixedSecurity&);
|
||||
std::string security_descriptor(const api::Security&);
|
||||
// std::string security_descriptor(const FixedSecurity&);
|
||||
|
||||
// Convert a fixedsecurity to a normal Security - caveat: spreads will be normalized
|
||||
Security convert_security(const FixedSecurity&);
|
||||
@@ -138,6 +139,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
|
||||
|
||||
SecurityType securitytype_of_contractcase(Security::ContractCase cc);
|
||||
SecurityType securitytype_of(const Security&);
|
||||
size_t num_legs(const Security&);
|
||||
|
||||
// Stack allocates a security of the given type
|
||||
template <class C>
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
#ifndef WIN32
|
||||
#include "SyntheticSecurityService.h"
|
||||
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/api/public/ServiceContext.h"
|
||||
#include "lib/api/services/PersistenceService.pb.h"
|
||||
#include "lib/md/common.h"
|
||||
#include "lib/md/SecurityIDRecorder.h"
|
||||
#include "lib/md/SecurityDB.h"
|
||||
#include "lib/md/get_symbols.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/utils/Event.h"
|
||||
#include <set>
|
||||
|
||||
namespace bts {
|
||||
namespace api {
|
||||
namespace synthetic {
|
||||
|
||||
struct SecurityLess {
|
||||
bool operator()(const api::Security& left, const api::Security& right) {
|
||||
return left.id() < right.id();
|
||||
}
|
||||
};
|
||||
|
||||
struct SyntheticSecurityService::Pimpl {
|
||||
Pimpl() :
|
||||
recorder(),
|
||||
timer(utils::Dispatcher::background(), 200, [&]() { recorder.do_work(); }),
|
||||
synthetics(), on_security(),
|
||||
meta(), on_metadata()
|
||||
{}
|
||||
md::SecurityIDRecorder recorder;
|
||||
utils::DispatcherTimer timer;
|
||||
std::unordered_map<SecurityKey, Security> synthetics;
|
||||
utils::Event<Security> on_security;
|
||||
std::unordered_map<SecurityKey, Metadata> meta;
|
||||
utils::Event<Metadata> on_metadata;
|
||||
};
|
||||
|
||||
SyntheticSecurityService::SyntheticSecurityService(api::ServiceContext& context)
|
||||
: SyntheticSecurityServiceBase(context),
|
||||
_me(new Pimpl()) {
|
||||
|
||||
// load securities
|
||||
std::set<md::UserSymbol> symbols;
|
||||
md::get_system_md_symbols(symbols);
|
||||
md::db::load_securities(symbols, [=](const api::Security& security) {
|
||||
if (security.Contract_case() == api::Security::kSynthetic) {
|
||||
auto key = SecurityKey(security.id());
|
||||
_me->synthetics.insert(std::make_pair(key, security));
|
||||
}
|
||||
});
|
||||
|
||||
// Load metadata
|
||||
auto metadata = md::db::load_metadata();
|
||||
for (const auto& entry : metadata) {
|
||||
_me->meta.insert(std::make_pair(entry.security_key(), entry));
|
||||
}
|
||||
|
||||
}
|
||||
SyntheticSecurityService::~SyntheticSecurityService() { }
|
||||
|
||||
|
||||
const std::unordered_map<SecurityKey, Security> SyntheticSecurityService::securities() {
|
||||
return _me->synthetics;
|
||||
}
|
||||
utils::EventSubscriber<Security>& SyntheticSecurityService::on_securities() {
|
||||
return _me->on_security;
|
||||
}
|
||||
const Metadata* SyntheticSecurityService::get_metadata(SecurityKey key) {
|
||||
auto iter = _me->meta.find(key);
|
||||
if (iter != _me->meta.end()) {
|
||||
return &iter->second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
utils::EventSubscriber<Metadata>& SyntheticSecurityService::on_metadata() {
|
||||
return _me->on_metadata;
|
||||
}
|
||||
|
||||
void send_reply(const api::Response<CreateResult>& fn, CreateResult::Error err, uint32_t key=md::invalid_security_key) {
|
||||
CreateResult r;
|
||||
r.set_key(key);
|
||||
r.set_error(err);
|
||||
fn(r);
|
||||
}
|
||||
|
||||
void SyntheticSecurityService::create_synthetic(
|
||||
const Security& security,
|
||||
const api::Response<CreateResult>& reply) {
|
||||
|
||||
CreateResult result;
|
||||
// Validate security is synthetic
|
||||
if (security.Contract_case() != Security::kSynthetic) {
|
||||
send_reply(reply, CreateResult::ERROR_NOT_SYNTHETIC);
|
||||
return;
|
||||
}
|
||||
if (security.id() != md::invalid_security_key) {
|
||||
send_reply(reply, CreateResult::ERROR_ID_ALREADY_ASSIGNED);
|
||||
return;
|
||||
}
|
||||
|
||||
// Assign security key
|
||||
_me->recorder.assign_key(security, [=](const api::Security& security) {
|
||||
// Notify listeners that a new security is added
|
||||
_me->synthetics.insert(std::make_pair(security.id(), security));
|
||||
_me->on_security.publish(security);
|
||||
// signal success
|
||||
send_reply(reply, CreateResult::ERROR_OK, security.id());
|
||||
});
|
||||
}
|
||||
|
||||
void SyntheticSecurityService::update_security_metadata(
|
||||
const Metadata& request,
|
||||
const api::Response<UpdateResult>& reply) {
|
||||
UpdateResult result;
|
||||
if (request.security_key() == md::invalid_security_key) {
|
||||
result.set_error(UpdateResult::ERROR_MISSING_KEY);
|
||||
reply(result);
|
||||
return;
|
||||
}
|
||||
auto key = SecurityKey(request.security_key());
|
||||
if (_me->synthetics.find(key) == _me->synthetics.end()) {
|
||||
result.set_error(UpdateResult::ERROR_UNKNOWN_SECURITY);
|
||||
reply(result);
|
||||
return;
|
||||
}
|
||||
// update the on disk database
|
||||
if (!md::db::store_metadata(request)) {
|
||||
result.set_error(UpdateResult::ERROR_DATABASE);
|
||||
reply(result);
|
||||
return;
|
||||
}
|
||||
// update the local database
|
||||
_me->meta[key] = request;
|
||||
// notify subscribers
|
||||
_me->on_metadata.publish(request);
|
||||
|
||||
reply(result);
|
||||
}
|
||||
|
||||
void SyntheticSecurityService::query_security_metadata(
|
||||
const MetadataQuery& request,
|
||||
const api::Response<MetadataCollection>& reply) {
|
||||
|
||||
MetadataCollection result;
|
||||
for (const auto k : request.security_key()) {
|
||||
SecurityKey key(k);
|
||||
auto iter = _me->meta.find(key);
|
||||
if (iter != _me->meta.end()) {
|
||||
result.mutable_data()->Add()->CopyFrom(iter->second);
|
||||
}
|
||||
}
|
||||
reply(result);
|
||||
}
|
||||
|
||||
|
||||
}}}//namespace
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,49 @@
|
||||
#pragma once
|
||||
#include "lib/md/common.h"
|
||||
#include "lib/api/services/SyntheticSecurityService.pb.h"
|
||||
#include "lib/api/services/SyntheticSecurityService.server.pb.h"
|
||||
#include "lib/domain_model/SecurityKey.h"
|
||||
#include "ISyntheticSecurityService.h"
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace bts {
|
||||
namespace api {
|
||||
namespace synthetic {
|
||||
|
||||
|
||||
class SyntheticSecurityService :
|
||||
private api::synthetic::SyntheticSecurityServiceBase,
|
||||
public ISyntheticSecurityService
|
||||
{
|
||||
struct Pimpl;
|
||||
std::unique_ptr<Pimpl> _me;
|
||||
|
||||
public:
|
||||
SyntheticSecurityService(api::ServiceContext& context);
|
||||
~SyntheticSecurityService();
|
||||
|
||||
const std::unordered_map<SecurityKey, api::Security> securities() override;
|
||||
utils::EventSubscriber<Security>& on_securities() override;
|
||||
const Metadata* get_metadata(SecurityKey) override;
|
||||
utils::EventSubscriber<Metadata>& on_metadata() override;
|
||||
|
||||
private:
|
||||
void create_synthetic(
|
||||
const Security& request,
|
||||
const api::Response<CreateResult>& reply) override;
|
||||
|
||||
void update_security_metadata(
|
||||
const Metadata& request,
|
||||
const api::Response<UpdateResult>& reply) override;
|
||||
|
||||
void query_security_metadata(
|
||||
const MetadataQuery& request,
|
||||
const api::Response<MetadataCollection>& reply) override;
|
||||
|
||||
};
|
||||
|
||||
}}}//namespace
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ namespace bts {
|
||||
os << sym.symbol() << "-" << api::SecurityType_Name(sym.type());
|
||||
return os;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace co2 {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "lib/utils/bts_export.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include "lib/domain_model/SecurityKey.h"
|
||||
|
||||
#include "lib/md/protos/co2.pb.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
|
||||
@@ -791,6 +791,7 @@ message CrossingHedgeOrder
|
||||
|
||||
message ExecutionStrategy
|
||||
{
|
||||
bool deleted = 1;
|
||||
map<uint32, LegSpreaderParams> leg_params_map = 20;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <list>
|
||||
#include <functional>
|
||||
#include <tuple>
|
||||
#include <mutex>
|
||||
|
||||
namespace bts {
|
||||
namespace utils {
|
||||
|
||||
template <class T>
|
||||
struct EventSubscriber {
|
||||
typedef int Handle;
|
||||
typedef std::function<void(const T&)> Callback;
|
||||
|
||||
EventSubscriber() : _id(0), _subscribers(), _lock() {}
|
||||
|
||||
Handle subscribe(const Callback& cb) {
|
||||
std::lock_guard<std::mutex> l(_lock);
|
||||
auto id = ++_id;
|
||||
_subscribers.push_back(std::make_tuple(id, cb));
|
||||
return id;
|
||||
}
|
||||
void unsubscribe(Handle h) {
|
||||
std::lock_guard<std::mutex> l(_lock);
|
||||
_subscribers.erase(
|
||||
std::remove_if(_subscribers.begin(),
|
||||
_subscribers.end(),
|
||||
[h](std::tuple<Handle,Callback>& t) { return std::get<Handle>(t) == h; }));
|
||||
}
|
||||
protected:
|
||||
void broadcast(const T& item) {
|
||||
std::lock_guard<std::mutex> l(_lock);
|
||||
for (const auto& t : _subscribers) {
|
||||
std::get<Callback>(t)(item);
|
||||
}
|
||||
}
|
||||
private:
|
||||
Handle _id;
|
||||
std::list<std::tuple<Handle, Callback>> _subscribers;
|
||||
std::mutex _lock;
|
||||
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct EventPublisher {
|
||||
typedef int Handle;
|
||||
virtual void publish(const T&) = 0;
|
||||
};
|
||||
|
||||
|
||||
template <class T>
|
||||
class Event :
|
||||
public EventSubscriber<T>,
|
||||
public EventPublisher<T> {
|
||||
|
||||
public:
|
||||
void publish(const T& item) {
|
||||
this->broadcast(item);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// template <class T>
|
||||
// class ReplayEvent {
|
||||
// typedef std::function<void(const T&)> Callback;
|
||||
// typedef int Handle;
|
||||
// Handle _id;
|
||||
// std::vector<T> _items;
|
||||
// std::list<std::tuple<Handle, Callback>> _subscribers;
|
||||
|
||||
// void broadcast(const T& item) {
|
||||
// for(const auto& t : _subscribers) {
|
||||
// std::get<Callback>(t)(item);
|
||||
// }
|
||||
// }
|
||||
|
||||
// public:
|
||||
// ReplayEvent() :
|
||||
// _id(0),
|
||||
// _items(),
|
||||
// _subscribers() {}
|
||||
// ~ReplayEvent() {}
|
||||
|
||||
// void publish(T&& item) {
|
||||
// _items.push_back(move(item));
|
||||
// broadcast(_items.back());
|
||||
// }
|
||||
// void publish(const T& item) {
|
||||
// _items.push_back(item);
|
||||
// broadcast(item);
|
||||
// }
|
||||
|
||||
// Handle subscribe(Callback& cb) {
|
||||
// auto id = ++_id;
|
||||
// _subscribers.push_back(std::make_tuple(id, cb));
|
||||
// for (const auto& item : _items) {
|
||||
// cb(item);
|
||||
// }
|
||||
// return id;
|
||||
// }
|
||||
// void unsubscribe(Handle h) {
|
||||
// _subscribers.erase(
|
||||
// std::remove_if(_subscribers.begin(),
|
||||
// _subscribers.end(),
|
||||
// [h](std::tuple<Handle,Callback>& t) { return std::get<Handle>(t) == h; }));
|
||||
// }
|
||||
// };
|
||||
|
||||
|
||||
}}//namespace
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "server/sparkd/md/run_md.h"
|
||||
#include "server/sparkd/exchange_sim/ExchangeSim.h"
|
||||
#include "server/sparkd/orderman2/server/OrderServiceServer2.h"
|
||||
#include "lib/md/SyntheticSecurityService.h"
|
||||
|
||||
#include "lib/md/relay/QueryServer.h"
|
||||
|
||||
@@ -37,6 +38,7 @@ int Run::execute(std::vector<std::string> const &)
|
||||
user::UserServiceServer login_server(sc);
|
||||
account::AccountServiceServer account_server(sc);
|
||||
md::run_market_datapublishers();
|
||||
api::synthetic::SyntheticSecurityService synthetic_service(sc);
|
||||
md::QueryServerPtr md_query_server(md::run_query_server(sc));
|
||||
orderman2::OrderServiceServer2 order_service_server(sc);
|
||||
|
||||
|
||||
@@ -82,8 +82,7 @@ MDP3Listener::MDP3Listener(
|
||||
MDProducer& producer,
|
||||
SecurityIDRecorder& recorder,
|
||||
co2::SettlementRecorder& settles,
|
||||
std::vector<MDProducerChannel> channels,
|
||||
utils::Dispatcher& disp) :
|
||||
std::vector<MDProducerChannel> channels) :
|
||||
_index(index),
|
||||
_producer(producer),
|
||||
_packet_arrived(),
|
||||
@@ -93,17 +92,13 @@ MDP3Listener::MDP3Listener(
|
||||
_symbol_filter_storage(),
|
||||
_symbol_filter(),
|
||||
_channels(channels),
|
||||
_security_recorder_input_timer(disp, 900, [&]() { dequeue_securities(); }),
|
||||
_security_recorder_input(),
|
||||
_concurrency_monitor(0),
|
||||
_book_updates()
|
||||
{
|
||||
_book_updates.reserve(32);
|
||||
_security_recorder_input_timer.start();
|
||||
}
|
||||
|
||||
MDP3Listener::~MDP3Listener() {
|
||||
_security_recorder_input_timer.stop();
|
||||
}
|
||||
|
||||
//
|
||||
@@ -568,35 +563,9 @@ void MDP3Listener::add_security(const FixedSecurity& security) {
|
||||
// Add it to the index so it can receive messages
|
||||
SecurityData* data = _index.add_security(security.numeric_id, handle);
|
||||
|
||||
// Computing the descriptor may be expensive so defer that processing to the background worker thread (see dequeue_securities)
|
||||
_security_recorder_input.push(data);
|
||||
}
|
||||
|
||||
void MDP3Listener::dequeue_securities() {
|
||||
// already running on '_background' thread
|
||||
vector<SecurityData*> defer;
|
||||
|
||||
SecurityData* data = nullptr;
|
||||
while (_security_recorder_input.try_pop(data)) {
|
||||
bts_assert(data->handle.security().key == invalid_security_key);
|
||||
|
||||
string descriptor = security_descriptor(data->handle.security());
|
||||
if (descriptor.empty()) {
|
||||
defer.push_back(data);
|
||||
}
|
||||
else {
|
||||
_security_recorder.get_key(descriptor, data->handle.security().type, [](void* pv_handle, security_key key) {
|
||||
SecurityData* pdata = (SecurityData*)pv_handle;
|
||||
pdata->handle.set_key(key);
|
||||
}, data);
|
||||
// Restore saved settlements
|
||||
_settlement_recorder.restore_settlement(data->handle);
|
||||
}
|
||||
}
|
||||
|
||||
// try again later (when spread legs have resolved)
|
||||
for (auto* retry : defer) {
|
||||
_security_recorder_input.push(retry);
|
||||
// Assign a security key
|
||||
if (data->handle.key() == invalid_security_key) {
|
||||
_security_recorder.ensure_key(data->handle);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ class MDP3Listener :
|
||||
public onixs::SecurityListener
|
||||
{
|
||||
public:
|
||||
MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>, utils::Dispatcher&);
|
||||
MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
|
||||
virtual ~MDP3Listener();
|
||||
|
||||
//
|
||||
@@ -91,9 +91,10 @@ public:
|
||||
void add_filter_symbol(const std::string& symbol);
|
||||
void heartbeat();
|
||||
|
||||
private:
|
||||
// Add a new security to the index
|
||||
void add_security(const FixedSecurity& security);
|
||||
void dequeue_securities();
|
||||
|
||||
private:
|
||||
enum SecurityFilterAction { ADD, SKIP, DELETE };
|
||||
template <class Sec>
|
||||
SecurityFilterAction filter_security(int exchange_id, const Sec& sec) {
|
||||
@@ -122,8 +123,6 @@ private:
|
||||
std::vector<std::string> _symbol_filter_storage;
|
||||
std::set<onixs::StrRef> _symbol_filter;
|
||||
std::vector<MDProducerChannel> _channels;
|
||||
utils::DispatcherTimer _security_recorder_input_timer;
|
||||
tbb::concurrent_queue<SecurityData*> _security_recorder_input;
|
||||
std::atomic<int> _concurrency_monitor;
|
||||
|
||||
struct BookData {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "MDP3Utils.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/md/coconut2.h"
|
||||
#include "lib/md/SecurityDB.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/utils/proto_helper.h"
|
||||
@@ -16,7 +17,7 @@
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::co2;
|
||||
// using namespace onixs;
|
||||
using std::size_t;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
@@ -317,11 +318,19 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
|
||||
producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
|
||||
}
|
||||
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, utils::Dispatcher::current()));
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
|
||||
for (auto& symbol : channel.symbols) {
|
||||
listener->add_filter_symbol(symbol.symbol());
|
||||
}
|
||||
|
||||
// Load the list of known securities and add them to the index for fast startup
|
||||
db::load_securities(channel.symbols, [&](const api::Security& security) {
|
||||
with_security(securitytype_of(security), num_legs(security), [&](FixedSecurity& fs) {
|
||||
convert_security(security, fs, true);
|
||||
listener->add_security(fs);
|
||||
});
|
||||
});
|
||||
|
||||
handler.registerListener(*listener);
|
||||
handler.registerMarketDataListener(*listener);
|
||||
handler.registerSecurityListener(*listener);
|
||||
|
||||
Reference in new issue
Block a user