From 2fc4c8bd6061c0f13393afe57039ca784c91574a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 14 Nov 2018 19:08:57 -0500 Subject: [PATCH 01/25] Move sqlite_utils.h out of utils:: namespace --- src/lib/md/SecurityDB.cpp | 23 ++++++++++++----------- src/lib/md/SettlementRecorder.cpp | 15 ++++++++------- src/lib/persistence/SqliteStore.cpp | 27 ++++++++++++++------------- src/lib/utils/Sqlite_utils.cpp | 17 +++++++++++------ src/lib/utils/Sqlite_utils.h | 15 +++++++-------- 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index d84fbe4d4..64b33cf11 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -34,16 +34,17 @@ sqlite3_ptr open_db() { } // Create table if it does not exist - auto dbptr = utils::open_db(db_path.c_str()); + auto dbptr = sqlite::open_db(db_path.c_str()); + bts_assert(dbptr); auto db = dbptr.get(); sqlite3_busy_timeout(db, 9999); - sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF"); - sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(db, "PRAGMA journal_mode=OFF"); + sqlite::exec_nofail(db, "PRAGMA syncronous=0"); - sqlite3_exec_nofail(db, + sqlite::exec_nofail(db, "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); - sqlite3_exec_nofail(db, + sqlite::exec_nofail(db, "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); return dbptr; @@ -67,7 +68,7 @@ size_t load_securities( // Load all existing securities and call the callback auto db = dbptr.get(); - auto select_stmt = prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); + auto select_stmt = sqlite::prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); auto select = select_stmt.get(); sqlite3_bind_int(select, 1, sec_expiration_cut_off().to_short()); auto result = sqlite3_step(select); @@ -112,8 +113,8 @@ size_t load_securities( void store_if_new(const std::vector& securities) { auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION"); + auto insert = sqlite::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + sqlite::exec_nofail(dbptr, "BEGIN TRANSACTION"); std::string buffer; for (const api::Security* psecurity : securities) { if (psecurity && psecurity->SerializeToString(&buffer)) { @@ -127,7 +128,7 @@ void store_if_new(const std::vector& securities) { sqlite3_reset(insert.get()); } } - sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION"); + sqlite::exec_nofail(dbptr, "COMMIT TRANSACTION"); } @@ -135,7 +136,7 @@ std::vector load_metadata() { std::vector values; auto dbptr = open_db(); auto db = dbptr.get(); - auto select_stmt = prepare_query(db, "select key,data from metadata"); + auto select_stmt = sqlite::prepare_query(db, "select key,data from metadata"); auto select = select_stmt.get(); auto result = sqlite3_step(select); while(result == SQLITE_ROW) { @@ -156,7 +157,7 @@ std::vector load_metadata() { } bool store_metadata(const api::synthetic::Metadata& data) { auto dbptr = open_db(); - auto insert = utils::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); + auto insert = sqlite::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); std::string buffer = data.SerializeAsString(); diff --git a/src/lib/md/SettlementRecorder.cpp b/src/lib/md/SettlementRecorder.cpp index dfa1eb549..0d7680bbd 100644 --- a/src/lib/md/SettlementRecorder.cpp +++ b/src/lib/md/SettlementRecorder.cpp @@ -94,13 +94,14 @@ SettlementRecorder::SettlementRecorder(Platform platform) : } // initialize database - _self->db = open_db(db_path.c_str()); + _self->db = sqlite::open_db(db_path.c_str()); + bts_assert_msg(_self->db, db_path.c_str()); sqlite3* db = _self->db.get(); sqlite3_busy_timeout(db, 9999); - sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF"); - sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(db, "PRAGMA journal_mode=OFF"); + sqlite::exec_nofail(db, "PRAGMA syncronous=0"); // key=(security_key+date) - sqlite3_exec_nofail(db, + sqlite::exec_nofail(db, "CREATE TABLE IF NOT EXISTS settlements2 (key INTEGER NOT NULL, date INTEGER NOT NULL, mantissa INTEGER NOT NULL, exponent INTEGER NOT NULL, PRIMARY KEY (key,date));"); // Delete any records > 10 days old @@ -108,11 +109,11 @@ SettlementRecorder::SettlementRecorder(Platform platform) : auto encoded = datetime::ll_to_date(datetime::ptime_to_ll(cutoff)).to_short(); char query[256]; sprintf(query, "delete from settlements2 where date < %i", encoded); - sqlite3_exec_nofail(db, query); + sqlite::exec_nofail(db, query); // statements - _self->select_stmt = prepare_query(db, "select mantissa,exponent,date from settlements2 where key=? order by date desc limit 2"); - _self->insert_stmt = prepare_query(db, "insert or replace into settlements2(key, date, mantissa,exponent) values(?,?,?,?)"); + _self->select_stmt = sqlite::prepare_query(db, "select mantissa,exponent,date from settlements2 where key=? order by date desc limit 2"); + _self->insert_stmt = sqlite::prepare_query(db, "insert or replace into settlements2(key, date, mantissa,exponent) values(?,?,?,?)"); } SettlementRecorder::~SettlementRecorder() { } diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index 814ae95f1..34c91ecd2 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -57,42 +57,43 @@ void SqliteStore::init_db() { LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", me->db, path.string()); - me->conn = utils::open_db(path.c_str()); + me->conn = sqlite::open_db(path.c_str()); + bts_assert_msg(me->conn, path.c_str()); const char *create_schema_sql = "create table store ( ns text, key text, value text)"; - utils::sqlite3_exec_nofail(me->conn, create_schema_sql); + sqlite::exec_nofail(me->conn, create_schema_sql); const char *key_index_sql = "create unique index key_index on store(ns,key)"; - utils::sqlite3_exec_nofail(me->conn, key_index_sql); + sqlite::exec_nofail(me->conn, key_index_sql); const char *ns_index_sql = "create index ns_index on store(ns)"; - utils::sqlite3_exec_nofail(me->conn, ns_index_sql); + sqlite::exec_nofail(me->conn, ns_index_sql); } else { - me->conn = utils::open_db(path.c_str()); + me->conn = sqlite::open_db(path.c_str()); } // Asynchronous for speed - utils::sqlite3_exec_nofail(me->conn, "PRAGMA synchronous = 0;"); - utils::sqlite3_exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;"); + sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); // prepare insert statement const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; - me->insert = utils::prepare_query(me->conn, insert_sql); + me->insert = sqlite::prepare_query(me->conn, insert_sql); // prepare update statement const char *update_sql = "update store set value = ? where ns = ? and key = ?"; - me->update = utils::prepare_query(me->conn, update_sql); + me->update = sqlite::prepare_query(me->conn, update_sql); // prepare remove statement const char *remove_all_sql = "delete from store where ns = ?"; - me->remove_all = utils::prepare_query(me->conn, remove_all_sql); + me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql); // prepare remove statement const char *remove_sql = "delete from store where ns = ? and key = ?"; - me->remove = utils::prepare_query(me->conn, remove_sql); + me->remove = sqlite::prepare_query(me->conn, remove_sql); } SqliteStore::~SqliteStore() {} @@ -100,7 +101,7 @@ SqliteStore::~SqliteStore() {} SqliteStore::Data SqliteStore::retrieve_all() { SqliteStore::Data result; sqlite3_stmt_ptr query = - utils::prepare_query(me->conn, "select ns,key,value from store"); + sqlite::prepare_query(me->conn, "select ns,key,value from store"); int sqlite_result; do { @@ -125,7 +126,7 @@ std::pair SqliteStore::retrieve_one(std::string const &ns, std::string const &k) { auto query_str = fmt::format( "select ns,key,value from store where ns='{}' and key='{}'", ns, k); - sqlite3_stmt_ptr query = utils::prepare_query(me->conn, query_str.c_str()); + sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str()); std::pair result{false, ""}; diff --git a/src/lib/utils/Sqlite_utils.cpp b/src/lib/utils/Sqlite_utils.cpp index 4d31926a3..f8cc0a013 100644 --- a/src/lib/utils/Sqlite_utils.cpp +++ b/src/lib/utils/Sqlite_utils.cpp @@ -3,22 +3,27 @@ #include "bts_assert.h" #include namespace bts { -namespace utils { +namespace sqlite { -void sqlite3_exec_nofail(sqlite3* db, const char* sql) { +void exec_nofail(sqlite3* db, const char* sql) { char* error_message = nullptr; int result = sqlite3_exec(db, sql, nullptr, nullptr, &error_message); bts_assert_msg(result == SQLITE_OK, error_message); } -void sqlite3_exec_nofail(sqlite3_ptr & db, const char * sql) { - return sqlite3_exec_nofail(db.get(), sql); +void exec_nofail(sqlite3_ptr & db, const char * sql) { + return exec_nofail(db.get(), sql); } sqlite3_ptr open_db(const char* path) { sqlite3* temp; - bts_assert(sqlite3_open(path, &temp) == SQLITE_OK); - return sqlite3_ptr(temp, sqlite3_close); + if (sqlite3_open(path, &temp) == SQLITE_OK) { + return sqlite3_ptr(temp, sqlite3_close); + } + else { + LOG(Gen, Error, "Failed to open database {}", std::string(path)); + return null_db(); + } } sqlite3_stmt_ptr prepare_query(sqlite3 * db, const char * query) { diff --git a/src/lib/utils/Sqlite_utils.h b/src/lib/utils/Sqlite_utils.h index 4cb8e6ae3..1bc3cdbb4 100644 --- a/src/lib/utils/Sqlite_utils.h +++ b/src/lib/utils/Sqlite_utils.h @@ -1,12 +1,9 @@ #pragma once #ifndef WIN32 -#include #include -#include "lib/utils/bts_export.h" +#include #include "lib/utils/bts_assert.h" -typedef struct sqlite3 sqlite3; -typedef struct sqlite3_stmt sqlite3_stmt; typedef std::unique_ptr sqlite3_ptr; typedef std::unique_ptr sqlite3_stmt_ptr; @@ -18,13 +15,15 @@ typedef std::unique_ptr sqlite3_stmt_ptr; } \ } while(0) -namespace bts { -namespace utils { -void sqlite3_exec_nofail(sqlite3* db, const char* sql); -void sqlite3_exec_nofail(sqlite3_ptr& db, const char* sql); +namespace bts { +namespace sqlite { + +void exec_nofail(sqlite3* db, const char* sql); +void exec_nofail(sqlite3_ptr& db, const char* sql); sqlite3_ptr open_db(const char* path); +inline sqlite3_ptr null_db() { return sqlite3_ptr(nullptr, sqlite3_close); } sqlite3_stmt_ptr prepare_query(sqlite3* db, const char* query); sqlite3_stmt_ptr prepare_query(sqlite3_ptr& db, const char* query); }}//namespace From 4b3cdf9ca4b640e07fed52d9b4f37182dec18e01 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 14 Nov 2018 19:58:31 -0500 Subject: [PATCH 02/25] Add tests for SecurityIDRecorder += SpreadResolver --- src/lib/md/SecurityIDRecorder.cpp | 44 ++++++----- src/lib/md/SecurityIDRecorder.h | 18 +++-- src/lib/md/Shm/FixedSecurity.cpp | 14 +++- src/lib/md/Shm/FixedSecurity.h | 2 + src/lib/md/SyntheticSecurityService.cpp | 2 +- src/lib/md/relay/RelayRecorder.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 78 +++++++++++++++++-- .../sparkd/md/platforms/celoxica/CXMain.cpp | 2 +- 8 files changed, 127 insertions(+), 35 deletions(-) diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 192ca68e2..d98fc948e 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -7,7 +7,7 @@ #include "lib/logging/logging.h" #include "lib/md/SecurityDB.h" #include -#include // for reservation +#include #include "lib/config/protos/MarketData.pb.h" #include "lib/config/protos/system.pb.h" @@ -22,6 +22,8 @@ using std::size_t; namespace bts { namespace md { +uint64_t exchange_id(const api::Security& security); + struct SecurityIDEntry { api::Security security; SecurityIDRecorder::Callback callback; @@ -58,17 +60,16 @@ struct SecurityIDRecorderState { security_key insert(const std::string& descriptor, SecurityType); }; -SecurityIDRecorder::SecurityIDRecorder(): +SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params): _self(new SecurityIDRecorderState()) { + if (params.db) { + _self->db = std::move(params.db); + } init(); -} - -SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher): - _self(new SecurityIDRecorderState()) -{ - init(); - _self->run(dispatcher); + if (params.dispatcher) { + _self->run(*params.dispatcher); + } } SecurityIDRecorder::~SecurityIDRecorder() { } @@ -81,13 +82,16 @@ void SecurityIDRecorder::init() { boost::filesystem::create_directories(db_dir); } - _self->db = open_db(db_path.c_str()); + if (!_self->db) { + _self->db = sqlite::open_db(db_path.c_str()); + bts_assert(_self->db); + } // Set a long busy timeout sqlite3_busy_timeout(_self->db.get(), 9999); - sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); - sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); - sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); + sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); + sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); + sqlite::exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { @@ -196,7 +200,7 @@ void SecurityIDRecorderState::work_db() { if (!new_entries.empty()) { - sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION"); + sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; @@ -205,7 +209,7 @@ void SecurityIDRecorderState::work_db() { security.set_id(key); } - sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION"); + sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } // record the securities @@ -231,10 +235,10 @@ void SecurityIDRecorderState::work_db() { void SecurityIDRecorder::bulk_insert(const std::vector& data, const char* prefix) { if (data.size() == 0) return; - sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); + sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); std::string descriptor; { - auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); + auto insert_stmt = sqlite::prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); for (const api::Security* sec : data) { descriptor = prefix + security_descriptor(*sec); auto type = securitytype_of(*sec); @@ -246,11 +250,11 @@ void SecurityIDRecorder::bulk_insert(const std::vector& da sqlite3_reset(insert_stmt.get()); } } - sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } security_key SecurityIDRecorderState::search(const std::string& descriptor) { - auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); + auto select_stmt = sqlite::prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { @@ -260,7 +264,7 @@ security_key SecurityIDRecorderState::search(const std::string& descriptor) { } security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) { - auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); + auto insert_stmt = sqlite::prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 2, (int)type); int result = sqlite3_step(insert_stmt.get()); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index f63731ff1..659f5a74a 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -3,11 +3,13 @@ #include "common.h" #include "MDProducerHandle.h" +#include "lib/utils/Sqlite_utils.h" #include #include #include #include -#include + +typedef struct sqlite3 sqlite3; namespace bts { namespace md { @@ -17,13 +19,19 @@ namespace md { * Provides a callback when ids are safely persisted. */ +struct SecurityIDRecorderParams { + SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {} + SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {} + SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {} + utils::Dispatcher* dispatcher; + sqlite3_ptr db; +}; struct SecurityIDRecorderState; + class SecurityIDRecorder { public: - SecurityIDRecorder(); - // Automatically call do_work on this dispatcher - SecurityIDRecorder(utils::Dispatcher&); + SecurityIDRecorder(SecurityIDRecorderParams); ~SecurityIDRecorder(); typedef std::function Callback; @@ -32,7 +40,7 @@ public: // Slow (copies security) - for use by services void assign_key(const api::Security& security, const Callback& key_assigned); - // Someone must call work periodically to do lookups and flush the reserved ids to disk. + // If a dispatcher is not provided you must call work periodically to do lookups and flush the reserved ids to disk. void do_work(); void bulk_insert(const std::vector&, const char* prefix); diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index cc8d304ea..a0405bdfe 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -250,6 +250,16 @@ size_t num_legs(const Security& sec) { } return 0; } +uint64_t exchange_id(const Security& security) { + switch(security.ExchangeData_case()) { + case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id(); + case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id(); + case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id(); + case Security::ExchangeDataCase::kIce: bts_assert(false); + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false); + } + return -1; +} void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { dest.key = set_security_key ? src.id() : invalid_security_key; @@ -272,7 +282,9 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit case Security::ExchangeDataCase::kCfe: dest.cboe_class_key = src.cfe().class_key(); break; - default: + case Security::ExchangeDataCase::kIce: + bts_assert_d(false); + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: break; } dest.price_break = src.contract_spec().price_break() / src.contract_spec().price_multiplier(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index b8fd82d55..80d44b287 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -180,6 +180,8 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit SecurityType securitytype_of_contractcase(Security::ContractCase cc); SecurityType securitytype_of(const Security&); size_t num_legs(const Security&); +uint64_t exchange_id(const Security& security); + // Stack allocates a security of the given type template diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index 8d6ca60aa..eeea9168b 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -25,7 +25,7 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher& fn) : - recorder(utils::Dispatcher::background()), + recorder({utils::Dispatcher::background()}), synthetics(), on_security(), meta(), on_metadata(), metadata_publisher(fn) { diff --git a/src/lib/md/relay/RelayRecorder.cpp b/src/lib/md/relay/RelayRecorder.cpp index 55ca64edb..a6f453e73 100644 --- a/src/lib/md/relay/RelayRecorder.cpp +++ b/src/lib/md/relay/RelayRecorder.cpp @@ -130,7 +130,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { CoconutReplayHeader header; size_t count = 0; #ifndef _WIN32 - md::SecurityIDRecorder idrec; + md::SecurityIDRecorder idrec {{}}; #endif if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) { co2::MDReply message; diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 9a3301311..ba81b8413 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -13,6 +13,7 @@ #include "lib/metrics/metrics.h" #include "lib/config/protos/MarketData.pb.h" +#include "lib/utils/Sqlite_utils.h" #include #include @@ -27,7 +28,6 @@ #include #include - using namespace std; namespace bts { @@ -151,6 +151,7 @@ void assign_ids(vector::iterator begin, vector(); + auto hot_header = std::make_shared(); + + // Create a spread with 2 legs + auto f1 = mkfuture("FO1", invalid_security_key, 2018, 1, 17); + f1.mutable_glbx()->set_exchange_id(100); + REQUIRE(f1.id() == invalid_security_key); + auto f1handle = std::make_unique (f1, cold_header, hot_header); + + auto f2 = mkfuture("FO2", invalid_security_key, 2018, 1, 20); + f2.mutable_glbx()->set_exchange_id(200); + auto f2handle = std::make_unique (f2, cold_header, hot_header); + + auto s1 = mkspread("FOO", invalid_security_key, { f1, f2 }); + auto s1handle = std::make_unique (s1, cold_header, hot_header); + + // Add spread to SecurityIDRecorder and work it + auto db = sqlite::open_db(":memory:"); + REQUIRE(db); + SecurityIDRecorder idsvc{{db}}; + auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); }; + // auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } }; + idsvc.ensure_key(s1handle->producer()); + workwork(); + + // Expect no key + REQUIRE(s1handle->consumer().key() == invalid_security_key); + + // Add leg spreads + idsvc.ensure_key(f1handle->producer()); + idsvc.ensure_key(f2handle->producer()); + workwork(); + + // Expect legs to have keys + REQUIRE(f1handle->consumer().key() != invalid_security_key); + REQUIRE(f2handle->consumer().key() != invalid_security_key); + + // Expect spread to have keys + REQUIRE(s1handle->consumer().key() != invalid_security_key); } #endif @@ -1002,7 +1064,11 @@ TEST_CASE("CO2Tests","[md]") { #ifndef WIN32 SECTION("test_security_id_recorder") { - co2_tests->test_security_id_recorder(); + co2_tests->test_security_id_recorder(":memory:"); + co2_tests->test_security_id_recorder("file::memory:?cache=shared"); + } + SECTION("test_security_id_recorder_spreads") { + co2_tests->test_security_id_recorder_spreads(); } #endif SECTION("test_price_size") { diff --git a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp index bfb96e727..1c39dcd72 100644 --- a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp +++ b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp @@ -43,7 +43,7 @@ void run_cx_server(UserSymbol* psyms, size_t symcount) CXMain::CXMain() : _producer(co2::PLATFORM_CELOXICA), - _security_recorder(), + _security_recorder({}), _settlement_recorder(co2::PLATFORM_CELOXICA), _listeners() { From 396ac023c44dd218627a6e8ac74124138b61c52e Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Thu, 15 Nov 2018 13:33:41 -0500 Subject: [PATCH 03/25] adding field to proto --- src/lib/api/services/AdminService.proto | 6 ++++++ src/web/ui/src/Access.fs | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/lib/api/services/AdminService.proto b/src/lib/api/services/AdminService.proto index 3e13152f1..a52585403 100644 --- a/src/lib/api/services/AdminService.proto +++ b/src/lib/api/services/AdminService.proto @@ -141,6 +141,12 @@ message BTSAccount{ AccountSecurityClipLimits security_clip_limits = 90; AccountExchangeConnections connections = 100; ExchangeAccountData exchange_account_data = 110; + // extra parameters for risk enforcement + RiskLimitEnforcement risk_limit_enforcement=120; +} + +message RiskLimitEnforcement{ + bool product_limit_in_net_terms = 10; } message AccountProductPositionLimitUnit{ diff --git a/src/web/ui/src/Access.fs b/src/web/ui/src/Access.fs index 9d3ba6411..b795990a3 100644 --- a/src/web/ui/src/Access.fs +++ b/src/web/ui/src/Access.fs @@ -102,6 +102,9 @@ module Accounts = clearingAccount = "" } } + riskLimitEnforcement={ + productLimitInNetTerms=true + } } From 500e5d508a57466f30a8244f816b79ab7e9d0463 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 15 Nov 2018 14:51:09 -0500 Subject: [PATCH 04/25] SPARK-1879: riskLimitEnforcement radio buttons --- src/web/ui/src/Access.fs | 6 +++--- src/web/ui/src/Accounts/State.fs | 2 ++ src/web/ui/src/Accounts/Types.fs | 1 + src/web/ui/src/Accounts/View.fs | 8 ++++++-- src/web/ui/src/Controls/Fields.fs | 20 ++++++++++++++++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/web/ui/src/Access.fs b/src/web/ui/src/Access.fs index b795990a3..4241744d2 100644 --- a/src/web/ui/src/Access.fs +++ b/src/web/ui/src/Access.fs @@ -102,9 +102,9 @@ module Accounts = clearingAccount = "" } } - riskLimitEnforcement={ - productLimitInNetTerms=true - } + riskLimitEnforcement = + { productLimitInNetTerms = true + } } diff --git a/src/web/ui/src/Accounts/State.fs b/src/web/ui/src/Accounts/State.fs index e1d11ae06..5a7580c37 100644 --- a/src/web/ui/src/Accounts/State.fs +++ b/src/web/ui/src/Accounts/State.fs @@ -269,6 +269,8 @@ let update msg (model:Model) : Model * Cmd = | DeleteProductLimit i -> let limits = erase model.scratch.productPositionLimits i { model with scratch = { model.scratch with productPositionLimits = limits } }, Cmd.Empty + | EnforceProductLimitInNetTerms value -> + {model with scratch = {model.scratch with riskLimitEnforcement = {productLimitInNetTerms = value}}}, Cmd.none | SecurityLimits limits -> { model with scratch = { model.scratch with securityPositionLimits = limits } }, Cmd.Empty | DeleteSecurityLimit i -> diff --git a/src/web/ui/src/Accounts/Types.fs b/src/web/ui/src/Accounts/Types.fs index 402f009e8..5e240f483 100644 --- a/src/web/ui/src/Accounts/Types.fs +++ b/src/web/ui/src/Accounts/Types.fs @@ -69,6 +69,7 @@ type Msg = | ProductLimits of AccountProductPositionLimitUnit * AccountProductPositionLimitUnit | AddProductLimit of AccountProductPositionLimitUnit | DeleteProductLimit of int + | EnforceProductLimitInNetTerms of bool | SecurityLimits of AccountSecurityPositionLimitUnit[] | DeleteSecurityLimit of int diff --git a/src/web/ui/src/Accounts/View.fs b/src/web/ui/src/Accounts/View.fs index ed74b8a34..5118a4b0f 100644 --- a/src/web/ui/src/Accounts/View.fs +++ b/src/web/ui/src/Accounts/View.fs @@ -512,6 +512,10 @@ let productLimits model dispatch = div [ ClassName "column is-size-7 has-text-grey-mid is-help-text" ] [str "This limits the net position for a product (i.e. accumulated position over all securities belonging to a product). This can be set for one product, or for all products within a security type by selecting \"All\" for symbol. (Note that \"All\" applies the same limits to each symbol, not the combiled position of all symbols)"] ] + div [][ + Fields.radioButton "Use Net Contracts" "" model.scratch.riskLimitEnforcement.productLimitInNetTerms [] (EnforceProductLimitInNetTerms >> dispatch) + Fields.radioButton "Use Absolute Contracts" "" (not model.scratch.riskLimitEnforcement.productLimitInNetTerms) [] (not >> EnforceProductLimitInNetTerms >> dispatch) + ] TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow) br [] ] @@ -527,8 +531,8 @@ let securityLimitsView model dispatch = TableControl.Cell (str "Security Type") TableControl.Cell (str "Symbol") TableControl.Cell (str "Security") - TableControl.Cell (str "Max Net Long Position") - TableControl.Cell (str "Max Net Short Position") + TableControl.Cell (str "Max Long Position") + TableControl.Cell (str "Max Short Position") ] className = "" header = None diff --git a/src/web/ui/src/Controls/Fields.fs b/src/web/ui/src/Controls/Fields.fs index 91961a3f6..a81122410 100644 --- a/src/web/ui/src/Controls/Fields.fs +++ b/src/web/ui/src/Controls/Fields.fs @@ -173,6 +173,26 @@ let checkbox (lbl:string) (description:string) (dv:bool) (attrs:IHTMLProp list) ] ] +let radioButton (lbl:string) (description:string) (dv:bool) (attrs:IHTMLProp list) (onchange:bool->unit) = + div [ ClassName "field" ] + [ + div [ ClassName "control" ] + [ + label [ ] [ + input ([ + ClassName "checkbox" + Type "radio" + Checked dv + OnChange (fun ev -> !!ev.target?``checked`` |> onchange) + ] @ attrs) + str (" " + lbl) + ] + nbsp.element + span [ClassName "is-size-7 has-text-grey-mid"][ str description] + ] + ] + + let mapSelect (lbl:string) (options:Map) (defaultText:string option) (defaultValue:string option) (onchange:'a->unit) = let optList:ReactElement list = match defaultText with From 29fbfd447912f2e686ad2b5261823ed3f2e7cd63 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 15 Nov 2018 16:35:14 -0500 Subject: [PATCH 05/25] SPARK-1877: product not security --- src/web/ui/src/Accounts/View.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/ui/src/Accounts/View.fs b/src/web/ui/src/Accounts/View.fs index 5118a4b0f..f0dfc9368 100644 --- a/src/web/ui/src/Accounts/View.fs +++ b/src/web/ui/src/Accounts/View.fs @@ -346,8 +346,8 @@ let productLimits model dispatch = TableControl.Cell (str "Exchange") TableControl.Cell (str "Security Type") TableControl.Cell (str "Symbol") - TableControl.Cell (str "Max Net Long Position") - TableControl.Cell (str "Max Net Short Position") + TableControl.Cell (str "Max Long Position") + TableControl.Cell (str "Max Short Position") ] className = "" header = None From 4a292c80303ca9c5ff17e94612383f53a5c8f53a Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Thu, 15 Nov 2018 16:21:13 -0500 Subject: [PATCH 06/25] limit enforcing logic added --- src/lib/orderman/limiter/LimiterUtils.h | 81 ++++++++++++++++--- src/lib/orderman/limiter/ProductLimitNode.cpp | 9 ++- src/lib/orderman/limiter/ProductLimitNode.h | 73 +++++++++++++---- src/lib/orderman/limiter/ProductLimitTree.cpp | 8 +- src/lib/orderman/limiter/SecurityLimitNode.h | 12 +-- 5 files changed, 147 insertions(+), 36 deletions(-) diff --git a/src/lib/orderman/limiter/LimiterUtils.h b/src/lib/orderman/limiter/LimiterUtils.h index 2b29cb9f2..d6f6b7068 100644 --- a/src/lib/orderman/limiter/LimiterUtils.h +++ b/src/lib/orderman/limiter/LimiterUtils.h @@ -9,10 +9,13 @@ namespace orderman{ class LimiterUtils{ public: - static void can_send_new_order(api::Side side, Quantity quantity, bool is_security_level, - int long_pos_limit, int short_pos_limit, - int net_pos, - Limiter::LimiterQueryResult& out_res){ + static void can_send_new_order_net(api::Side side, + Quantity quantity, + bool is_security_level, + int long_pos_limit, + int short_pos_limit, + int net_pos, + Limiter::LimiterQueryResult& out_res){ out_res.ok = true; bts_assert(long_pos_limit >= 0); bts_assert(short_pos_limit <= 0); @@ -35,13 +38,15 @@ public: else out_res.error_msg = fmt::format("Product Short Position (Max {} Current {})", short_pos_limit, net_pos); } - } - static void can_update_order (api::Side side, - Quantity from, Quantity to, - bool is_security_level, - int long_pos_limit, int short_pos_limit, - int net_pos, - Limiter::LimiterQueryResult& out_res){ + } + static void can_update_order_net (api::Side side, + Quantity from, + Quantity to, + bool is_security_level, + int long_pos_limit, + int short_pos_limit, + int net_pos, + Limiter::LimiterQueryResult& out_res){ out_res.ok = true; if (to.value() > from.value()){ int net_pos_new = 0; @@ -65,6 +70,60 @@ public: out_res.error_msg = fmt::format("Product Short Position (Max {} Current {})", short_pos_limit, net_pos); } } + } + static void can_send_new_order_abs(api::Side side, + Quantity quantity, + uint32_t long_pos_limit, + uint32_t short_pos_limit, + uint32_t long_pos, + uint32_t short_pos, + Limiter::LimiterQueryResult& out_res){ + out_res.ok = true; + if(side == api::Side::SIDE_BUY){ + if(long_pos + quantity.value() > long_pos_limit){ + out_res.error_msg = fmt::format("Long Position (Abs) " + "(Max {} Current {})", + long_pos_limit,long_pos); + out_res.ok = false; + } + } + else{ + if(short_pos + quantity.value() > short_pos_limit){ + out_res.error_msg = fmt::format("Short Position (Abs) " + "(Max {} Current {})", + long_pos_limit,long_pos); + out_res.ok = false; + } + } + } + static void can_update_order_abs (api::Side side, + Quantity from, + Quantity to, + uint32_t long_pos_limit, + uint32_t short_pos_limit, + uint32_t long_pos, + uint32_t short_pos, + Limiter::LimiterQueryResult& out_res){ + out_res.ok = true; + if (to.value() > from.value()){ + uint32_t diff = to.value() - from.value(); + if(side == api::Side::SIDE_BUY){ + if(long_pos + diff > long_pos_limit){ + out_res.error_msg = fmt::format("Long Position (Abs) " + "(Max {} Current {})", + long_pos_limit,long_pos); + out_res.ok = false; + } + } + else{ + if(short_pos + diff > short_pos_limit){ + out_res.error_msg = fmt::format("Short Position (Abs) " + "(Max {} Current {})", + long_pos_limit,long_pos); + out_res.ok = false; + } + } + } } static api::Side invert_side(api::Side side){ return side == api::SIDE_BUY ? api::SIDE_SELL : api::SIDE_BUY; diff --git a/src/lib/orderman/limiter/ProductLimitNode.cpp b/src/lib/orderman/limiter/ProductLimitNode.cpp index d35e3ce42..a6ee36784 100644 --- a/src/lib/orderman/limiter/ProductLimitNode.cpp +++ b/src/lib/orderman/limiter/ProductLimitNode.cpp @@ -5,14 +5,17 @@ namespace orderman{ ProductLimitNode::ProductLimitNode(uint32_t account_id, uint32_t, // parent_account_id, - const bts::api::admin::AccountProductPositionLimitUnit& limit) + const bts::api::admin::AccountProductPositionLimitUnit& limit, + bool enforce_net) : _account_id(account_id), //_parent_account_id(parent_account_id), _limit(limit), + _enforce_net(enforce_net), _long_pos_limit (limit.long_position_limit()), - _short_pos_limit (-1 * limit.short_position_limit()), - _net_pos(0) + _short_pos_limit (limit.short_position_limit()), + _long_pos(0), + _short_pos(0) {} }} diff --git a/src/lib/orderman/limiter/ProductLimitNode.h b/src/lib/orderman/limiter/ProductLimitNode.h index 0ed567adf..b6ce3d88d 100644 --- a/src/lib/orderman/limiter/ProductLimitNode.h +++ b/src/lib/orderman/limiter/ProductLimitNode.h @@ -20,49 +20,92 @@ class ProductLimitNode{ private: const uint32_t _account_id; // , _parent_account_id; const bts::api::admin::AccountProductPositionLimitUnit& _limit; - const int _long_pos_limit, _short_pos_limit; - int _net_pos; + const bool _enforce_net; + const uint32_t _long_pos_limit, _short_pos_limit; + // this is the only state here + uint32_t _long_pos, _short_pos; public: ProductLimitNode(uint32_t account_id, uint32_t parent_account_id, - const bts::api::admin::AccountProductPositionLimitUnit& limit); + const bts::api::admin::AccountProductPositionLimitUnit& limit, + bool enforce_net); // accessors //////////// uint32_t account_id() const {return _account_id;} - int net_pos() const {return _net_pos;} + int net_pos() const { + return static_cast(_long_pos)-static_cast(_short_pos); + } // position updates /////////////////// void copy_positions_from(const ProductLimitNode& source){ - _net_pos = source.net_pos(); + _long_pos=source._long_pos; + _short_pos=source._short_pos; } void add_position(api::Side side, uint32_t qty){ if (side==api::Side::SIDE_BUY) - _net_pos += qty; + _long_pos += qty; else - _net_pos -= qty; + _short_pos += qty; } void remove_position(api::Side side, uint32_t qty){ if (side==api::Side::SIDE_BUY) - _net_pos -= qty; + _long_pos = _subtract_unsigned(_long_pos, qty); else - _net_pos += qty; + _short_pos = _subtract_unsigned(_short_pos, qty); } // risk limit checks //////////////////// Limiter::LimiterQueryResult can_send_new_order(api::Side side, Quantity quantity) const{ Limiter::LimiterQueryResult result; - LimiterUtils::can_send_new_order (side, quantity, false, - _long_pos_limit, _short_pos_limit, - _net_pos, result); + if (_enforce_net) + LimiterUtils::can_send_new_order_net (side, + quantity, + false, + _long_pos_limit, + -1 * _short_pos_limit, + net_pos(), + result); + else + LimiterUtils::can_send_new_order_abs (side, + quantity, + _long_pos_limit, + _short_pos_limit, + _long_pos, + _short_pos, + result); return result; } Limiter::LimiterQueryResult can_update_order(api::Side side, Quantity from, Quantity to) const{ Limiter::LimiterQueryResult result; - LimiterUtils::can_update_order (side, from, to, false, - _long_pos_limit, _short_pos_limit, - _net_pos, result); + if(_enforce_net) + LimiterUtils::can_update_order_net (side, + from, + to, + false, + _long_pos_limit, + -1 * _short_pos_limit, + net_pos(), + result); + else + LimiterUtils::can_update_order_abs (side, + from, + to, + _long_pos_limit, + _short_pos_limit, + _long_pos, + _short_pos, + result); return result; } + uint32_t _subtract_unsigned(uint32_t n1, uint32_t n2){ + if(n1 >= n2) + return n1-n2; + else{ + LOG(OrderMan, Error, "ProductLimitNode._subtract_unsigned {0}-{1} ", + n1, n2); + return 0; + } + } // test only functions ////////////////////// const bts::api::admin::AccountProductPositionLimitUnit& test_only_func_limit() const {return _limit;} diff --git a/src/lib/orderman/limiter/ProductLimitTree.cpp b/src/lib/orderman/limiter/ProductLimitTree.cpp index 836ade264..bab765af6 100644 --- a/src/lib/orderman/limiter/ProductLimitTree.cpp +++ b/src/lib/orderman/limiter/ProductLimitTree.cpp @@ -10,8 +10,14 @@ ProductLimitTree::ProductLimitTree(const api::Product& product, _product (product) { for(const auto& account : admin_data.accounts()){ + bool enforce_net=true; // default is net + if(account.has_risk_limit_enforcement()) + enforce_net=account.risk_limit_enforcement().product_limit_in_net_terms(); const bts::api::admin::AccountProductPositionLimitUnit* limit = bts::admin::AdminServiceUtils::find_account_product_position_limit(account, _product); - _tree.emplace_back(account.id(), bts::admin::AdminServiceUtils::has_id(account.parent_account_id()) ? account.parent_account_id() : 0, limit != nullptr ? *limit : zero_limit); + _tree.emplace_back(account.id(), + bts::admin::AdminServiceUtils::has_id(account.parent_account_id()) ? account.parent_account_id() : 0, + limit != nullptr ? *limit : zero_limit, + enforce_net); } } }} \ No newline at end of file diff --git a/src/lib/orderman/limiter/SecurityLimitNode.h b/src/lib/orderman/limiter/SecurityLimitNode.h index 5cafd044e..230171aaa 100644 --- a/src/lib/orderman/limiter/SecurityLimitNode.h +++ b/src/lib/orderman/limiter/SecurityLimitNode.h @@ -71,9 +71,9 @@ public: return result; } if (_apply_sec_position_limits) - LimiterUtils::can_send_new_order (side, quantity, true, - _long_pos_limit, _short_pos_limit, - _net_pos, result); + LimiterUtils::can_send_new_order_net (side, quantity, true, + _long_pos_limit, _short_pos_limit, + _net_pos, result); if (result.ok) result = _product_node.can_send_new_order(side, quantity); return result; @@ -86,9 +86,9 @@ public: return result; } if (_apply_sec_position_limits) - LimiterUtils::can_update_order (side, from, to, true, - _long_pos_limit, _short_pos_limit, - _net_pos, result); + LimiterUtils::can_update_order_net (side, from, to, true, + _long_pos_limit, _short_pos_limit, + _net_pos, result); if (result.ok) result = _product_node.can_update_order(side, from, to); return result; From 6e4ed3f3280ffa328c92ef38409fe01a6090bcb1 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Thu, 15 Nov 2018 17:25:01 -0500 Subject: [PATCH 07/25] for the first time at start up, net pos is enforced --- src/lib/orderman/limiter/LimiterUtils.h | 4 ++-- src/server/sparkd/user/AdminServiceServer.cpp | 8 +++++++- src/server/sparkd/user/AdminServiceServer.h | 7 ++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/orderman/limiter/LimiterUtils.h b/src/lib/orderman/limiter/LimiterUtils.h index d6f6b7068..97bdc4fcc 100644 --- a/src/lib/orderman/limiter/LimiterUtils.h +++ b/src/lib/orderman/limiter/LimiterUtils.h @@ -91,7 +91,7 @@ public: if(short_pos + quantity.value() > short_pos_limit){ out_res.error_msg = fmt::format("Short Position (Abs) " "(Max {} Current {})", - long_pos_limit,long_pos); + short_pos_limit,short_pos); out_res.ok = false; } } @@ -119,7 +119,7 @@ public: if(short_pos + diff > short_pos_limit){ out_res.error_msg = fmt::format("Short Position (Abs) " "(Max {} Current {})", - long_pos_limit,long_pos); + short_pos_limit,short_pos); out_res.ok = false; } } diff --git a/src/server/sparkd/user/AdminServiceServer.cpp b/src/server/sparkd/user/AdminServiceServer.cpp index bb5c07fe1..b23803299 100644 --- a/src/server/sparkd/user/AdminServiceServer.cpp +++ b/src/server/sparkd/user/AdminServiceServer.cpp @@ -44,7 +44,7 @@ AdminServiceServer::AdminServiceServer(bts::api::ServiceContext& context) if (_state.ParseFromString(ser_data)) { data_found = true; - AdminServiceUtils::fill_connections_from_config(_state); + _check_data_on_start(); _update_and_publish(false); } } @@ -250,4 +250,10 @@ void AdminServiceServer::_publish_md_permissions(){ _md_perm_publisher.add_or_update(p); _md_perm_publisher.flush(); } +void AdminServiceServer::_check_data_on_start (){ + AdminServiceUtils::fill_connections_from_config(_state); + for(int i=0; i<_state.accounts().size(); i++) + if(!_state.mutable_accounts(i)->has_risk_limit_enforcement()) + _state.mutable_accounts(i)->mutable_risk_limit_enforcement()->set_product_limit_in_net_terms(true); +} }} diff --git a/src/server/sparkd/user/AdminServiceServer.h b/src/server/sparkd/user/AdminServiceServer.h index 32bbe126c..618a20a38 100644 --- a/src/server/sparkd/user/AdminServiceServer.h +++ b/src/server/sparkd/user/AdminServiceServer.h @@ -68,12 +68,17 @@ public: const api::Response& reply) override; private: + // this is mainly does 2 things : + // 1. some data is in partially in config, this copies + // that to here + // 2. for new data fields, it initializes them here + void _check_data_on_start(); void _update_and_publish (bool is_insert); void _publish_md_entitlements(); void _publish_md_permissions(); void _add_live_order(map>& agg, const OrderRecord& order); void _cancel_orders(const std::vector& accounts, - const std::vector& users); + const std::vector& users); }; }} From 25f749bb0c2b3561a20bebe85678825b66c730a6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 15 Nov 2018 12:48:07 -0500 Subject: [PATCH 08/25] Simplify md security key generation for spreads. --- .vscode/launch.json | 3 +- src/lib/md/MDProducerHandle.cpp | 12 --- src/lib/md/MDProducerHandle.h | 3 +- src/lib/md/SecurityIDRecorder.cpp | 76 ++++++++++++++++-- src/lib/md/SecurityIDRecorder.h | 12 ++- src/lib/md/SecurityIndex.h | 54 +------------ src/lib/md/Shm/FixedSecurity.cpp | 7 +- src/lib/md/Shm/FixedSecurity.h | 5 ++ src/lib/md/SpreadResolver.h | 79 ------------------- src/lib/md/test/md2_tests.cpp | 57 ++----------- .../md/platforms/mdp3_onixs/MDP3Listener.h | 6 +- .../md/platforms/mdp3_onixs/MDP3Main.cpp | 8 +- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 1 + 13 files changed, 108 insertions(+), 215 deletions(-) delete mode 100644 src/lib/md/SpreadResolver.h diff --git a/.vscode/launch.json b/.vscode/launch.json index 0eaae7c1f..163cad3eb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,8 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - // "args": ["test","Co2RelayTests"], - "args": ["md", "run", "10"], + "args": ["test","Co2Tests"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDProducerHandle.cpp b/src/lib/md/MDProducerHandle.cpp index 4e3cb476d..cd22abe83 100644 --- a/src/lib/md/MDProducerHandle.cpp +++ b/src/lib/md/MDProducerHandle.cpp @@ -61,18 +61,6 @@ void MDProducerHandle::set_key(security_key key) { } } -void MDProducerHandle::set_legs(const std::vector& legs) { - bts_assert(cdata().security.type == api::SECURITYTYPE_SPREAD); - FixedSpreadSecurityL* fss = (FixedSpreadSecurityL*)&cdata().security; - bts_assert(fss->leg_count() == legs.size()); - for (size_t x = 0; x < legs.size(); ++x) { - security_key key = legs[x]; - fss->legs[x].key = key; - // attempt to validate that this FixedSpreadLeg is valid - bts_assert_d(fss->legs[x].leg_quantity > 0); - } -} - void MDProducerHandle::update_bbo(TimeStamp time, const PriceSize bid, const PriceSize ask, uint16_t seq) { if (!has_depth()) { auto oldtime = hdata().time(); diff --git a/src/lib/md/MDProducerHandle.h b/src/lib/md/MDProducerHandle.h index 5f5e5f52b..2e21384da 100644 --- a/src/lib/md/MDProducerHandle.h +++ b/src/lib/md/MDProducerHandle.h @@ -23,7 +23,7 @@ public: MDProducerHandle& operator=(MDProducerHandle const& other); const FixedSecurity& security() const { return cdata().security; } - FixedSecurity* mutable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } + FixedSecurity* mootable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } bool has_depth() const { return cheader().has_depth; } void set_key(security_key); @@ -32,7 +32,6 @@ public: bts_assert_d(key == cdata().security.key); return key; } - void set_legs(const std::vector& legs); void update_bbo(TimeStamp time, const PriceSize bid_price, const PriceSize ask_price, uint16_t seq=0); void update_direct(TimeStamp time, const std::array& bids, const std::array& asks, uint32_t seq=0) { diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index d98fc948e..9302fa69d 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -30,38 +30,52 @@ struct SecurityIDEntry { }; struct SecurityIDRecorderState { - SecurityIDRecorderState() : + SecurityIDRecorderState(SecurityIDRecorderParams& p) : work(), pending_handle(), + thread_id(), db(nullptr, sqlite3_close), timer(), - id((size_t)this) + lookup_key(p.lookup_key), + unresolved_legs() {} + tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; + std::thread::id thread_id; sqlite3_ptr db; unique_ptr timer; - size_t id; + + KeyLookupOption lookup_key; + std::map> unresolved_legs; + + // Oh no, I got methods in my state void run(utils::Dispatcher& dispatcher) { timer = std::make_unique( - dispatcher, 500, + dispatcher, 200, [=]() { do_work(); }); timer->start(); } void do_work() { + thread_id = std::this_thread::get_id(); + work_spreads(); work_handles(); work_db(); } + void work_spreads(); void work_handles(); void work_db(); + + void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void assign_key(const api::Security& security, const SecurityIDRecorder::Callback& key_assigned); + void add_unresolved_spread(MDProducerHandle& handle); security_key search(const std::string& descriptor); security_key insert(const std::string& descriptor, SecurityType); }; SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params): - _self(new SecurityIDRecorderState()) + _self(new SecurityIDRecorderState(params)) { if (params.db) { _self->db = std::move(params.db); @@ -99,6 +113,15 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { } void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { + if (security.Contract_case() == Security::ContractCase::kSpread) { + for(const auto& leg : security.spread().legs()) { + if (leg.security().id() == invalid_security_key) { + LOG(md,Error, "Security {} has legs which will never be resolved.", security.has_glbx() ? security.glbx().security_desc() : security.symbol()); + // No point in trying to add this to the queue + return; + } + } + } _self->assign_key(security, callback); } @@ -115,6 +138,43 @@ void SecurityIDRecorder::do_work() { _self->do_work(); } +void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle& handle) { + bts_assert_d(lookup_key); + FixedSpreadSecurityXL* pspread = static_cast(handle.mootable_security()); + if (pspread != nullptr) { + for (size_t x=0; x < pspread->leg_count(); ++x) { + auto& leg = pspread->legs[x]; + bts_assert_d(leg.exchange_key.value() != 0); + if (leg.key == invalid_security_key) { + // Because leg is a reference to shared memory we know it will not move + for (const auto* existing : unresolved_legs[leg.exchange_key]) { + if (existing == &leg) goto next; + } + unresolved_legs[leg.exchange_key].push_back(&leg); + } + next:; + } + } +} + +void SecurityIDRecorderState::work_spreads() { + check_access(); + if (!lookup_key) return; + KeyLookup& lookup = *lookup_key; + int count = 0; + for (auto iter = unresolved_legs.begin(); count < 50 && iter != unresolved_legs.end(); iter++) { + const auto& pair = *iter; + ++count; + SecurityKey key = lookup(pair.first); + if (key.value() != invalid_security_key) { + for (FixedSpreadLeg* leg : pair.second) { + leg->key = key.value(); + } + unresolved_legs.erase(iter); + } + } +} + bool has_all_leg_keys(const MDProducerHandle& handle) { if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) { const FixedSpreadSecurityXL* pspread = static_cast(&handle.security()); @@ -124,11 +184,12 @@ bool has_all_leg_keys(const MDProducerHandle& handle) { } } } - // SyntheticFixedSecurity is guarunteed to have all of its legs + // SyntheticFixedSecurity is guaranteed to have all of its legs return true; } void SecurityIDRecorderState::work_handles() { + check_access(); Security security; MDProducerHandle handle; vector retry; @@ -137,6 +198,8 @@ void SecurityIDRecorderState::work_handles() { security.Clear(); // if a spread does not have all of its legs we need to wait if (!has_all_leg_keys(handle)) { + // add + add_unresolved_spread(handle); retry.push_back(handle); } else { @@ -154,6 +217,7 @@ void SecurityIDRecorderState::work_handles() { } void SecurityIDRecorderState::work_db() { + check_access(); // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 659f5a74a..19895d84c 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -8,6 +8,7 @@ #include #include #include +#include typedef struct sqlite3 sqlite3; @@ -19,12 +20,17 @@ namespace md { * Provides a callback when ids are safely persisted. */ +typedef const std::function KeyLookup; +typedef std::optional KeyLookupOption; struct SecurityIDRecorderParams { - SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {} - SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {} - SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {} + SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()), lookup_key() {} + SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()), lookup_key() {} + SecurityIDRecorderParams(utils::Dispatcher& d, KeyLookup& get_key): dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {} + SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)), lookup_key() {} + SecurityIDRecorderParams(sqlite3_ptr& db, KeyLookup& get_key): dispatcher(), db(std::move(db)), lookup_key(get_key) {} utils::Dispatcher* dispatcher; sqlite3_ptr db; + KeyLookupOption lookup_key; }; struct SecurityIDRecorderState; diff --git a/src/lib/md/SecurityIndex.h b/src/lib/md/SecurityIndex.h index 6ea7eea4b..1caea89c2 100644 --- a/src/lib/md/SecurityIndex.h +++ b/src/lib/md/SecurityIndex.h @@ -4,7 +4,6 @@ #include "lib/logging/logging.h" #include "MDProducerHandle.h" #include "common.h" -#include "SpreadResolver.h" #include #include #include @@ -26,8 +25,7 @@ class SimpleSecurityIndex { public: SimpleSecurityIndex(): - _map(), - _spread_resolver() + _map() {} ~SimpleSecurityIndex() {} @@ -79,35 +77,11 @@ public: return add_security(key, std::move(data)); } - // - // Spread resolve logic - - void add_spread_legs(ExchangeKey key, const std::vector& legs) { - _spread_resolver.add(key, legs); - } - - // Resolves spreads and returns the number spreads resolved - size_t resolve_spreads() { - return _spread_resolver.resolve( - [&](ExchangeKey key) { return get_key(key); }, - [&](ExchangeKey& key, std::vector& legs) { on_spread_resolved(key, legs); }); - } - -protected: - void on_spread_resolved(ExchangeKey& key, std::vector& legs) { - auto* entry = get_security(key); - bts_assert_d(entry); - if (entry) { - entry->handle.set_legs(legs); - } - } - private: BTS_NO_COPY(SimpleSecurityIndex); typedef std::unordered_map> MapType; MapType _map; mutable tbb::queuing_rw_mutex _mutex; - SpreadResolver _spread_resolver; }; @@ -120,8 +94,7 @@ class ConcurrentSecurityIndex { public: ConcurrentSecurityIndex() : - _map(), - _spread_resolver() + _map() {} ~ConcurrentSecurityIndex() {} @@ -171,33 +144,10 @@ public: return add_security(key, std::move(data)); } - // - // Spread resolve logic - - void add_spread_legs(ExchangeKey key, const std::vector& legs) { - _spread_resolver.add(key, legs); - } - - // Resolves spreads and returns the number spreads resolved - size_t resolve_spreads() { - return _spread_resolver.resolve( - [&](ExchangeKey key) { return get_key(key); }, - [&](ExchangeKey& key, std::vector& legs) { on_spread_resolved(key, legs); }); - } - -protected: - void on_spread_resolved(ExchangeKey& key, std::vector& legs) { - auto* entry = get_security(key); - bts_assert_d(entry); - if (entry) { - entry->handle.set_legs(legs); - } - } private: BTS_NO_COPY(ConcurrentSecurityIndex); typedef tbb::concurrent_hash_map MapType; MapType _map; - SpreadResolver _spread_resolver; }; }}//namespace diff --git a/src/lib/md/Shm/FixedSecurity.cpp b/src/lib/md/Shm/FixedSecurity.cpp index a0405bdfe..55b020b89 100644 --- a/src/lib/md/Shm/FixedSecurity.cpp +++ b/src/lib/md/Shm/FixedSecurity.cpp @@ -250,13 +250,17 @@ size_t num_legs(const Security& sec) { } return 0; } + uint64_t exchange_id(const Security& security) { switch(security.ExchangeData_case()) { case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id(); case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id(); case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id(); case Security::ExchangeDataCase::kIce: bts_assert(false); - case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false); + // Debatable whether this should be an error or not. + // If is possible that we might create a spread with no exchange in the future + // and if we did this would be a runtime error. + case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: return 0; } return -1; } @@ -319,6 +323,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit for (int x = 0; x < src.spread().legs_size(); ++x) { const auto& src_leg = src.spread().legs(x); spread.legs[x].key = set_security_key ? src_leg.security().id() : invalid_security_key; + spread.legs[x].exchange_key = ExchangeKey(exchange_id(src_leg.security())); spread.legs[x].leg_side = src_leg.side(); spread.legs[x].leg_quantity = src_leg.quantity(); spread.legs[x].leg_price = src_leg.price_multiplier(); diff --git a/src/lib/md/Shm/FixedSecurity.h b/src/lib/md/Shm/FixedSecurity.h index 80d44b287..0ad9e4ffa 100644 --- a/src/lib/md/Shm/FixedSecurity.h +++ b/src/lib/md/Shm/FixedSecurity.h @@ -6,6 +6,7 @@ #include "lib/utils/bts_array_size.h" #include "lib/utils/bts_assert.h" #include "lib/utils/bts_no_copy.h" +#include "lib/utils/ValueWrapper.h" #include #include @@ -13,6 +14,9 @@ namespace bts { namespace md { +using SecurityKey = utils::ValueWrapper; +using ExchangeKey = utils::ValueWrapper; + GCC_DIAG_OFF("-Weffc++") // No Virtual Destructors by design #define STRING_PROPERTY(NAME) const char* NAME() const { return _##NAME; } \ @@ -95,6 +99,7 @@ struct BTS_EXPORT FixedSpreadLeg { FixedSpreadLeg(); security_key key; + ExchangeKey exchange_key; Side leg_side; double leg_quantity; double leg_price; diff --git a/src/lib/md/SpreadResolver.h b/src/lib/md/SpreadResolver.h deleted file mode 100644 index 8885bb189..000000000 --- a/src/lib/md/SpreadResolver.h +++ /dev/null @@ -1,79 +0,0 @@ -#pragma once - -#include "lib/utils/bts_no_copy.h" -#include -#include - -namespace bts { -namespace md { - - -/* - * SpreadResolved accepts exchange spread definitions and resolves the legs - */ - -template -class SpreadResolver { - struct ResolverEntry { - ResolverEntry(ExchangeKey key) : exchange_key(key), legs() {} - ResolverEntry() : exchange_key(), legs() {} - ExchangeKey exchange_key; - std::vector legs; - }; -public: - SpreadResolver() : _queue() {} - template - void add(ExchangeKey key, LegContainer legs) { - ResolverEntry entry(key); - entry.legs.insert(end(entry.legs), begin(legs), end(legs)); - _queue.push(entry); - } - - // callback should accept const ExchangeKey& key, vector& legs) - // When the callback is called the legs will be completely resolved - template - size_t resolve(MapKey keymap, CallBack on_resolved) { - // Drain the queue and call keymap to resolve each leg - size_t result = 0; - std::vector unresolved; - std::vector legs; - while(!_queue.empty()) { - legs.clear(); - ResolverEntry entry; - if(_queue.try_pop(entry)) { - // Attempt to look up each spread leg - for(ExchangeKey key : entry.legs) { - security_key result = keymap(key); - if (result == invalid_security_key) { - unresolved.push_back(entry); - break; - } - else { - legs.push_back(result); - } - } - if (legs.size() == entry.legs.size()) { - ++result; - on_resolved(entry.exchange_key, legs); - } - } - } - - // If any legs remain unresolved push the entry back in the queue for later - for(auto& entry : unresolved) { - _queue.push(entry); - } - - return result; - } - - size_t count() const { return _queue.unsafe_size(); } -private: - BTS_NO_COPY(SpreadResolver); - tbb::concurrent_queue _queue; -}; - - - -}}//namespace - diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index ba81b8413..9da770f46 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -231,9 +231,15 @@ public: // Add spread to SecurityIDRecorder and work it auto db = sqlite::open_db(":memory:"); REQUIRE(db); - SecurityIDRecorder idsvc{{db}}; + + SecurityIDRecorder idsvc { {db, [&](ExchangeKey k) -> SecurityKey { + for (const auto fut : { f1handle->producer(), f2handle->producer() }) { + if (k.value() == fut.security().numeric_id) return SecurityKey(fut.key()); + } + return SecurityKey(invalid_security_key); + } } }; + auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); }; - // auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } }; idsvc.ensure_key(s1handle->producer()); workwork(); @@ -580,50 +586,6 @@ public: REQUIRE(bbo_count > 0); } - void test_spread_resolver() { - SpreadResolver resolver; - vector legs; - legs.push_back("1"); - legs.push_back("2"); - legs.push_back("3"); - legs.push_back("4"); - resolver.add("1234", legs); - legs.clear(); - legs.push_back("4"); - legs.push_back("5"); - resolver.add("45", legs); - auto key_map = [](string& key) -> security_key { - if (key == "1") return 1; - if (key == "2") return 2; - if (key == "3") return 3; - if (key == "4") return 4; - if (key == "5") return invalid_security_key; - REQUIRE(false); - return 0; - }; - auto key_map2 = [](string& key) -> security_key { - if (key == "4") return 4; - if (key == "5") return 5; - REQUIRE(false); - return 0; - }; - - // 2 unresolved spreads - REQUIRE(resolver.count() == 2); - resolver.resolve(key_map, [&](const string& key, const vector& legs) { - REQUIRE(key == "1234"); - REQUIRE(legs.size() == 4); - }); - // 1 spread remains - REQUIRE(resolver.count() == 1); - resolver.resolve(key_map2, [&](const string& key, const vector& legs) { - REQUIRE(key == "45"); - REQUIRE(legs.size() == 2); - }); - REQUIRE(resolver.count() == 0); - } - - void test_no_double_security_callback() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); @@ -1092,9 +1054,6 @@ TEST_CASE("CO2Tests","[md]") { SECTION("test_consumer_defer_handle2") { co2_tests->test_consumer_defer_handle2(); } - SECTION("test_spread_resolver") { - co2_tests->test_spread_resolver(); - } SECTION("test_no_double_security_callback") { co2_tests->test_no_double_security_callback(); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 1a54c9bd0..c29a1d01c 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -312,8 +312,6 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr FixedSpreadSecurityL fsec; set_security_common(sec, fsec); - std::vector leg_exchange_ids; - leg_exchange_ids.reserve(sec.legs().size()); onixs::Decimal min_tick = tick_rule(sec); @@ -341,10 +339,10 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr } fsec.legs[x].leg_quantity = leg.legRatioQty(); fsec.legs[x].leg_side = leg.legSide() == onixs::LegSide::BuySide ? api::SIDE_BUY : api::SIDE_SELL; - leg_exchange_ids.push_back(leg.legSecurityId()); + fsec.legs[x].exchange_key = ExchangeKey(leg.legSecurityId()); + fsec.legs[x].key = invalid_security_key; } add_security(fsec); - _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids); } template diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 679f92f3c..425a3e8d3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -26,7 +26,9 @@ namespace bts { namespace md { MDP3Main::MDP3Main() : - _producer(PLATFORM_MDP3ONIXS, { utils::Dispatcher::current() }), + _index(), + _recorder ({ utils::Dispatcher::current(), [=](ExchangeKey k) { return SecurityKey(_index.get_key(k.value())); } }), + _producer(PLATFORM_MDP3ONIXS, _recorder ), // _settlement_recorder(PLATFORM_MDP3ONIXS), _engines(), _handlers(), @@ -152,10 +154,6 @@ void MDP3Main::run(std::vector& symbols) { // Timers auto& disp = utils::Dispatcher::current(); - utils::DispatcherTimer spread_resolve_timer(disp, 1000, [&]() { - _index.resolve_spreads(); - }); - spread_resolve_timer.start(); // utils::DispatcherTimer settlement_timer(disp, 100, [&]() { // _settlement_recorder.do_work(); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index a14825fdc..4eb1fbf62 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -38,6 +38,7 @@ private: BTS_NO_COPY(MDP3Main); CMESecurityIndex _index; + SecurityIDRecorder _recorder; MDProducer _producer; // co2::SettlementRecorder _settlement_recorder; std::map> _engines; From 38668f8de14666acdbb4a715aff44570c764fc78 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 16 Nov 2018 13:09:05 -0500 Subject: [PATCH 09/25] Code review feedback --- src/lib/md/MDProducerHandle.h | 2 +- src/lib/md/SecurityIDRecorder.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/md/MDProducerHandle.h b/src/lib/md/MDProducerHandle.h index a846d6fdc..da5afcf2d 100644 --- a/src/lib/md/MDProducerHandle.h +++ b/src/lib/md/MDProducerHandle.h @@ -23,7 +23,7 @@ public: MDProducerHandle& operator=(MDProducerHandle const& other); const FixedSecurity& security() const { return cdata().security; } - FixedSecurity* mootable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } + FixedSecurity* mutable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } bool has_depth() const { return cheader().has_depth; } void set_key(security_key); diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 9302fa69d..bd1b82466 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -140,7 +140,7 @@ void SecurityIDRecorder::do_work() { void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle& handle) { bts_assert_d(lookup_key); - FixedSpreadSecurityXL* pspread = static_cast(handle.mootable_security()); + FixedSpreadSecurityXL* pspread = static_cast(handle.mutable_security()); if (pspread != nullptr) { for (size_t x=0; x < pspread->leg_count(); ++x) { auto& leg = pspread->legs[x]; From 3b371f3588a8e841bffcf77d37cf81ef67d48fdb Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 19 Nov 2018 13:54:48 +0000 Subject: [PATCH 10/25] Fixed bug in quickbus handling of broken connections. --- src/lib/quickbus/network/QBConnection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/quickbus/network/QBConnection.cpp b/src/lib/quickbus/network/QBConnection.cpp index 7debc1e47..c774be446 100644 --- a/src/lib/quickbus/network/QBConnection.cpp +++ b/src/lib/quickbus/network/QBConnection.cpp @@ -167,7 +167,7 @@ void QBConnection::recv_msg_body() handle_msg(&m); } else - recv_msg_body(); + close_connection(error.message()); }); } From 2cfbd655cbdef732ba351c23b11aba1a77ba57b9 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 19 Nov 2018 15:43:44 +0000 Subject: [PATCH 11/25] Trimmed account name from fill record. At some point in the past this was saved with spaces to the side of it, and that messes up matching with the account names. The implicit assumption is that we will never want account names with spaces at the end. --- src/lib/orderman/actors/order_manager/OrderManager.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/orderman/actors/order_manager/OrderManager.cpp b/src/lib/orderman/actors/order_manager/OrderManager.cpp index a827073ff..fe06f382d 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.cpp +++ b/src/lib/orderman/actors/order_manager/OrderManager.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace bts { namespace orderman { @@ -123,10 +124,11 @@ void OrderManager::on_fill_recovery( update.events->size()); me->state = State::WAIT_ORDERS; - for (auto const &ev : *(update.events)) - me->services_data.limiter.add(*ev.security, ev.fill.account(), - ev.fill.side(), + for (auto const &ev : *(update.events)) { + auto trimmed_account_name = boost::algorithm::trim_copy(ev.fill.account()); + me->services_data.limiter.add(*ev.security, trimmed_account_name,ev.fill.side(), Quantity(ev.fill.last_shares())); + } me->stored_order_sub = subscribe( [this](auto const &orders) { this->on_order_recovery(orders); }); From d9e967610ede1ca62a1a80a46ac7ce371c7baf66 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Mon, 19 Nov 2018 10:44:03 -0500 Subject: [PATCH 12/25] static casting uint to int before passing it to a function that takes in ints --- src/lib/orderman/limiter/ProductLimitNode.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/orderman/limiter/ProductLimitNode.h b/src/lib/orderman/limiter/ProductLimitNode.h index b6ce3d88d..c6f675454 100644 --- a/src/lib/orderman/limiter/ProductLimitNode.h +++ b/src/lib/orderman/limiter/ProductLimitNode.h @@ -61,8 +61,8 @@ public: LimiterUtils::can_send_new_order_net (side, quantity, false, - _long_pos_limit, - -1 * _short_pos_limit, + static_cast(_long_pos_limit), + -1 * static_cast(_short_pos_limit), net_pos(), result); else @@ -82,8 +82,8 @@ public: from, to, false, - _long_pos_limit, - -1 * _short_pos_limit, + static_cast(_long_pos_limit), + -1 * static_cast(_short_pos_limit), net_pos(), result); else From 8d9697d06de1fb3f9867d7601293fe44feb8d14a Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 19 Nov 2018 13:46:05 -0600 Subject: [PATCH 13/25] SPARK-1804 --- .../spark/view/Controls/UniformTabGrid.cs | 0 .../Controls/UniformTabGrid/UniformTabGrid.cs | 50 +++++++++++++++++++ .../spark/view/Shell/FloatingWindow.xaml | 18 +++++-- src/client/spark/view/Shell/TabDocument.xaml | 3 +- 4 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 src/client/spark/view/Controls/UniformTabGrid.cs create mode 100644 src/client/spark/view/Controls/UniformTabGrid/UniformTabGrid.cs diff --git a/src/client/spark/view/Controls/UniformTabGrid.cs b/src/client/spark/view/Controls/UniformTabGrid.cs new file mode 100644 index 000000000..e69de29bb diff --git a/src/client/spark/view/Controls/UniformTabGrid/UniformTabGrid.cs b/src/client/spark/view/Controls/UniformTabGrid/UniformTabGrid.cs new file mode 100644 index 000000000..017e16a45 --- /dev/null +++ b/src/client/spark/view/Controls/UniformTabGrid/UniformTabGrid.cs @@ -0,0 +1,50 @@ +using bts; +using mbts.spark.filters; +using mbts.ui.controls.adorners; +using System.Collections.Generic; +using System.Linq; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Controls.Primitives; + +namespace mbts.ui.controls { + public class UniformTabGrid : UniformGrid { + public UniformTabGrid () { + this.IsItemsHost = true; + this.Rows = 1; + } + + protected override Size MeasureOverride(Size constraint) { + var child_tabs = this.Children.OfType(); + var child_tabs_max_width = child_tabs.Sum(t => t.MaxWidth); + // I noticed when debugging that this can sometimes be infinty, + // especially when a TabItem has not been set with a MaxWidth. + if (!double.IsInfinity(child_tabs_max_width)) { + // if the window width is greater than the total tab width, then + // there is more than enough space for all the tabs. If this + // is set to the default of Stretch and the window has only 1 + // tab or few tabs, the tab appears in the middle, which is not good. + // i.e. we want tabs to added left to right, so align left + // + // When the window is less than the total tab width, then align + // can be set to stretch b/c the UniformGrid will take care of + // sizing each tab accordingly. + this.HorizontalAlignment = (constraint.Width > child_tabs_max_width) + ? HorizontalAlignment.Left + : HorizontalAlignment.Stretch; + + // this bit here is necessary because when the window is expanded + // beyond the maxwidth of all the tabs, we switch to Left and the + // headers snap to some smaller than MaxWidth size, which is not + // the behavior we want + foreach (var child in child_tabs) { + child.Width = this.HorizontalAlignment == HorizontalAlignment.Left + ? child.MaxWidth + : double.NaN; + } + } + + return base.MeasureOverride(constraint); + } + } +} \ No newline at end of file diff --git a/src/client/spark/view/Shell/FloatingWindow.xaml b/src/client/spark/view/Shell/FloatingWindow.xaml index a214e9073..ebf0ec5b8 100644 --- a/src/client/spark/view/Shell/FloatingWindow.xaml +++ b/src/client/spark/view/Shell/FloatingWindow.xaml @@ -5,6 +5,7 @@ xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:diag="clr-namespace:System.Diagnostics;assembly=WindowsBase" xmlns:ui="clr-namespace:mbts.ui" + xmlns:c="clr-namespace:mbts.ui.controls" mc:Ignorable="d" AllowDrop="True" Style="{StaticResource BTSFloatingWindow}" d:DesignHeight="300" d:DesignWidth="300" @@ -25,18 +26,25 @@ - + + - + Background="Transparent" + ui:WindowStyle.IsNonClientArea="True" + > + + - + +