#include "lib/domain_model/Security.h" #ifndef WIN32 #include "SecurityIDRecorder.h" #include "lib/config/config.h" #include "lib/logging/logging.h" #include "lib/utils/Sqlite_utils.h" #include #include #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system.pb.h" #include #include using namespace std; using namespace bts::utils; using std::size_t; namespace bts { namespace md { uint64_t exchange_id(const api::Security &security); struct SecurityIDEntry { api::Security security; SecurityIDRecorder::Callback callback; }; struct SecurityIDRecorderState { SecurityIDRecorderState(SecurityIDRecorderParams &p) : work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(), lookup_key(p.lookup_key), unresolved_legs() {} tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; std::thread::id thread_id; sqlite3_ptr db; unique_ptr timer; KeyLookupOption lookup_key; std::map> unresolved_legs; // Oh no, I got methods in my state void run(utils::Dispatcher &dispatcher) { timer = std::make_unique( dispatcher, 200, [=]() { do_work(); }); timer->start(); } void do_work() { thread_id = std::this_thread::get_id(); work_spreads(); work_handles(); work_db(); } void work_spreads(); void work_handles(); void work_db(); void check_access() { bts_assert(thread_id == std::this_thread::get_id()); } void assign_key( const api::Security &security, const SecurityIDRecorder::Callback &key_assigned); void add_unresolved_spread(MDProducerHandle &handle); security_key search(const std::string &descriptor); security_key insert(const std::string &descriptor, SecurityType); }; SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params) : _self(new SecurityIDRecorderState(params)) { if (params.db) { _self->db = std::move(params.db); } init(); if (params.dispatcher) { _self->run(*params.dispatcher); } } SecurityIDRecorder::~SecurityIDRecorder() {} void SecurityIDRecorder::init() { LOG(md, Info, "SecurityIDRecorder::init"); string db_dir = config::get().data_path() + "databases/md/"; string db_path = db_dir + "/security_keys.sqlite"; if (!boost::filesystem::exists(db_dir)) { boost::filesystem::create_directories(db_dir); } if (!_self->db) { _self->db = sqlite::open_db(db_path.c_str()); bts_assert(_self->db); } // Set a long busy timeout sqlite3_busy_timeout(_self->db.get(), 9999); sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0"); sqlite::exec_nofail( _self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING " "NOT NULL PRIMARY KEY, type int);"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}", std::string(handle.security().security_desc()), handle.security().key, handle.security().numeric_id); _self->pending_handle.push(handle); } void SecurityIDRecorder::assign_key( const api::Security &security, const SecurityIDRecorder::Callback &callback) { LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}", security.DebugString()); if (security.Contract_case() == Security::ContractCase::kSpread) { for (const auto &leg : security.spread().legs()) { if (leg.security().id() == invalid_security_key) { LOG(md, Error, "Security {} has legs which will never be resolved.", security.has_glbx() ? security.glbx().security_desc() : security.symbol()); // No point in trying to add this to the queue return; } } } _self->assign_key(security, callback); } void SecurityIDRecorderState::assign_key( const api::Security &security, const SecurityIDRecorder::Callback &callback) { LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}", bts::securities::descriptor(security)); if (security.id() == invalid_security_key) { auto work = std::make_shared(); work->security = security; work->callback = callback; LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}", bts::securities::descriptor(work->security)); this->work.push(work); } } void SecurityIDRecorder::do_work() { _self->do_work(); } void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle &handle) { bts_assert_d(lookup_key); FixedSpreadSecurityXL *pspread = static_cast(handle.mutable_security()); if (pspread != nullptr) { for (size_t x = 0; x < pspread->leg_count(); ++x) { auto &leg = pspread->legs[x]; bts_assert_d(leg.exchange_key.value() != 0); if (leg.key == invalid_security_key) { // Because leg is a reference to shared memory we know it will not // move for (const auto *existing : unresolved_legs[leg.exchange_key]) { if (existing == &leg) goto next; } unresolved_legs[leg.exchange_key].push_back(&leg); } next:; } } } void SecurityIDRecorderState::work_spreads() { check_access(); if (!lookup_key) return; KeyLookup &lookup = *lookup_key; int count = 0; for (auto iter = unresolved_legs.begin(); count < 500 && iter != unresolved_legs.end();) { const auto &pair = *iter; ++count; SecurityKey key = lookup(pair.first); if (key.value() != invalid_security_key) { for (FixedSpreadLeg *leg : pair.second) { leg->key = key.value(); } iter = unresolved_legs.erase(iter); } else { ++iter; } } } bool has_all_leg_keys(const MDProducerHandle &handle) { if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) { const FixedSpreadSecurityXL *pspread = static_cast(&handle.security()); for (size_t x = 0; x < pspread->leg_count(); ++x) { if (pspread->legs[x].key == invalid_security_key) { return false; } } } // SyntheticFixedSecurity is guaranteed to have all of its legs return true; } void SecurityIDRecorderState::work_handles() { check_access(); Security security; MDProducerHandle handle; vector retry; while (pending_handle.try_pop(handle)) { security.Clear(); // if a spread does not have all of its legs we need to wait if (!has_all_leg_keys(handle)) { // add add_unresolved_spread(handle); retry.push_back(handle); } else { convert_security(handle.security(), &security); assign_key(security, [handle](const api::Security &security) mutable { handle.set_key(security.id()); }); } } for (const auto &h : retry) { pending_handle.push(h); } } void SecurityIDRecorderState::work_db() { check_access(); // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list // Then persist that list // Once a security has a key it is public so record it in the security // database std::shared_ptr temp; vector> new_entries; // SPARK-1446 Need to try to add existing securities to the db in case we // deleted them by hand vector> existing_securities; vector> defer; vector new_descriptors; size_t count = 0; string descriptor; while (work.try_pop(temp)) { descriptor = security_descriptor(temp->security); // If we're unable to create a descriptor try again later if (descriptor == "") { defer.push_back(std::move(temp)); continue; } security_key key = search(descriptor); if (key != invalid_security_key) { temp->security.set_id(key); temp->callback(temp->security); existing_securities.push_back(std::move(temp)); } else { // always push in parallel new_entries.push_back(std::move(temp)); new_descriptors.push_back(descriptor); ++count; // Limit the amount of time we spend writing to sqlite during startup // to avoid blocking the heartbeat if (count > 200) { break; } } } for (auto &d : defer) { LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}", bts::securities::descriptor(d->security)); work.push(std::move(d)); } if (!new_entries.empty()) { sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security &security = new_entries[x]->security; security_key key = insert(new_descriptors[x], securitytype_of(security)); bts_assert_d(key); security.set_id(key); } sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } if (!new_entries.empty()) { for (size_t x = 0; x < new_entries.size(); ++x) { Security &security = new_entries[x]->security; new_entries[x]->callback(security); } } } void SecurityIDRecorder::bulk_insert( const std::vector &data, const char *prefix) { LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size()); if (data.size() == 0) return; sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); std::string descriptor; { auto insert_stmt = sqlite::prepare_query( _self->db, "INSERT INTO security_keys (rowid, descriptor, type) " "values (?, ?, ?)"); for (const api::Security *sec : data) { descriptor = prefix + security_descriptor(*sec); auto type = securitytype_of(*sec); sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); sqlite3_bind_text( insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 3, (int)type); sqlite3_step(insert_stmt.get()); sqlite3_reset(insert_stmt.get()); } } sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } security_key SecurityIDRecorderState::search(const std::string &descriptor) { LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor); auto select_stmt = sqlite::prepare_query( db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); sqlite3_bind_text( select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_column_int(select_stmt.get(), 0); } return invalid_security_key; } security_key SecurityIDRecorderState::insert( const std::string &descriptor, SecurityType type) { LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor); auto insert_stmt = sqlite::prepare_query( db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); sqlite3_bind_text( insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 2, (int)type); int result = sqlite3_step(insert_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_last_insert_rowid(db.get()); } // Just in case our insert failed due to duplicate row auto key = search(descriptor); if (key == invalid_security_key) { LOG(md, Error, "Failed to assign security key. SQLite return code: {}", result); } return key; } } // namespace md } // namespace bts #endif