#ifndef WIN32 #include "SecurityIDRecorder.h" #include "lib/utils/defer.h" #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" #include "lib/md/SecurityDB.h" #include #include // for reservation #include "lib/config/protos/MarketData.pb.h" #include "lib/config/protos/system.pb.h" #include #include using namespace std; using namespace bts::utils; using std::size_t; namespace bts { namespace md { struct SecurityIDEntry { api::Security security; SecurityIDRecorder::Callback callback; }; struct SecurityIDRecorderState { SecurityIDRecorderState() : work(), pending_handle(), db(nullptr, sqlite3_close), select_stmt(nullptr, sqlite3_finalize), insert_stmt(nullptr, sqlite3_finalize) {} tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; sqlite3_stmt_ptr select_stmt; sqlite3_stmt_ptr insert_stmt; }; SecurityIDRecorder::SecurityIDRecorder(): _self(new SecurityIDRecorderState()) { init(); } SecurityIDRecorder::~SecurityIDRecorder() { } void SecurityIDRecorder::init() { string db_dir = config::get().data_path() + "databases/md/"; string db_path = db_dir + "/security_keys.sqlite"; if (!boost::filesystem::exists(db_dir)) { boost::filesystem::create_directories(db_dir); } _self->db = open_db(db_path.c_str()); // Set a long busy timeout sqlite3_busy_timeout(_self->db.get(), 9999); sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); // Prepare statements _self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { if (security.id() == invalid_security_key) { auto work = std::make_shared(); work->security = security; work->callback = callback; _self->work.push(work); } } void SecurityIDRecorder::do_work() { work_handles(); work_db(); } bool has_all_leg_keys(const MDProducerHandle& handle) { if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) { const FixedSpreadSecurityXL* pspread = static_cast(&handle.security()); for (size_t x = 0; x < pspread->leg_count(); ++x) { if (pspread->legs[x].key == invalid_security_key) { return false; } } } // SyntheticFixedSecurity is guarunteed to have all of its legs return true; } void SecurityIDRecorder::work_handles() { Security security; MDProducerHandle handle; vector retry; while (_self->pending_handle.try_pop(handle)) { security.Clear(); // if a spread does not have all of its legs we need to wait if (!has_all_leg_keys(handle)) { retry.push_back(handle); } else { convert_security(handle.security(), &security); assign_key(security, [handle](const api::Security& security) mutable { handle.set_key(security.id()); }); } } for(const auto& h : retry) { _self->pending_handle.push(h); } } void SecurityIDRecorder::work_db() { // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list // Then persist that list // Once a security has a key it is public so record it in the security database std::shared_ptr temp; vector> new_entries; vector> defer; vector new_descriptors; size_t count = 0; string descriptor; while (_self->work.try_pop(temp)) { descriptor = security_descriptor(temp->security); // If we're unable to create a descriptor try again later if (descriptor == "") { defer.push_back(move(temp)); continue; } security_key key = search(descriptor); if (key != invalid_security_key) { temp->security.set_id(key); temp->callback(temp->security); } else { // always push in parallel new_entries.push_back(move(temp)); new_descriptors.push_back(descriptor); ++count; // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat if (count > 500) { break; } } } for (auto& d : defer) { _self->work.push(move(d)); } if (new_entries.size() > 0) { sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; security_key key = insert(new_descriptors[x], securitytype_of(security)); bts_assert_d(key); security.set_id(key); } sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); // record the securities std::vector save_these; save_these.reserve(new_entries.size()); for (const auto& e : new_entries) { save_these.push_back(&e->security); } db::store_if_new(save_these); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; new_entries[x]->callback(security); } } } void SecurityIDRecorder::bulk_insert(const std::vector& data, const char* prefix) { if (data.size() == 0) return; sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); std::string descriptor; { auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)"); for (const api::Security* sec : data) { descriptor = prefix + security_descriptor(*sec); auto type = securitytype_of(*sec); sqlite3_bind_int(insert_stmt.get(), 1, sec->id()); sqlite3_bind_text(insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 3, (int)type); sqlite3_step(insert_stmt.get()); sqlite3_reset(insert_stmt.get()); } } sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } security_key SecurityIDRecorder::search(const std::string& descriptor) { sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); DEFER(sqlite3_reset(_self->select_stmt.get())); int result = sqlite3_step(_self->select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_column_int(_self->select_stmt.get(), 0); } return invalid_security_key; } security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) { sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type); DEFER(sqlite3_reset(_self->insert_stmt.get())); int result = sqlite3_step(_self->insert_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_last_insert_rowid(_self->db.get()); } // Just in case our insert failed due to duplicate row auto key = search(descriptor); if (key == invalid_security_key) { LOG(md,Error,"Failed to assign security key. SQLite return code: {}",result); } return key; } }}//namespace #endif