#ifndef WIN32 #include "SecurityIDRecorder.h" #include "lib/utils/defer.h" #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" #include #include // for reservation #include "lib/config/protos/MarketData.pb.h" #include "lib/config/protos/system.pb.h" #include #include using namespace std; using namespace bts::utils; namespace bts { namespace md2 { //GCC_DIAG_OFF("-Weffc++") // Plain Struct struct SecurityIDTask { std::string descriptor; SecurityType type; SecurityIDRecorder::callback cb; void* arg; }; //GCC_DIAG_RESTORE() struct SecurityIDRecorderState { SecurityIDRecorderState() : work(), pending_handle(), db(nullptr, sqlite3_close), select_stmt(nullptr, sqlite3_finalize), insert_stmt(nullptr, sqlite3_finalize) {} tbb::concurrent_queue work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; sqlite3_stmt_ptr select_stmt; sqlite3_stmt_ptr insert_stmt; }; SecurityIDRecorder::SecurityIDRecorder(): _self(new SecurityIDRecorderState()) { init(); } SecurityIDRecorder::~SecurityIDRecorder() { } void SecurityIDRecorder::init() { string db_dir = config::get().data_path() + "/md/"; string db_path = db_dir + "/security_keys.sqlite"; if (!boost::filesystem::exists(db_dir)) { boost::filesystem::create_directories(db_dir); } _self->db = open_db(db_path.c_str()); // Set a long busy timeout sqlite3_busy_timeout(_self->db.get(), 9999); sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); // Prepare statements _self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); } void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, callback cb, void* arg) { SecurityIDTask task; task.descriptor = descriptor; task.type = type; task.cb = cb; task.arg = arg; _self->work.push(task); } void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, Callback cb) { Callback* pcb = new Callback(cb); get_key(descriptor, type, [](void* data, security_key key) { Callback* pcb = static_cast(data); DEFER(if (pcb) delete pcb); // exception safe (*pcb)(key); }, pcb); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } bool SecurityIDRecorder::do_work() { // Check if any handles need work first work_pending_handles(); // Drain the work queue // Query the database for each security. If it has an ID return it // If it does not have an ID add it to a list // Then persist that list SecurityIDTask temp; vector new_securities; size_t count = 0; bool incomplete = false; while (_self->work.try_pop(temp)) { security_key key = search(temp.descriptor); if (key != invalid_security_key) { temp.cb(temp.arg, key); } else { new_securities.push_back(temp); ++count; // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat if (count > 500) { incomplete = true; break; } } } if (new_securities.size() > 0) { sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); for (SecurityIDTask& security : new_securities) { security_key key = insert(security.descriptor, security.type); bts_assert_d(key); security.cb(security.arg, key); } sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } return incomplete; } void SecurityIDRecorder::work_pending_handles() { MDProducerHandle handle; vector defer; while(_self->pending_handle.try_pop(handle)) { // A spread will not have a descriptor until all of its leg keys are available string descriptor = security_descriptor(handle.security()); if (descriptor.empty()) { defer.push_back(handle); continue; } MDProducerHandle* phandle = new MDProducerHandle(handle); DEFER(if (phandle) delete phandle); get_key(descriptor, handle.security().type, [](void* arg, security_key key) { auto* phandle = static_cast(arg); if (phandle) { phandle->set_key(key); delete phandle; } }, phandle); phandle = nullptr; } for (auto& handle : defer) { _self->pending_handle.push(handle); } } security_key SecurityIDRecorder::search(const std::string& descriptor) { sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); DEFER(sqlite3_reset(_self->select_stmt.get())); int result = sqlite3_step(_self->select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_column_int(_self->select_stmt.get(), 0); } return invalid_security_key; } security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) { sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type); DEFER(sqlite3_reset(_self->insert_stmt.get())); int result = sqlite3_step(_self->insert_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_last_insert_rowid(_self->db.get()); } // Just in case our insert failed due to duplicate row auto key = search(descriptor); if (key == invalid_security_key) { LOG(MarketData,Error,"Failed to assign security key. SQLite return code: {}",result); } return key; } }}//namespace #endif