#include "SecurityDB.h" #ifndef WIN32 #include #include #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" #include "lib/md/Shm/FixedSecurity.h" #include "lib/config/protos/MarketData.pb.h" #include "lib/domain_model/Security.h" #include #include #include #include #include using std::string; using namespace bts::utils; using namespace SQLite; namespace bts { namespace md { namespace db { /* * Schema: * key, created_date, expiration_date */ std::unique_ptr open_db() { // Directory location on disk auto db_dir = config::get().data_path() + "databases/md/"; auto db_path = db_dir + "/securities.sqlite"; if (!boost::filesystem::exists(db_dir)) { boost::filesystem::create_directories(db_dir); } // Create table if it does not exist auto db = std::make_unique(db_path, OPEN_READWRITE | OPEN_CREATE); db->setBusyTimeout(9999); db->exec("PRAGMA journal_mode=WAL"); db->exec("PRAGMA syncronous=0"); db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); return db; } datetime::Date sec_expiration_cut_off() { boost::gregorian::date_duration max_exp_age (config::get().max_days_after_expiration()); auto pt = (datetime::now() - max_exp_age); return datetime::Date(pt.date().year(),pt.date().month(),pt.date().day()); } void apply_compatibilit_mutations(Security& security) { // Migrate from old tick stuff if (security.has_glbx()) { if (security.has_glbx()) { if (security.contract_spec().main_fraction() == 0) { security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); } if (security.contract_spec().sub_fraction() == 0) { security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); } if (security.contract_spec().price_display_format() == 0) { security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); } } } } size_t load_securities( const std::set& symbols, const std::function& callback) { size_t count = 0; auto db = open_db(); // Load all existing securities and call the callback SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp"); select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); Security security; while (select_stmt.tryExecuteStep() == SQLITE_ROW) { security_key security_key = select_stmt.getColumn("key"); std::string symbol = select_stmt.getColumn("symbol"); api::Venue venue = (api::Venue)(int) select_stmt.getColumn("venue"); auto type = (api::SecurityType)(int) select_stmt.getColumn("type"); UserSymbol us(symbol, venue, type); if (symbols.find(us) != symbols.end()) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); } else { LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); } } } return count; } SecurityPtr find_security(SecurityKey key) { try { auto db = open_db(); SQLite::Statement select(*db, "select security_blob from securities where key = :key"); select.bind(":key", key.value()); if(select.tryExecuteStep() == SQLITE_ROW) { auto result = std::make_shared(); auto blob = select.getColumn(0); if(result->ParseFromArray(blob, blob.getBytes())) { apply_compatibilit_mutations(*result); return result; } } } catch(const SQLite::Exception& e) { LOG(md,Error, "Error reading security database. {}", std::string(e.what())); } return nullptr; } Result insert_or_replace(const std::vector& securities) { try { auto db = open_db(); SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); SQLite::Transaction transaction(*db); std::string buffer; for (const auto& psecurity : securities) { if (psecurity && psecurity->SerializeToString(&buffer)) { insert.bind(1, psecurity->id()); insert.bind(2, psecurity->symbol()); insert.bind(3, psecurity->exchange()); insert.bind(4, securitytype_of(*psecurity)); insert.bind(5, buffer); auto greg_date = securities::expiration(*psecurity); if (greg_date == boost::gregorian::date()) { insert.bind(6, datetime::Date(9999,1,1).to_short()); } else { ushort year = greg_date.year(); ushort month = greg_date.month(); ushort day = greg_date.day(); insert.bind(6, datetime::Date(year,month,day).to_short()); } insert.exec(); insert.reset(); } } transaction.commit(); } catch(const SQLite::Exception& e) { LOG(md,Warn, "Error writing to security db (retry pending): {}", std::string(e.what())); return Result::error; } return Result::ok; } std::vector load_metadata() { std::vector values; auto db = open_db(); SQLite::Statement select(*db, "select key,data from metadata"); while(select.tryExecuteStep() == SQLITE_ROW) { uint32_t key = select.getColumn(0); auto blob = select.getColumn(1); api::synthetic::Metadata meta; if (meta.ParseFromArray(blob, blob.getBytes())) { bts_assert(key == meta.security_key()); values.push_back(meta); } else { LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); } } return values; } bool store_metadata(const api::synthetic::Metadata& data) { auto db = open_db(); SQLite::Statement insert(*db, "insert or replace into metadata (key, data) values (?,?)"); insert.bind(1, data.security_key()); std::string buffer = data.SerializeAsString(); insert.bind(2, buffer); auto result = insert.tryExecuteStep(); if (result != SQLITE_DONE) { LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); return false; } return true; } #else namespace bts { namespace md { namespace db { size_t load_securities( const std::set& symbols, const std::function& callback) { return 0; } Result insert_or_replace(const std::vector& handles) { return Result::error; } SecurityPtr find_security(SecurityKey) { return nullptr; } #endif }}}//namespace