#include "SecurityDB.h" #include "lib/logging/logging.h" #ifndef _WIN32 #include #include #include "lib/config/config.h" #include "lib/config/protos/market_data.pb.h" #include "lib/config/protos/system.pb.h" #include "lib/domain_model/Security.h" #include "lib/md/Shm/FixedSecurity.h" #include #include #include #include #include using std::string; using namespace bts::utils; using namespace SQLite; namespace bts { namespace md { namespace db { /* * Schema: * key, created_date, expiration_date */ std::unique_ptr open_db() { // Directory location on disk auto db_dir = config::get().data_path() + "databases/md/"; auto db_path = db_dir + "/securities.sqlite"; if (!boost::filesystem::exists(db_dir)) { boost::filesystem::create_directories(db_dir); } // Create table if it does not exist auto db = std::make_unique(db_path, OPEN_READWRITE | OPEN_CREATE); db->setBusyTimeout(9999); db->exec("PRAGMA journal_mode=WAL"); db->exec("PRAGMA syncronous=0"); db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, " "symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT " "NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, " "data BLOB NOT NULL);"); return db; } datetime::Date sec_expiration_cut_off() { boost::gregorian::date_duration max_exp_age( config::get().max_days_after_expiration()); auto pt = (datetime::now() - max_exp_age); return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day()); } void apply_compatibilit_mutations(Security &security) { // Migrate from old tick stuff if (security.has_glbx()) { if (security.has_glbx()) { if (security.contract_spec().main_fraction() == 0) { security.mutable_contract_spec()->set_main_fraction( security.glbx().do_not_use_main_fraction()); } if (security.contract_spec().sub_fraction() == 0) { security.mutable_contract_spec()->set_sub_fraction( security.glbx().do_not_use_sub_fraction()); } if (security.contract_spec().price_display_format() == 0) { security.mutable_contract_spec()->set_price_display_format( security.glbx().do_not_use_price_display_format()); } } } } size_t load_securities( std::function const &sym_filter, const std::function &callback) { size_t count = 0; auto db = open_db(); // Load all existing securities and call the callback SQLite::Statement select_stmt( *db, "select key,symbol,venue,type,security_blob from securities where " "expiration > :exp"); select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); Security security; while (select_stmt.tryExecuteStep() == SQLITE_ROW) { security_key security_key = select_stmt.getColumn("key"); std::string symbol = select_stmt.getColumn("symbol"); api::Venue venue = (api::Venue)(int)select_stmt.getColumn("venue"); auto type = (api::SecurityType)(int)select_stmt.getColumn("type"); UserSymbol us(symbol, venue, type); if (sym_filter(us)) { auto blob = select_stmt.getColumn("security_blob"); if (security.ParseFromArray(blob, blob.getBytes())) { apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); } else { LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); } } } LOG(md, Info, "SecurityDB::load_securities count: {}", count); return count; } size_t load_securities( const std::set &symbols, const std::function &callback) { std::string log_str = ""; for (const auto &s : symbols) { log_str += s.symbol() + "-" + SecurityType_Name(s.type()) + "-" + Venue_Name(s.exchange()) + ", "; } LOG(md, Info, "SecurityDB::load_securities size: {} symbols: {}", symbols.size(), log_str); return load_securities( [&symbols](UserSymbol const &symbol) { return symbols.find(symbol) != symbols.end(); }, callback); } SecurityPtr find_security(SecurityKey key) { LOG(md, Info, "SecurityDB::find_security key: {}", key); try { auto db = open_db(); SQLite::Statement select( *db, "select security_blob from securities where key = :key"); select.bind(":key", key.value()); if (select.tryExecuteStep() == SQLITE_ROW) { auto result = std::make_shared(); auto blob = select.getColumn(0); if (result->ParseFromArray(blob, blob.getBytes())) { apply_compatibilit_mutations(*result); return result; } } } catch (const SQLite::Exception &e) { LOG(md, Error, "Error reading security database. {}", std::string(e.what())); } return nullptr; } Result insert_or_replace(const std::vector &securities) { std::string log_str = ""; for (const auto &s : securities) { log_str += bts::securities::descriptor(*s) + ", "; } LOG(md, Info, "SecurityDB::insert_or_replace size: {} securities: {}", securities.size(), log_str); try { auto db = open_db(); SQLite::Statement insert( *db, "insert or replace into securities (key, symbol, venue, type, " "security_blob, expiration) values (?,?,?,?,?,?)"); SQLite::Transaction transaction(*db); std::string buffer; for (const auto &psecurity : securities) { if (psecurity && psecurity->SerializeToString(&buffer)) { insert.bind(1, psecurity->id()); insert.bind(2, psecurity->symbol()); insert.bind(3, psecurity->exchange()); insert.bind(4, securitytype_of(*psecurity)); insert.bind(5, buffer); auto greg_date = securities::expiration(*psecurity); if (greg_date == boost::gregorian::date()) { insert.bind(6, datetime::Date(9999, 1, 1).to_short()); } else { ushort year = greg_date.year(); ushort month = greg_date.month(); ushort day = greg_date.day(); insert.bind(6, datetime::Date(year, month, day).to_short()); } insert.exec(); insert.reset(); } } transaction.commit(); } catch (const SQLite::Exception &e) { LOG(md, Warn, "Error writing to security db (retry pending): {}", std::string(e.what())); return Result::error; } return Result::ok; } std::vector load_metadata() { std::vector values; auto db = open_db(); SQLite::Statement select(*db, "select key,data from metadata"); while (select.tryExecuteStep() == SQLITE_ROW) { uint32_t key = select.getColumn(0); auto blob = select.getColumn(1); api::synthetic::Metadata meta; if (meta.ParseFromArray(blob, blob.getBytes())) { bts_assert(key == meta.security_key()); values.push_back(meta); } else { LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); } } return values; } bool store_metadata(const api::synthetic::Metadata &data) { auto db = open_db(); SQLite::Statement insert( *db, "insert or replace into metadata (key, data) values (?,?)"); insert.bind(1, data.security_key()); std::string buffer = data.SerializeAsString(); insert.bind(2, buffer); auto result = insert.tryExecuteStep(); if (result != SQLITE_DONE) { LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); return false; } return true; } #else namespace bts { namespace md { namespace db { size_t load_securities( const std::set &symbols, const std::function &callback) { return 0; } Result insert_or_replace(const std::vector &handles) { return Result::error; } SecurityPtr find_security(SecurityKey) { return nullptr; } #endif } } } // namespace