This is a prerequisite to synchronizing security ids stored via IDGenerator (the "new" way to store ids, added when we implemented Deribit), and SecurityDB (the original way from spark days). The problem was that these 2 implementations were using separate databases. We need to unify the interfaces between these storage services to avoid breaking existing code.
234 lines
7.8 KiB
C++
234 lines
7.8 KiB
C++
#include "SecurityDB.h"
|
|
#include "lib/logging/logging.h"
|
|
#ifndef _WIN32
|
|
|
|
#include <SQLiteCpp/SQLiteCpp.h>
|
|
#include <sqlite/sqlite3.h>
|
|
|
|
#include "lib/config/config.h"
|
|
#include "lib/config/protos/market_data.pb.h"
|
|
#include "lib/config/protos/system_config.pb.h"
|
|
#include "lib/domain_model/Security.h"
|
|
#include "lib/md/Shm/FixedSecurity.h"
|
|
|
|
#include <boost/container/flat_set.hpp>
|
|
#include <boost/date_time/gregorian/gregorian.hpp>
|
|
#include <boost/date_time/posix_time/posix_time.hpp>
|
|
#include <boost/filesystem.hpp>
|
|
#include <string>
|
|
|
|
using std::string;
|
|
using namespace bts::utils;
|
|
using namespace SQLite;
|
|
|
|
namespace bts {
|
|
namespace md {
|
|
namespace db {
|
|
|
|
/*
|
|
* Schema:
|
|
* key, created_date, expiration_date
|
|
*/
|
|
|
|
std::unique_ptr<SQLite::Database> open_db() {
|
|
// Directory location on disk
|
|
auto db_dir =
|
|
config::get<config::SystemConfig>().data_path() + "databases/md/";
|
|
auto db_path = db_dir + "/securities.sqlite";
|
|
if (!boost::filesystem::exists(db_dir)) {
|
|
boost::filesystem::create_directories(db_dir);
|
|
}
|
|
|
|
// Create table if it does not exist
|
|
auto db = std::make_unique<Database>(db_path, OPEN_READWRITE | OPEN_CREATE);
|
|
db->setBusyTimeout(9999);
|
|
db->exec("PRAGMA journal_mode=WAL");
|
|
db->exec("PRAGMA syncronous=0");
|
|
db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, "
|
|
"symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT "
|
|
"NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);");
|
|
db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, "
|
|
"data BLOB NOT NULL);");
|
|
|
|
return db;
|
|
}
|
|
|
|
datetime::Date sec_expiration_cut_off() {
|
|
boost::gregorian::date_duration max_exp_age(
|
|
config::get<SecurityExpirationCutoff>().max_days_after_expiration());
|
|
|
|
auto pt = (datetime::now() - max_exp_age);
|
|
return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day());
|
|
}
|
|
|
|
size_t load_securities(
|
|
std::function<bool(const UserSymbol &symbo)> const &sym_filter,
|
|
const std::function<void(const api::Security &)> &callback) {
|
|
|
|
size_t count = 0;
|
|
auto db = open_db();
|
|
|
|
// Load all existing securities and call the callback
|
|
SQLite::Statement select_stmt(
|
|
*db, "select key,symbol,venue,type,security_blob from securities where "
|
|
"expiration > :exp");
|
|
select_stmt.bind(":exp", sec_expiration_cut_off().to_short());
|
|
|
|
Security security;
|
|
while (select_stmt.tryExecuteStep() == SQLITE_ROW) {
|
|
security_key security_key = select_stmt.getColumn("key");
|
|
std::string symbol = select_stmt.getColumn("symbol");
|
|
api::Venue venue = (api::Venue)(int)select_stmt.getColumn("venue");
|
|
auto type = (api::SecurityType)(int)select_stmt.getColumn("type");
|
|
UserSymbol us(symbol, venue, type);
|
|
if (sym_filter(us)) {
|
|
auto blob = select_stmt.getColumn("security_blob");
|
|
if (security.ParseFromArray(blob, blob.getBytes())) {
|
|
bts_assert(security_key == security.id());
|
|
count++;
|
|
callback(security);
|
|
} else {
|
|
LOG(md, Warn, "Failed to parse security from database. id = {}",
|
|
security_key);
|
|
}
|
|
}
|
|
}
|
|
LOG(md, Info, "SecurityDB::load_securities count: {}", count);
|
|
return count;
|
|
}
|
|
|
|
size_t load_securities(
|
|
const std::set<UserSymbol> &symbols,
|
|
const std::function<void(const api::Security &)> &callback) {
|
|
std::string log_str = "";
|
|
for (const auto &s : symbols) {
|
|
log_str += s.symbol() + "-" + SecurityType_Name(s.type()) + "-" +
|
|
Venue_Name(s.exchange()) + ", ";
|
|
}
|
|
LOG(md, Info, "SecurityDB::load_securities size: {} symbols: {}",
|
|
symbols.size(), log_str);
|
|
return load_securities(
|
|
[&symbols](UserSymbol const &symbol) {
|
|
return symbols.find(symbol) != symbols.end();
|
|
},
|
|
callback);
|
|
}
|
|
|
|
SecurityPtr find_security(SecurityKey key) {
|
|
LOG(md, Info, "SecurityDB::find_security key: {}", key);
|
|
try {
|
|
auto db = open_db();
|
|
SQLite::Statement select(
|
|
*db, "select security_blob from securities where key = :key");
|
|
select.bind(":key", key.value());
|
|
if (select.tryExecuteStep() == SQLITE_ROW) {
|
|
auto result = std::make_shared<Security>();
|
|
auto blob = select.getColumn(0);
|
|
if (result->ParseFromArray(blob, blob.getBytes())) {
|
|
return result;
|
|
}
|
|
}
|
|
} catch (const SQLite::Exception &e) {
|
|
LOG(md, Error, "Error reading security database. {}",
|
|
std::string(e.what()));
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
Result insert_or_replace(const std::vector<SecurityPtr> &securities) {
|
|
std::string log_str = "";
|
|
for (const auto &s : securities) {
|
|
log_str += bts::securities::descriptor(*s) + ", ";
|
|
}
|
|
LOG(md, Info, "SecurityDB::insert_or_replace size: {} securities: {}",
|
|
securities.size(), log_str);
|
|
try {
|
|
auto db = open_db();
|
|
SQLite::Statement insert(
|
|
*db, "insert or replace into securities (key, symbol, venue, type, "
|
|
"security_blob, expiration) values (?,?,?,?,?,?)");
|
|
SQLite::Transaction transaction(*db);
|
|
std::string buffer;
|
|
for (const auto &psecurity : securities) {
|
|
if (psecurity && psecurity->SerializeToString(&buffer)) {
|
|
insert.bind(1, psecurity->id());
|
|
insert.bind(2, psecurity->symbol());
|
|
insert.bind(3, psecurity->exchange());
|
|
insert.bind(4, securitytype_of(*psecurity));
|
|
insert.bind(5, buffer);
|
|
auto greg_date = securities::expiration(*psecurity);
|
|
if (greg_date == boost::gregorian::date()) {
|
|
insert.bind(6, datetime::Date(9999, 1, 1).to_short());
|
|
} else {
|
|
ushort year = greg_date.year();
|
|
ushort month = greg_date.month();
|
|
ushort day = greg_date.day();
|
|
insert.bind(6, datetime::Date(year, month, day).to_short());
|
|
}
|
|
insert.exec();
|
|
insert.reset();
|
|
}
|
|
}
|
|
transaction.commit();
|
|
} catch (const SQLite::Exception &e) {
|
|
LOG(md, Warn, "Error writing to security db (retry pending): {}",
|
|
std::string(e.what()));
|
|
return Result::error;
|
|
}
|
|
return Result::ok;
|
|
}
|
|
|
|
std::vector<api::synthetic::Metadata> load_metadata() {
|
|
std::vector<api::synthetic::Metadata> values;
|
|
auto db = open_db();
|
|
SQLite::Statement select(*db, "select key,data from metadata");
|
|
while (select.tryExecuteStep() == SQLITE_ROW) {
|
|
uint32_t key = select.getColumn(0);
|
|
auto blob = select.getColumn(1);
|
|
api::synthetic::Metadata meta;
|
|
if (meta.ParseFromArray(blob, blob.getBytes())) {
|
|
bts_assert(key == meta.security_key());
|
|
values.push_back(meta);
|
|
} else {
|
|
LOG(md, Warn, "Failed to parse security metadata for security_key={}",
|
|
key);
|
|
}
|
|
}
|
|
return values;
|
|
}
|
|
bool store_metadata(const api::synthetic::Metadata &data) {
|
|
auto db = open_db();
|
|
SQLite::Statement insert(
|
|
*db, "insert or replace into metadata (key, data) values (?,?)");
|
|
|
|
insert.bind(1, data.security_key());
|
|
std::string buffer = data.SerializeAsString();
|
|
insert.bind(2, buffer);
|
|
auto result = insert.tryExecuteStep();
|
|
if (result != SQLITE_DONE) {
|
|
LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
#else
|
|
namespace bts {
|
|
namespace md {
|
|
namespace db {
|
|
|
|
size_t load_securities(
|
|
const std::set<UserSymbol> &symbols,
|
|
const std::function<void(const api::Security &)> &callback) {
|
|
return 0;
|
|
}
|
|
Result insert_or_replace(const std::vector<SecurityPtr> &handles) {
|
|
return Result::error;
|
|
}
|
|
SecurityPtr find_security(SecurityKey) { return nullptr; }
|
|
|
|
#endif
|
|
}
|
|
}
|
|
} // namespace
|