Files
spark/src/lib/md/SecurityDB.cpp
T

217 lines
7.5 KiB
C++

#include "SecurityDB.h"
#ifndef WIN32
#include <SQLiteCpp/SQLiteCpp.h>
#include <sqlite/sqlite3.h>
#include "lib/config/config.h"
#include "lib/config/protos/system.pb.h"
#include "lib/md/Shm/FixedSecurity.h"
#include "lib/config/protos/MarketData.pb.h"
#include "lib/domain_model/Security.h"
#include <boost/container/flat_set.hpp>
#include <boost/filesystem.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <string>
using std::string;
using namespace bts::utils;
using namespace SQLite;
namespace bts {
namespace md {
namespace db {
/*
* Schema:
* key, created_date, expiration_date
*/
std::unique_ptr<SQLite::Database> open_db() {
// Directory location on disk
auto db_dir = config::get<config::System>().data_path() + "databases/md/";
auto db_path = db_dir + "/securities.sqlite";
if (!boost::filesystem::exists(db_dir)) {
boost::filesystem::create_directories(db_dir);
}
// Create table if it does not exist
auto db = std::make_unique<Database>(db_path, OPEN_READWRITE | OPEN_CREATE);
db->setBusyTimeout(9999);
db->exec("PRAGMA journal_mode=WAL");
db->exec("PRAGMA syncronous=0");
db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);");
db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);");
return db;
}
datetime::Date sec_expiration_cut_off()
{
boost::gregorian::date_duration max_exp_age
(config::get<SecurityExpirationCutoff>().max_days_after_expiration());
auto pt = (datetime::now() - max_exp_age);
return datetime::Date(pt.date().year(),pt.date().month(),pt.date().day());
}
void apply_compatibilit_mutations(Security& security) {
// Migrate from old tick stuff
if (security.has_glbx()) {
if (security.has_glbx()) {
if (security.contract_spec().main_fraction() == 0) {
security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction());
}
if (security.contract_spec().sub_fraction() == 0) {
security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction());
}
if (security.contract_spec().price_display_format() == 0) {
security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format());
}
}
}
}
size_t load_securities(
const std::set<UserSymbol>& symbols,
const std::function<void(const api::Security&)>& callback) {
size_t count = 0;
auto db = open_db();
// Load all existing securities and call the callback
SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp");
select_stmt.bind(":exp", sec_expiration_cut_off().to_short());
Security security;
while (select_stmt.tryExecuteStep() == SQLITE_ROW) {
security_key security_key = select_stmt.getColumn("key");
std::string symbol = select_stmt.getColumn("symbol");
api::Venue venue = (api::Venue)(int) select_stmt.getColumn("venue");
auto type = (api::SecurityType)(int) select_stmt.getColumn("type");
UserSymbol us(symbol, venue, type);
if (symbols.find(us) != symbols.end()) {
auto blob = select_stmt.getColumn("security_blob");
if (security.ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(security);
bts_assert(security_key == security.id());
count++;
callback(security);
}
else {
LOG(md, Warn, "Failed to parse security from database. id = {}", security_key);
}
}
}
return count;
}
SecurityPtr find_security(SecurityKey key) {
try {
auto db = open_db();
SQLite::Statement select(*db, "select security_blob from securities where key = :key");
select.bind(":key", key.value());
if(select.tryExecuteStep() == SQLITE_ROW) {
auto result = std::make_shared<Security>();
auto blob = select.getColumn(0);
if(result->ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(*result);
return result;
}
}
}
catch(const SQLite::Exception& e) {
LOG(md,Error, "Error reading security database. {}", std::string(e.what()));
}
return nullptr;
}
Result insert_or_replace(const std::vector<SecurityPtr>& securities) {
try {
auto db = open_db();
SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
SQLite::Transaction transaction(*db);
std::string buffer;
for (const auto& psecurity : securities) {
if (psecurity && psecurity->SerializeToString(&buffer)) {
insert.bind(1, psecurity->id());
insert.bind(2, psecurity->symbol());
insert.bind(3, psecurity->exchange());
insert.bind(4, securitytype_of(*psecurity));
insert.bind(5, buffer);
auto greg_date = securities::expiration(*psecurity);
if (greg_date == boost::gregorian::date()) {
insert.bind(6, datetime::Date(9999,1,1).to_short());
} else {
ushort year = greg_date.year();
ushort month = greg_date.month();
ushort day = greg_date.day();
insert.bind(6, datetime::Date(year,month,day).to_short());
}
insert.exec();
insert.reset();
}
}
transaction.commit();
}
catch(const SQLite::Exception& e) {
LOG(md,Warn, "Error writing to security db (retry pending): {}", std::string(e.what()));
return Result::error;
}
return Result::ok;
}
std::vector<api::synthetic::Metadata> load_metadata() {
std::vector<api::synthetic::Metadata> values;
auto db = open_db();
SQLite::Statement select(*db, "select key,data from metadata");
while(select.tryExecuteStep() == SQLITE_ROW) {
uint32_t key = select.getColumn(0);
auto blob = select.getColumn(1);
api::synthetic::Metadata meta;
if (meta.ParseFromArray(blob, blob.getBytes())) {
bts_assert(key == meta.security_key());
values.push_back(meta);
}
else {
LOG(md, Warn, "Failed to parse security metadata for security_key={}", key);
}
}
return values;
}
bool store_metadata(const api::synthetic::Metadata& data) {
auto db = open_db();
SQLite::Statement insert(*db, "insert or replace into metadata (key, data) values (?,?)");
insert.bind(1, data.security_key());
std::string buffer = data.SerializeAsString();
insert.bind(2, buffer);
auto result = insert.tryExecuteStep();
if (result != SQLITE_DONE) {
LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result);
return false;
}
return true;
}
#else
namespace bts {
namespace md {
namespace db {
size_t load_securities(
const std::set<UserSymbol>& symbols,
const std::function<void(const api::Security&)>& callback) {
return 0;
}
Result insert_or_replace(const std::vector<SecurityPtr>& handles) { return Result::error; }
SecurityPtr find_security(SecurityKey) { return nullptr; }
#endif
}}}//namespace