217 lines
7.5 KiB
C++
217 lines
7.5 KiB
C++
#include "SecurityDB.h"
|
|
#ifndef WIN32
|
|
|
|
#include <SQLiteCpp/SQLiteCpp.h>
|
|
#include <sqlite/sqlite3.h>
|
|
|
|
#include "lib/config/config.h"
|
|
#include "lib/config/protos/system.pb.h"
|
|
#include "lib/md/Shm/FixedSecurity.h"
|
|
#include "lib/config/protos/MarketData.pb.h"
|
|
#include "lib/domain_model/Security.h"
|
|
|
|
#include <boost/container/flat_set.hpp>
|
|
#include <boost/filesystem.hpp>
|
|
#include <boost/date_time/posix_time/posix_time.hpp>
|
|
#include <boost/date_time/gregorian/gregorian.hpp>
|
|
#include <string>
|
|
|
|
using std::string;
|
|
using namespace bts::utils;
|
|
using namespace SQLite;
|
|
|
|
namespace bts {
|
|
namespace md {
|
|
namespace db {
|
|
|
|
/*
|
|
* Schema:
|
|
* key, created_date, expiration_date
|
|
*/
|
|
|
|
std::unique_ptr<SQLite::Database> open_db() {
|
|
// Directory location on disk
|
|
auto db_dir = config::get<config::System>().data_path() + "databases/md/";
|
|
auto db_path = db_dir + "/securities.sqlite";
|
|
if (!boost::filesystem::exists(db_dir)) {
|
|
boost::filesystem::create_directories(db_dir);
|
|
}
|
|
|
|
// Create table if it does not exist
|
|
auto db = std::make_unique<Database>(db_path, OPEN_READWRITE | OPEN_CREATE);
|
|
db->setBusyTimeout(9999);
|
|
db->exec("PRAGMA journal_mode=WAL");
|
|
db->exec("PRAGMA syncronous=0");
|
|
db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);");
|
|
db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);");
|
|
|
|
return db;
|
|
}
|
|
|
|
datetime::Date sec_expiration_cut_off()
|
|
{
|
|
boost::gregorian::date_duration max_exp_age
|
|
(config::get<SecurityExpirationCutoff>().max_days_after_expiration());
|
|
|
|
auto pt = (datetime::now() - max_exp_age);
|
|
return datetime::Date(pt.date().year(),pt.date().month(),pt.date().day());
|
|
}
|
|
|
|
void apply_compatibilit_mutations(Security& security) {
|
|
// Migrate from old tick stuff
|
|
if (security.has_glbx()) {
|
|
if (security.has_glbx()) {
|
|
if (security.contract_spec().main_fraction() == 0) {
|
|
security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction());
|
|
}
|
|
if (security.contract_spec().sub_fraction() == 0) {
|
|
security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction());
|
|
}
|
|
if (security.contract_spec().price_display_format() == 0) {
|
|
security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
size_t load_securities(
|
|
const std::set<UserSymbol>& symbols,
|
|
const std::function<void(const api::Security&)>& callback) {
|
|
|
|
size_t count = 0;
|
|
auto db = open_db();
|
|
|
|
// Load all existing securities and call the callback
|
|
SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp");
|
|
select_stmt.bind(":exp", sec_expiration_cut_off().to_short());
|
|
|
|
Security security;
|
|
while (select_stmt.tryExecuteStep() == SQLITE_ROW) {
|
|
security_key security_key = select_stmt.getColumn("key");
|
|
std::string symbol = select_stmt.getColumn("symbol");
|
|
api::Venue venue = (api::Venue)(int) select_stmt.getColumn("venue");
|
|
auto type = (api::SecurityType)(int) select_stmt.getColumn("type");
|
|
UserSymbol us(symbol, venue, type);
|
|
if (symbols.find(us) != symbols.end()) {
|
|
auto blob = select_stmt.getColumn("security_blob");
|
|
if (security.ParseFromArray(blob, blob.getBytes())) {
|
|
apply_compatibilit_mutations(security);
|
|
bts_assert(security_key == security.id());
|
|
count++;
|
|
callback(security);
|
|
}
|
|
else {
|
|
LOG(md, Warn, "Failed to parse security from database. id = {}", security_key);
|
|
}
|
|
}
|
|
}
|
|
|
|
return count;
|
|
}
|
|
|
|
SecurityPtr find_security(SecurityKey key) {
|
|
try {
|
|
auto db = open_db();
|
|
SQLite::Statement select(*db, "select security_blob from securities where key = :key");
|
|
select.bind(":key", key.value());
|
|
if(select.tryExecuteStep() == SQLITE_ROW) {
|
|
auto result = std::make_shared<Security>();
|
|
auto blob = select.getColumn(0);
|
|
if(result->ParseFromArray(blob, blob.getBytes())) {
|
|
apply_compatibilit_mutations(*result);
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
catch(const SQLite::Exception& e) {
|
|
LOG(md,Error, "Error reading security database. {}", std::string(e.what()));
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
Result insert_or_replace(const std::vector<SecurityPtr>& securities) {
|
|
try {
|
|
auto db = open_db();
|
|
SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
|
|
SQLite::Transaction transaction(*db);
|
|
std::string buffer;
|
|
for (const auto& psecurity : securities) {
|
|
if (psecurity && psecurity->SerializeToString(&buffer)) {
|
|
insert.bind(1, psecurity->id());
|
|
insert.bind(2, psecurity->symbol());
|
|
insert.bind(3, psecurity->exchange());
|
|
insert.bind(4, securitytype_of(*psecurity));
|
|
insert.bind(5, buffer);
|
|
auto greg_date = securities::expiration(*psecurity);
|
|
if (greg_date == boost::gregorian::date()) {
|
|
insert.bind(6, datetime::Date(9999,1,1).to_short());
|
|
} else {
|
|
ushort year = greg_date.year();
|
|
ushort month = greg_date.month();
|
|
ushort day = greg_date.day();
|
|
insert.bind(6, datetime::Date(year,month,day).to_short());
|
|
}
|
|
insert.exec();
|
|
insert.reset();
|
|
}
|
|
}
|
|
transaction.commit();
|
|
}
|
|
catch(const SQLite::Exception& e) {
|
|
LOG(md,Warn, "Error writing to security db (retry pending): {}", std::string(e.what()));
|
|
return Result::error;
|
|
}
|
|
return Result::ok;
|
|
}
|
|
|
|
|
|
std::vector<api::synthetic::Metadata> load_metadata() {
|
|
std::vector<api::synthetic::Metadata> values;
|
|
auto db = open_db();
|
|
SQLite::Statement select(*db, "select key,data from metadata");
|
|
while(select.tryExecuteStep() == SQLITE_ROW) {
|
|
uint32_t key = select.getColumn(0);
|
|
auto blob = select.getColumn(1);
|
|
api::synthetic::Metadata meta;
|
|
if (meta.ParseFromArray(blob, blob.getBytes())) {
|
|
bts_assert(key == meta.security_key());
|
|
values.push_back(meta);
|
|
}
|
|
else {
|
|
LOG(md, Warn, "Failed to parse security metadata for security_key={}", key);
|
|
}
|
|
}
|
|
return values;
|
|
}
|
|
bool store_metadata(const api::synthetic::Metadata& data) {
|
|
auto db = open_db();
|
|
SQLite::Statement insert(*db, "insert or replace into metadata (key, data) values (?,?)");
|
|
|
|
insert.bind(1, data.security_key());
|
|
std::string buffer = data.SerializeAsString();
|
|
insert.bind(2, buffer);
|
|
auto result = insert.tryExecuteStep();
|
|
if (result != SQLITE_DONE) {
|
|
LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
#else
|
|
namespace bts {
|
|
namespace md {
|
|
namespace db {
|
|
|
|
size_t load_securities(
|
|
const std::set<UserSymbol>& symbols,
|
|
const std::function<void(const api::Security&)>& callback) {
|
|
return 0;
|
|
}
|
|
Result insert_or_replace(const std::vector<SecurityPtr>& handles) { return Result::error; }
|
|
SecurityPtr find_security(SecurityKey) { return nullptr; }
|
|
|
|
#endif
|
|
|
|
}}}//namespace
|