Files
spark/src/lib/md/SecurityIDRecorder.cpp
T

372 lines
12 KiB
C++

#include "lib/domain_model/Security.h"
#ifndef WIN32
#include "SecurityIDRecorder.h"
#include "lib/config/config.h"
#include "lib/logging/logging.h"
#include "lib/utils/Sqlite_utils.h"
#include <boost/filesystem.hpp>
#include <tbb/concurrent_queue.h>
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system.pb.h"
#include <sqlite/sqlite3.h>
#include <string>
using namespace std;
using namespace bts::utils;
using std::size_t;
namespace bts {
namespace md {
uint64_t exchange_id(const api::Security &security);
struct SecurityIDEntry {
api::Security security;
SecurityIDRecorder::Callback callback;
};
struct SecurityIDRecorderState {
SecurityIDRecorderState(SecurityIDRecorderParams &p)
: work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close),
timer(), lookup_key(p.lookup_key), unresolved_legs() {}
tbb::concurrent_queue<std::shared_ptr<SecurityIDEntry>> work;
tbb::concurrent_queue<MDProducerHandle> pending_handle;
std::thread::id thread_id;
sqlite3_ptr db;
unique_ptr<DispatcherTimer> timer;
KeyLookupOption lookup_key;
std::map<ExchangeKey, std::vector<FixedSpreadLeg *>> unresolved_legs;
// Oh no, I got methods in my state
void run(utils::Dispatcher &dispatcher) {
timer = std::make_unique<utils::DispatcherTimer>(
dispatcher, 200, [=]() { do_work(); });
timer->start();
}
void do_work() {
thread_id = std::this_thread::get_id();
work_spreads();
work_handles();
work_db();
}
void work_spreads();
void work_handles();
void work_db();
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
void assign_key(
const api::Security &security,
const SecurityIDRecorder::Callback &key_assigned);
void add_unresolved_spread(MDProducerHandle &handle);
security_key search(const std::string &descriptor);
security_key insert(const std::string &descriptor, SecurityType);
};
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
: _self(new SecurityIDRecorderState(params)) {
if (params.db) {
_self->db = std::move(params.db);
}
init();
if (params.dispatcher) {
_self->run(*params.dispatcher);
}
}
SecurityIDRecorder::~SecurityIDRecorder() {}
void SecurityIDRecorder::init() {
LOG(md, Info, "SecurityIDRecorder::init");
string db_dir = config::get<config::System>().data_path() + "databases/md/";
string db_path = db_dir + "/security_keys.sqlite";
if (!boost::filesystem::exists(db_dir)) {
boost::filesystem::create_directories(db_dir);
}
if (!_self->db) {
_self->db = sqlite::open_db(db_path.c_str());
bts_assert(_self->db);
}
// Set a long busy timeout
sqlite3_busy_timeout(_self->db.get(), 9999);
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite::exec_nofail(
_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING "
"NOT NULL PRIMARY KEY, type int);");
}
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
std::string(handle.security().security_desc()), handle.security().key,
handle.security().numeric_id);
_self->pending_handle.push(handle);
}
void SecurityIDRecorder::assign_key(
const api::Security &security,
const SecurityIDRecorder::Callback &callback) {
LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
security.DebugString());
if (security.Contract_case() == Security::ContractCase::kSpread) {
for (const auto &leg : security.spread().legs()) {
if (leg.security().id() == invalid_security_key) {
LOG(md, Error, "Security {} has legs which will never be resolved.",
security.has_glbx() ? security.glbx().security_desc()
: security.symbol());
// No point in trying to add this to the queue
return;
}
}
}
_self->assign_key(security, callback);
}
void SecurityIDRecorderState::assign_key(
const api::Security &security,
const SecurityIDRecorder::Callback &callback) {
LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
bts::securities::descriptor(security));
if (security.id() == invalid_security_key) {
auto work = std::make_shared<SecurityIDEntry>();
work->security = security;
work->callback = callback;
LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}",
bts::securities::descriptor(work->security));
this->work.push(work);
}
}
void SecurityIDRecorder::do_work() { _self->do_work(); }
void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle &handle) {
bts_assert_d(lookup_key);
FixedSpreadSecurityXL *pspread =
static_cast<FixedSpreadSecurityXL *>(handle.mutable_security());
if (pspread != nullptr) {
for (size_t x = 0; x < pspread->leg_count(); ++x) {
auto &leg = pspread->legs[x];
bts_assert_d(leg.exchange_key.value() != 0);
if (leg.key == invalid_security_key) {
// Because leg is a reference to shared memory we know it will not
// move
for (const auto *existing : unresolved_legs[leg.exchange_key]) {
if (existing == &leg)
goto next;
}
unresolved_legs[leg.exchange_key].push_back(&leg);
}
next:;
}
}
}
void SecurityIDRecorderState::work_spreads() {
check_access();
if (!lookup_key)
return;
KeyLookup &lookup = *lookup_key;
int count = 0;
for (auto iter = unresolved_legs.begin();
count < 500 && iter != unresolved_legs.end();) {
const auto &pair = *iter;
++count;
SecurityKey key = lookup(pair.first);
if (key.value() != invalid_security_key) {
for (FixedSpreadLeg *leg : pair.second) {
leg->key = key.value();
}
iter = unresolved_legs.erase(iter);
} else {
++iter;
}
}
}
bool has_all_leg_keys(const MDProducerHandle &handle) {
if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) {
const FixedSpreadSecurityXL *pspread =
static_cast<const FixedSpreadSecurityXL *>(&handle.security());
for (size_t x = 0; x < pspread->leg_count(); ++x) {
if (pspread->legs[x].key == invalid_security_key) {
return false;
}
}
}
// SyntheticFixedSecurity is guaranteed to have all of its legs
return true;
}
void SecurityIDRecorderState::work_handles() {
check_access();
Security security;
MDProducerHandle handle;
vector<MDProducerHandle> retry;
while (pending_handle.try_pop(handle)) {
security.Clear();
// if a spread does not have all of its legs we need to wait
if (!has_all_leg_keys(handle)) {
// add
add_unresolved_spread(handle);
retry.push_back(handle);
} else {
convert_security(handle.security(), &security);
assign_key(security, [handle](const api::Security &security) mutable {
handle.set_key(security.id());
});
}
}
for (const auto &h : retry) {
pending_handle.push(h);
}
}
void SecurityIDRecorderState::work_db() {
check_access();
// Drain the work queue
// Query the database for each security. If it has an ID set it
// If it does not have an ID add it to a list
// Then persist that list
// Once a security has a key it is public so record it in the security
// database
std::shared_ptr<SecurityIDEntry> temp;
vector<std::shared_ptr<SecurityIDEntry>> new_entries;
// SPARK-1446 Need to try to add existing securities to the db in case we
// deleted them by hand
vector<std::shared_ptr<SecurityIDEntry>> existing_securities;
vector<std::shared_ptr<SecurityIDEntry>> defer;
vector<string> new_descriptors;
size_t count = 0;
string descriptor;
while (work.try_pop(temp)) {
descriptor = security_descriptor(temp->security);
// If we're unable to create a descriptor try again later
if (descriptor == "") {
defer.push_back(std::move(temp));
continue;
}
security_key key = search(descriptor);
if (key != invalid_security_key) {
temp->security.set_id(key);
temp->callback(temp->security);
existing_securities.push_back(std::move(temp));
} else {
// always push in parallel
new_entries.push_back(std::move(temp));
new_descriptors.push_back(descriptor);
++count;
// Limit the amount of time we spend writing to sqlite during startup
// to avoid blocking the heartbeat
if (count > 200) {
break;
}
}
}
for (auto &d : defer) {
LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
bts::securities::descriptor(d->security));
work.push(std::move(d));
}
if (!new_entries.empty()) {
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
for (size_t x = 0; x < new_entries.size(); ++x) {
Security &security = new_entries[x]->security;
security_key key =
insert(new_descriptors[x], securitytype_of(security));
bts_assert_d(key);
security.set_id(key);
}
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
}
if (!new_entries.empty()) {
for (size_t x = 0; x < new_entries.size(); ++x) {
Security &security = new_entries[x]->security;
new_entries[x]->callback(security);
}
}
}
void SecurityIDRecorder::bulk_insert(
const std::vector<const api::Security *> &data, const char *prefix) {
LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
if (data.size() == 0)
return;
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
std::string descriptor;
{
auto insert_stmt = sqlite::prepare_query(
_self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
"values (?, ?, ?)");
for (const api::Security *sec : data) {
descriptor = prefix + security_descriptor(*sec);
auto type = securitytype_of(*sec);
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
sqlite3_bind_text(
insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
sqlite3_step(insert_stmt.get());
sqlite3_reset(insert_stmt.get());
}
}
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
}
security_key SecurityIDRecorderState::search(const std::string &descriptor) {
LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
auto select_stmt = sqlite::prepare_query(
db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
sqlite3_bind_text(
select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
int result = sqlite3_step(select_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_column_int(select_stmt.get(), 0);
}
return invalid_security_key;
}
security_key SecurityIDRecorderState::insert(
const std::string &descriptor, SecurityType type) {
LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
auto insert_stmt = sqlite::prepare_query(
db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
sqlite3_bind_text(
insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
int result = sqlite3_step(insert_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_last_insert_rowid(db.get());
}
// Just in case our insert failed due to duplicate row
auto key = search(descriptor);
if (key == invalid_security_key) {
LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
result);
}
return key;
}
} // namespace md
} // namespace bts
#endif