Now both old and new (domain) md sources index securities by the same string format.
395 lines
13 KiB
C++
395 lines
13 KiB
C++
#ifndef _WIN32
|
|
#include "SecurityIDRecorder.h"
|
|
#include "lib/domain_model/Security.h"
|
|
#include <boost/pool/object_pool.hpp>
|
|
|
|
#include "lib/config/config.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/utils/Sqlite_utils.h"
|
|
#include <boost/filesystem.hpp>
|
|
#include <tbb/concurrent_queue.h>
|
|
|
|
#include "lib/config/protos/market_data.pb.h"
|
|
#include "lib/config/protos/system_config.pb.h"
|
|
|
|
#include <sqlite/sqlite3.h>
|
|
#include <string>
|
|
|
|
using namespace std;
|
|
using namespace bts::utils;
|
|
using std::size_t;
|
|
|
|
namespace bts {
|
|
namespace md {
|
|
|
|
using bts::api::Security;
|
|
using bts::api::SecurityType;
|
|
using bts::api::Venue;
|
|
using bts::config::SystemConfig;
|
|
|
|
uint64_t exchange_id(const api::Security &security);
|
|
|
|
struct SecurityIdRequest {
|
|
Venue venue;
|
|
SecurityType type;
|
|
std::string name;
|
|
std::function<void(Venue, SecurityType, const std::string &, std::uint32_t)>
|
|
callback;
|
|
};
|
|
|
|
struct SecurityIDRecorderState {
|
|
SecurityIDRecorderState(SecurityIDRecorderParams &p)
|
|
: pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(),
|
|
lookup_key(p.lookup_key), unresolved_legs() {}
|
|
|
|
tbb::concurrent_queue<std::shared_ptr<SecurityIdRequest>> id_request_queue;
|
|
tbb::concurrent_queue<MDProducerHandle> pending_handle;
|
|
std::thread::id thread_id;
|
|
sqlite3_ptr db;
|
|
unique_ptr<DispatcherTimer> timer;
|
|
|
|
KeyLookupOption lookup_key;
|
|
std::map<ExchangeKey, std::vector<FixedSpreadLeg *>> unresolved_legs;
|
|
|
|
// Oh no, I got methods in my state
|
|
|
|
void run(utils::Dispatcher &dispatcher) {
|
|
timer = std::make_unique<utils::DispatcherTimer>(
|
|
dispatcher, 200, [=, this]() { do_work(); });
|
|
timer->start();
|
|
}
|
|
void do_work() {
|
|
thread_id = std::this_thread::get_id();
|
|
work_spreads();
|
|
work_handles();
|
|
work_db();
|
|
}
|
|
void work_spreads();
|
|
void work_handles();
|
|
void work_db();
|
|
|
|
void assign_key(
|
|
const api::Security &security,
|
|
const SecurityIDRecorder::Callback &key_assigned);
|
|
void assign_key(
|
|
api::Venue venue,
|
|
api::SecurityType type,
|
|
const std::string &name,
|
|
std::function<void(
|
|
api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
|
key_assigned);
|
|
|
|
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
|
|
void add_unresolved_spread(MDProducerHandle &handle);
|
|
security_key search(const std::string &descriptor);
|
|
security_key insert(const std::string &descriptor, SecurityType);
|
|
};
|
|
|
|
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
|
|
: _self(new SecurityIDRecorderState(params)) {
|
|
if (params.db) {
|
|
_self->db = std::move(params.db);
|
|
}
|
|
init();
|
|
if (params.dispatcher) {
|
|
_self->run(*params.dispatcher);
|
|
}
|
|
}
|
|
|
|
SecurityIDRecorder::~SecurityIDRecorder() {}
|
|
|
|
void SecurityIDRecorder::init() {
|
|
LOG(md, Info, "SecurityIDRecorder::init");
|
|
string db_dir =
|
|
config::get<config::SystemConfig>().data_path() + "databases/md/";
|
|
string db_path = db_dir + "/security_keys.sqlite";
|
|
|
|
if (!boost::filesystem::exists(db_dir)) {
|
|
boost::filesystem::create_directories(db_dir);
|
|
}
|
|
|
|
if (!_self->db) {
|
|
_self->db = sqlite::open_db(db_path.c_str());
|
|
bts_assert(_self->db);
|
|
}
|
|
|
|
// Set a long busy timeout
|
|
sqlite3_busy_timeout(_self->db.get(), 9999);
|
|
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
|
|
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
|
|
sqlite::exec_nofail(
|
|
_self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING "
|
|
"NOT NULL PRIMARY KEY, type int);");
|
|
}
|
|
|
|
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
|
|
LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
|
|
std::string(handle.security().security_desc()), handle.security().key,
|
|
handle.security().numeric_id);
|
|
_self->pending_handle.push(handle);
|
|
}
|
|
|
|
void SecurityIDRecorder::assign_key(
|
|
const api::Security &security, const Callback &callback) {
|
|
assign_key(
|
|
security.exchange(), bts::securities::security_type(security),
|
|
bts::securities::descriptor(security),
|
|
[&callback, msec = security](
|
|
api::Venue, api::SecurityType, const std::string &,
|
|
uint32_t id) mutable {
|
|
msec.set_id(id);
|
|
callback(msec);
|
|
});
|
|
}
|
|
|
|
void SecurityIDRecorderState::assign_key(
|
|
api::Venue venue,
|
|
api::SecurityType type,
|
|
const std::string &name,
|
|
std::function<
|
|
void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
|
callback) {
|
|
LOG(md, Info, "SecurityIDRecorder::assign_key {} {} {}", venue, type, name);
|
|
auto id_req = std::make_shared<SecurityIdRequest>();
|
|
id_req->venue = venue;
|
|
id_req->type = type;
|
|
id_req->name = name;
|
|
id_req->callback = callback;
|
|
id_request_queue.push(id_req);
|
|
}
|
|
|
|
void SecurityIDRecorder::assign_key(
|
|
api::Venue venue,
|
|
api::SecurityType type,
|
|
const std::string &name,
|
|
std::function<
|
|
void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
|
callback) {
|
|
_self->assign_key(venue, type, name, std::move(callback));
|
|
}
|
|
|
|
void SecurityIDRecorder::do_work() { _self->do_work(); }
|
|
|
|
void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle &handle) {
|
|
bts_assert_d(lookup_key);
|
|
FixedSpreadSecurityXL *pspread =
|
|
static_cast<FixedSpreadSecurityXL *>(handle.mutable_security());
|
|
if (pspread != nullptr) {
|
|
for (size_t x = 0; x < pspread->leg_count(); ++x) {
|
|
auto &leg = pspread->legs[x];
|
|
bts_assert_d(leg.exchange_key.value() != 0);
|
|
if (leg.key == invalid_security_key) {
|
|
// Because leg is a reference to shared memory we know it will not
|
|
// move
|
|
for (const auto *existing : unresolved_legs[leg.exchange_key]) {
|
|
if (existing == &leg)
|
|
goto next;
|
|
}
|
|
unresolved_legs[leg.exchange_key].push_back(&leg);
|
|
}
|
|
next:;
|
|
}
|
|
}
|
|
}
|
|
|
|
void SecurityIDRecorderState::work_spreads() {
|
|
check_access();
|
|
if (!lookup_key)
|
|
return;
|
|
KeyLookup &lookup = *lookup_key;
|
|
int count = 0;
|
|
for (auto iter = unresolved_legs.begin();
|
|
count < 500 && iter != unresolved_legs.end();) {
|
|
const auto &pair = *iter;
|
|
++count;
|
|
SecurityKey key = lookup(pair.first);
|
|
if (key.value() != invalid_security_key) {
|
|
for (FixedSpreadLeg *leg : pair.second) {
|
|
leg->key = key.value();
|
|
}
|
|
iter = unresolved_legs.erase(iter);
|
|
} else {
|
|
++iter;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool has_all_leg_keys(const MDProducerHandle &handle) {
|
|
if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) {
|
|
const FixedSpreadSecurityXL *pspread =
|
|
static_cast<const FixedSpreadSecurityXL *>(&handle.security());
|
|
for (size_t x = 0; x < pspread->leg_count(); ++x) {
|
|
if (pspread->legs[x].key == invalid_security_key) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
// SyntheticFixedSecurity is guaranteed to have all of its legs
|
|
return true;
|
|
}
|
|
|
|
void SecurityIDRecorderState::work_handles() {
|
|
check_access();
|
|
Security security;
|
|
MDProducerHandle handle;
|
|
vector<MDProducerHandle> retry;
|
|
while (pending_handle.try_pop(handle)) {
|
|
security.Clear();
|
|
// if a spread does not have all of its legs we need to wait
|
|
if (!has_all_leg_keys(handle)) {
|
|
// add
|
|
add_unresolved_spread(handle);
|
|
retry.push_back(handle);
|
|
} else {
|
|
convert_security(handle.security(), &security);
|
|
assign_key(
|
|
security.exchange(), bts::securities::security_type(security),
|
|
bts::securities::descriptor(security),
|
|
[handle](
|
|
api::Venue, api::SecurityType, const std::string &,
|
|
uint32_t id) mutable { handle.set_key(id); });
|
|
}
|
|
}
|
|
|
|
for (const auto &h : retry) {
|
|
pending_handle.push(h);
|
|
}
|
|
}
|
|
|
|
void SecurityIDRecorderState::work_db() {
|
|
check_access();
|
|
// Drain the work queue
|
|
// Query the database for each security. If it has an ID set it
|
|
// If it does not have an ID add it to a list
|
|
// Then persist that list
|
|
// Once a security has a key it is public so record it in the security
|
|
// database
|
|
std::shared_ptr<SecurityIdRequest> temp;
|
|
vector<std::pair<uint32_t, std::shared_ptr<SecurityIdRequest>>> new_entries;
|
|
// SPARK-1446 Need to try to add existing securities to the db in case we
|
|
// deleted them by hand
|
|
vector<std::shared_ptr<SecurityIdRequest>> existing_securities;
|
|
vector<std::shared_ptr<SecurityIdRequest>> defer;
|
|
vector<string> new_descriptors;
|
|
size_t count = 0;
|
|
string descriptor;
|
|
while (id_request_queue.try_pop(temp)) {
|
|
descriptor = temp->name;
|
|
|
|
// If we're unable to create a descriptor try again later
|
|
if (descriptor == "") {
|
|
defer.push_back(std::move(temp));
|
|
continue;
|
|
}
|
|
|
|
security_key key = search(descriptor);
|
|
if (key != invalid_security_key) {
|
|
temp->callback(temp->venue, temp->type, temp->name, key);
|
|
existing_securities.push_back(std::move(temp));
|
|
} else {
|
|
// always push in parallel
|
|
new_entries.push_back(std::make_pair(key, std::move(temp)));
|
|
new_descriptors.push_back(descriptor);
|
|
++count;
|
|
// Limit the amount of time we spend writing to sqlite during startup
|
|
// to avoid blocking the heartbeat
|
|
if (count > 200) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (auto &d : defer) {
|
|
LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
|
|
d->name);
|
|
id_request_queue.push(std::move(d));
|
|
}
|
|
|
|
if (!new_entries.empty()) {
|
|
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
|
|
|
|
for (size_t x = 0; x < new_entries.size(); ++x) {
|
|
auto &entry = new_entries[x];
|
|
entry.first = insert(new_descriptors[x], entry.second->type);
|
|
bts_assert_d(entry.first != invalid_security_key);
|
|
}
|
|
|
|
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
|
|
}
|
|
|
|
if (!new_entries.empty()) {
|
|
for (size_t x = 0; x < new_entries.size(); ++x) {
|
|
auto &[id, req] = new_entries[x];
|
|
req->callback(req->venue, req->type, req->name, id);
|
|
}
|
|
}
|
|
}
|
|
|
|
void SecurityIDRecorder::bulk_insert(
|
|
const std::vector<const api::Security *> &data, const char *prefix) {
|
|
LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
|
|
if (data.size() == 0)
|
|
return;
|
|
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
|
std::string descriptor;
|
|
{
|
|
auto insert_stmt = sqlite::prepare_query(
|
|
_self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
|
|
"values (?, ?, ?)");
|
|
for (const api::Security *sec : data) {
|
|
descriptor = prefix + bts::securities::unique_descriptor(*sec);
|
|
auto type = securitytype_of(*sec);
|
|
|
|
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
|
|
sqlite3_bind_text(
|
|
insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
|
|
SQLITE_STATIC);
|
|
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
|
|
sqlite3_step(insert_stmt.get());
|
|
sqlite3_reset(insert_stmt.get());
|
|
}
|
|
}
|
|
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
|
}
|
|
|
|
security_key SecurityIDRecorderState::search(const std::string &descriptor) {
|
|
LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
|
|
auto select_stmt = sqlite::prepare_query(
|
|
db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
|
|
sqlite3_bind_text(
|
|
select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
|
|
SQLITE_STATIC);
|
|
int result = sqlite3_step(select_stmt.get());
|
|
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
|
return sqlite3_column_int(select_stmt.get(), 0);
|
|
}
|
|
return invalid_security_key;
|
|
}
|
|
|
|
security_key SecurityIDRecorderState::insert(
|
|
const std::string &descriptor, SecurityType type) {
|
|
LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
|
|
auto insert_stmt = sqlite::prepare_query(
|
|
db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
|
|
sqlite3_bind_text(
|
|
insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
|
|
SQLITE_STATIC);
|
|
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
|
|
int result = sqlite3_step(insert_stmt.get());
|
|
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
|
return sqlite3_last_insert_rowid(db.get());
|
|
}
|
|
// Just in case our insert failed due to duplicate row
|
|
auto key = search(descriptor);
|
|
if (key == invalid_security_key) {
|
|
LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
|
|
result);
|
|
}
|
|
return key;
|
|
}
|
|
|
|
} // namespace md
|
|
} // namespace bts
|
|
|
|
#endif
|