200 lines
6.0 KiB
C++
200 lines
6.0 KiB
C++
#ifndef WIN32
|
|
#include "SecurityIDRecorder.h"
|
|
|
|
#include "lib/utils/defer.h"
|
|
|
|
#include "lib/config/config.h"
|
|
#include "lib/utils/Sqlite_utils.h"
|
|
#include "lib/logging/logging.h"
|
|
#include <boost/filesystem.hpp>
|
|
#include <tbb/queuing_mutex.h> // for reservation
|
|
|
|
#include "lib/config/protos/MarketData.pb.h"
|
|
#include "lib/config/protos/system.pb.h"
|
|
|
|
#include <string>
|
|
#include <sqlite/sqlite3.h>
|
|
|
|
using namespace std;
|
|
using namespace bts::utils;
|
|
|
|
namespace bts {
|
|
namespace md2 {
|
|
|
|
//GCC_DIAG_OFF("-Weffc++") // Plain Struct
|
|
struct SecurityIDTask {
|
|
std::string descriptor;
|
|
SecurityType type;
|
|
SecurityIDRecorder::callback cb;
|
|
void* arg;
|
|
};
|
|
//GCC_DIAG_RESTORE()
|
|
struct SecurityIDRecorderState {
|
|
SecurityIDRecorderState() :
|
|
work(),
|
|
pending_handle(),
|
|
db(nullptr, sqlite3_close),
|
|
select_stmt(nullptr, sqlite3_finalize),
|
|
insert_stmt(nullptr, sqlite3_finalize)
|
|
{}
|
|
tbb::concurrent_queue<SecurityIDTask> work;
|
|
tbb::concurrent_queue<MDProducerHandle> pending_handle;
|
|
sqlite3_ptr db;
|
|
sqlite3_stmt_ptr select_stmt;
|
|
sqlite3_stmt_ptr insert_stmt;
|
|
};
|
|
|
|
SecurityIDRecorder::SecurityIDRecorder():
|
|
_self(new SecurityIDRecorderState())
|
|
{
|
|
init();
|
|
}
|
|
|
|
SecurityIDRecorder::~SecurityIDRecorder()
|
|
{
|
|
}
|
|
|
|
void SecurityIDRecorder::init() {
|
|
string db_dir = config::get<config::System>().data_path() + "/md/";
|
|
string db_path = db_dir + "/security_keys.sqlite";
|
|
|
|
if (!boost::filesystem::exists(db_dir)) {
|
|
boost::filesystem::create_directories(db_dir);
|
|
}
|
|
|
|
_self->db = open_db(db_path.c_str());
|
|
|
|
// Set a long busy timeout
|
|
sqlite3_busy_timeout(_self->db.get(), 9999);
|
|
sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
|
|
sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0");
|
|
sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
|
|
|
|
// Prepare statements
|
|
_self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
|
|
_self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
|
|
}
|
|
|
|
void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, callback cb, void* arg) {
|
|
SecurityIDTask task;
|
|
task.descriptor = descriptor;
|
|
task.type = type;
|
|
task.cb = cb;
|
|
task.arg = arg;
|
|
_self->work.push(task);
|
|
}
|
|
|
|
void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, Callback cb) {
|
|
Callback* pcb = new Callback(cb);
|
|
get_key(descriptor, type, [](void* data, security_key key) {
|
|
Callback* pcb = static_cast<Callback*>(data);
|
|
DEFER(if (pcb) delete pcb); // exception safe
|
|
(*pcb)(key);
|
|
}, pcb);
|
|
}
|
|
|
|
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
|
|
_self->pending_handle.push(handle);
|
|
}
|
|
|
|
bool SecurityIDRecorder::do_work() {
|
|
// Check if any handles need work first
|
|
work_pending_handles();
|
|
|
|
// Drain the work queue
|
|
// Query the database for each security. If it has an ID return it
|
|
// If it does not have an ID add it to a list
|
|
// Then persist that list
|
|
SecurityIDTask temp;
|
|
vector<SecurityIDTask> new_securities;
|
|
size_t count = 0;
|
|
bool incomplete = false;
|
|
while (_self->work.try_pop(temp)) {
|
|
security_key key = search(temp.descriptor);
|
|
if (key != invalid_security_key) {
|
|
temp.cb(temp.arg, key);
|
|
}
|
|
else {
|
|
new_securities.push_back(temp);
|
|
++count;
|
|
// Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat
|
|
if (count > 500) {
|
|
incomplete = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
if (new_securities.size() > 0) {
|
|
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
|
|
|
for (SecurityIDTask& security : new_securities) {
|
|
security_key key = insert(security.descriptor, security.type);
|
|
bts_assert_d(key);
|
|
security.cb(security.arg, key);
|
|
}
|
|
|
|
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
|
}
|
|
return incomplete;
|
|
}
|
|
|
|
void SecurityIDRecorder::work_pending_handles() {
|
|
MDProducerHandle handle;
|
|
vector<MDProducerHandle> defer;
|
|
while(_self->pending_handle.try_pop(handle)) {
|
|
// A spread will not have a descriptor until all of its leg keys are available
|
|
string descriptor = security_descriptor(handle.security());
|
|
if (descriptor.empty()) {
|
|
defer.push_back(handle);
|
|
continue;
|
|
}
|
|
|
|
MDProducerHandle* phandle = new MDProducerHandle(handle);
|
|
DEFER(if (phandle) delete phandle);
|
|
get_key(descriptor, handle.security().type, [](void* arg, security_key key) {
|
|
auto* phandle = static_cast<MDProducerHandle*>(arg);
|
|
if (phandle) {
|
|
phandle->set_key(key);
|
|
delete phandle;
|
|
}
|
|
}, phandle);
|
|
phandle = nullptr;
|
|
}
|
|
|
|
for (auto& handle : defer) {
|
|
_self->pending_handle.push(handle);
|
|
}
|
|
}
|
|
|
|
|
|
security_key SecurityIDRecorder::search(const std::string& descriptor) {
|
|
sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
|
|
DEFER(sqlite3_reset(_self->select_stmt.get()));
|
|
int result = sqlite3_step(_self->select_stmt.get());
|
|
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
|
return sqlite3_column_int(_self->select_stmt.get(), 0);
|
|
}
|
|
return invalid_security_key;
|
|
}
|
|
|
|
security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) {
|
|
sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
|
|
sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type);
|
|
DEFER(sqlite3_reset(_self->insert_stmt.get()));
|
|
int result = sqlite3_step(_self->insert_stmt.get());
|
|
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
|
return sqlite3_last_insert_rowid(_self->db.get());
|
|
}
|
|
// Just in case our insert failed due to duplicate row
|
|
auto key = search(descriptor);
|
|
if (key == invalid_security_key) {
|
|
LOG(MarketData,Error,"Failed to assign security key. SQLite return code: {}",result);
|
|
}
|
|
return key;
|
|
}
|
|
|
|
}}//namespace
|
|
|
|
#endif |