Files
spark/src/lib/md/SecurityIDRecorder.cpp
T
2017-09-21 16:51:48 -04:00

200 lines
6.0 KiB
C++

#ifndef WIN32
#include "SecurityIDRecorder.h"
#include "lib/utils/defer.h"
#include "lib/config/config.h"
#include "lib/utils/Sqlite_utils.h"
#include "lib/logging/logging.h"
#include <boost/filesystem.hpp>
#include <tbb/queuing_mutex.h> // for reservation
#include "lib/config/protos/MarketData.pb.h"
#include "lib/config/protos/system.pb.h"
#include <string>
#include <sqlite/sqlite3.h>
using namespace std;
using namespace bts::utils;
namespace bts {
namespace md2 {
//GCC_DIAG_OFF("-Weffc++") // Plain Struct
struct SecurityIDTask {
std::string descriptor;
SecurityType type;
SecurityIDRecorder::callback cb;
void* arg;
};
//GCC_DIAG_RESTORE()
struct SecurityIDRecorderState {
SecurityIDRecorderState() :
work(),
pending_handle(),
db(nullptr, sqlite3_close),
select_stmt(nullptr, sqlite3_finalize),
insert_stmt(nullptr, sqlite3_finalize)
{}
tbb::concurrent_queue<SecurityIDTask> work;
tbb::concurrent_queue<MDProducerHandle> pending_handle;
sqlite3_ptr db;
sqlite3_stmt_ptr select_stmt;
sqlite3_stmt_ptr insert_stmt;
};
SecurityIDRecorder::SecurityIDRecorder():
_self(new SecurityIDRecorderState())
{
init();
}
SecurityIDRecorder::~SecurityIDRecorder()
{
}
void SecurityIDRecorder::init() {
string db_dir = config::get<config::System>().data_path() + "/md/";
string db_path = db_dir + "/security_keys.sqlite";
if (!boost::filesystem::exists(db_dir)) {
boost::filesystem::create_directories(db_dir);
}
_self->db = open_db(db_path.c_str());
// Set a long busy timeout
sqlite3_busy_timeout(_self->db.get(), 9999);
sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
// Prepare statements
_self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
_self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
}
void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, callback cb, void* arg) {
SecurityIDTask task;
task.descriptor = descriptor;
task.type = type;
task.cb = cb;
task.arg = arg;
_self->work.push(task);
}
void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType type, Callback cb) {
Callback* pcb = new Callback(cb);
get_key(descriptor, type, [](void* data, security_key key) {
Callback* pcb = static_cast<Callback*>(data);
DEFER(if (pcb) delete pcb); // exception safe
(*pcb)(key);
}, pcb);
}
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
_self->pending_handle.push(handle);
}
bool SecurityIDRecorder::do_work() {
// Check if any handles need work first
work_pending_handles();
// Drain the work queue
// Query the database for each security. If it has an ID return it
// If it does not have an ID add it to a list
// Then persist that list
SecurityIDTask temp;
vector<SecurityIDTask> new_securities;
size_t count = 0;
bool incomplete = false;
while (_self->work.try_pop(temp)) {
security_key key = search(temp.descriptor);
if (key != invalid_security_key) {
temp.cb(temp.arg, key);
}
else {
new_securities.push_back(temp);
++count;
// Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat
if (count > 500) {
incomplete = true;
break;
}
}
}
if (new_securities.size() > 0) {
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
for (SecurityIDTask& security : new_securities) {
security_key key = insert(security.descriptor, security.type);
bts_assert_d(key);
security.cb(security.arg, key);
}
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
}
return incomplete;
}
void SecurityIDRecorder::work_pending_handles() {
MDProducerHandle handle;
vector<MDProducerHandle> defer;
while(_self->pending_handle.try_pop(handle)) {
// A spread will not have a descriptor until all of its leg keys are available
string descriptor = security_descriptor(handle.security());
if (descriptor.empty()) {
defer.push_back(handle);
continue;
}
MDProducerHandle* phandle = new MDProducerHandle(handle);
DEFER(if (phandle) delete phandle);
get_key(descriptor, handle.security().type, [](void* arg, security_key key) {
auto* phandle = static_cast<MDProducerHandle*>(arg);
if (phandle) {
phandle->set_key(key);
delete phandle;
}
}, phandle);
phandle = nullptr;
}
for (auto& handle : defer) {
_self->pending_handle.push(handle);
}
}
security_key SecurityIDRecorder::search(const std::string& descriptor) {
sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
DEFER(sqlite3_reset(_self->select_stmt.get()));
int result = sqlite3_step(_self->select_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_column_int(_self->select_stmt.get(), 0);
}
return invalid_security_key;
}
security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) {
sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type);
DEFER(sqlite3_reset(_self->insert_stmt.get()));
int result = sqlite3_step(_self->insert_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_last_insert_rowid(_self->db.get());
}
// Just in case our insert failed due to duplicate row
auto key = search(descriptor);
if (key == invalid_security_key) {
LOG(MarketData,Error,"Failed to assign security key. SQLite return code: {}",result);
}
return key;
}
}}//namespace
#endif