refactor: Return row id in IStore::retrieve_one

This is a prerequisite to synchronizing security ids stored via IDGenerator
(the "new" way to store ids, added when we implemented Deribit), and
SecurityDB (the original way from spark days). The problem was that these
2 implementations were using separate databases. We need to unify the
interfaces between these storage services to avoid breaking existing code.
This commit is contained in:
John Colvin committed 2024-11-20 22:17:08 +00:00
1 parent cb187e6997
commit bd8c97244a
28 files changed
+719 -566

No files matched your search

+10
View File
@@ -50,7 +50,9 @@ class TimeSpan {
constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
@@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() < rhs.nanos();
}
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() <= rhs.nanos();
}
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() > rhs.nanos();
}
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() >= rhs.nanos();
}
} // namespace bts::core
FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());
+1 -76
View File
@@ -53,82 +53,7 @@ string dump_header(const ColdHeader &header) {
}
string ShmDumper::dump_security(const FixedSecurity &security) {
ostringstream res;
res << "{";
res << "'key': " << security.key << ", ";
res << "'symbol': '" << security.symbol() << "', ";
res << "'type': '" << api::SecurityType_Name(security.type) << "', ";
res << "'exchange': '" << api::Venue_Name(security.exchange) << "', ";
res << "'alt_symbol': '" << security.alt_symbol() << "', ";
res << "'cfi_code': '" << security.cfi_code() << "', ";
res << "'security_desc': '" << security.security_desc() << "', ";
res << "'platform_id': '" << security.platform_id() << "', ";
res << "'display_factor': " << (int)security.display_factor << ", ";
res << "'numeric_id': " << security.numeric_id << ", ";
res << "'cboe_class_key': " << security.cboe_class_key << ", ";
res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
res << "'price_format': " << (int)security.cme_price_display_format << ", ";
res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";
res << "'price_break': " << security.price_break << ", ";
res << "'low_tick': " << security.low_tick << ", ";
res << "'high_tick': " << security.high_tick << ", ";
res << "'cpp': " << security.cash_per_point << ", ";
res << "'market_description ': " << security.ice_market_description()
<< ", ";
res << "'cleared_alias ': " << security.ice_cleared_alias() << ", ";
res << "'product_description ': " << security.ice_product_name() << ", ";
res << "'hub_alias ': " << security.ice_hub_alias() << ", ";
res << "'strip_name': " << security.ice_strip_name() << ", ";
res << "'mic_code': " << security.mic_code() << ", ";
switch (security.type) {
case SECURITYTYPE_SPOT:
break;
case SECURITYTYPE_FUTURE: {
auto *psec = static_cast<const FixedFutureSecurity *>(&security);
res << "'expiration': '";
securities::write_date_string(
psec->expiration_year, psec->expiration_month, psec->expiration_day,
res);
res << "' ";
break;
}
case SECURITYTYPE_OPTION: {
auto *psec = static_cast<const FixedOptionSecurity *>(&security);
res << "'expiration': '";
securities::write_date_string(
psec->expiration_year, psec->expiration_month, psec->expiration_day,
res);
res << "', ";
res << "'strike': " << psec->strike << ", ";
res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' ";
break;
}
case SECURITYTYPE_SPREAD: {
auto *psec = static_cast<const FixedSpreadSecurityXL *>(&security);
res << "'legs': [";
for (size_t x = 0; x < psec->leg_count(); ++x) {
auto &leg = psec->legs[x];
if (x > 0)
res << ",";
res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key
<< ", 'quantity': " << leg.leg_quantity
<< ", 'leg_price': " << leg.leg_price
<< ", 'delta': " << leg.leg_delta << ", 'side': '"
<< api::Side_Name(leg.leg_side) << "' }";
}
res << "]";
break;
}
default:
break;
}
res << "}";
return res.str();
return JsonDumper::dump(security);
}
string dump(datetime::Date date) {
-22
View File
@@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() {
return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day());
}
void apply_compatibilit_mutations(Security &security) {
// Migrate from old tick stuff
if (security.has_glbx()) {
if (security.has_glbx()) {
if (security.contract_spec().main_fraction() == 0) {
security.mutable_contract_spec()->set_main_fraction(
security.glbx().do_not_use_main_fraction());
}
if (security.contract_spec().sub_fraction() == 0) {
security.mutable_contract_spec()->set_sub_fraction(
security.glbx().do_not_use_sub_fraction());
}
if (security.contract_spec().price_display_format() == 0) {
security.mutable_contract_spec()->set_price_display_format(
security.glbx().do_not_use_price_display_format());
}
}
}
}
size_t load_securities(
std::function<bool(const UserSymbol &symbo)> const &sym_filter,
const std::function<void(const api::Security &)> &callback) {
@@ -104,7 +84,6 @@ size_t load_securities(
if (sym_filter(us)) {
auto blob = select_stmt.getColumn("security_blob");
if (security.ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(security);
bts_assert(security_key == security.id());
count++;
callback(security);
@@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) {
auto result = std::make_shared<Security>();
auto blob = select.getColumn(0);
if (result->ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(*result);
return result;
}
}
+77 -112
View File
@@ -1,19 +1,25 @@
#ifndef _WIN32
#include "SecurityIDRecorder.h"
#include "lib/domain_model/Security.h"
#include "lib/persistence/IStore.h"
#include "lib/persistence/SqliteStore.h"
#include <boost/pool/object_pool.hpp>
#include "lib/config/config.h"
#include "lib/logging/logging.h"
#include "lib/utils/Sqlite_utils.h"
#include "lib/utils/checked_arithmetic.h"
#include <boost/filesystem.hpp>
#include <tbb/concurrent_queue.h>
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include <memory>
#include <sqlite/sqlite3.h>
#include <stdexcept>
#include <string>
#include <tbb/concurrent_queue.h>
using namespace std;
using namespace bts::utils;
@@ -26,9 +32,32 @@ using bts::api::Security;
using bts::api::SecurityType;
using bts::api::Venue;
using bts::config::SystemConfig;
using bts::persistence::IStore;
using bts::persistence::SqliteStore;
uint64_t exchange_id(const api::Security &security);
static std::unique_ptr<IStore> get_default_store() {
return std::make_unique<SqliteStore>("key_ids");
}
static persistence::StoreRow get_row(
Venue venue,
SecurityType type,
const std::string &name,
const std::string &prefix = "") {
return persistence::StoreRow{
fmt::format(
"{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6),
SecurityType_Name(type).substr(13)),
name, ""};
}
static persistence::StoreRow get_row(
const api::Security &s, const std::string &prefix = "") {
return get_row(s.exchange(), bts::securities::security_type(s), prefix);
}
struct SecurityIdRequest {
Venue venue;
SecurityType type;
@@ -38,14 +67,16 @@ struct SecurityIdRequest {
};
struct SecurityIDRecorderState {
SecurityIDRecorderState(SecurityIDRecorderParams &p)
: pending_handle(), thread_id(), db(nullptr, sqlite3_close), timer(),
lookup_key(p.lookup_key), unresolved_legs() {}
SecurityIDRecorderState(
std::unique_ptr<IStore> store, KeyLookupOption lookup_key)
: store(store == nullptr ? get_default_store() : std::move(store)),
pending_handle(), thread_id(), timer(), lookup_key(lookup_key),
unresolved_legs() {}
std::unique_ptr<IStore> store{nullptr};
tbb::concurrent_queue<std::shared_ptr<SecurityIdRequest>> id_request_queue;
tbb::concurrent_queue<MDProducerHandle> pending_handle;
std::thread::id thread_id;
sqlite3_ptr db;
unique_ptr<DispatcherTimer> timer;
KeyLookupOption lookup_key;
@@ -81,46 +112,28 @@ struct SecurityIDRecorderState {
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
void add_unresolved_spread(MDProducerHandle &handle);
security_key search(const std::string &descriptor);
security_key insert(const std::string &descriptor, SecurityType);
security_key search(const persistence::StoreRow &row);
security_key insert(const persistence::StoreRow &row);
};
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
: _self(new SecurityIDRecorderState(params)) {
if (params.db) {
_self->db = std::move(params.db);
}
SecurityIDRecorder::SecurityIDRecorder(
std::unique_ptr<persistence::IStore> store,
KeyLookupOption lookup_key,
utils::Dispatcher *dispatcher)
: _self(std::make_unique<SecurityIDRecorderState>(
store == nullptr
? std::make_unique<persistence::SqliteStore>("key_ids")
: std::move(store),
lookup_key)) {
init();
if (params.dispatcher) {
_self->run(*params.dispatcher);
if (dispatcher != nullptr) {
_self->run(*dispatcher);
}
}
SecurityIDRecorder::~SecurityIDRecorder() {}
void SecurityIDRecorder::init() {
LOG(md, Info, "SecurityIDRecorder::init");
string db_dir =
config::get<config::SystemConfig>().data_path() + "databases/md/";
string db_path = db_dir + "/security_keys.sqlite";
if (!boost::filesystem::exists(db_dir)) {
boost::filesystem::create_directories(db_dir);
}
if (!_self->db) {
_self->db = sqlite::open_db(db_path.c_str());
bts_assert(_self->db);
}
// Set a long busy timeout
sqlite3_busy_timeout(_self->db.get(), 9999);
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite::exec_nofail(
_self->db, "CREATE TABLE IF NOT EXISTS store (descriptor STRING "
"NOT NULL PRIMARY KEY, type int);");
}
void SecurityIDRecorder::init() {}
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
@@ -134,7 +147,7 @@ void SecurityIDRecorder::assign_key(
assign_key(
security.exchange(), bts::securities::security_type(security),
bts::securities::descriptor(security),
[&callback, msec = security](
[callback, msec = security](
api::Venue, api::SecurityType, const std::string &,
uint32_t id) mutable {
msec.set_id(id);
@@ -269,27 +282,25 @@ void SecurityIDRecorderState::work_db() {
// SPARK-1446 Need to try to add existing securities to the db in case we
// deleted them by hand
vector<std::shared_ptr<SecurityIdRequest>> existing_securities;
vector<std::shared_ptr<SecurityIdRequest>> defer;
vector<string> new_descriptors;
vector<persistence::StoreRow> new_rows;
size_t count = 0;
string descriptor;
while (id_request_queue.try_pop(temp)) {
descriptor = temp->name;
// If we're unable to create a descriptor try again later
if (descriptor == "") {
defer.push_back(std::move(temp));
continue;
security_key key = invalid_security_key;
auto row = get_row(temp->venue, temp->type, temp->name);
try {
key = search(row);
} catch (const std::overflow_error &ex) {
LOG(persist, Error,
"SecurityIDRecorderState::work_db. Row search threw ex: {}",
std::string{ex.what()});
}
security_key key = search(descriptor);
if (key != invalid_security_key) {
temp->callback(temp->venue, temp->type, temp->name, key);
existing_securities.push_back(std::move(temp));
} else {
// always push in parallel
new_entries.push_back(std::make_pair(key, std::move(temp)));
new_descriptors.push_back(descriptor);
new_rows.push_back(std::move(row));
++count;
// Limit the amount of time we spend writing to sqlite during startup
// to avoid blocking the heartbeat
@@ -299,22 +310,14 @@ void SecurityIDRecorderState::work_db() {
}
}
for (auto &d : defer) {
LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
d->name);
id_request_queue.push(std::move(d));
}
if (!new_entries.empty()) {
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
store->insert_many_resolve_ids(new_rows);
for (size_t x = 0; x < new_entries.size(); ++x) {
auto &entry = new_entries[x];
entry.first = insert(new_descriptors[x], entry.second->type);
auto &row = new_rows[x];
entry.first = row.id;
bts_assert_d(entry.first != invalid_security_key);
}
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
}
if (!new_entries.empty()) {
@@ -327,65 +330,27 @@ void SecurityIDRecorderState::work_db() {
void SecurityIDRecorder::bulk_insert(
const std::vector<const api::Security *> &data, const char *prefix) {
LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
if (data.size() == 0)
return;
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
std::string descriptor;
{
auto insert_stmt = sqlite::prepare_query(
_self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
"values (?, ?, ?)");
for (const api::Security *sec : data) {
descriptor = prefix + bts::securities::unique_descriptor(*sec);
auto type = securitytype_of(*sec);
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
sqlite3_bind_text(
insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
sqlite3_step(insert_stmt.get());
sqlite3_reset(insert_stmt.get());
}
std::vector<persistence::StoreRow> rows;
rows.reserve(data.size());
for (const auto *s : data) {
rows.push_back(get_row(*s, prefix));
}
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
_self->store->insert_many_resolve_ids(rows);
}
security_key SecurityIDRecorderState::search(const std::string &descriptor) {
LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
auto select_stmt = sqlite::prepare_query(
db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
sqlite3_bind_text(
select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
int result = sqlite3_step(select_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_column_int(select_stmt.get(), 0);
security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) {
auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key);
if (exists) {
if (!val.empty()) {
bts_assert(stol(val) == rowid);
}
return system::math::checked_ltou(rowid);
}
return invalid_security_key;
}
security_key SecurityIDRecorderState::insert(
const std::string &descriptor, SecurityType type) {
LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
auto insert_stmt = sqlite::prepare_query(
db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
sqlite3_bind_text(
insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
int result = sqlite3_step(insert_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_last_insert_rowid(db.get());
}
// Just in case our insert failed due to duplicate row
auto key = search(descriptor);
if (key == invalid_security_key) {
LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
result);
}
return key;
security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) {
return store->insert(row.ns, row.key, row.value);
}
} // namespace md
+6 -17
View File
@@ -2,7 +2,7 @@
#ifndef _WIN32
#include "MDProducerHandle.h"
#include "common.h"
#include "lib/persistence/IStore.h"
#include "lib/utils/Sqlite_utils.h"
#include <atomic>
#include <functional>
@@ -22,26 +22,15 @@ namespace md {
typedef const std::function<SecurityKey(ExchangeKey)> KeyLookup;
typedef std::optional<KeyLookup> KeyLookupOption;
struct SecurityIDRecorderParams {
SecurityIDRecorderParams()
: dispatcher(), db(sqlite::null_db()), lookup_key() {}
SecurityIDRecorderParams(utils::Dispatcher &d)
: dispatcher(&d), db(sqlite::null_db()), lookup_key() {}
SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key)
: dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {}
SecurityIDRecorderParams(sqlite3_ptr &db)
: dispatcher(), db(std::move(db)), lookup_key() {}
SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key)
: dispatcher(), db(std::move(db)), lookup_key(get_key) {}
utils::Dispatcher *dispatcher;
sqlite3_ptr db;
KeyLookupOption lookup_key;
};
struct SecurityIDRecorderState;
class SecurityIDRecorder {
public:
SecurityIDRecorder(SecurityIDRecorderParams);
SecurityIDRecorder(
std::unique_ptr<persistence::IStore> store = nullptr,
KeyLookupOption lookup_key = std::nullopt,
utils::Dispatcher *dispatcher = nullptr);
~SecurityIDRecorder();
typedef std::function<void(const api::Security &)> Callback;
+5 -19
View File
@@ -217,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) {
get_contract_spec(source, *dest.mutable_contract_spec());
}
SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
switch (cc) {
case Security::ContractCase::kFuture:
return SECURITYTYPE_FUTURE;
case Security::ContractCase::kOption:
return SECURITYTYPE_OPTION;
case Security::ContractCase::kSpot:
return api::SECURITYTYPE_SPOT;
case Security::ContractCase::kSpread:
return SECURITYTYPE_SPREAD;
case Security::ContractCase::kSynthetic:
return SECURITYTYPE_SYNTHETIC;
case Security::ContractCase::CONTRACT_NOT_SET:
bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET);
}
bts_assert(false);
}
SecurityType securitytype_of(const Security &s) {
return securitytype_of_contractcase(s.Contract_case());
auto type = bts::securities::security_type(s.Contract_case());
bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET);
return type;
}
size_t num_legs(const Security &sec) {
if (sec.Contract_case() == api::Security::ContractCase::kSpread) {
return sec.spread().legs_size();
@@ -284,7 +270,7 @@ uint64_t exchange_id(const Security &security) {
void convert_security(
const Security &src, FixedSecurity &dest, bool set_security_key) {
dest.key = set_security_key ? src.id() : invalid_security_key;
dest.type = securitytype_of_contractcase(src.Contract_case());
dest.type = bts::securities::security_type(src.Contract_case());
dest.symbol(src.symbol());
dest.currency(src.currency());
dest.exchange = src.exchange();
+3 -4
View File
@@ -215,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *);
void convert_security(
const Security &src, FixedSecurity &dest, bool set_security_key = false);
SecurityType securitytype_of_contractcase(Security::ContractCase cc);
SecurityType securitytype_of(const Security &);
size_t num_legs(const Security &);
uint64_t exchange_id(const Security &security);
SecurityType securitytype_of(const api::Security &);
size_t num_legs(const api::Security &);
uint64_t exchange_id(const api::Security &security);
void to_json(nlohmann::json &j, const FixedSpreadLeg &f);
+3 -2
View File
@@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher
struct SyntheticSecurityService::Pimpl {
Pimpl(const MetadataStatefulPublisher &fn)
: recorder(utils::Dispatcher::background()), synthetics(), on_security(),
meta(), on_metadata(), metadata_publisher(fn) {}
: recorder(nullptr, std::nullopt, &utils::Dispatcher::background()),
synthetics(), on_security(), meta(), on_metadata(),
metadata_publisher(fn) {}
SecurityIDRecorder recorder;
std::unordered_map<SecurityKey, Security> synthetics;
utils::Event<Security> on_security;
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "lib/os/spark_data_directory.h"
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
namespace bts::md {
struct Mdp3DataDirectory {
boost::filesystem::path log_dir() const { return root.log_dir() / "onixs"; }
boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; }
boost::filesystem::path config_file() const {
return data_dir() / "config.xml";
}
boost::filesystem::path templates_file() const {
return data_dir() / "templates.xml";
}
void create_dirs() {
boost::filesystem::create_directories(data_dir());
boost::filesystem::create_directories(log_dir());
}
private:
spark::SparkDataDirectory root{};
};
} // namespace bts::md
+14 -15
View File
@@ -9,6 +9,7 @@
#include "lib/config/config.h"
#include "lib/metrics/metrics.h"
#include "lib/os/os.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/Shuffle.h"
#include "lib/utils/bts_chrono.h"
@@ -160,12 +161,10 @@ class Co2Tests {
#ifndef _WIN32
void test_security_id_recorder(const std::string &db_name) {
// Verify callbacks happen after work complete
auto db1 = sqlite::open_db(db_name.c_str());
REQUIRE(db1);
auto db2 = sqlite::open_db(db_name.c_str());
REQUIRE(db2);
SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
SecurityIDRecorder idsvc{
std::make_unique<persistence::SqliteStore>(db_name)};
SecurityIDRecorder idsvc2{
std::make_unique<persistence::SqliteStore>(db_name)};
security_key result = invalid_security_key;
Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
@@ -215,18 +214,18 @@ class Co2Tests {
std::make_unique<Co2TestInstrument>(s1, cold_header, hot_header);
// Add spread to SecurityIDRecorder and work it
auto db = sqlite::open_db(":memory:");
auto db = std::make_unique<persistence::SqliteStore>(":memory:");
REQUIRE(db);
SecurityIDRecorder idsvc{
{db, [&](ExchangeKey k) -> SecurityKey {
for (const auto &fut :
{f1handle->producer(), f2handle->producer()}) {
if (k.value() == fut.security().numeric_id)
return SecurityKey(fut.key());
}
return SecurityKey(invalid_security_key);
}}};
std::move(db), [&](ExchangeKey k) -> SecurityKey {
for (const auto &fut :
{f1handle->producer(), f2handle->producer()}) {
if (k.value() == fut.security().numeric_id)
return SecurityKey(fut.key());
}
return SecurityKey(invalid_security_key);
}};
auto workwork = [&]() {
idsvc.do_work();
+45
View File
@@ -0,0 +1,45 @@
#include "spark_data_directory.h"
#include "lib/config/config.h"
#include "lib/config/protos/system_config.pb.h"
#include <boost/filesystem/path.hpp>
#include <fmt/format.h>
#include <string>
namespace bts::spark {
SparkDataDirectory::SparkDataDirectory()
: SparkDataDirectory(config::get<config::SystemConfig>().data_path()) {}
SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {}
boost::filesystem::path SparkDataDirectory::root() const { return root_; }
boost::filesystem::path SparkDataDirectory::log_dir() const {
return root_ / "logging";
}
boost::filesystem::path SparkDataDirectory::log_file() const {
return log_dir() / "log";
}
boost::filesystem::path SparkDataDirectory::log_cache_file() const {
return log_dir() / "log_cache";
}
boost::filesystem::path SparkDataDirectory::database_dir(
const std::string &name) const {
return root_ / "databases" / (name + "/");
}
boost::filesystem::path SparkDataDirectory::database_file(
const std::string &name) const {
return root_ / "databases" / name / (name + ".sqlite");
}
boost::filesystem::path SparkDataDirectory::database_file(
const std::string &dir_name, const std::string &db_name) const {
return root_ / "databases" / dir_name / (db_name + ".sqlite");
}
} // namespace bts::spark
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include <boost/filesystem/path.hpp>
#include <string>
namespace bts::spark {
class SparkDataDirectory {
public:
SparkDataDirectory();
explicit SparkDataDirectory(const std::string &root);
boost::filesystem::path root() const;
boost::filesystem::path log_dir() const;
boost::filesystem::path log_file() const;
boost::filesystem::path log_cache_file() const;
boost::filesystem::path database_dir(const std::string &name) const;
boost::filesystem::path database_file(const std::string &name) const;
boost::filesystem::path database_file(
const std::string &dir_name, const std::string &db_name) const;
private:
boost::filesystem::path root_;
};
} // namespace bts::spark
+5 -15
View File
@@ -60,24 +60,21 @@ std::optional<uint64_t> IDGenerator::get_id(
bool add_if_new) {
std::string ns = fmt::format(
"{}.{}", Venue_Name(venue).substr(6),
SecurityType_Name(type).substr(12));
SecurityType_Name(type).substr(14));
return get_id(ns, name, add_if_new);
}
std::optional<uint64_t> IDGenerator::get_id(
const std::string &ns, const std::string &key, bool add_if_new) {
std::lock_guard<std::mutex> myLock(_instance_mx);
auto [exists, val] = me->kv_store->retrieve_one(ns, key);
auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key);
if (exists) {
uint64_t rv = str_to_id(val);
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv);
return std::optional<uint64_t>(rv);
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid);
return std::optional<uint64_t>(rowid);
} else {
if (add_if_new) {
auto all = me->kv_store->retrieve_all();
uint64_t new_id = get_next_id(all);
me->kv_store->insert(ns, key, std::to_string(new_id));
auto new_id = me->kv_store->insert(ns, key, "");
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key,
new_id);
return std::optional<uint64_t>(new_id);
@@ -88,13 +85,6 @@ std::optional<uint64_t> IDGenerator::get_id(
}
}
std::uint64_t IDGenerator::get_or_add(
const std::string &ns, const std::string &key) {
auto id = get_id(ns, key, true);
bts_assert(id.has_value());
return *id;
}
std::uint64_t IDGenerator::get_or_add(
api::domain::Venue venue,
api::domain::SecurityType type,
+36 -3
View File
@@ -1,8 +1,13 @@
#pragma once
#include "lib/persistence/store_row.h"
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <tuple>
#include <vector>
namespace bts {
namespace persistence {
@@ -11,17 +16,45 @@ class IStore {
public:
virtual std::map<std::string, std::map<std::string, std::string>>
retrieve_all() = 0;
virtual std::pair<bool, std::string>
retrieve_one(std::string const &ns, std::string const &k) = 0;
/**
* Finds the row with the specified namespace and key.
*
* @return std::tuple<bool, std::int64, std::string> The first item indicates
* whether or not a row was found. For successful queries, the second and third
* items are the rowid and value, respectively.
*/
virtual std::tuple<bool, std::int64_t, std::string> retrieve_one(
std::string const &ns, std::string const &k) = 0;
virtual void
remove(std::string const &ns, std::set<std::string> const &keys) = 0;
virtual void remove_all(std::string const &ns) = 0;
virtual void insert(
virtual std::int64_t insert(
std::string const &ns, std::string const &k, std::string const &v) = 0;
void insert_many(const std::vector<StoreRow> &rows);
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
virtual void update(
std::string const &ns, std::string const &k, std::string const &v) = 0;
virtual ~IStore();
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline void IStore::insert_many(const std::vector<StoreRow> &rows) {
for (const auto &row : rows) {
insert(row.ns, row.key, row.value);
}
}
inline void IStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
for (auto &row : rows) {
row.id = insert(row.ns, row.key, row.value);
}
}
} // namespace persistence
} // namespace bts
+225 -117
View File
@@ -35,26 +35,28 @@ static boost::filesystem::path get_db_path(const std::string &str) {
}
struct SqliteStore::PImpl {
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn{nullptr, sqlite3_close};
sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr update{nullptr, sqlite3_finalize};
PImpl(const boost::filesystem::path &db_path)
: db(db_path.stem().string()), db_path(db_path),
conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize),
remove(nullptr, sqlite3_finalize),
remove_all(nullptr, sqlite3_finalize),
update(nullptr, sqlite3_finalize) {
: db(db_path.stem().string()), db_path(db_path) {
bts_assert(db.size());
}
PImpl(std::string const &store_root, std::string const &db_name)
: PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {}
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn;
sqlite3_stmt_ptr insert;
sqlite3_stmt_ptr remove;
sqlite3_stmt_ptr remove_all;
sqlite3_stmt_ptr update;
BTS_NO_COPY(PImpl);
explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {}
PImpl(const PImpl &) = delete;
PImpl &operator=(const PImpl &) = delete;
};
SqliteStore::SqliteStore(std::string const &db)
@@ -64,54 +66,64 @@ SqliteStore::SqliteStore(std::string const &db)
init_db();
}
SqliteStore::SqliteStore(sqlite3_ptr db)
: me(std::make_unique<PImpl>(std::move(db))) {
LOG(persist, Info, "SqliteStore ctr. db: <imported ptr>");
init_db();
}
void SqliteStore::init_db() {
auto path = me->db_path;
auto parent = path.parent_path();
const auto path = me->db_path;
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
me->conn = sqlite::open_db(path.c_str());
bts_assert_msg(me->conn, path.c_str());
const char *create_schema_sql =
"create table store ( ns text, key text, value text)";
sqlite::exec_nofail(me->conn, create_schema_sql);
const char *key_index_sql =
"create unique index key_index on store(ns,key)";
sqlite::exec_nofail(me->conn, key_index_sql);
const char *ns_index_sql = "create index ns_index on store(ns)";
sqlite::exec_nofail(me->conn, ns_index_sql);
} else {
me->conn = sqlite::open_db(path.c_str());
if (!path.empty()) {
if (!boost::filesystem::is_directory(path.parent_path())) {
bts_assert(boost::filesystem::create_directories(path.parent_path()));
}
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
}
}
auto &conn = me->conn;
if (conn == nullptr) {
conn = sqlite::open_db(path.c_str());
bts_assert(conn != nullptr);
}
sqlite::exec_nofail(
conn,
"CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)");
sqlite::exec_nofail(
conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)");
sqlite::exec_nofail(
conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)");
conn = sqlite::open_db(path.c_str());
// Asynchronous for speed
sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL");
// prepare insert statement
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
me->insert = sqlite::prepare_query(me->conn, insert_sql);
me->insert = sqlite::prepare_query(conn, insert_sql);
// prepare insert_with_id statement
const char *insert_with_id_sql =
"insert into store (rowid, ns, key, value) values (?,?,?,?)";
me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql);
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
me->update = sqlite::prepare_query(me->conn, update_sql);
me->update = sqlite::prepare_query(conn, update_sql);
// prepare remove statement
const char *remove_all_sql = "delete from store where ns = ?";
me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
me->remove_all = sqlite::prepare_query(conn, remove_all_sql);
// prepare remove statement
const char *remove_sql = "delete from store where ns = ? and key = ?";
me->remove = sqlite::prepare_query(me->conn, remove_sql);
me->remove = sqlite::prepare_query(conn, remove_sql);
}
SqliteStore::~SqliteStore() {}
@@ -140,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() {
return result;
}
std::pair<bool, std::string> SqliteStore::retrieve_one(
std::tuple<bool, std::int64_t, std::string> SqliteStore::retrieve_one(
std::string const &ns, std::string const &k) {
auto query_str = fmt::format(
"select ns,key,value from store where ns='{}' and key='{}'", ns, k);
"select rowid,ns,key,value from store where ns='{}' and key='{}'", ns,
k);
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
std::pair<bool, std::string> result{false, ""};
auto result = std::make_tuple<bool, std::int64_t, std::string>(false, 0, "");
auto pq = query.get();
auto sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::int64_t rowid = sqlite3_column_int64(pq, 0);
std::string ns(
(char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0));
std::string key(
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
std::string key(
(char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2));
std::string val(
(char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2));
result.first = true;
result.second = std::move(val);
(char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3));
std::get<0>(result) = true;
std::get<1>(result) = rowid;
std::get<2>(result) = std::move(val);
}
return result;
}
@@ -167,91 +181,185 @@ void SqliteStore::remove(
std::string const &ns, std::set<std::string> const &keys) {
bts_assert(ns.size());
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete row in store. Will retry once. error_code: {} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
try {
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
"Could not delete row in store. Will retry once. error_code: "
"{} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::remove_all(std::string const &ns) {
bts_assert(ns.size());
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete all rows in store. Will retry once. error_code: {} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
"Could not delete all rows in store. Will retry once. error_code: "
"{} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove_all threw. what: {}",
std::string{ex.what()});
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
}
void SqliteStore::insert(
std::int64_t SqliteStore::insert(
std::string const &ns, std::string const &k, std::string const &v) {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not insert row in store. Will retry once. error_code: {} ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: {}",
r);
bts_assert(r == SQLITE_DONE);
"Could not insert row in store. Will retry once. error_code: {} "
"ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
const auto rowid = sqlite3_last_insert_rowid(me->conn.get());
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return rowid;
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert threw. what: {}",
std::string{ex.what()});
}
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return 0;
}
void SqliteStore::insert_many(const std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size());
if (rows.empty()) {
return;
}
try {
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (const auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
sqlite3_last_insert_rowid(conn_rawp);
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert_many threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size());
if (rows.empty()) {
return;
}
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
if (row.id > 0) {
bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id);
} else {
row.id = sqlite3_last_insert_rowid(conn_rawp);
}
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
}
void SqliteStore::update(
+11 -2
View File
@@ -1,7 +1,12 @@
#pragma once
#include "IStore.h"
#include "lib/persistence/store_row.h"
#include "lib/utils/Sqlite_utils.h"
#include <cstdint>
#include <memory>
#include <vector>
namespace bts {
namespace persistence {
@@ -9,22 +14,26 @@ namespace persistence {
class SqliteStore : public IStore {
public:
SqliteStore(std::string const &db);
explicit SqliteStore(sqlite3_ptr db);
~SqliteStore();
static SqliteStore from_path(const std::string &full_path);
typedef std::map<std::string, std::map<std::string, std::string>> Data;
Data retrieve_all() override;
std::pair<bool, std::string> retrieve_one(
std::tuple<bool, std::int64_t, std::string> retrieve_one(
std::string const &ns, std::string const &k) override;
void remove(
std::string const &ns, std::set<std::string> const &keys) override;
void remove_all(std::string const &ns) override;
void insert(
std::int64_t insert(
std::string const &ns,
std::string const &k,
std::string const &v) override;
void insert_many(const std::vector<StoreRow> &rows);
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
void update(
std::string const &ns,
std::string const &k,
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <cstdint>
#include <string>
namespace bts::persistence {
struct StoreRow {
std::string ns;
std::string key;
std::string value;
/**
* Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that
* the rowid isn't known, and should either be generated by the db (for inserts),
* or resolved by finding the row with matching ns, key, value (for retrieves).
*/
std::int64_t id{0};
};
} // namespace bts::persistence
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include <cstdint>
#include <limits>
#include <stdexcept>
namespace bts::system::math {
static std::uint32_t checked_ltou(std::int64_t l);
static std::uint64_t checked_ltoul(std::int64_t l);
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::uint32_t checked_ltou(std::int64_t l) {
if (l < 0) {
throw std::overflow_error{
"Failed to convert int64 to uint32 (value < 0)"};
}
if (l >
static_cast<std::int64_t>(std::numeric_limits<std::uint32_t>::max())) {
throw std::overflow_error{
"Failed to convert int64 to uint32. value > max)"};
}
return static_cast<std::uint32_t>(l);
}
inline std::uint64_t checked_ltoul(std::int64_t l) {
if (l < 0) {
throw std::overflow_error{
"Failed to convert int64 to uint64 (value < 0)"};
}
return static_cast<std::uint64_t>(l);
}
} // namespace bts::system::math
@@ -52,7 +52,7 @@ void add_or_update_to_table(
const std::string &ns,
const std::string &key,
const std::string &val) {
auto [exists, existing_val] = store->retrieve_one(ns, key);
auto [exists, rowid, existing_val] = store->retrieve_one(ns, key);
LOG(Gen, Info,
"add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
key, val, exists, existing_val);
@@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report(
const api::Response<::bts::api::ResultMessage> &resp) {
Report report = report_arg;
std::string key = std::to_string(report.id());
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
if (!present) {
int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
report.set_id(new_id);
@@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report(
const bts::api::analytics::IntWrapper &id,
const api::Response<::bts::api::ResultMessage> &resp) {
std::string key = std::to_string(id.data());
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
api::ResultMessage res;
res.set_success(false);
if (!present) {
@@ -581,10 +581,11 @@ void negate_id(api::Security *sec) {
}
void ExchangeSimApplication::persist_book(OrderBook const &b) {
auto existing = me->store->retrieve_one("books", b.sec_desc());
auto [exists, rowid, oldval] =
me->store->retrieve_one("books", b.sec_desc());
auto val = utils::pb::to_string(b);
if (existing.first)
if (exists)
me->store->update("books", b.sec_desc(), val);
else
me->store->insert("books", b.sec_desc(), val);
@@ -18,10 +18,11 @@ struct EmumbaDeribitData {
EmumbaDeribitData()
: sec_index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
},
&utils::Dispatcher::current()),
md_producer(recorder), listener(sec_index, md_producer) {}
};
} // namespace bts::md
@@ -43,10 +43,11 @@ struct EmumbaMain::PImpl {
PImpl()
: sec_index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), listener(sec_index, producer) {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
@@ -1,70 +1,66 @@
#include "ICEMain.h"
#include "lib/config/config.h"
#include "lib/md/SecurityDB.h"
#include "lib/md/coconut2.h"
#include "lib/md/ice/ICEUtils.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/md/vendor/OnixsUtils.h"
#include "lib/os/os.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/helper_macros.h"
#include "lib/utils/proto_helper.h"
#include "lib/config/protos/deployment_type.pb.h"
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/utils/thread_manager.h"
#include "lib/md/ice/ICEUtils.h"
#include "lib/md/vendor/OnixsUtils.h"
#include <OnixS/ICE/iMpact/MarketData.h>
#include <boost/filesystem.hpp>
#include <memory>
#include <set>
void recursive_copy(
const boost::filesystem::path &src, const boost::filesystem::path &dst) {
if (boost::filesystem::exists(dst))
throw std::runtime_error(dst.generic_string() + " exists");
if (boost::filesystem::is_directory(src)) {
boost::filesystem::create_directories(dst);
for (boost::filesystem::directory_entry &item :
boost::filesystem::directory_iterator(src))
recursive_copy(item.path(), dst / item.path().filename());
} else if (boost::filesystem::is_regular_file(src))
boost::filesystem::copy(src, dst);
else
throw std::runtime_error(dst.generic_string() + " not dir or file");
}
using namespace std;
using namespace bts::co2;
using std::size_t;
namespace bts {
namespace md {
namespace ice {
namespace bts::md::ice {
using bts::config::SystemConfig;
using Path = boost::filesystem::path;
ICEMain::ICEMain()
: _index(), _recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(_index.get_key(k.value()));
}}),
_producer(_recorder), _handlers(), _listeners() {}
struct ICEMain::PImpl {
ICEOptions options;
std::string config_file;
ICESecurityIndex index;
SecurityIDRecorder recorder;
MDProducer producer;
std::vector<std::pair<ICEChannel, std::unique_ptr<onixs_ice::Handler>>>
handlers;
std::vector<std::unique_ptr<ICEListener>> listeners;
ICEMain::~ICEMain() {}
PImpl(ICEOptions options)
: options(options),
config_file((Path{config::get<SystemConfig>().data_path()} /
"onixs_ice" / this->options.ice_config_file())
.string()),
index(), recorder(
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), handlers(), listeners() {}
const IceChannelConfig channel_settings_for(const ICEChannel &channel) {
const auto &config = config::get<ICEOptions>();
for (const auto &ch : config.channel()) {
if (ch.name() == channel.name)
return ch;
IceChannelConfig channel_settings_for(const ICEChannel &channel) {
for (const auto &ch : options.channel()) {
if (ch.name() == channel.name)
return ch;
}
return {};
}
return {};
};
ICEMain::ICEMain() : ICEMain(config::get<ICEOptions>()) {}
ICEMain::ICEMain(ICEOptions options)
: me_(std::make_unique<PImpl>(std::move(options))) {
me_->options = std::move(options);
}
ICEMain::~ICEMain() = default;
onixs_ice::MarketSubscriptions market_subcription_for_channel(
const ICEChannel &chan) {
onixs_ice::MarketSubscriptions subs;
@@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel(
return subs;
}
void ICEMain::start(const std::vector<UserSymbol> &symbols) {
auto ice_conf_location =
Path{config::get<SystemConfig>().data_path()} / "onixs_ice/";
if (!boost::filesystem::is_directory(ice_conf_location)) {
auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/";
recursive_copy(Path{conf_path}, ice_conf_location);
static void validate_config_file(const std::string &config_file) {
if (!boost::filesystem::exists(config_file)) {
const std::string config_url =
config::get<config::DeploymentType>().type() == "dev"
? "https://community.theice.com/docs/DOC-19989"
: "https://drive.google.com/"
"open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp";
bts_assert_msgf(
false, "Ice config file not found '{}'. Please download from {}",
config_file, config_url);
}
}
void ICEMain::start(const std::vector<UserSymbol> &symbols) {
await_ice_config();
validate_config_file(me_->config_file);
auto license = onixs_create_temporary_license();
const auto &channels = parse_ice_channels(symbols);
for (const auto &channel : channels) {
unique_ptr<onixs_ice::Handler> phandler =
prepare_handler(channel, license.path.string());
_listeners.push_back(prepare_listener(*phandler, channel));
_handlers.push_back(std::make_pair(channel, std::move(phandler)));
me_->listeners.push_back(prepare_listener(*phandler, channel));
me_->handlers.push_back(std::make_pair(channel, std::move(phandler)));
std::stringstream symstr;
for (const auto &sym : channel.symbols)
@@ -115,7 +118,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
symstr.str());
}
for (auto &pair : _handlers) {
for (auto &pair : me_->handlers) {
LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string());
sleep_for(16s);
onixs_ice::MarketSubscriptions subs =
@@ -132,7 +135,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
}
void ICEMain::send_heartbeats() {
for (const auto &listener : _listeners)
for (const auto &listener : me_->listeners)
listener->heartbeat();
}
@@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() {
// Initialize HandlerSettings and allocate the channel Handler
unique_ptr<onixs_ice::Handler> ICEMain::prepare_handler(
const ICEChannel &channel, const std::string &license) {
const auto &options = channel_settings_for(channel);
const auto &options = me_->channel_settings_for(channel);
LOG(md, Info, "Loaded channel options for channel {}:\n{}",
channel.to_string(), options.DebugString());
onixs_ice::HandlerSettings settings;
settings.licenseDirectory = license;
//
// Logging
settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode();
settings.logDirectory = ice_log_path();
//
// Config file
settings.connectivityConfiguration = ice_config_path();
settings.logDirectory =
(boost::filesystem::path{config::get<SystemConfig>().data_path()} /
"logging" / "onixs_ice/")
.string();
settings.connectivityConfiguration = me_->config_file;
//
// Network settings
@@ -231,12 +232,12 @@ unique_ptr<ICEListener> ICEMain::prepare_listener(
// heartbeats
vector<MDProducerChannel> producer_channels;
for (auto &symbol : channel.symbols) {
producer_channels.push_back(_producer.get_channel(
producer_channels.push_back(me_->producer.get_channel(
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
unique_ptr<ICEListener> listener(new ICEListener(
_index, _producer, /*_settlement_recorder,*/ producer_channels));
me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels));
for (auto &symbol : channel.symbols)
listener->add_filter_symbol(symbol.symbol());
@@ -254,6 +255,4 @@ unique_ptr<ICEListener> ICEMain::prepare_listener(
return listener;
}
} // namespace ice
} // namespace md
} // namespace bts
} // namespace bts::md::ice
@@ -1,21 +1,12 @@
#pragma once
#include "ICEListener.h"
#include "ICESecurityIndex.h"
#include "lib/md/coconut2.h"
#include "lib/utils/bts_no_copy.h"
#include "onixs.h"
#include "server/sparkd/md/platforms/IMDPlatform.h"
#include <map>
#include <memory>
#include <vector>
namespace bts {
namespace md {
namespace ice {
// void run_onixs_ice_server(UserSymbol* sym, size_t symcount);
namespace bts::md::ice {
struct ICEChannel;
@@ -28,27 +19,22 @@ struct ICEChannel;
class ICEMain : public IMDPlatform {
public:
ICEMain();
explicit ICEMain(ICEOptions options);
ICEMain(const ICEMain &) = delete;
ICEMain &operator=(const ICEMain &) = delete;
~ICEMain();
void start(const std::vector<UserSymbol> &symbols);
void send_heartbeats() override;
std::unique_ptr<onixs_ice::Handler>
prepare_handler(const ICEChannel &channel, const std::string &license);
std::unique_ptr<ICEListener>
prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel);
std::unique_ptr<onixs_ice::Handler> prepare_handler(
const ICEChannel &channel, const std::string &license);
std::unique_ptr<ICEListener> prepare_listener(
onixs_ice::Handler &handler, const ICEChannel &channel);
private:
BTS_NO_COPY(ICEMain);
ICESecurityIndex _index;
SecurityIDRecorder _recorder;
MDProducer _producer;
std::vector<std::pair<ICEChannel, std::unique_ptr<onixs_ice::Handler>>>
_handlers;
std::vector<std::unique_ptr<ICEListener>> _listeners;
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace ice
} // namespace md
} // namespace bts
} // namespace bts::md::ice
@@ -1,24 +1,18 @@
#include "MDP3Main.h"
#include "lib/api/md/GlbxMarketDataConfig.pb.h"
#include "lib/config/config.h"
#include "lib/config/protos/deployment_type.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/MBOProcessor.h"
#include "lib/md/SecurityDB.h"
#include "lib/md/cme/MDP3Utils.h"
#include "lib/md/coconut2.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/md/cme/mdp3_data_directory.h"
#include "lib/md/vendor/OnixsUtils.h"
#include "lib/os/os.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/proto_helper.h"
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/os/spark_data_directory.h"
#include "lib/utils/thread_manager.h"
#include <boost/filesystem.hpp>
#include <memory>
#include <optional>
#include <set>
#include <sstream>
@@ -29,16 +23,33 @@ using std::size_t;
namespace bts {
namespace md {
MDP3Main::MDP3Main()
: _index(), _recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(_index.get_key(k.value()));
}}),
_producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() {
}
#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x
MDP3Main::~MDP3Main() {}
using Path = boost::filesystem::path;
struct MDP3Main::PImpl {
Mdp3DataDirectory data_dir;
CMESecurityIndex index;
SecurityIDRecorder recorder;
MDProducer producer;
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> engines;
std::map<std::string, std::unique_ptr<onixs::Handler>> handlers;
std::vector<std::unique_ptr<MDP3Listener>> listeners;
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> recovery;
PImpl()
: index(), recorder(
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), engines(), handlers(), listeners(), recovery() {}
};
MDP3Main::MDP3Main()
: me_(std::make_unique<PImpl>()) {}
MDP3Main::~MDP3Main() = default;
const CmeChannelConfig channel_settings_for(const string &id) {
const auto &config = config::get<CMEOptions>();
@@ -173,11 +184,11 @@ void MDP3Main::start(
auto license = onixs_create_temporary_license();
initialize(license.path.string());
const auto &channels = parse_channels(symbols);
_engines = create_feed_engines(channels);
me_->engines = create_feed_engines(channels);
for (const auto &channel : channels) {
unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
_listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
_handlers.insert(std::make_pair(channel.id, std::move(phandler)));
me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
me_->handlers.insert(std::make_pair(channel.id, std::move(phandler)));
std::stringstream symstr;
for (const auto &sym : channel.symbols) {
symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " ";
@@ -186,7 +197,7 @@ void MDP3Main::start(
symstr.str());
}
for (auto &pair : _handlers) {
for (auto &pair : me_->handlers) {
onixs::SessionSettings settings =
session_settings_for_channel(pair.first, mbo);
pair.second->start(settings);
@@ -196,7 +207,7 @@ void MDP3Main::start(
}
void MDP3Main::send_heartbeats() {
for (const auto &listener : _listeners) {
for (const auto &listener : me_->listeners) {
listener->heartbeat();
}
}
@@ -219,7 +230,8 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
onixs::HandlerSettings settings;
settings.channel(std::stoul(channel_id));
settings.feeds().connectivityConfigurationFile(config_path());
settings.feeds().connectivityConfigurationFile(
(me_->data_dir.config_file()).string());
// Market data options
settings.bookManagement().directBooks().maintain(true);
@@ -306,7 +318,8 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
logger = new onixs::NullLogger();
} else {
onixs::FileLoggerSettings loggerSettings;
loggerSettings.filename(log_path() + "/log_" + channel_id);
loggerSettings.filename(
(me_->data_dir.log_dir() / ("log_" + channel_id)).string());
loggerSettings.severityLevel(
(onixs::LogSeverity::Enum)options.onixs_log_mode());
logger = new onixs::FileLogger(loggerSettings);
@@ -338,19 +351,19 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
try {
auto handler = unique_ptr<onixs::Handler>(new onixs::Handler(settings));
auto iter = _engines.find(options.feed_engine());
if (iter == _engines.end()) {
auto iter = me_->engines.find(options.feed_engine());
if (iter == me_->engines.end()) {
LOG(md, Fatal,
"Missing feed engine definition '{}' for channel id '{}'. Add a "
"feed_engine entry with name:'{}' to the CMEOptions config key.",
options.feed_engine(), channel_id, options.feed_engine());
bts_assert_msg(
iter != _engines.end(), "Missing feed engine definition.");
iter != me_->engines.end(), "Missing feed engine definition.");
}
handler->bindFeedEngine(*(iter->second));
if (recovery) {
handler->bindTcpRecovery(*recovery);
_recovery.push_back(std::move(recovery));
me_->recovery.push_back(std::move(recovery));
}
if (logger) {
handler->bindLogger(*logger);
@@ -375,12 +388,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
// heartbeats
vector<MDProducerChannel> producer_channels;
for (auto &symbol : channel.symbols) {
producer_channels.push_back(_producer.get_channel(
producer_channels.push_back(me_->producer.get_channel(
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
unique_ptr<MDP3Listener> listener(new MDP3Listener(
_index, _producer, mbo, producer_channels, trade_summary_dispatcher));
me_->index, me_->producer, mbo, producer_channels,
trade_summary_dispatcher));
for (auto &symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
@@ -29,7 +29,9 @@ struct CMEChannel;
*/
class MDP3Main : public IMDPlatform {
public:
MDP3Main();
explicit MDP3Main();
MDP3Main(const MDP3Main &) = delete;
MDP3Main &operator=(const MDP3Main &) = delete;
~MDP3Main();
void start(
@@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform {
void send_heartbeats() override;
void initialize(const std::string &license_store);
std::unique_ptr<onixs::Handler>
prepare_handler(const std::string &channel, MBOWriter *);
std::unique_ptr<onixs::Handler> prepare_handler(
const std::string &channel, MBOWriter *);
std::unique_ptr<MDP3Listener> prepare_listener(
onixs::Handler &handler,
const CMEChannel &channel,
@@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform {
TradeSummarySignalDispatcher *);
private:
BTS_NO_COPY(MDP3Main);
CMESecurityIndex _index;
SecurityIDRecorder _recorder;
MDProducer _producer;
// co2::SettlementRecorder _settlement_recorder;
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> _engines;
std::map<std::string, std::unique_ptr<onixs::Handler>> _handlers;
std::vector<std::unique_ptr<MDP3Listener>> _listeners;
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> _recovery;
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace md
@@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill(
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
@@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill(
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
@@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters(
req.op_params_size());
for (const auto &p : req.op_params()) {
std::string key_str = std::to_string(p.security_key());
auto db_p =
auto [exists, rowid, val] =
me->security_params_store->retrieve_one(security_params_ns, key_str);
auto found = db_p.first;
if (!found) {
if (!exists) {
me->security_params_store->insert(
security_params_ns, key_str, utils::pb::to_string(p));
} else {
+4 -2
View File
@@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {}
AdminServiceDB::~AdminServiceDB() {}
std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
auto [has_table, rowid, data] =
me->store.retrieve_one(table_name, unique_key);
if (!has_table)
return {};
@@ -33,7 +34,8 @@ std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
}
bool AdminServiceDB::update(api::admin::AdminData const &state) {
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
auto [has_table, rowid, data] =
me->store.retrieve_one(table_name, unique_key);
UNUSED(data);
std::string ser_state;