This new structure provides several benefits:
* Allows us to install multiple CMake projects in the same destination
dir (without include path conflicts).
- This makes shipping the trading algo sdk a trivial excercise. Simply
copy the lib and include dirs from the install dest. That's it!
* Puts us in a position to break up the monolithic libspark into smaller
modules, which are easier to manage.
Windows specific notes:
* Get rid of unnecessary script vs_filters.cmake
* Define WebsocketConnectionGateway directly in cs (instead of proto)
- The proto would no longer be in the include path passed to protoc.
Moving it to c# because there wasn't a compelling reason for it to
be a proto in the first place.
* Set spark output prefix to "lib" on windows
- So we get "libspark.dll" instead of "spark.dll", which conflicts
with the csharp lib, "Spark.dll".
* Copy libspark.dll to net8.0 & net8.0-windows in a single cmake build
target
fix build: Copy libspark to both dirs net8.0, net8.0-windows
403 lines
13 KiB
C++
403 lines
13 KiB
C++
#include "SqliteStore.h"
|
|
|
|
#include "spark/config/config.h"
|
|
#include "spark/config/protos/system_config.pb.h"
|
|
#include "spark/logging/logging.h"
|
|
#include "spark/os/os.h"
|
|
#include "spark/persistence/persistence_utils.h"
|
|
#include "spark/utils/Sqlite_utils.h"
|
|
#include "spark/utils/bts_assert.h"
|
|
#include "spark/utils/bts_chrono.h"
|
|
#include "spark/utils/bts_no_copy.h"
|
|
#include "spark/utils/defer.h"
|
|
#include "spark/utils/file_utils.h"
|
|
|
|
#include <boost/filesystem.hpp>
|
|
#include <sqlite/sqlite3.h>
|
|
|
|
namespace bts {
|
|
namespace persistence {
|
|
|
|
using bts::config::SystemConfig;
|
|
|
|
static boost::filesystem::path get_db_path(const std::string &str) {
|
|
for (char c : str) {
|
|
switch (c) {
|
|
case '.':
|
|
case '/':
|
|
case '\\':
|
|
// Looks like 'str' is already a path
|
|
return {str};
|
|
}
|
|
}
|
|
return boost::filesystem::path{config::get<SystemConfig>().data_path()} /
|
|
"databases" / str / (str + ".sqlite");
|
|
}
|
|
|
|
struct SqliteStore::PImpl {
|
|
std::string const db;
|
|
boost::filesystem::path db_path;
|
|
sqlite3_ptr conn{nullptr, sqlite3_close};
|
|
sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize};
|
|
sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize};
|
|
sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize};
|
|
sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize};
|
|
sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize};
|
|
sqlite3_stmt_ptr update{nullptr, sqlite3_finalize};
|
|
|
|
PImpl(const boost::filesystem::path &db_path)
|
|
: db(db_path.stem().string()), db_path(db_path) {
|
|
bts_assert(db.size());
|
|
}
|
|
|
|
PImpl(std::string const &store_root, std::string const &db_name)
|
|
: PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {}
|
|
|
|
explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {}
|
|
|
|
PImpl(const PImpl &) = delete;
|
|
PImpl &operator=(const PImpl &) = delete;
|
|
};
|
|
|
|
SqliteStore::SqliteStore(std::string const &db)
|
|
: me(std::make_unique<PImpl>(get_db_path(db))) {
|
|
LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db,
|
|
me->db_path.string());
|
|
init_db();
|
|
}
|
|
|
|
SqliteStore::SqliteStore(sqlite3_ptr db)
|
|
: me(std::make_unique<PImpl>(std::move(db))) {
|
|
LOG(persist, Info, "SqliteStore ctr. db: <imported ptr>");
|
|
init_db();
|
|
}
|
|
|
|
void SqliteStore::init_db() {
|
|
const auto path = me->db_path;
|
|
|
|
if (!path.empty()) {
|
|
if (!boost::filesystem::is_directory(path.parent_path())) {
|
|
bts_assert(boost::filesystem::create_directories(path.parent_path()));
|
|
}
|
|
if (!boost::filesystem::exists(path)) {
|
|
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
|
|
me->db, path.string());
|
|
}
|
|
}
|
|
|
|
auto &conn = me->conn;
|
|
if (conn == nullptr) {
|
|
conn = sqlite::open_db(path.c_str());
|
|
bts_assert(conn != nullptr);
|
|
}
|
|
sqlite::exec_nofail(
|
|
conn,
|
|
"CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)");
|
|
sqlite::exec_nofail(
|
|
conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)");
|
|
sqlite::exec_nofail(
|
|
conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)");
|
|
conn = sqlite::open_db(path.c_str());
|
|
|
|
// Asynchronous for speed
|
|
sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;");
|
|
sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL");
|
|
|
|
// prepare insert statement
|
|
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
|
|
me->insert = sqlite::prepare_query(conn, insert_sql);
|
|
|
|
// prepare insert_with_id statement
|
|
const char *insert_with_id_sql =
|
|
"insert into store (rowid, ns, key, value) values (?,?,?,?)";
|
|
me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql);
|
|
|
|
// prepare update statement
|
|
const char *update_sql =
|
|
"update store set value = ? where ns = ? and key = ?";
|
|
me->update = sqlite::prepare_query(conn, update_sql);
|
|
|
|
// prepare remove statement
|
|
const char *remove_all_sql = "delete from store where ns = ?";
|
|
me->remove_all = sqlite::prepare_query(conn, remove_all_sql);
|
|
|
|
// prepare remove statement
|
|
const char *remove_sql = "delete from store where ns = ? and key = ?";
|
|
me->remove = sqlite::prepare_query(conn, remove_sql);
|
|
}
|
|
|
|
SqliteStore::~SqliteStore() {}
|
|
|
|
SqliteStore::Data SqliteStore::retrieve_all() {
|
|
SqliteStore::Data result;
|
|
sqlite3_stmt_ptr query =
|
|
sqlite::prepare_query(me->conn, "select ns,key,value from store");
|
|
|
|
int sqlite_result;
|
|
do {
|
|
auto pq = query.get();
|
|
sqlite_result = sqlite3_step(pq);
|
|
if (sqlite_result == SQLITE_ROW) {
|
|
std::string ns(
|
|
(char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0));
|
|
std::string key(
|
|
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
|
|
std::string val(
|
|
(char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2));
|
|
auto &r = result[ns][key];
|
|
r = std::move(val);
|
|
}
|
|
} while (sqlite_result == SQLITE_ROW);
|
|
|
|
return result;
|
|
}
|
|
|
|
std::tuple<bool, std::int64_t, std::string> SqliteStore::retrieve_one(
|
|
std::string const &ns, std::string const &k) {
|
|
auto query_str = fmt::format(
|
|
"select rowid,ns,key,value from store where ns='{}' and key='{}'", ns,
|
|
k);
|
|
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
|
|
|
|
auto result = std::make_tuple<bool, std::int64_t, std::string>(false, 0, "");
|
|
auto pq = query.get();
|
|
auto sqlite_result = sqlite3_step(pq);
|
|
if (sqlite_result == SQLITE_ROW) {
|
|
std::int64_t rowid = sqlite3_column_int64(pq, 0);
|
|
std::string ns(
|
|
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
|
|
std::string key(
|
|
(char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2));
|
|
std::string val(
|
|
(char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3));
|
|
std::get<0>(result) = true;
|
|
std::get<1>(result) = rowid;
|
|
std::get<2>(result) = std::move(val);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void SqliteStore::remove(
|
|
std::string const &ns, std::set<std::string> const &keys) {
|
|
bts_assert(ns.size());
|
|
|
|
try {
|
|
for (auto &k : keys) {
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->remove.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not delete row in store. Will retry once. error_code: "
|
|
"{} "
|
|
"ns:{}, k: {}",
|
|
r, ns, k);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->remove.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to delete row. Bailing out. "
|
|
"error_code: "
|
|
"{}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
|
|
}
|
|
} catch (const std::exception &ex) {
|
|
LOG(persist, Error, "SqliteStore::remove threw. what: {}",
|
|
std::string{ex.what()});
|
|
}
|
|
}
|
|
|
|
void SqliteStore::remove_all(std::string const &ns) {
|
|
bts_assert(ns.size());
|
|
try {
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->remove_all.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not delete all rows in store. Will retry once. error_code: "
|
|
"{} "
|
|
"ns:{}",
|
|
r, ns);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->remove_all.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to delete all rows. Bailing out. "
|
|
"error_code: "
|
|
"{}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
|
|
} catch (const std::exception &ex) {
|
|
LOG(persist, Error, "SqliteStore::remove_all threw. what: {}",
|
|
std::string{ex.what()});
|
|
}
|
|
}
|
|
|
|
std::int64_t SqliteStore::insert(
|
|
std::string const &ns, std::string const &k, std::string const &v) {
|
|
try {
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->insert.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not insert row in store. Will retry once. error_code: {} "
|
|
"ns: "
|
|
"{}, k: {}, v: {}",
|
|
r, ns, k, v);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->insert.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to insert row. Bailing out. error_code: "
|
|
"{}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
const auto rowid = sqlite3_last_insert_rowid(me->conn.get());
|
|
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
|
|
return rowid;
|
|
} catch (const std::exception &ex) {
|
|
LOG(persist, Error, "SqliteStore::insert threw. what: {}",
|
|
std::string{ex.what()});
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void SqliteStore::insert_many(const std::vector<StoreRow> &rows) {
|
|
LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size());
|
|
if (rows.empty()) {
|
|
return;
|
|
}
|
|
try {
|
|
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
|
|
sqlite3 *conn_rawp = me->conn.get();
|
|
sqlite3_stmt *insert_rawp = me->insert.get();
|
|
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
|
|
for (const auto &row : rows) {
|
|
int col_idx = 1;
|
|
sqlite3_stmt *stmt;
|
|
if (row.id > 0) {
|
|
stmt = insert_with_id_rawp;
|
|
sqlite3_bind_int(stmt, col_idx++, row.id);
|
|
} else {
|
|
stmt = insert_rawp;
|
|
}
|
|
sqlite3_bind_text(
|
|
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
|
|
sqlite3_bind_text(
|
|
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
|
|
int result = sqlite3_step(stmt);
|
|
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
|
sqlite3_last_insert_rowid(conn_rawp);
|
|
}
|
|
sqlite3_reset(stmt);
|
|
}
|
|
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
|
|
} catch (const std::exception &ex) {
|
|
LOG(persist, Error, "SqliteStore::insert_many threw. what: {}",
|
|
std::string{ex.what()});
|
|
}
|
|
}
|
|
|
|
void SqliteStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
|
|
LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size());
|
|
if (rows.empty()) {
|
|
return;
|
|
}
|
|
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
|
|
sqlite3 *conn_rawp = me->conn.get();
|
|
sqlite3_stmt *insert_rawp = me->insert.get();
|
|
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
|
|
for (auto &row : rows) {
|
|
int col_idx = 1;
|
|
sqlite3_stmt *stmt;
|
|
if (row.id > 0) {
|
|
stmt = insert_with_id_rawp;
|
|
sqlite3_bind_int(stmt, col_idx++, row.id);
|
|
} else {
|
|
stmt = insert_rawp;
|
|
}
|
|
sqlite3_bind_text(
|
|
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
|
|
sqlite3_bind_text(
|
|
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
|
|
int result = sqlite3_step(stmt);
|
|
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
|
if (row.id > 0) {
|
|
bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id);
|
|
} else {
|
|
row.id = sqlite3_last_insert_rowid(conn_rawp);
|
|
}
|
|
}
|
|
sqlite3_reset(stmt);
|
|
}
|
|
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
|
|
}
|
|
|
|
void SqliteStore::update(
|
|
std::string const &ns, std::string const &k, std::string const &v) {
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->update.get(), 1, v.c_str(), v.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->update.get(), 2, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
SQLITE_CHK(
|
|
sqlite3_bind_text(
|
|
me->update.get(), 3, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
|
me->conn);
|
|
auto r = sqlite3_step(me->update.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Could not update row in store. Will retry once. error_code: {} ns: "
|
|
"{}, k: {}, v: {}",
|
|
r, ns, k, v);
|
|
sleep_for(2s);
|
|
r = sqlite3_step(me->update.get());
|
|
if (r != SQLITE_DONE) {
|
|
LOG(persist, Error,
|
|
"Failed a second time to update row. Bailing out. error_code: {}",
|
|
r);
|
|
bts_assert(r == SQLITE_DONE);
|
|
}
|
|
}
|
|
SQLITE_CHK(sqlite3_reset(me->update.get()), me->conn);
|
|
}
|
|
|
|
std::string SqliteStore::storage_location(std::string const &db) {
|
|
return persistence::storage_location(db);
|
|
}
|
|
|
|
} // namespace persistence
|
|
} // namespace bts
|