Files
John Colvin 5163ca4cae refactor: Organize libspark proj w/ stuttering include
This new structure provides several benefits:

* Allows us to install multiple CMake projects in the same destination
  dir (without include path conflicts).
  - This makes shipping the trading algo sdk a trivial excercise. Simply
    copy the lib and include dirs from the install dest. That's it!
* Puts us in a position to break up the monolithic libspark into smaller
  modules, which are easier to manage.

Windows specific notes:

* Get rid of unnecessary script vs_filters.cmake
* Define WebsocketConnectionGateway directly in cs (instead of proto)
  - The proto would no longer be in the include path passed to protoc.
    Moving it to c# because there wasn't a compelling reason for it to
    be a proto in the first place.
* Set spark output prefix to "lib" on windows
  - So we get "libspark.dll" instead of "spark.dll", which conflicts
    with the csharp lib, "Spark.dll".
* Copy libspark.dll to net8.0 & net8.0-windows in a single cmake build
  target

fix build: Copy libspark to both dirs net8.0, net8.0-windows
2025-06-16 11:08:53 -04:00

403 lines
13 KiB
C++

#include "SqliteStore.h"
#include "spark/config/config.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/logging/logging.h"
#include "spark/os/os.h"
#include "spark/persistence/persistence_utils.h"
#include "spark/utils/Sqlite_utils.h"
#include "spark/utils/bts_assert.h"
#include "spark/utils/bts_chrono.h"
#include "spark/utils/bts_no_copy.h"
#include "spark/utils/defer.h"
#include "spark/utils/file_utils.h"
#include <boost/filesystem.hpp>
#include <sqlite/sqlite3.h>
namespace bts {
namespace persistence {
using bts::config::SystemConfig;
static boost::filesystem::path get_db_path(const std::string &str) {
for (char c : str) {
switch (c) {
case '.':
case '/':
case '\\':
// Looks like 'str' is already a path
return {str};
}
}
return boost::filesystem::path{config::get<SystemConfig>().data_path()} /
"databases" / str / (str + ".sqlite");
}
struct SqliteStore::PImpl {
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn{nullptr, sqlite3_close};
sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr update{nullptr, sqlite3_finalize};
PImpl(const boost::filesystem::path &db_path)
: db(db_path.stem().string()), db_path(db_path) {
bts_assert(db.size());
}
PImpl(std::string const &store_root, std::string const &db_name)
: PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {}
explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {}
PImpl(const PImpl &) = delete;
PImpl &operator=(const PImpl &) = delete;
};
SqliteStore::SqliteStore(std::string const &db)
: me(std::make_unique<PImpl>(get_db_path(db))) {
LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db,
me->db_path.string());
init_db();
}
SqliteStore::SqliteStore(sqlite3_ptr db)
: me(std::make_unique<PImpl>(std::move(db))) {
LOG(persist, Info, "SqliteStore ctr. db: <imported ptr>");
init_db();
}
void SqliteStore::init_db() {
const auto path = me->db_path;
if (!path.empty()) {
if (!boost::filesystem::is_directory(path.parent_path())) {
bts_assert(boost::filesystem::create_directories(path.parent_path()));
}
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
}
}
auto &conn = me->conn;
if (conn == nullptr) {
conn = sqlite::open_db(path.c_str());
bts_assert(conn != nullptr);
}
sqlite::exec_nofail(
conn,
"CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)");
sqlite::exec_nofail(
conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)");
sqlite::exec_nofail(
conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)");
conn = sqlite::open_db(path.c_str());
// Asynchronous for speed
sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL");
// prepare insert statement
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
me->insert = sqlite::prepare_query(conn, insert_sql);
// prepare insert_with_id statement
const char *insert_with_id_sql =
"insert into store (rowid, ns, key, value) values (?,?,?,?)";
me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql);
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
me->update = sqlite::prepare_query(conn, update_sql);
// prepare remove statement
const char *remove_all_sql = "delete from store where ns = ?";
me->remove_all = sqlite::prepare_query(conn, remove_all_sql);
// prepare remove statement
const char *remove_sql = "delete from store where ns = ? and key = ?";
me->remove = sqlite::prepare_query(conn, remove_sql);
}
SqliteStore::~SqliteStore() {}
SqliteStore::Data SqliteStore::retrieve_all() {
SqliteStore::Data result;
sqlite3_stmt_ptr query =
sqlite::prepare_query(me->conn, "select ns,key,value from store");
int sqlite_result;
do {
auto pq = query.get();
sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::string ns(
(char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0));
std::string key(
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
std::string val(
(char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2));
auto &r = result[ns][key];
r = std::move(val);
}
} while (sqlite_result == SQLITE_ROW);
return result;
}
std::tuple<bool, std::int64_t, std::string> SqliteStore::retrieve_one(
std::string const &ns, std::string const &k) {
auto query_str = fmt::format(
"select rowid,ns,key,value from store where ns='{}' and key='{}'", ns,
k);
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
auto result = std::make_tuple<bool, std::int64_t, std::string>(false, 0, "");
auto pq = query.get();
auto sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::int64_t rowid = sqlite3_column_int64(pq, 0);
std::string ns(
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
std::string key(
(char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2));
std::string val(
(char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3));
std::get<0>(result) = true;
std::get<1>(result) = rowid;
std::get<2>(result) = std::move(val);
}
return result;
}
void SqliteStore::remove(
std::string const &ns, std::set<std::string> const &keys) {
bts_assert(ns.size());
try {
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete row in store. Will retry once. error_code: "
"{} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
}
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::remove_all(std::string const &ns) {
bts_assert(ns.size());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete all rows in store. Will retry once. error_code: "
"{} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove_all threw. what: {}",
std::string{ex.what()});
}
}
std::int64_t SqliteStore::insert(
std::string const &ns, std::string const &k, std::string const &v) {
try {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not insert row in store. Will retry once. error_code: {} "
"ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
const auto rowid = sqlite3_last_insert_rowid(me->conn.get());
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return rowid;
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert threw. what: {}",
std::string{ex.what()});
}
return 0;
}
void SqliteStore::insert_many(const std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size());
if (rows.empty()) {
return;
}
try {
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (const auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
sqlite3_last_insert_rowid(conn_rawp);
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert_many threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size());
if (rows.empty()) {
return;
}
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
if (row.id > 0) {
bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id);
} else {
row.id = sqlite3_last_insert_rowid(conn_rawp);
}
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
}
void SqliteStore::update(
std::string const &ns, std::string const &k, std::string const &v) {
SQLITE_CHK(
sqlite3_bind_text(
me->update.get(), 1, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->update.get(), 2, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->update.get(), 3, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->update.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not update row in store. Will retry once. error_code: {} ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->update.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to update row. Bailing out. error_code: {}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->update.get()), me->conn);
}
std::string SqliteStore::storage_location(std::string const &db) {
return persistence::storage_location(db);
}
} // namespace persistence
} // namespace bts