diff --git a/src/lib/orderman/actors/OrderStore.cpp b/src/lib/orderman/actors/OrderStore.cpp index e6c13e5bc..1c51b6d22 100644 --- a/src/lib/orderman/actors/OrderStore.cpp +++ b/src/lib/orderman/actors/OrderStore.cpp @@ -1,15 +1,15 @@ #include "OrderStore.h" #include "lib/api/services/OrderService.pb.h" -#include "lib/config/protos/orders_fills_cleanup.pb.h" #include "lib/config/config.h" +#include "lib/config/protos/orders_fills_cleanup.pb.h" #include "lib/config/protos/toggles.pb.h" -#include "lib/utils/PBUtils.h" +#include "lib/orderman/actors/SODPosition.h" #include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" #include "lib/persistence/SqliteStore.h" -#include "lib/orderman/actors/SODPosition.h" +#include "lib/utils/PBUtils.h" #include @@ -24,8 +24,8 @@ struct OrderStore::PImpl { PImpl() : state(State::WAIT_SEC), known_orders(), store(new persistence::SqliteStore("orders")), order_stream(), - soft_cleanup_enabled (config::get().enabled()) - {} + soft_cleanup_enabled( + config::get().enabled()) {} State state; const GatewayName gw_name; @@ -75,7 +75,7 @@ void OrderStore::init() { auto all = me->store->retrieve_all()[orders_ns]; LOG(persist, Info, "Finished loading all {} orders", all.size()); - me->cutoff_time=SODPosition::get_order_cutoff_time(all); + me->cutoff_time = SODPosition::get_order_cutoff_time(all); for (auto const &kvp : all) if (utils::pb::from_string( @@ -83,7 +83,9 @@ void OrderStore::init() { auto ok = OrderKey(std::stoi(kvp.first)); auto sk = SecurityKey(rec.def().security_key()); - if (config::get ().enabled() && + if (config::get< + config::toggles::FilterOrdersWithDanglingSecurities>() + .enabled() && sec_idx.find(sk) == sec_idx.end()) { LOG(OrderMan, Diag, "Order {} refers to unresolved security {}", @@ -141,15 +143,13 @@ void OrderStore::on_order_state_update( } } -bool OrderStore::_order_cutoff(const api::orderman::OrderRecord& rec){ +bool OrderStore::_order_cutoff(const api::orderman::OrderRecord &rec) { // TODO: move this to orderutils - bool working= - (rec.state().status()== - api::orderman::OrderStatus::ORDER_STATUS_WORKING) || - (rec.state().status()== - api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING); - return me->soft_cleanup_enabled && - !working && + bool working = (rec.state().status() == + api::orderman::OrderStatus::ORDER_STATUS_WORKING) || + (rec.state().status() == + api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING); + return me->soft_cleanup_enabled && !working && rec.state().created().seconds() < me->cutoff_time.tv_sec; } diff --git a/src/lib/persistence/SqliteStore.cpp b/src/lib/persistence/SqliteStore.cpp index 30ee8d135..a145f0dda 100644 --- a/src/lib/persistence/SqliteStore.cpp +++ b/src/lib/persistence/SqliteStore.cpp @@ -1,12 +1,12 @@ #include "SqliteStore.h" -#include "lib/utils/bts_assert.h" -#include "lib/utils/file_utils.h" -#include "lib/utils/bts_no_copy.h" -#include "lib/os/os.h" #include "lib/logging/logging.h" +#include "lib/os/os.h" #include "lib/utils/Sqlite_utils.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_no_copy.h" #include "lib/utils/defer.h" +#include "lib/utils/file_utils.h" #include #include @@ -14,74 +14,70 @@ namespace bts { namespace persistence { -boost::filesystem::path db_location (std::string const&store_root,std::string const &db) -{ - std::string base_path = store_root; - boost::filesystem::path result = utils::normalize_data_path (base_path); +boost::filesystem::path db_location(std::string const &store_root, + std::string const &db) { + std::string base_path = store_root; + boost::filesystem::path result = utils::normalize_data_path(base_path); result /= "databases"; result /= db; result /= db + ".sqlite"; return result; } -struct SqliteStore::PImpl -{ - PImpl(std::string const&store_root, - std::string const &db_name) - : db(db_name), - db_path (db_location(store_root,db)), - conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), remove(nullptr, sqlite3_finalize), - remove_all(nullptr, sqlite3_finalize), update(nullptr, sqlite3_finalize) - { +struct SqliteStore::PImpl { + PImpl(std::string const &store_root, std::string const &db_name) + : db(db_name), db_path(db_location(store_root, db)), + conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize), + remove(nullptr, sqlite3_finalize), + remove_all(nullptr, sqlite3_finalize), + update(nullptr, sqlite3_finalize) { bts_assert(db.size()); } std::string const db; - boost::filesystem::path db_path; + boost::filesystem::path db_path; sqlite3_ptr conn; sqlite3_stmt_ptr insert; sqlite3_stmt_ptr remove; sqlite3_stmt_ptr remove_all; sqlite3_stmt_ptr update; - BTS_NO_COPY (PImpl); + BTS_NO_COPY(PImpl); }; -SqliteStore::SqliteStore (std::string const &db) - : me (new PImpl(os::bts_data_path (),db)) -{init_db();} - -SqliteStore::SqliteStore (std::string const &root_path, std::string const &db) - : me (new PImpl(root_path,db)) -{init_db();} +SqliteStore::SqliteStore(std::string const &db) + : me(new PImpl(os::bts_data_path(), db)) { + init_db(); +} +SqliteStore::SqliteStore(std::string const &root_path, std::string const &db) + : me(new PImpl(root_path, db)) { + init_db(); +} void SqliteStore::init_db() { auto path = me->db_path; auto parent = path.parent_path(); - - if (!boost::filesystem::is_directory(parent)) + + if (!boost::filesystem::is_directory(parent)) bts_assert(boost::filesystem::create_directories(parent)); - if(!boost::filesystem::exists(path)) { - LOG (persist,Warn, "{} database does not exist yet. Creating it at {}", - me->db, path.string()); + if (!boost::filesystem::exists(path)) { + LOG(persist, Warn, "{} database does not exist yet. Creating it at {}", + me->db, path.string()); me->conn = utils::open_db(path.c_str()); - - const char* create_schema_sql= - "create table store ( ns text, key text, value text)"; + const char *create_schema_sql = + "create table store ( ns text, key text, value text)"; utils::sqlite3_exec_nofail(me->conn, create_schema_sql); - const char* key_index_sql = - "create unique index key_index on store(ns,key)"; + const char *key_index_sql = + "create unique index key_index on store(ns,key)"; utils::sqlite3_exec_nofail(me->conn, key_index_sql); - const char* ns_index_sql = - "create index ns_index on store(ns)"; + const char *ns_index_sql = "create index ns_index on store(ns)"; utils::sqlite3_exec_nofail(me->conn, ns_index_sql); - } - else { + } else { me->conn = utils::open_db(path.c_str()); } @@ -90,110 +86,132 @@ void SqliteStore::init_db() { utils::sqlite3_exec_nofail(me->conn, "PRAGMA journal_mode=WAL"); // prepare insert statement - const char *insert_sql = - "insert into store (ns, key, value) values(?,?,?)"; + const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)"; me->insert = utils::prepare_query(me->conn, insert_sql); // prepare update statement const char *update_sql = - "update store set value = ? where ns = ? and key = ?"; + "update store set value = ? where ns = ? and key = ?"; me->update = utils::prepare_query(me->conn, update_sql); // prepare remove statement - const char *remove_all_sql = - "delete from store where ns = ?"; + const char *remove_all_sql = "delete from store where ns = ?"; me->remove_all = utils::prepare_query(me->conn, remove_all_sql); // prepare remove statement - const char *remove_sql = - "delete from store where ns = ? and key = ?"; + const char *remove_sql = "delete from store where ns = ? and key = ?"; me->remove = utils::prepare_query(me->conn, remove_sql); } - -SqliteStore::~SqliteStore () { } +SqliteStore::~SqliteStore() {} SqliteStore::Data SqliteStore::retrieve_all() { SqliteStore::Data result; - sqlite3_stmt_ptr query = utils::prepare_query(me->conn, "select ns,key,value from store"); + sqlite3_stmt_ptr query = + utils::prepare_query(me->conn, "select ns,key,value from store"); int sqlite_result; do { auto pq = query.get(); sqlite_result = sqlite3_step(pq); - if (sqlite_result == SQLITE_ROW ) { - std::string ns((char*)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key((char*)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); - std::string val((char*)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); - auto& r = result[ns][key]; + if (sqlite_result == SQLITE_ROW) { + std::string ns((char *)sqlite3_column_text(pq, 0), + sqlite3_column_bytes(pq, 0)); + std::string key((char *)sqlite3_column_text(pq, 1), + sqlite3_column_bytes(pq, 1)); + std::string val((char *)sqlite3_column_blob(pq, 2), + sqlite3_column_bytes(pq, 2)); + auto &r = result[ns][key]; r = std::move(val); } - } while(sqlite_result == SQLITE_ROW); + } while (sqlite_result == SQLITE_ROW); return result; } -std::pair SqliteStore::retrieve_one(std::string const&ns, std::string const&k) -{ - auto query_str = fmt::format ("select ns,key,value from store where ns='{}' and key='{}'", ns,k); - sqlite3_stmt_ptr query = utils::prepare_query(me->conn,query_str.c_str()); +std::pair SqliteStore::retrieve_one(std::string const &ns, + std::string const &k) { + auto query_str = fmt::format( + "select ns,key,value from store where ns='{}' and key='{}'", ns, k); + sqlite3_stmt_ptr query = utils::prepare_query(me->conn, query_str.c_str()); + + std::pair result{false, ""}; - std::pair result {false,""}; - auto pq = query.get(); auto sqlite_result = sqlite3_step(pq); - if (sqlite_result == SQLITE_ROW ) { - std::string ns((char*)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0)); - std::string key((char*)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1)); - std::string val((char*)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2)); + if (sqlite_result == SQLITE_ROW) { + std::string ns((char *)sqlite3_column_text(pq, 0), + sqlite3_column_bytes(pq, 0)); + std::string key((char *)sqlite3_column_text(pq, 1), + sqlite3_column_bytes(pq, 1)); + std::string val((char *)sqlite3_column_blob(pq, 2), + sqlite3_column_bytes(pq, 2)); result.first = true; result.second = std::move(val); } return result; } -void SqliteStore::remove (std::string const &ns, std::set const &keys) -{ +void SqliteStore::remove(std::string const &ns, + std::set const &keys) { bts_assert(ns.size()); for (auto &k : keys) { - SQLITE_CHK (sqlite3_bind_text(me->remove.get(), 1,ns.c_str(),ns.size(),SQLITE_TRANSIENT),me->conn); - SQLITE_CHK (sqlite3_bind_text(me->remove.get(), 2,k.c_str(),k.size(),SQLITE_TRANSIENT),me->conn); - bts_assert (sqlite3_step(me->remove.get()) == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->remove.get()),me->conn); + SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 1, ns.c_str(), ns.size(), + SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK(sqlite3_bind_text(me->remove.get(), 2, k.c_str(), k.size(), + SQLITE_TRANSIENT), + me->conn); + bts_assert(sqlite3_step(me->remove.get()) == SQLITE_DONE); + SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn); } } -void SqliteStore::remove_all (std::string const &ns) { +void SqliteStore::remove_all(std::string const &ns) { bts_assert(ns.size()); - SQLITE_CHK (sqlite3_bind_text(me->remove_all.get(), 1,ns.c_str(),ns.size(),SQLITE_TRANSIENT),me->conn); - bts_assert (sqlite3_step(me->remove_all.get()) == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->remove_all.get()),me->conn); + SQLITE_CHK(sqlite3_bind_text(me->remove_all.get(), 1, ns.c_str(), ns.size(), + SQLITE_TRANSIENT), + me->conn); + bts_assert(sqlite3_step(me->remove_all.get()) == SQLITE_DONE); + SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn); } -void SqliteStore::insert (std::string const &ns, std::string const&k,std::string const&v) -{ - SQLITE_CHK (sqlite3_bind_text(me->insert.get(), 1,ns.c_str(),ns.size(),SQLITE_TRANSIENT),me->conn); - SQLITE_CHK (sqlite3_bind_text(me->insert.get(), 2,k.c_str(),k.size(),SQLITE_TRANSIENT),me->conn); - SQLITE_CHK (sqlite3_bind_text(me->insert.get(), 3,v.c_str(),v.size(),SQLITE_TRANSIENT),me->conn); - auto r = sqlite3_step(me->insert.get()) ; - bts_assert ( r == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->insert.get()),me->conn); +void SqliteStore::insert(std::string const &ns, std::string const &k, + std::string const &v) { + SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 1, ns.c_str(), ns.size(), + SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 2, k.c_str(), k.size(), + SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK(sqlite3_bind_text(me->insert.get(), 3, v.c_str(), v.size(), + SQLITE_TRANSIENT), + me->conn); + auto r = sqlite3_step(me->insert.get()); + bts_assert(r == SQLITE_DONE); + SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn); } -void SqliteStore::update (std::string const &ns, std::string const&k,std::string const&v) -{ - SQLITE_CHK (sqlite3_bind_text(me->update.get(), 1,v.c_str(),v.size(),SQLITE_TRANSIENT),me->conn); - SQLITE_CHK (sqlite3_bind_text(me->update.get(), 2,ns.c_str(),ns.size(),SQLITE_TRANSIENT),me->conn); - SQLITE_CHK (sqlite3_bind_text(me->update.get(), 3,k.c_str(),k.size(),SQLITE_TRANSIENT),me->conn); +void SqliteStore::update(std::string const &ns, std::string const &k, + std::string const &v) { + SQLITE_CHK(sqlite3_bind_text(me->update.get(), 1, v.c_str(), v.size(), + SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK(sqlite3_bind_text(me->update.get(), 2, ns.c_str(), ns.size(), + SQLITE_TRANSIENT), + me->conn); + SQLITE_CHK(sqlite3_bind_text(me->update.get(), 3, k.c_str(), k.size(), + SQLITE_TRANSIENT), + me->conn); auto r = sqlite3_step(me->update.get()); - bts_assert (r == SQLITE_DONE); - SQLITE_CHK (sqlite3_reset (me->update.get()),me->conn); + bts_assert(r == SQLITE_DONE); + SQLITE_CHK(sqlite3_reset(me->update.get()), me->conn); } -std::string SqliteStore::storage_location(std::string const&db) -{ - return db_location(os::bts_data_path(),db).string(); +std::string SqliteStore::storage_location(std::string const &db) { + return db_location(os::bts_data_path(), db).string(); } -}} +} // namespace persistence +} // namespace bts