Files
spark/src/lib/orderman/actors/OrderStore.cpp
T
John Colvin 5db89e1e8d diag: Tone down noisy order management logs
* EMNativeOrder
* OrderRouter - only log routing record result if there are errors
* OrderServiceServer
* OrderStore - also avoid calling pb::to_string twice per change (it's
  expensive)
* ServiceBridge
2025-03-17 00:51:47 +00:00

216 lines
7.7 KiB
C++

#include "OrderStore.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/config/config.h"
#include "lib/config/protos/orders_fills_cleanup.pb.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/ExchangeSecurityKey.h"
#include "lib/orderman/actors/SODPosition.h"
#include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
#include "lib/orderman/events/OrderStateUpdateEvent.h"
#include "lib/orderman/events/StoredOrderEvent.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
#include <google/protobuf/timestamp.pb.h>
#include <unordered_set>
namespace bts {
namespace orderman {
static auto const orders_ns = "orders";
struct OrderStore::PImpl {
PImpl(md::SecurityOracle o)
: known_orders(), store(new persistence::SqliteStore("orders")),
order_stream(),
soft_cleanup_enabled(
config::get<config::toggles::OrdersSoftCleanup>().enabled()),
oracle(o) {}
const GatewayName gw_name;
struct ORInfo {
api::orderman::OrderRecord rec;
bool is_saved;
};
std::unordered_map<OrderKey, ORInfo> known_orders;
std::unique_ptr<persistence::IStore> store;
mercury::Stream<StoredOrderEvent> order_stream;
bool soft_cleanup_enabled;
md::SecurityOracle oracle;
timeval cutoff_time;
bool clear_all_orders = true;
};
OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
OrderStore::~OrderStore() {}
void OrderStore::init() {
// subscribe to order updates so we can store them
subscribe<OrderStateUpdateEvent>(
[this](auto const &ev) { this->on_order_state_update(ev); });
subscribe<OrderDefinitionUpdateEvent>(
[this](auto const &ev) { this->on_order_definition_update(ev); });
// now load orders from disk, for each order check if
// it reffers to a valid security. if so publish it.
api::orderman::OrderRecord rec;
std::vector<StoredOrderEvent> stored_batch;
LOG(persist, Info, "Loading all orders");
auto all = me->store->retrieve_all()[orders_ns];
LOG(persist, Info, "Finished loading all {} orders", all.size());
me->cutoff_time = SODPosition::get_order_cutoff_time(all);
auto load_orders = [&]() {
for (auto const &kvp : all) {
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto ok = OrderKey(std::stoi(kvp.first));
auto sk = SecurityKey(rec.def().security_key());
auto sec = me->oracle.get_security(sk);
if (config::get<
config::toggles::FilterOrdersWithDanglingSecurities>()
.enabled() &&
sec == nullptr) {
LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
ok.value(), sk.value());
} else {
if (!_order_cutoff(rec))
stored_batch.emplace_back(rec);
me->known_orders[ok] = PImpl::ORInfo{rec, true};
}
} else
LOG(OrderMan, Error,
"Could not recover order {} from storage. "
"This should be investigated",
kvp.first);
}
};
me->order_stream = make_stream<StoredOrderEvent>();
LOG(OrderMan, Info,
"Order Store published {} orders recoved from disk out of {} "
"total orders",
stored_batch.size(), all.size());
if (me->clear_all_orders) {
// clear orders
me->store->remove_all(orders_ns);
} else {
load_orders();
}
publish_batch(me->order_stream, stored_batch);
}
void OrderStore::on_order_state_update(
mercury::Update<OrderStateUpdateEvent> const &u) {
std::set<OrderKey> modified_orders;
if (u.events)
for (auto const &ev : *u.events) {
LOG(OrderMan, Info,
"OrderStore::on_order_state_update k: {} p: {} q: {} fq: {} "
"key_exists: "
"{}",
ev.key, ev.price, ev.quantity, ev.filled_quantity,
me->known_orders.find(ev.key) != me->known_orders.end());
auto &rec_info = me->known_orders[ev.key];
LOG(iLink3, Info, "Received order state update for {}",
ev.key.value());
auto &rec = rec_info.rec;
bts_assert(rec.has_def());
rec.set_key(ev.key.value());
rec.set_parent_key(ev.parent_key.value());
modified_orders.insert(ev.key);
const auto now = utils::pb::pb_now();
if (rec.has_state()) {
google::protobuf::Timestamp created;
if (rec.state().has_created()) {
created = rec.state().created();
}
rec.mutable_state()->CopyFrom(ev.rebuild_state());
*rec.mutable_state()->mutable_created() =
created.seconds() > 0 || created.nanos() > 0 ? created : now;
// if filled qty went from 0.99 -> 1, it was copying the int part and not clearing the fraction part,
// so changing MergeFrom to CopyFrom
} else {
*rec.mutable_state() = ev.rebuild_state();
*rec.mutable_state()->mutable_created() = now;
}
*rec.mutable_state()->mutable_last_change() = now;
LOG(OrderMan, Diag, "OrderStore::on_order_state_update k: {} rec: {}",
ev.key, utils::pb::to_string(rec));
}
store_and_publish_if_complete(modified_orders);
}
void OrderStore::on_order_definition_update(
mercury::Update<OrderDefinitionUpdateEvent> const &u) {
std::set<OrderKey> modified_orders;
if (u.events)
for (auto const &ev : *u.events) {
LOG(iLink3, Info, "Received order definition update for {}",
ev.key.value());
auto &rec_info = me->known_orders[ev.key];
auto &rec = rec_info.rec;
rec.set_key(ev.key.value());
rec_info.is_saved = rec.has_def();
*rec.mutable_def() = *ev.def;
modified_orders.insert(ev.key);
}
store_and_publish_if_complete(modified_orders);
}
void OrderStore::store_and_publish_if_complete(
std::set<OrderKey> const &changed) {
for (auto k : changed) {
auto &rec_info = me->known_orders[k];
auto &rec = rec_info.rec;
if (rec.has_def() && rec.has_state()) {
const std::string rec_str = utils::pb::to_string(rec);
const std::string key_str = std::to_string(k.value());
LOG(OrderMan, Diag,
"OrderStore::store_and_publish_if_complete rec: {}", rec_str);
publish(me->order_stream, rec);
if (rec_info.is_saved) {
me->store->update(orders_ns, key_str, rec_str);
LOG(OrderMan, Info,
"OrderStore::store_and_publish_if_complete updated key: {}", k);
} else {
me->store->insert(orders_ns, key_str, rec_str);
rec_info.is_saved = true;
LOG(OrderMan, Info,
"OrderStore::store_and_publish_if_complete added key: {}", k);
}
} else {
LOG(OrderMan, Warn,
"OrderStore::store_and_publish_if_complete failed to add key: {} "
"has_def: {} has_state: {}",
k.value(), rec.has_def(), rec.has_state());
}
}
}
bool OrderStore::_order_cutoff(const api::orderman::OrderRecord &rec) {
// TODO: move this to orderutils
bool working = (rec.state().status() ==
api::orderman::OrderStatus::ORDER_STATUS_WORKING) ||
(rec.state().status() ==
api::orderman::OrderStatus::ORDER_STATUS_DISCONNECTED) ||
(rec.state().status() ==
api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING);
return me->soft_cleanup_enabled && !working &&
rec.state().created().seconds() < me->cutoff_time.tv_sec;
}
} // namespace orderman
} // namespace bts