#include "OrderStore.h" #include "lib/api/services/OrderService.pb.h" #include "lib/config/config.h" #include "lib/config/protos/orders_fills_cleanup.pb.h" #include "lib/config/protos/toggles.pb.h" #include "lib/logging/logging.h" #include "lib/md/ExchangeSecurityKey.h" #include "lib/orderman/actors/SODPosition.h" #include "lib/orderman/events/OrderDefinitionUpdateEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" #include "lib/persistence/SqliteStore.h" #include "lib/utils/PBUtils.h" #include #include namespace bts { namespace orderman { static auto const orders_ns = "orders"; struct OrderStore::PImpl { PImpl(md::SecurityOracle o) : known_orders(), store(new persistence::SqliteStore("orders")), order_stream(), soft_cleanup_enabled( config::get().enabled()), oracle(o) {} const GatewayName gw_name; struct ORInfo { api::orderman::OrderRecord rec; bool is_saved; }; std::unordered_map known_orders; std::unique_ptr store; mercury::Stream order_stream; bool soft_cleanup_enabled; md::SecurityOracle oracle; timeval cutoff_time; bool clear_all_orders = true; }; OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {} OrderStore::~OrderStore() {} void OrderStore::init() { // subscribe to order updates so we can store them subscribe( [this](auto const &ev) { this->on_order_state_update(ev); }); subscribe( [this](auto const &ev) { this->on_order_definition_update(ev); }); // now load orders from disk, for each order check if // it reffers to a valid security. if so publish it. api::orderman::OrderRecord rec; std::vector stored_batch; LOG(persist, Info, "Loading all orders"); auto all = me->store->retrieve_all()[orders_ns]; LOG(persist, Info, "Finished loading all {} orders", all.size()); me->cutoff_time = SODPosition::get_order_cutoff_time(all); auto load_orders = [&]() { for (auto const &kvp : all) { if (utils::pb::from_string( kvp.second, &rec)) { auto ok = OrderKey(std::stoi(kvp.first)); auto sk = SecurityKey(rec.def().security_key()); auto sec = me->oracle.get_security(sk); if (config::get< config::toggles::FilterOrdersWithDanglingSecurities>() .enabled() && sec == nullptr) { LOG(OrderMan, Diag, "Order {} refers to unresolved security {}", ok.value(), sk.value()); } else { if (!_order_cutoff(rec)) stored_batch.emplace_back(rec); me->known_orders[ok] = PImpl::ORInfo{rec, true}; } } else LOG(OrderMan, Error, "Could not recover order {} from storage. " "This should be investigated", kvp.first); } }; me->order_stream = make_stream(); LOG(OrderMan, Info, "Order Store published {} orders recoved from disk out of {} " "total orders", stored_batch.size(), all.size()); if (me->clear_all_orders) { // clear orders me->store->remove_all(orders_ns); } else { load_orders(); } publish_batch(me->order_stream, stored_batch); } void OrderStore::on_order_state_update( mercury::Update const &u) { std::set modified_orders; if (u.events) for (auto const &ev : *u.events) { LOG(OrderMan, Info, "OrderStore::on_order_state_update k: {} p: {} q: {} fq: {} " "key_exists: " "{}", ev.key, ev.price, ev.quantity, ev.filled_quantity, me->known_orders.find(ev.key) != me->known_orders.end()); auto &rec_info = me->known_orders[ev.key]; LOG(iLink3, Info, "Received order state update for {}", ev.key.value()); auto &rec = rec_info.rec; bts_assert(rec.has_def()); rec.set_key(ev.key.value()); rec.set_parent_key(ev.parent_key.value()); modified_orders.insert(ev.key); const auto now = utils::pb::pb_now(); if (rec.has_state()) { google::protobuf::Timestamp created; if (rec.state().has_created()) { created = rec.state().created(); } rec.mutable_state()->CopyFrom(ev.rebuild_state()); *rec.mutable_state()->mutable_created() = created.seconds() > 0 || created.nanos() > 0 ? created : now; // if filled qty went from 0.99 -> 1, it was copying the int part and not clearing the fraction part, // so changing MergeFrom to CopyFrom } else { *rec.mutable_state() = ev.rebuild_state(); *rec.mutable_state()->mutable_created() = now; } *rec.mutable_state()->mutable_last_change() = now; LOG(OrderMan, Info, "OrderStore::on_order_state_update k: {} rec: {}", ev.key, utils::pb::to_string(rec)); } store_and_publish_if_complete(modified_orders); } void OrderStore::on_order_definition_update( mercury::Update const &u) { std::set modified_orders; if (u.events) for (auto const &ev : *u.events) { LOG(iLink3, Info, "Received order definition update for {}", ev.key.value()); auto &rec_info = me->known_orders[ev.key]; auto &rec = rec_info.rec; rec.set_key(ev.key.value()); rec_info.is_saved = rec.has_def(); *rec.mutable_def() = *ev.def; modified_orders.insert(ev.key); } store_and_publish_if_complete(modified_orders); } void OrderStore::store_and_publish_if_complete( std::set const &changed) { for (auto k : changed) { auto &rec_info = me->known_orders[k]; auto &rec = rec_info.rec; if (rec.has_def() && rec.has_state()) { LOG(OrderMan, Diag, "OrderStore::store_and_publish_if_complete rec: {}", utils::pb::to_string(rec)); publish(me->order_stream, rec); if (rec_info.is_saved) { me->store->update( orders_ns, std::to_string(k.value()), utils::pb::to_string(rec)); LOG(OrderMan, Info, "OrderStore::store_and_publish_if_complete updated key: {}", k); } else { me->store->insert( orders_ns, std::to_string(k.value()), utils::pb::to_string(rec)); rec_info.is_saved = true; LOG(OrderMan, Info, "OrderStore::store_and_publish_if_complete added key: {}", k); } } else { LOG(OrderMan, Warn, "OrderStore::store_and_publish_if_complete failed to add key: {} " "has_def: {} has_state: {}", k.value(), rec.has_def(), rec.has_state()); } } } bool OrderStore::_order_cutoff(const api::orderman::OrderRecord &rec) { // TODO: move this to orderutils bool working = (rec.state().status() == api::orderman::OrderStatus::ORDER_STATUS_WORKING) || (rec.state().status() == api::orderman::OrderStatus::ORDER_STATUS_DISCONNECTED) || (rec.state().status() == api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING); return me->soft_cleanup_enabled && !working && rec.state().created().seconds() < me->cutoff_time.tv_sec; } } // namespace orderman } // namespace bts