Orders in DB are deleted on server bounce

This commit is contained in:
Taha Afzal committed 2025-03-10 15:42:12 +00:00
1 parent 2072105c0f
commit 0ebc8e5b4b
2 files changed
+171 -45

No files matched your search

+136 -24
View File
@@ -2,7 +2,10 @@
#include "lib/api/services/OrderService.pb.h"
#include "lib/config/config.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/core/TimeStamp.h"
#include "lib/domain_model/Security.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/SecurityKey.h"
#include "lib/logging/logging.h"
#include "lib/mercury/events/Update.h"
#include "lib/orderman/actors/SODPosition.h"
@@ -14,10 +17,13 @@
#include "lib/os/os.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/overloaded.h"
#include <boost/date_time/posix_time/posix_time.hpp>
#include <google/protobuf/timestamp.pb.h>
#include <string>
#include <sys/time.h>
@@ -32,6 +38,67 @@ namespace orderman {
auto const fills_ns = "fills";
struct FillAggregator {
struct SecPosition {
private:
Fill agg_fill;
public:
double agg_price = 0, agg_qty = 0;
SecPosition(const Fill &fill) : agg_fill(fill) {
const auto &fill_key = fill.fill_key();
std::string new_fill_key = "aggpos_" + fill_key;
agg_fill.set_fill_key(new_fill_key);
auto now_ns = bts::core::TimeStamp::now_ns();
agg_fill.mutable_timestamp()->set_nanos(now_ns);
add_fill(fill);
}
void add_fill(const Fill &fill) {
bts_assert(fill.security_key() == agg_fill.security_key());
double mult = fill.side() == api::SIDE_BUY ? 1.0 : -1.0;
double fp = fill.avg_price();
double fq = mult * fill.last_shares();
double new_agg_price =
((agg_price * agg_qty) + (fp * fq)) / (agg_qty + fq);
double new_agg_qty = agg_qty + fq;
agg_price = new_agg_price;
agg_qty = new_agg_qty;
}
public:
Fill fill() const {
Fill rv = agg_fill;
rv.set_avg_price(agg_price);
rv.set_last_shares(std::abs(agg_qty));
rv.set_side(::bts::api::Side::SIDE_BUY);
if (agg_qty < 0) {
rv.set_side(::bts::api::Side::SIDE_SELL);
}
return rv;
}
};
std::map<SecurityKey, SecPosition> pos_map = {};
void add_fill(const Fill &fill) {
SecurityKey k = SecurityKey(fill.security_key());
auto it = pos_map.find(k);
if (it != pos_map.end()) {
SecPosition &sec_pos = it->second;
sec_pos.add_fill(fill);
} else {
pos_map.insert({k, {fill}});
}
}
std::vector<Fill> get_aggregated_fills() const {
std::vector<Fill> rv = {};
for (const auto &kv : pos_map) {
rv.push_back(kv.second.fill());
}
return rv;
}
};
struct FillStore::UnresolvedFills {
std::vector<api::orderman::Fill> fills;
api::Security const *security;
@@ -48,6 +115,8 @@ struct FillStore::PImpl {
std::unordered_map<OrderKey, UnresolvedFills> fills_with_unresolved_accounts;
mercury::Stream<StoredFillEvent> fill_stream;
md::SecurityOracle oracle;
FillAggregator fill_aggregator = {};
bool aggregate_pos = false;
};
FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {
@@ -100,40 +169,83 @@ void FillStore::init() {
os::get_thread_id(), os::get_thread_name());
LOG(OrderMan, Info, "FillStore::init oracle n_secs: {}",
me->oracle.get_securities().size());
// Subscribe to NewExecutionReportEvent for new fills
subscribe<NewExecutionReportEvent>(
[this](auto const &ev) { this->on_execution_report(ev); });
// Load stored fills, resolve their securities and publish the lot
api::orderman::Fill fill;
std::vector<StoredFillEvent> stored_batch;
LOG(persist, Info, "Loading load all fills");
auto all = me->store->retrieve_all()[fills_ns];
LOG(persist, Info, "Finished loading all {} fills", all.size());
SODPosition sod_helper;
for (auto const &kvp : all)
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
if (sod_helper.is_sod(fill.timestamp())) {
auto fill_kind = sod_fill_kind(fill);
fill.set_fill_kind(fill_kind);
auto aggregate_clean_and_publish = [&]() {
api::orderman::Fill fill;
for (auto const &kvp : all) {
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
me->fill_aggregator.add_fill(fill);
} else {
LOG(persist, Error,
"Could not recover fill {} from storage. "
"This should be investigated",
kvp.first);
}
auto sec = me->oracle.get_security(SecurityKey(fill.security_key()));
}
// 1. go through all fills in aggregator and make StoredFillEvent
auto aggregated_fills = me->fill_aggregator.get_aggregated_fills();
for (auto &f : aggregated_fills) {
auto sec = me->oracle.get_security(SecurityKey(f.security_key()));
if (sec) {
auto fk = kvp.first;
me->known_fills.insert(fk);
if (!config::get<config::toggles::FilterExpiredFills>().enabled() ||
!securities::is_expired(*sec)) {
stored_batch.emplace_back(fill, *sec);
stored_batch.emplace_back(f, *sec);
} else {
LOG(persist, Error,
"Stored fill {} contains unresolved security {}. ",
utils::pb::to_string(f), fill.security_key());
}
}
// 2 delete all fills in the DB
me->store->remove_all(fills_ns);
// 3. add the aggregated fills back
for (const auto &e : stored_batch) {
const Fill &f = e.fill;
me->store->insert(fills_ns, f.fill_key(), utils::pb::to_string(f));
}
};
auto publish_with_sod_marking = [&]() {
api::orderman::Fill fill;
SODPosition sod_helper;
for (auto const &kvp : all) {
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
if (sod_helper.is_sod(fill.timestamp())) {
auto fill_kind = sod_fill_kind(fill);
fill.set_fill_kind(fill_kind);
}
auto sec =
me->oracle.get_security(SecurityKey(fill.security_key()));
if (sec) {
auto fk = kvp.first;
me->known_fills.insert(fk);
if (!config::get<config::toggles::FilterExpiredFills>()
.enabled() ||
!securities::is_expired(*sec)) {
stored_batch.emplace_back(fill, *sec);
}
} else
LOG(persist, Error,
"Stored fill {} contains unresolved security {}. ",
kvp.first, fill.security_key());
} else
LOG(persist, Error,
"Stored fill {} contains unresolved security {}. ", kvp.first,
fill.security_key());
} else
LOG(persist, Error,
"Could not recover fill {} from storage. "
"This should be investigated",
kvp.first);
"Could not recover fill {} from storage. "
"This should be investigated",
kvp.first);
}
};
// Subscribe to NewExecutionReportEvent for new fills
subscribe<NewExecutionReportEvent>(
[this](auto const &ev) { this->on_execution_report(ev); });
if (me->aggregate_pos) {
aggregate_clean_and_publish();
} else {
publish_with_sod_marking();
}
me->fill_stream = make_stream<StoredFillEvent>();
LOG(OrderMan, Info,
"Fill Store published {} fills recoved from disk out of {} "
+35 -21
View File
@@ -5,6 +5,7 @@
#include "lib/config/protos/orders_fills_cleanup.pb.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/ExchangeSecurityKey.h"
#include "lib/orderman/actors/SODPosition.h"
#include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
#include "lib/orderman/events/OrderStateUpdateEvent.h"
@@ -39,6 +40,7 @@ struct OrderStore::PImpl {
bool soft_cleanup_enabled;
md::SecurityOracle oracle;
timeval cutoff_time;
bool clear_all_orders = true;
};
OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
@@ -64,34 +66,46 @@ void OrderStore::init() {
me->cutoff_time = SODPosition::get_order_cutoff_time(all);
for (auto const &kvp : all)
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto ok = OrderKey(std::stoi(kvp.first));
auto sk = SecurityKey(rec.def().security_key());
auto sec = me->oracle.get_security(sk);
auto load_orders = [&]() {
for (auto const &kvp : all) {
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto ok = OrderKey(std::stoi(kvp.first));
auto sk = SecurityKey(rec.def().security_key());
auto sec = me->oracle.get_security(sk);
if (config::get<config::toggles::FilterOrdersWithDanglingSecurities>()
.enabled() &&
sec == nullptr) {
LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
ok.value(), sk.value());
} else {
if (!_order_cutoff(rec))
stored_batch.emplace_back(rec);
me->known_orders[ok] = PImpl::ORInfo{rec, true};
}
} else
LOG(OrderMan, Error,
"Could not recover order {} from storage. "
"This should be investigated",
kvp.first);
if (config::get<
config::toggles::FilterOrdersWithDanglingSecurities>()
.enabled() &&
sec == nullptr) {
LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
ok.value(), sk.value());
} else {
if (!_order_cutoff(rec))
stored_batch.emplace_back(rec);
me->known_orders[ok] = PImpl::ORInfo{rec, true};
}
} else
LOG(OrderMan, Error,
"Could not recover order {} from storage. "
"This should be investigated",
kvp.first);
}
};
me->order_stream = make_stream<StoredOrderEvent>();
LOG(OrderMan, Info,
"Order Store published {} orders recoved from disk out of {} "
"total orders",
stored_batch.size(), all.size());
if (me->clear_all_orders) {
// clear orders
me->store->remove_all(orders_ns);
} else {
load_orders();
}
publish_batch(me->order_stream, stored_batch);
}