Orders in DB are deleted on server bounce
This commit is contained in:
1 parent
2072105c0f
commit
0ebc8e5b4b
2 files changed
+171
-45
No files matched your search
@@ -2,7 +2,10 @@
|
||||
#include "lib/api/services/OrderService.pb.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/toggles.pb.h"
|
||||
#include "lib/core/TimeStamp.h"
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/SecurityKey.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/mercury/events/Update.h"
|
||||
#include "lib/orderman/actors/SODPosition.h"
|
||||
@@ -14,10 +17,13 @@
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/persistence/SqliteStore.h"
|
||||
#include "lib/utils/PBUtils.h"
|
||||
#include "lib/utils/TimeStamp.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
#include "lib/utils/overloaded.h"
|
||||
|
||||
#include <boost/date_time/posix_time/posix_time.hpp>
|
||||
#include <google/protobuf/timestamp.pb.h>
|
||||
#include <string>
|
||||
#include <sys/time.h>
|
||||
|
||||
@@ -32,6 +38,67 @@ namespace orderman {
|
||||
|
||||
auto const fills_ns = "fills";
|
||||
|
||||
struct FillAggregator {
|
||||
struct SecPosition {
|
||||
private:
|
||||
Fill agg_fill;
|
||||
|
||||
public:
|
||||
double agg_price = 0, agg_qty = 0;
|
||||
SecPosition(const Fill &fill) : agg_fill(fill) {
|
||||
const auto &fill_key = fill.fill_key();
|
||||
std::string new_fill_key = "aggpos_" + fill_key;
|
||||
agg_fill.set_fill_key(new_fill_key);
|
||||
auto now_ns = bts::core::TimeStamp::now_ns();
|
||||
agg_fill.mutable_timestamp()->set_nanos(now_ns);
|
||||
add_fill(fill);
|
||||
}
|
||||
void add_fill(const Fill &fill) {
|
||||
bts_assert(fill.security_key() == agg_fill.security_key());
|
||||
double mult = fill.side() == api::SIDE_BUY ? 1.0 : -1.0;
|
||||
double fp = fill.avg_price();
|
||||
double fq = mult * fill.last_shares();
|
||||
double new_agg_price =
|
||||
((agg_price * agg_qty) + (fp * fq)) / (agg_qty + fq);
|
||||
double new_agg_qty = agg_qty + fq;
|
||||
agg_price = new_agg_price;
|
||||
agg_qty = new_agg_qty;
|
||||
}
|
||||
|
||||
public:
|
||||
Fill fill() const {
|
||||
Fill rv = agg_fill;
|
||||
rv.set_avg_price(agg_price);
|
||||
rv.set_last_shares(std::abs(agg_qty));
|
||||
rv.set_side(::bts::api::Side::SIDE_BUY);
|
||||
if (agg_qty < 0) {
|
||||
rv.set_side(::bts::api::Side::SIDE_SELL);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
};
|
||||
std::map<SecurityKey, SecPosition> pos_map = {};
|
||||
|
||||
void add_fill(const Fill &fill) {
|
||||
SecurityKey k = SecurityKey(fill.security_key());
|
||||
auto it = pos_map.find(k);
|
||||
if (it != pos_map.end()) {
|
||||
SecPosition &sec_pos = it->second;
|
||||
sec_pos.add_fill(fill);
|
||||
} else {
|
||||
pos_map.insert({k, {fill}});
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Fill> get_aggregated_fills() const {
|
||||
std::vector<Fill> rv = {};
|
||||
for (const auto &kv : pos_map) {
|
||||
rv.push_back(kv.second.fill());
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
};
|
||||
|
||||
struct FillStore::UnresolvedFills {
|
||||
std::vector<api::orderman::Fill> fills;
|
||||
api::Security const *security;
|
||||
@@ -48,6 +115,8 @@ struct FillStore::PImpl {
|
||||
std::unordered_map<OrderKey, UnresolvedFills> fills_with_unresolved_accounts;
|
||||
mercury::Stream<StoredFillEvent> fill_stream;
|
||||
md::SecurityOracle oracle;
|
||||
FillAggregator fill_aggregator = {};
|
||||
bool aggregate_pos = false;
|
||||
};
|
||||
|
||||
FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {
|
||||
@@ -100,40 +169,83 @@ void FillStore::init() {
|
||||
os::get_thread_id(), os::get_thread_name());
|
||||
LOG(OrderMan, Info, "FillStore::init oracle n_secs: {}",
|
||||
me->oracle.get_securities().size());
|
||||
// Subscribe to NewExecutionReportEvent for new fills
|
||||
subscribe<NewExecutionReportEvent>(
|
||||
[this](auto const &ev) { this->on_execution_report(ev); });
|
||||
|
||||
// Load stored fills, resolve their securities and publish the lot
|
||||
api::orderman::Fill fill;
|
||||
std::vector<StoredFillEvent> stored_batch;
|
||||
LOG(persist, Info, "Loading load all fills");
|
||||
auto all = me->store->retrieve_all()[fills_ns];
|
||||
LOG(persist, Info, "Finished loading all {} fills", all.size());
|
||||
SODPosition sod_helper;
|
||||
for (auto const &kvp : all)
|
||||
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
|
||||
if (sod_helper.is_sod(fill.timestamp())) {
|
||||
auto fill_kind = sod_fill_kind(fill);
|
||||
fill.set_fill_kind(fill_kind);
|
||||
auto aggregate_clean_and_publish = [&]() {
|
||||
api::orderman::Fill fill;
|
||||
for (auto const &kvp : all) {
|
||||
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
|
||||
me->fill_aggregator.add_fill(fill);
|
||||
} else {
|
||||
LOG(persist, Error,
|
||||
"Could not recover fill {} from storage. "
|
||||
"This should be investigated",
|
||||
kvp.first);
|
||||
}
|
||||
auto sec = me->oracle.get_security(SecurityKey(fill.security_key()));
|
||||
}
|
||||
// 1. go through all fills in aggregator and make StoredFillEvent
|
||||
auto aggregated_fills = me->fill_aggregator.get_aggregated_fills();
|
||||
for (auto &f : aggregated_fills) {
|
||||
auto sec = me->oracle.get_security(SecurityKey(f.security_key()));
|
||||
if (sec) {
|
||||
auto fk = kvp.first;
|
||||
me->known_fills.insert(fk);
|
||||
if (!config::get<config::toggles::FilterExpiredFills>().enabled() ||
|
||||
!securities::is_expired(*sec)) {
|
||||
stored_batch.emplace_back(fill, *sec);
|
||||
stored_batch.emplace_back(f, *sec);
|
||||
} else {
|
||||
LOG(persist, Error,
|
||||
"Stored fill {} contains unresolved security {}. ",
|
||||
utils::pb::to_string(f), fill.security_key());
|
||||
}
|
||||
}
|
||||
// 2 delete all fills in the DB
|
||||
me->store->remove_all(fills_ns);
|
||||
// 3. add the aggregated fills back
|
||||
for (const auto &e : stored_batch) {
|
||||
const Fill &f = e.fill;
|
||||
me->store->insert(fills_ns, f.fill_key(), utils::pb::to_string(f));
|
||||
}
|
||||
};
|
||||
auto publish_with_sod_marking = [&]() {
|
||||
api::orderman::Fill fill;
|
||||
SODPosition sod_helper;
|
||||
for (auto const &kvp : all) {
|
||||
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
|
||||
if (sod_helper.is_sod(fill.timestamp())) {
|
||||
auto fill_kind = sod_fill_kind(fill);
|
||||
fill.set_fill_kind(fill_kind);
|
||||
}
|
||||
auto sec =
|
||||
me->oracle.get_security(SecurityKey(fill.security_key()));
|
||||
if (sec) {
|
||||
auto fk = kvp.first;
|
||||
me->known_fills.insert(fk);
|
||||
if (!config::get<config::toggles::FilterExpiredFills>()
|
||||
.enabled() ||
|
||||
!securities::is_expired(*sec)) {
|
||||
stored_batch.emplace_back(fill, *sec);
|
||||
}
|
||||
} else
|
||||
LOG(persist, Error,
|
||||
"Stored fill {} contains unresolved security {}. ",
|
||||
kvp.first, fill.security_key());
|
||||
} else
|
||||
LOG(persist, Error,
|
||||
"Stored fill {} contains unresolved security {}. ", kvp.first,
|
||||
fill.security_key());
|
||||
} else
|
||||
LOG(persist, Error,
|
||||
"Could not recover fill {} from storage. "
|
||||
"This should be investigated",
|
||||
kvp.first);
|
||||
"Could not recover fill {} from storage. "
|
||||
"This should be investigated",
|
||||
kvp.first);
|
||||
}
|
||||
};
|
||||
|
||||
// Subscribe to NewExecutionReportEvent for new fills
|
||||
subscribe<NewExecutionReportEvent>(
|
||||
[this](auto const &ev) { this->on_execution_report(ev); });
|
||||
if (me->aggregate_pos) {
|
||||
aggregate_clean_and_publish();
|
||||
} else {
|
||||
publish_with_sod_marking();
|
||||
}
|
||||
|
||||
me->fill_stream = make_stream<StoredFillEvent>();
|
||||
LOG(OrderMan, Info,
|
||||
"Fill Store published {} fills recoved from disk out of {} "
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "lib/config/protos/orders_fills_cleanup.pb.h"
|
||||
#include "lib/config/protos/toggles.pb.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/md/ExchangeSecurityKey.h"
|
||||
#include "lib/orderman/actors/SODPosition.h"
|
||||
#include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
|
||||
#include "lib/orderman/events/OrderStateUpdateEvent.h"
|
||||
@@ -39,6 +40,7 @@ struct OrderStore::PImpl {
|
||||
bool soft_cleanup_enabled;
|
||||
md::SecurityOracle oracle;
|
||||
timeval cutoff_time;
|
||||
bool clear_all_orders = true;
|
||||
};
|
||||
|
||||
OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
|
||||
@@ -64,34 +66,46 @@ void OrderStore::init() {
|
||||
|
||||
me->cutoff_time = SODPosition::get_order_cutoff_time(all);
|
||||
|
||||
for (auto const &kvp : all)
|
||||
if (utils::pb::from_string<api::orderman::OrderRecord>(
|
||||
kvp.second, &rec)) {
|
||||
auto ok = OrderKey(std::stoi(kvp.first));
|
||||
auto sk = SecurityKey(rec.def().security_key());
|
||||
auto sec = me->oracle.get_security(sk);
|
||||
auto load_orders = [&]() {
|
||||
for (auto const &kvp : all) {
|
||||
if (utils::pb::from_string<api::orderman::OrderRecord>(
|
||||
kvp.second, &rec)) {
|
||||
auto ok = OrderKey(std::stoi(kvp.first));
|
||||
auto sk = SecurityKey(rec.def().security_key());
|
||||
auto sec = me->oracle.get_security(sk);
|
||||
|
||||
if (config::get<config::toggles::FilterOrdersWithDanglingSecurities>()
|
||||
.enabled() &&
|
||||
sec == nullptr) {
|
||||
LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
|
||||
ok.value(), sk.value());
|
||||
} else {
|
||||
if (!_order_cutoff(rec))
|
||||
stored_batch.emplace_back(rec);
|
||||
me->known_orders[ok] = PImpl::ORInfo{rec, true};
|
||||
}
|
||||
} else
|
||||
LOG(OrderMan, Error,
|
||||
"Could not recover order {} from storage. "
|
||||
"This should be investigated",
|
||||
kvp.first);
|
||||
if (config::get<
|
||||
config::toggles::FilterOrdersWithDanglingSecurities>()
|
||||
.enabled() &&
|
||||
sec == nullptr) {
|
||||
LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
|
||||
ok.value(), sk.value());
|
||||
} else {
|
||||
if (!_order_cutoff(rec))
|
||||
stored_batch.emplace_back(rec);
|
||||
me->known_orders[ok] = PImpl::ORInfo{rec, true};
|
||||
}
|
||||
} else
|
||||
LOG(OrderMan, Error,
|
||||
"Could not recover order {} from storage. "
|
||||
"This should be investigated",
|
||||
kvp.first);
|
||||
}
|
||||
};
|
||||
|
||||
me->order_stream = make_stream<StoredOrderEvent>();
|
||||
LOG(OrderMan, Info,
|
||||
"Order Store published {} orders recoved from disk out of {} "
|
||||
"total orders",
|
||||
stored_batch.size(), all.size());
|
||||
|
||||
if (me->clear_all_orders) {
|
||||
// clear orders
|
||||
me->store->remove_all(orders_ns);
|
||||
} else {
|
||||
load_orders();
|
||||
}
|
||||
|
||||
publish_batch(me->order_stream, stored_batch);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user