This is a prerequisite to synchronizing security ids stored via IDGenerator (the "new" way to store ids, added when we implemented Deribit), and SecurityDB (the original way from spark days). The problem was that these 2 implementations were using separate databases. We need to unify the interfaces between these storage services to avoid breaking existing code.
409 lines
15 KiB
C++
409 lines
15 KiB
C++
#include "PositionServiceServer.h"
|
|
#include "PositionMap.h"
|
|
#include "PositionServicePNLHelper.h"
|
|
#include "lib/api/public/ServiceContext.h"
|
|
#include "lib/api/service_helpers/AdminServiceUtils.h"
|
|
#include "lib/api/service_helpers/PositionUtilities.h"
|
|
#include "lib/api/services/AdminService.client.pb.h"
|
|
#include "lib/api/services/AdminService.pb.h"
|
|
#include "lib/api/services/OrderService.client.pb.h"
|
|
#include "lib/api/services/OrderService.pb.h"
|
|
#include "lib/api/services/PositionService.pb.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/orderman/actors/SODPosition.h"
|
|
#include "lib/persistence/SqliteStore.h"
|
|
#include "lib/utils/PBUtils.h"
|
|
#include "lib/utils/Puid.h"
|
|
|
|
#include <boost/algorithm/string.hpp>
|
|
#include <boost/tokenizer.hpp>
|
|
#include <fmt/format.h>
|
|
#include <map>
|
|
#include <set>
|
|
#include <utility>
|
|
|
|
namespace bts {
|
|
namespace position {
|
|
|
|
using namespace bts::api::position;
|
|
using namespace bts::md;
|
|
|
|
static auto const manual_fills_ns = "manual_fills";
|
|
static auto const security_params_ns = "security_parameters";
|
|
|
|
struct PositionServiceServer::PImpl {
|
|
bts::orderman::PositionMap state;
|
|
bts::api::ServiceContext &context;
|
|
bts::api::orderman::OrderService order_service;
|
|
PositionServiceBase::Position_dataStatefulPublisher pos_publisher;
|
|
int id;
|
|
// manual fills management
|
|
std::unique_ptr<persistence::IStore> manual_fills_store;
|
|
std::unique_ptr<persistence::IStore> security_params_store;
|
|
PositionServiceBase::Manual_fillsStatefulPublisher manual_fills_publisher;
|
|
// pnl
|
|
PositionServiceBase::Pnl_dataStatefulPublisher pnl_publisher;
|
|
PositionServiceBase::Pnl_account_dataStatefulPublisher pnl_account_publisher;
|
|
PositionServicePNLHelper pnl_helper;
|
|
int64_t poll_period_ms;
|
|
// admin data needed for pnl
|
|
bts::api::admin::AdminService admin_service;
|
|
api::admin::AdminData admin_data;
|
|
std::map<std::string, uint32_t> account_map;
|
|
std::map<uint32_t, double> cpp_override_map;
|
|
|
|
PImpl(
|
|
bts::api::ServiceContext &context,
|
|
const PositionServiceBase::Position_dataStatefulPublisher &pos_publisher,
|
|
const PositionServiceBase::Pnl_dataStatefulPublisher &pnl_publisher,
|
|
const PositionServiceBase::Pnl_account_dataStatefulPublisher
|
|
&pnl_account_publisher,
|
|
const PositionServiceBase::Manual_fillsStatefulPublisher
|
|
&manual_fills_publisher,
|
|
bts::md::MDClient &md_client)
|
|
: // pos
|
|
context(context),
|
|
order_service(context.locate<bts::api::orderman::OrderService>()),
|
|
pos_publisher(pos_publisher),
|
|
// manual fills
|
|
manual_fills_store(
|
|
std::make_unique<persistence::SqliteStore>("manual_fills")),
|
|
security_params_store(
|
|
std::make_unique<persistence::SqliteStore>("security_parameters")),
|
|
manual_fills_publisher(manual_fills_publisher),
|
|
// pnl
|
|
pnl_publisher(pnl_publisher),
|
|
pnl_account_publisher(pnl_account_publisher), pnl_helper(md_client),
|
|
poll_period_ms(300),
|
|
// admin
|
|
admin_service(context.locate<bts::api::admin::AdminService>()),
|
|
admin_data(), account_map(), cpp_override_map() {}
|
|
};
|
|
|
|
PositionServiceServer::PositionServiceServer(
|
|
bts::api::ServiceContext &context, bts::md::MDClient &md_client)
|
|
: PositionServiceBase(context),
|
|
me(new PImpl(
|
|
context,
|
|
make_position_data_stateful_publisher(),
|
|
make_pnl_data_stateful_publisher(),
|
|
make_pnl_account_data_stateful_publisher(),
|
|
make_manual_fills_stateful_publisher(),
|
|
md_client)) {
|
|
_delete_presod_local_on_start();
|
|
// incremental fill update
|
|
me->order_service.subscribe_fills() |
|
|
rxcpp::operators::subscribe<api::orderman::FillsUpdate>(
|
|
[this](auto const &update) {
|
|
for (const auto &f : update.removed())
|
|
me->pos_publisher.add_or_update(_on_fill(*f, false));
|
|
for (const auto &f : update.added())
|
|
me->pos_publisher.add_or_update(_on_fill(*f, true));
|
|
for (const auto &f : update.changed()) {
|
|
me->pos_publisher.add_or_update(
|
|
_on_fill(*f.previous(), false));
|
|
me->pos_publisher.add_or_update(_on_fill(*f.current(), true));
|
|
}
|
|
me->pos_publisher.flush();
|
|
_compute_pnl_and_publish();
|
|
});
|
|
// admin data
|
|
me->admin_service.subscribe_admin_data() |
|
|
rxcpp::operators::subscribe<api::admin::Admin_dataUpdate>(
|
|
[this](auto const &update) {
|
|
for (const auto &r : update.added())
|
|
_set_admin_data(*r);
|
|
for (const auto &r : update.changed())
|
|
_set_admin_data(*r.current());
|
|
});
|
|
// manual fills
|
|
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
|
|
LOG(persist, Info, "Finished loading all {} manual fills", all.size());
|
|
ManualFill rec;
|
|
for (auto const &kvp : all)
|
|
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
|
|
me->manual_fills_publisher.add_or_update(rec);
|
|
me->pos_publisher.add_or_update(_on_manual_fill(rec, true));
|
|
} else {
|
|
LOG(OrderMan, Error,
|
|
"Could not recover manual fill {} from storage. "
|
|
"This should be investigated",
|
|
kvp.first);
|
|
}
|
|
me->manual_fills_publisher.flush();
|
|
me->pos_publisher.flush();
|
|
me->context.post([this]() { _periodic_poll(); }, me->poll_period_ms);
|
|
}
|
|
|
|
void PositionServiceServer::_delete_presod_local_on_start() const {
|
|
std::set<std::string> rm_ids;
|
|
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
|
|
ManualFill rec;
|
|
for (auto const &kvp : all) {
|
|
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
|
|
if (rec.fill_type() == ManualFill::LOCAL &&
|
|
me->state.is_sod(rec.timestamp())) {
|
|
LOG(OrderMan, Info,
|
|
"PositionServiceServer::_delete_presod_local_on_start "
|
|
"fill: {}",
|
|
kvp.second);
|
|
auto id = std::to_string(rec.record_id());
|
|
rm_ids.insert(id);
|
|
}
|
|
} else
|
|
LOG(OrderMan, Error,
|
|
"Could not recover manual fill {} from storage. "
|
|
"This should be investigated",
|
|
kvp.first);
|
|
}
|
|
me->manual_fills_store->remove(manual_fills_ns, rm_ids);
|
|
}
|
|
|
|
const PositionUnit &PositionServiceServer::_on_fill(
|
|
const Fill &f, bool is_add) {
|
|
return me->state.on_fill(f, is_add);
|
|
}
|
|
|
|
const PositionUnit &PositionServiceServer::_on_manual_fill(
|
|
const ManualFill &mf, bool is_add) {
|
|
return me->state.on_manual_fill(mf, is_add);
|
|
}
|
|
|
|
void PositionServiceServer::_set_admin_data(
|
|
const api::admin::AdminData &admin_data) {
|
|
me->admin_data = admin_data;
|
|
for (const auto &a : admin_data.accounts())
|
|
me->account_map[a.name()] = a.id();
|
|
me->state.set_admin_data(admin_data);
|
|
_clear_and_publish_pos();
|
|
}
|
|
|
|
template <class T, class TKey>
|
|
void _diff(
|
|
const std::map<TKey, T const *> &new_state,
|
|
const std::map<TKey, T const *> &prev_state,
|
|
std::vector<T const *> &added,
|
|
std::vector<T const *> &updated,
|
|
std::vector<T const *> &deleted) {
|
|
for (const auto &p : new_state) {
|
|
if (prev_state.find(p.first) == prev_state.end())
|
|
added.push_back(p.second);
|
|
else
|
|
updated.push_back(p.second);
|
|
}
|
|
for (const auto &p : prev_state)
|
|
if (new_state.find(p.first) == new_state.end())
|
|
deleted.push_back(p.second);
|
|
}
|
|
|
|
void PositionServiceServer::_clear_and_publish_pos() {
|
|
std::map<int, PositionUnit const *> new_state;
|
|
me->state.get_pos(new_state);
|
|
// entries are never deleted from new_state
|
|
for (const auto &p : new_state)
|
|
me->pos_publisher.add_or_update(*p.second);
|
|
me->pos_publisher.flush();
|
|
}
|
|
|
|
void PositionServiceServer::_load_security_params() {
|
|
me->cpp_override_map.clear();
|
|
auto all = me->security_params_store->retrieve_all()[security_params_ns];
|
|
SecurityParameters rec;
|
|
for (auto const &kvp : all)
|
|
if (utils::pb::from_string<SecurityParameters>(kvp.second, &rec)) {
|
|
if (rec.has_cpp_override()) {
|
|
LOG(OrderMan, Info, "Overriding cpp, key:{} cpp:{}",
|
|
rec.security_key(), rec.cpp_override().value());
|
|
me->cpp_override_map[rec.security_key()] =
|
|
rec.cpp_override().value();
|
|
}
|
|
} else {
|
|
LOG(OrderMan, Error,
|
|
"Could not recover security parameter for security {} from "
|
|
"storage. "
|
|
"This should be investigated",
|
|
kvp.first);
|
|
}
|
|
}
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
/// Manual Fills API //////////////////////////////////////////////////////////
|
|
void PositionServiceServer::create_manual_fill(
|
|
const ManualFillBatch &batch,
|
|
const api::Response<PositionServiceResponse> &r) {
|
|
PositionServiceResponse resp;
|
|
for (auto mf : batch.manual_fill()) {
|
|
auto id = utils::gen_id("manual_fills");
|
|
mf.set_record_id(id);
|
|
struct timeval tv;
|
|
gettimeofday(&tv, NULL);
|
|
mf.mutable_timestamp()->set_seconds(tv.tv_sec);
|
|
mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
|
|
|
|
me->manual_fills_store->insert(
|
|
manual_fills_ns, std::to_string(id), utils::pb::to_string(mf));
|
|
resp.add_response()->set_ok(true);
|
|
me->manual_fills_publisher.add_or_update(mf);
|
|
me->pos_publisher.add_or_update(_on_manual_fill(mf, true));
|
|
}
|
|
r(resp);
|
|
me->manual_fills_publisher.flush();
|
|
me->pos_publisher.flush();
|
|
}
|
|
|
|
void PositionServiceServer::reset_position(
|
|
const ::bts::api::position::ResetPositionParameters &reset_params,
|
|
const api::Response<::bts::api::position::PositionServiceResponse> &r) {
|
|
LOG(OrderMan, Info,
|
|
"PositionServiceServer::reset_position acct: {} n_recs: {}",
|
|
reset_params.account(),
|
|
reset_params.total_position().manual_fill().size());
|
|
for (const auto &f : reset_params.total_position().manual_fill()) {
|
|
LOG(OrderMan, Info,
|
|
"PositionServiceServer::reset_position acct: {} fill: {}",
|
|
reset_params.account(), utils::pb::to_string(f));
|
|
}
|
|
|
|
// find existing fills for this account and remove them
|
|
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
|
|
ManualFillBatch fills_to_remove;
|
|
ManualFill rec;
|
|
for (auto const &kvp : all)
|
|
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
|
|
if (rec.account_name() == reset_params.account()) {
|
|
fills_to_remove.add_manual_fill()->CopyFrom(rec);
|
|
}
|
|
}
|
|
remove_manual_fill(fills_to_remove, [&](const auto &) {});
|
|
// now that existing fills have been removed, add the fills represeting total
|
|
// positions in the function arg
|
|
create_manual_fill(reset_params.total_position(), [&](const auto &) {});
|
|
// bust all fills for this account by calling into OrderService
|
|
api::orderman::ManualBustParameters mb_params;
|
|
mb_params.set_account(reset_params.account());
|
|
me->order_service.manually_bust_all_fills(mb_params, [&](const auto &) {});
|
|
bts::api::position::PositionServiceResponse resp;
|
|
r(resp);
|
|
}
|
|
|
|
void PositionServiceServer::update_manual_fill(
|
|
const ManualFillBatch &batch,
|
|
const api::Response<PositionServiceResponse> &r) {
|
|
PositionServiceResponse resp;
|
|
for (auto mf : batch.manual_fill()) {
|
|
auto *this_resp = resp.add_response();
|
|
auto id = std::to_string(mf.record_id());
|
|
|
|
auto [exists, rowid, val] =
|
|
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
|
|
if (!exists) {
|
|
this_resp->set_ok(false);
|
|
this_resp->set_msg("Could not find record");
|
|
} else {
|
|
me->manual_fills_store->update(
|
|
manual_fills_ns, id, utils::pb::to_string(mf));
|
|
this_resp->set_ok(true);
|
|
|
|
// Retrieve old record so we can remove it from position
|
|
ManualFill const *old_ptr = me->state.get_manual_fill(mf.record_id());
|
|
bts_assert(old_ptr != nullptr);
|
|
// remove old
|
|
me->pos_publisher.add_or_update(_on_manual_fill(*old_ptr, false));
|
|
// add new
|
|
me->pos_publisher.add_or_update(_on_manual_fill(mf, true));
|
|
me->manual_fills_publisher.add_or_update(mf);
|
|
}
|
|
}
|
|
r(resp);
|
|
me->pos_publisher.flush();
|
|
me->manual_fills_publisher.flush();
|
|
}
|
|
|
|
void PositionServiceServer::remove_manual_fill(
|
|
const ManualFillBatch &batch,
|
|
const api::Response<PositionServiceResponse> &r) {
|
|
PositionServiceResponse resp;
|
|
for (auto mf : batch.manual_fill()) {
|
|
auto *this_resp = resp.add_response();
|
|
auto id = std::to_string(mf.record_id());
|
|
|
|
auto [exists, rowid, val] =
|
|
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
|
|
if (!exists) {
|
|
this_resp->set_ok(false);
|
|
this_resp->set_msg("Could not find record");
|
|
} else {
|
|
// Retrieve old record so we can remove it from position
|
|
ManualFill const *old_ptr = me->state.get_manual_fill(mf.record_id());
|
|
bts_assert(old_ptr != nullptr);
|
|
me->pos_publisher.add_or_update(_on_manual_fill(*old_ptr, false));
|
|
|
|
std::set<std::string> to_r;
|
|
to_r.insert(id);
|
|
me->manual_fills_store->remove(manual_fills_ns, to_r);
|
|
this_resp->set_ok(true);
|
|
me->manual_fills_publisher.erase(mf.record_id());
|
|
}
|
|
}
|
|
r(resp);
|
|
me->pos_publisher.flush();
|
|
me->manual_fills_publisher.flush();
|
|
}
|
|
|
|
void PositionServiceServer::set_base_position(
|
|
const BasePositionBatch &req,
|
|
const api::Response<SetBasePositionResponse> &) {
|
|
LOG(persist, Info, "PositionServiceServer::set_base_position bp count:{}",
|
|
req.positions_size());
|
|
me->state.set_base_position(req);
|
|
_clear_and_publish_pos();
|
|
}
|
|
|
|
void PositionServiceServer::update_security_parameters(
|
|
const SecurityParametersBatchOp &req,
|
|
const api::Response<SecurityParametersBatchOpResponse> &) {
|
|
// TODO
|
|
LOG(persist, Info,
|
|
"PositionServiceServer::update_security_parameters bp count:{}",
|
|
req.op_params_size());
|
|
for (const auto &p : req.op_params()) {
|
|
std::string key_str = std::to_string(p.security_key());
|
|
auto [exists, rowid, val] =
|
|
me->security_params_store->retrieve_one(security_params_ns, key_str);
|
|
if (!exists) {
|
|
me->security_params_store->insert(
|
|
security_params_ns, key_str, utils::pb::to_string(p));
|
|
} else {
|
|
me->security_params_store->update(
|
|
security_params_ns, key_str, utils::pb::to_string(p));
|
|
}
|
|
}
|
|
_load_security_params();
|
|
_compute_pnl_and_publish();
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
/// PNL helpers ///////////////////////////////////////////////////////////////
|
|
void PositionServiceServer::_compute_pnl_and_publish() {
|
|
std::map<int, PositionUnit const *> in_pos;
|
|
me->state.get_pos(in_pos);
|
|
std::vector<PNLUnit> pnl;
|
|
std::vector<AccountPNL> account_pnl;
|
|
me->pnl_helper.compute_pnl(
|
|
in_pos, me->account_map, me->cpp_override_map, pnl, account_pnl);
|
|
for (const auto &u : pnl)
|
|
me->pnl_publisher.add_or_update(u);
|
|
for (const auto &u : account_pnl)
|
|
me->pnl_account_publisher.add_or_update(u);
|
|
me->pnl_publisher.flush();
|
|
me->pnl_account_publisher.flush();
|
|
}
|
|
|
|
void PositionServiceServer::_periodic_poll() {
|
|
_compute_pnl_and_publish();
|
|
me->context.post([this]() { _periodic_poll(); }, me->poll_period_ms);
|
|
}
|
|
|
|
PositionServiceServer::~PositionServiceServer() {}
|
|
} // namespace position
|
|
} // namespace bts
|