Files
spark/src/server/sparkd/position/PositionServiceServer.cpp
T
John Colvin bd8c97244a refactor: Return row id in IStore::retrieve_one
This is a prerequisite to synchronizing security ids stored via IDGenerator
(the "new" way to store ids, added when we implemented Deribit), and
SecurityDB (the original way from spark days). The problem was that these
2 implementations were using separate databases. We need to unify the
interfaces between these storage services to avoid breaking existing code.
2024-11-20 22:17:08 +00:00

409 lines
15 KiB
C++

#include "PositionServiceServer.h"
#include "PositionMap.h"
#include "PositionServicePNLHelper.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/api/service_helpers/AdminServiceUtils.h"
#include "lib/api/service_helpers/PositionUtilities.h"
#include "lib/api/services/AdminService.client.pb.h"
#include "lib/api/services/AdminService.pb.h"
#include "lib/api/services/OrderService.client.pb.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/api/services/PositionService.pb.h"
#include "lib/logging/logging.h"
#include "lib/orderman/actors/SODPosition.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Puid.h"
#include <boost/algorithm/string.hpp>
#include <boost/tokenizer.hpp>
#include <fmt/format.h>
#include <map>
#include <set>
#include <utility>
namespace bts {
namespace position {
using namespace bts::api::position;
using namespace bts::md;
static auto const manual_fills_ns = "manual_fills";
static auto const security_params_ns = "security_parameters";
struct PositionServiceServer::PImpl {
bts::orderman::PositionMap state;
bts::api::ServiceContext &context;
bts::api::orderman::OrderService order_service;
PositionServiceBase::Position_dataStatefulPublisher pos_publisher;
int id;
// manual fills management
std::unique_ptr<persistence::IStore> manual_fills_store;
std::unique_ptr<persistence::IStore> security_params_store;
PositionServiceBase::Manual_fillsStatefulPublisher manual_fills_publisher;
// pnl
PositionServiceBase::Pnl_dataStatefulPublisher pnl_publisher;
PositionServiceBase::Pnl_account_dataStatefulPublisher pnl_account_publisher;
PositionServicePNLHelper pnl_helper;
int64_t poll_period_ms;
// admin data needed for pnl
bts::api::admin::AdminService admin_service;
api::admin::AdminData admin_data;
std::map<std::string, uint32_t> account_map;
std::map<uint32_t, double> cpp_override_map;
PImpl(
bts::api::ServiceContext &context,
const PositionServiceBase::Position_dataStatefulPublisher &pos_publisher,
const PositionServiceBase::Pnl_dataStatefulPublisher &pnl_publisher,
const PositionServiceBase::Pnl_account_dataStatefulPublisher
&pnl_account_publisher,
const PositionServiceBase::Manual_fillsStatefulPublisher
&manual_fills_publisher,
bts::md::MDClient &md_client)
: // pos
context(context),
order_service(context.locate<bts::api::orderman::OrderService>()),
pos_publisher(pos_publisher),
// manual fills
manual_fills_store(
std::make_unique<persistence::SqliteStore>("manual_fills")),
security_params_store(
std::make_unique<persistence::SqliteStore>("security_parameters")),
manual_fills_publisher(manual_fills_publisher),
// pnl
pnl_publisher(pnl_publisher),
pnl_account_publisher(pnl_account_publisher), pnl_helper(md_client),
poll_period_ms(300),
// admin
admin_service(context.locate<bts::api::admin::AdminService>()),
admin_data(), account_map(), cpp_override_map() {}
};
PositionServiceServer::PositionServiceServer(
bts::api::ServiceContext &context, bts::md::MDClient &md_client)
: PositionServiceBase(context),
me(new PImpl(
context,
make_position_data_stateful_publisher(),
make_pnl_data_stateful_publisher(),
make_pnl_account_data_stateful_publisher(),
make_manual_fills_stateful_publisher(),
md_client)) {
_delete_presod_local_on_start();
// incremental fill update
me->order_service.subscribe_fills() |
rxcpp::operators::subscribe<api::orderman::FillsUpdate>(
[this](auto const &update) {
for (const auto &f : update.removed())
me->pos_publisher.add_or_update(_on_fill(*f, false));
for (const auto &f : update.added())
me->pos_publisher.add_or_update(_on_fill(*f, true));
for (const auto &f : update.changed()) {
me->pos_publisher.add_or_update(
_on_fill(*f.previous(), false));
me->pos_publisher.add_or_update(_on_fill(*f.current(), true));
}
me->pos_publisher.flush();
_compute_pnl_and_publish();
});
// admin data
me->admin_service.subscribe_admin_data() |
rxcpp::operators::subscribe<api::admin::Admin_dataUpdate>(
[this](auto const &update) {
for (const auto &r : update.added())
_set_admin_data(*r);
for (const auto &r : update.changed())
_set_admin_data(*r.current());
});
// manual fills
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
LOG(persist, Info, "Finished loading all {} manual fills", all.size());
ManualFill rec;
for (auto const &kvp : all)
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
me->manual_fills_publisher.add_or_update(rec);
me->pos_publisher.add_or_update(_on_manual_fill(rec, true));
} else {
LOG(OrderMan, Error,
"Could not recover manual fill {} from storage. "
"This should be investigated",
kvp.first);
}
me->manual_fills_publisher.flush();
me->pos_publisher.flush();
me->context.post([this]() { _periodic_poll(); }, me->poll_period_ms);
}
void PositionServiceServer::_delete_presod_local_on_start() const {
std::set<std::string> rm_ids;
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
ManualFill rec;
for (auto const &kvp : all) {
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
if (rec.fill_type() == ManualFill::LOCAL &&
me->state.is_sod(rec.timestamp())) {
LOG(OrderMan, Info,
"PositionServiceServer::_delete_presod_local_on_start "
"fill: {}",
kvp.second);
auto id = std::to_string(rec.record_id());
rm_ids.insert(id);
}
} else
LOG(OrderMan, Error,
"Could not recover manual fill {} from storage. "
"This should be investigated",
kvp.first);
}
me->manual_fills_store->remove(manual_fills_ns, rm_ids);
}
const PositionUnit &PositionServiceServer::_on_fill(
const Fill &f, bool is_add) {
return me->state.on_fill(f, is_add);
}
const PositionUnit &PositionServiceServer::_on_manual_fill(
const ManualFill &mf, bool is_add) {
return me->state.on_manual_fill(mf, is_add);
}
void PositionServiceServer::_set_admin_data(
const api::admin::AdminData &admin_data) {
me->admin_data = admin_data;
for (const auto &a : admin_data.accounts())
me->account_map[a.name()] = a.id();
me->state.set_admin_data(admin_data);
_clear_and_publish_pos();
}
template <class T, class TKey>
void _diff(
const std::map<TKey, T const *> &new_state,
const std::map<TKey, T const *> &prev_state,
std::vector<T const *> &added,
std::vector<T const *> &updated,
std::vector<T const *> &deleted) {
for (const auto &p : new_state) {
if (prev_state.find(p.first) == prev_state.end())
added.push_back(p.second);
else
updated.push_back(p.second);
}
for (const auto &p : prev_state)
if (new_state.find(p.first) == new_state.end())
deleted.push_back(p.second);
}
void PositionServiceServer::_clear_and_publish_pos() {
std::map<int, PositionUnit const *> new_state;
me->state.get_pos(new_state);
// entries are never deleted from new_state
for (const auto &p : new_state)
me->pos_publisher.add_or_update(*p.second);
me->pos_publisher.flush();
}
void PositionServiceServer::_load_security_params() {
me->cpp_override_map.clear();
auto all = me->security_params_store->retrieve_all()[security_params_ns];
SecurityParameters rec;
for (auto const &kvp : all)
if (utils::pb::from_string<SecurityParameters>(kvp.second, &rec)) {
if (rec.has_cpp_override()) {
LOG(OrderMan, Info, "Overriding cpp, key:{} cpp:{}",
rec.security_key(), rec.cpp_override().value());
me->cpp_override_map[rec.security_key()] =
rec.cpp_override().value();
}
} else {
LOG(OrderMan, Error,
"Could not recover security parameter for security {} from "
"storage. "
"This should be investigated",
kvp.first);
}
}
///////////////////////////////////////////////////////////////////////////////
/// Manual Fills API //////////////////////////////////////////////////////////
void PositionServiceServer::create_manual_fill(
const ManualFillBatch &batch,
const api::Response<PositionServiceResponse> &r) {
PositionServiceResponse resp;
for (auto mf : batch.manual_fill()) {
auto id = utils::gen_id("manual_fills");
mf.set_record_id(id);
struct timeval tv;
gettimeofday(&tv, NULL);
mf.mutable_timestamp()->set_seconds(tv.tv_sec);
mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
me->manual_fills_store->insert(
manual_fills_ns, std::to_string(id), utils::pb::to_string(mf));
resp.add_response()->set_ok(true);
me->manual_fills_publisher.add_or_update(mf);
me->pos_publisher.add_or_update(_on_manual_fill(mf, true));
}
r(resp);
me->manual_fills_publisher.flush();
me->pos_publisher.flush();
}
void PositionServiceServer::reset_position(
const ::bts::api::position::ResetPositionParameters &reset_params,
const api::Response<::bts::api::position::PositionServiceResponse> &r) {
LOG(OrderMan, Info,
"PositionServiceServer::reset_position acct: {} n_recs: {}",
reset_params.account(),
reset_params.total_position().manual_fill().size());
for (const auto &f : reset_params.total_position().manual_fill()) {
LOG(OrderMan, Info,
"PositionServiceServer::reset_position acct: {} fill: {}",
reset_params.account(), utils::pb::to_string(f));
}
// find existing fills for this account and remove them
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
ManualFillBatch fills_to_remove;
ManualFill rec;
for (auto const &kvp : all)
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
if (rec.account_name() == reset_params.account()) {
fills_to_remove.add_manual_fill()->CopyFrom(rec);
}
}
remove_manual_fill(fills_to_remove, [&](const auto &) {});
// now that existing fills have been removed, add the fills represeting total
// positions in the function arg
create_manual_fill(reset_params.total_position(), [&](const auto &) {});
// bust all fills for this account by calling into OrderService
api::orderman::ManualBustParameters mb_params;
mb_params.set_account(reset_params.account());
me->order_service.manually_bust_all_fills(mb_params, [&](const auto &) {});
bts::api::position::PositionServiceResponse resp;
r(resp);
}
void PositionServiceServer::update_manual_fill(
const ManualFillBatch &batch,
const api::Response<PositionServiceResponse> &r) {
PositionServiceResponse resp;
for (auto mf : batch.manual_fill()) {
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
me->manual_fills_store->update(
manual_fills_ns, id, utils::pb::to_string(mf));
this_resp->set_ok(true);
// Retrieve old record so we can remove it from position
ManualFill const *old_ptr = me->state.get_manual_fill(mf.record_id());
bts_assert(old_ptr != nullptr);
// remove old
me->pos_publisher.add_or_update(_on_manual_fill(*old_ptr, false));
// add new
me->pos_publisher.add_or_update(_on_manual_fill(mf, true));
me->manual_fills_publisher.add_or_update(mf);
}
}
r(resp);
me->pos_publisher.flush();
me->manual_fills_publisher.flush();
}
void PositionServiceServer::remove_manual_fill(
const ManualFillBatch &batch,
const api::Response<PositionServiceResponse> &r) {
PositionServiceResponse resp;
for (auto mf : batch.manual_fill()) {
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
// Retrieve old record so we can remove it from position
ManualFill const *old_ptr = me->state.get_manual_fill(mf.record_id());
bts_assert(old_ptr != nullptr);
me->pos_publisher.add_or_update(_on_manual_fill(*old_ptr, false));
std::set<std::string> to_r;
to_r.insert(id);
me->manual_fills_store->remove(manual_fills_ns, to_r);
this_resp->set_ok(true);
me->manual_fills_publisher.erase(mf.record_id());
}
}
r(resp);
me->pos_publisher.flush();
me->manual_fills_publisher.flush();
}
void PositionServiceServer::set_base_position(
const BasePositionBatch &req,
const api::Response<SetBasePositionResponse> &) {
LOG(persist, Info, "PositionServiceServer::set_base_position bp count:{}",
req.positions_size());
me->state.set_base_position(req);
_clear_and_publish_pos();
}
void PositionServiceServer::update_security_parameters(
const SecurityParametersBatchOp &req,
const api::Response<SecurityParametersBatchOpResponse> &) {
// TODO
LOG(persist, Info,
"PositionServiceServer::update_security_parameters bp count:{}",
req.op_params_size());
for (const auto &p : req.op_params()) {
std::string key_str = std::to_string(p.security_key());
auto [exists, rowid, val] =
me->security_params_store->retrieve_one(security_params_ns, key_str);
if (!exists) {
me->security_params_store->insert(
security_params_ns, key_str, utils::pb::to_string(p));
} else {
me->security_params_store->update(
security_params_ns, key_str, utils::pb::to_string(p));
}
}
_load_security_params();
_compute_pnl_and_publish();
}
///////////////////////////////////////////////////////////////////////////////
/// PNL helpers ///////////////////////////////////////////////////////////////
void PositionServiceServer::_compute_pnl_and_publish() {
std::map<int, PositionUnit const *> in_pos;
me->state.get_pos(in_pos);
std::vector<PNLUnit> pnl;
std::vector<AccountPNL> account_pnl;
me->pnl_helper.compute_pnl(
in_pos, me->account_map, me->cpp_override_map, pnl, account_pnl);
for (const auto &u : pnl)
me->pnl_publisher.add_or_update(u);
for (const auto &u : account_pnl)
me->pnl_account_publisher.add_or_update(u);
me->pnl_publisher.flush();
me->pnl_account_publisher.flush();
}
void PositionServiceServer::_periodic_poll() {
_compute_pnl_and_publish();
me->context.post([this]() { _periodic_poll(); }, me->poll_period_ms);
}
PositionServiceServer::~PositionServiceServer() {}
} // namespace position
} // namespace bts