Files
spark/src/server/sparkd/position/PositionServiceServer.cpp
T

404 lines
15 KiB
C++

#include "PositionServiceServer.h"
#include "PositionMap.h"
#include "PositionServicePNLHelper.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/api/service_helpers/AdminServiceUtils.h"
#include "lib/api/service_helpers/PositionUtilities.h"
#include "lib/api/services/AdminService.client.pb.h"
#include "lib/api/services/AdminService.pb.h"
#include "lib/api/services/OrderService.client.pb.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/api/services/PositionService.pb.h"
#include "lib/logging/logging.h"
#include "lib/orderman/actors/SODPosition.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Puid.h"
#include <boost/algorithm/string.hpp>
#include <boost/tokenizer.hpp>
#include <fmt/format.h>
#include <map>
#include <set>
#include <utility>
namespace bts {
namespace position {
using namespace bts::api::position;
using namespace bts::md;
static auto const manual_fills_ns = "manual_fills";
static auto const security_params_ns = "security_parameters";
struct PositionServiceServer::PImpl {
bts::orderman::PositionMap state;
bts::api::ServiceContext &context;
bts::api::orderman::OrderService order_service;
PositionServiceBase::Position_dataStatefulPublisher pos_publisher;
int id;
// manual fills management
std::unique_ptr<persistence::IStore> manual_fills_store;
std::unique_ptr<persistence::IStore> security_params_store;
PositionServiceBase::Manual_fillsStatefulPublisher manual_fills_publisher;
// pnl
PositionServiceBase::Pnl_dataStatefulPublisher pnl_publisher;
PositionServiceBase::Pnl_account_dataStatefulPublisher pnl_account_publisher;
PositionServicePNLHelper pnl_helper;
int64_t poll_period_ms;
// admin data needed for pnl
bts::api::admin::AdminService admin_service;
api::admin::AdminData admin_data;
std::map<std::string, uint32_t> account_map;
std::map<uint32_t, double> cpp_override_map;
PImpl(
bts::api::ServiceContext &context,
const PositionServiceBase::Position_dataStatefulPublisher &pos_publisher,
const PositionServiceBase::Pnl_dataStatefulPublisher &pnl_publisher,
const PositionServiceBase::Pnl_account_dataStatefulPublisher
&pnl_account_publisher,
const PositionServiceBase::Manual_fillsStatefulPublisher
&manual_fills_publisher,
bts::md::MDClient &md_client)
: // pos
context(context),
order_service(context.locate<bts::api::orderman::OrderService>()),
pos_publisher(pos_publisher),
// manual fills
manual_fills_store(
std::make_unique<persistence::SqliteStore>("manual_fills")),
security_params_store(
std::make_unique<persistence::SqliteStore>("security_parameters")),
manual_fills_publisher(manual_fills_publisher),
// pnl
pnl_publisher(pnl_publisher),
pnl_account_publisher(pnl_account_publisher), pnl_helper(md_client),
poll_period_ms(300),
// admin
admin_service(context.locate<bts::api::admin::AdminService>()),
admin_data(), account_map(), cpp_override_map() {}
};
PositionServiceServer::PositionServiceServer(
bts::api::ServiceContext &context, bts::md::MDClient &md_client)
: PositionServiceBase(context),
me(new PImpl(
context,
make_position_data_stateful_publisher(),
make_pnl_data_stateful_publisher(),
make_pnl_account_data_stateful_publisher(),
make_manual_fills_stateful_publisher(),
md_client)) {
_delete_presod_local_on_start();
// incremental fill update
me->order_service.subscribe_fills() |
rxcpp::operators::subscribe<api::orderman::FillsUpdate>(
[this](auto const &update) {
for (const auto &f : update.removed())
me->pos_publisher.add_or_update(_on_fill(*f, false));
for (const auto &f : update.added())
me->pos_publisher.add_or_update(_on_fill(*f, true));
for (const auto &f : update.changed()) {
me->pos_publisher.add_or_update(
_on_fill(*f.previous(), false));
me->pos_publisher.add_or_update(_on_fill(*f.current(), true));
}
me->pos_publisher.flush();
_compute_pnl_and_publish();
});
// admin data
me->admin_service.subscribe_admin_data() |
rxcpp::operators::subscribe<api::admin::Admin_dataUpdate>(
[this](auto const &update) {
for (const auto &r : update.added())
_set_admin_data(*r);
for (const auto &r : update.changed())
_set_admin_data(*r.current());
});
// manual fills
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
LOG(persist, Info, "Finished loading all {} manual fills", all.size());
ManualFill rec;
for (auto const &kvp : all)
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
me->manual_fills_publisher.add_or_update(rec);
me->pos_publisher.add_or_update(_on_manual_fill(rec, true));
} else {
LOG(OrderMan, Error,
"Could not recover manual fill {} from storage. "
"This should be investigated",
kvp.first);
}
me->manual_fills_publisher.flush();
me->pos_publisher.flush();
me->context.post([this]() { _periodic_poll(); }, me->poll_period_ms);
}
void PositionServiceServer::_delete_presod_local_on_start() const {
std::set<std::string> rm_ids;
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
ManualFill rec;
for (auto const &kvp : all) {
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
if (rec.fill_type() == ManualFill::LOCAL &&
me->state.is_sod(rec.timestamp())) {
LOG(OrderMan, Info,
"PositionServiceServer::_delete_presod_local_on_start "
"fill: {}",
kvp.second);
auto id = std::to_string(rec.record_id());
rm_ids.insert(id);
}
} else
LOG(OrderMan, Error,
"Could not recover manual fill {} from storage. "
"This should be investigated",
kvp.first);
}
me->manual_fills_store->remove(manual_fills_ns, rm_ids);
}
const PositionUnit &PositionServiceServer::_on_fill(
const Fill &f, bool is_add) {
return me->state.on_fill(f, is_add);
}
const PositionUnit &PositionServiceServer::_on_manual_fill(
const ManualFill &mf, bool is_add) {
return me->state.on_manual_fill(mf, is_add);
}
void PositionServiceServer::_set_admin_data(
const api::admin::AdminData &admin_data) {
me->admin_data = admin_data;
for (const auto &a : admin_data.accounts())
me->account_map[a.name()] = a.id();
me->state.set_admin_data(admin_data);
_clear_and_publish_pos();
}
template <class T, class TKey>
void _diff(
const std::map<TKey, T const *> &new_state,
const std::map<TKey, T const *> &prev_state,
std::vector<T const *> &added,
std::vector<T const *> &updated,
std::vector<T const *> &deleted) {
for (const auto &p : new_state) {
if (prev_state.find(p.first) == prev_state.end())
added.push_back(p.second);
else
updated.push_back(p.second);
}
for (const auto &p : prev_state)
if (new_state.find(p.first) == new_state.end())
deleted.push_back(p.second);
}
void PositionServiceServer::_clear_and_publish_pos() {
std::map<int, PositionUnit const *> new_state;
me->state.get_pos(new_state);
// entries are never deleted from new_state
for (const auto &p : new_state)
me->pos_publisher.add_or_update(*p.second);
me->pos_publisher.flush();
}
void PositionServiceServer::_load_security_params() {
me->cpp_override_map.clear();
auto all = me->security_params_store->retrieve_all()[security_params_ns];
SecurityParameters rec;
for (auto const &kvp : all)
if (utils::pb::from_string<SecurityParameters>(kvp.second, &rec)) {
if (rec.has_cpp_override()) {
LOG(OrderMan, Info, "Overriding cpp, key:{} cpp:{}",
rec.security_key(), rec.cpp_override().value());
me->cpp_override_map[rec.security_key()] =
rec.cpp_override().value();
}
} else {
LOG(OrderMan, Error,
"Could not recover security parameter for security {} from "
"storage. "
"This should be investigated",
kvp.first);
}
}
///////////////////////////////////////////////////////////////////////////////
/// Manual Fills API //////////////////////////////////////////////////////////
void PositionServiceServer::create_manual_fill(
const ManualFillBatch &batch,
const api::Response<PositionServiceResponse> &r) {
PositionServiceResponse resp;
for (auto mf : batch.manual_fill()) {
auto id = utils::gen_id("manual_fills");
mf.set_record_id(id);
struct timeval tv;
gettimeofday(&tv, NULL);
mf.mutable_timestamp()->set_seconds(tv.tv_sec);
mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
me->manual_fills_store->insert(
manual_fills_ns, std::to_string(id), utils::pb::to_string(mf));
resp.add_response()->set_ok(true);
me->manual_fills_publisher.add_or_update(mf);
me->pos_publisher.add_or_update(_on_manual_fill(mf, true));
}
r(resp);
me->manual_fills_publisher.flush();
me->pos_publisher.flush();
}
void PositionServiceServer::reset_position(
const ::bts::api::position::ResetPositionParameters &reset_params,
const api::Response<::bts::api::position::PositionServiceResponse> &r) {
LOG(OrderMan, Info,
"PositionServiceServer::reset_position acct: {} n_recs: {}",
reset_params.account(),
reset_params.total_position().manual_fill().size());
// find existing fills for this account and remove them
auto all = me->manual_fills_store->retrieve_all()[manual_fills_ns];
ManualFillBatch fills_to_remove;
ManualFill rec;
for (auto const &kvp : all)
if (utils::pb::from_string<ManualFill>(kvp.second, &rec)) {
if (rec.account_name() == reset_params.account()) {
fills_to_remove.add_manual_fill()->CopyFrom(rec);
}
}
remove_manual_fill(fills_to_remove, [&](const auto &) {});
// now that existing fills have been removed, add the fills represeting total
// positions in the function arg
create_manual_fill(reset_params.total_position(), [&](const auto &) {});
// bust all fills for this account by calling into OrderService
api::orderman::ManualBustParameters mb_params;
mb_params.set_account(reset_params.account());
me->order_service.manually_bust_all_fills(mb_params, [&](const auto &) {});
bts::api::position::PositionServiceResponse resp;
r(resp);
}
void PositionServiceServer::update_manual_fill(
const ManualFillBatch &batch,
const api::Response<PositionServiceResponse> &r) {
PositionServiceResponse resp;
for (auto mf : batch.manual_fill()) {
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
me->manual_fills_store->update(
manual_fills_ns, id, utils::pb::to_string(mf));
this_resp->set_ok(true);
// Retrieve old record so we can remove it from position
ManualFill const *old_ptr = me->state.get_manual_fill(mf.record_id());
bts_assert(old_ptr != nullptr);
// remove old
me->pos_publisher.add_or_update(_on_manual_fill(*old_ptr, false));
// add new
me->pos_publisher.add_or_update(_on_manual_fill(mf, true));
me->manual_fills_publisher.add_or_update(mf);
}
}
r(resp);
me->pos_publisher.flush();
me->manual_fills_publisher.flush();
}
void PositionServiceServer::remove_manual_fill(
const ManualFillBatch &batch,
const api::Response<PositionServiceResponse> &r) {
PositionServiceResponse resp;
for (auto mf : batch.manual_fill()) {
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
// Retrieve old record so we can remove it from position
ManualFill const *old_ptr = me->state.get_manual_fill(mf.record_id());
bts_assert(old_ptr != nullptr);
me->pos_publisher.add_or_update(_on_manual_fill(*old_ptr, false));
std::set<std::string> to_r;
to_r.insert(id);
me->manual_fills_store->remove(manual_fills_ns, to_r);
this_resp->set_ok(true);
me->manual_fills_publisher.erase(mf.record_id());
}
}
r(resp);
me->pos_publisher.flush();
me->manual_fills_publisher.flush();
}
void PositionServiceServer::set_base_position(
const BasePositionBatch &req,
const api::Response<SetBasePositionResponse> &) {
LOG(persist, Info, "PositionServiceServer::set_base_position bp count:{}",
req.positions_size());
me->state.set_base_position(req);
_clear_and_publish_pos();
}
void PositionServiceServer::update_security_parameters(
const SecurityParametersBatchOp &req,
const api::Response<SecurityParametersBatchOpResponse> &) {
// TODO
LOG(persist, Info,
"PositionServiceServer::update_security_parameters bp count:{}",
req.op_params_size());
for (const auto &p : req.op_params()) {
std::string key_str = std::to_string(p.security_key());
auto db_p =
me->security_params_store->retrieve_one(security_params_ns, key_str);
auto found = db_p.first;
if (!found) {
me->security_params_store->insert(
security_params_ns, key_str, utils::pb::to_string(p));
} else {
me->security_params_store->update(
security_params_ns, key_str, utils::pb::to_string(p));
}
}
_load_security_params();
_compute_pnl_and_publish();
}
///////////////////////////////////////////////////////////////////////////////
/// PNL helpers ///////////////////////////////////////////////////////////////
void PositionServiceServer::_compute_pnl_and_publish() {
std::map<int, PositionUnit const *> in_pos;
me->state.get_pos(in_pos);
std::vector<PNLUnit> pnl;
std::vector<AccountPNL> account_pnl;
me->pnl_helper.compute_pnl(
in_pos, me->account_map, me->cpp_override_map, pnl, account_pnl);
for (const auto &u : pnl)
me->pnl_publisher.add_or_update(u);
for (const auto &u : account_pnl)
me->pnl_account_publisher.add_or_update(u);
me->pnl_publisher.flush();
me->pnl_account_publisher.flush();
}
void PositionServiceServer::_periodic_poll() {
_compute_pnl_and_publish();
me->context.post([this]() { _periodic_poll(); }, me->poll_period_ms);
}
PositionServiceServer::~PositionServiceServer() {}
} // namespace position
} // namespace bts