Adding meaningful correlation ids

This commit is contained in:
Taha Afzal committed 2023-06-06 18:30:38 +00:00
1 parent 1025881ec8
commit aba478f3dc
6 files changed
+134 -114

No files matched your search

+90 -78
View File
@@ -16,6 +16,7 @@
#include "lib/utils/bts_assert.h"
#include <bits/stdint-uintn.h>
#include <boost/lockfree/spsc_queue.hpp>
#include <ccapi_cpp/ccapi_element.h>
#include <ccapi_cpp/ccapi_event.h>
#include <ccapi_cpp/ccapi_macro.h>
#include <ccapi_cpp/ccapi_message.h>
@@ -32,6 +33,9 @@
using namespace std;
namespace bts::orderman {
// This is the interface we need to pass to CCAPI to get the callbacks
// For threading reasons, it just queues the message which is then dequeued
// in dispatch_incoming_messages
class OrderEventHandler : public ccapi::EventHandler {
boost::lockfree::spsc_queue<ccapi::Event> &event_queue;
std::set<ccapi::Event::Type> event_types;
@@ -57,6 +61,7 @@ class OrderEventHandler : public ccapi::EventHandler {
}
};
// Sec state for the order and trade subs
struct SubscriptionStatus {
const uint32_t sec_id;
const std::string order_sub_corr_id;
@@ -78,7 +83,7 @@ struct SubscriptionStatus {
struct CCAPIDeribitOrderMux::PImpl {
const std::string missing_corrid_placeholder = "MISSING_CORRID";
md::SecurityOracle md_oracle;
md::SecurityOracle sec_oracle;
ccapi::SessionOptions session_options = {};
ccapi::SessionConfigs session_configs;
ccapi::Session *session = nullptr;
@@ -100,13 +105,13 @@ struct CCAPIDeribitOrderMux::PImpl {
// we get subcriptions confirmation
std::vector<CreateOrderMsg> buffered_creates = {};
PImpl(const md::SecurityOracle &md_oracle)
: md_oracle(md_oracle), incoming_event_queue(100),
: sec_oracle(md_oracle), incoming_event_queue(100),
event_handler(
incoming_event_queue,
{ccapi::Event::Type::RESPONSE,
ccapi::Event::Type::SUBSCRIPTION_DATA,
ccapi::Event::Type::SUBSCRIPTION_STATUS}) {
// Test
// TODO: get it from the outside
session_configs.setCredential({
{"grant_type", "client_signature"},
{CCAPI_DERIBIT_CLIENT_ID, "ZmIKL6fb"},
@@ -124,7 +129,7 @@ CCAPIDeribitOrderMux::CCAPIDeribitOrderMux(const md::SecurityOracle &sec_oracle)
"CCAPIDeribitOrderMux::CCAPIDeribitOrderMux constructor t_id: {} "
"t_name: {} ",
os::get_thread_id(), os::get_application_name());
_subscribe_to_order_callbacks();
//_subscribe_to_order_callbacks();
}
CCAPIDeribitOrderMux::~CCAPIDeribitOrderMux() {}
@@ -185,21 +190,25 @@ bool CCAPIDeribitOrderMux::update_order(
order_key, sec_desc);
return false;
}
const CCAPIOrderState &ord_state = it->second.order_state();
CCAPISingleOrderManager &ord_manager = it->second;
const CCAPIOrderState &ord_state = ord_manager.order_state();
double new_qty = cxl_repl_msg.create_fields.quantity.full_value();
double new_price = cxl_repl_msg.create_fields.price.value();
std::string query = "amount=" + std::to_string(new_qty) +
"&order_id=" + ord_state.cc_order_key +
"&price=" + std::to_string(new_price);
std::string corr_id = "OrderMod:" + ord_state.cc_order_key + ":" +
std::to_string(ord_state.n_outgoing_mods);
ccapi::Request request(
ccapi::Request::Operation::GENERIC_PRIVATE_REQUEST,
CCAPI_EXCHANGE_NAME_DERIBIT, sec_desc);
CCAPI_EXCHANGE_NAME_DERIBIT, sec_desc, corr_id);
request.appendParam(
{// {CCAPI_HTTP_PROTOCOL, "https"},
{CCAPI_HTTP_METHOD, "GET"},
{CCAPI_HTTP_PATH, "/api/v2/private/edit"},
{CCAPI_HTTP_QUERY_STRING, query},
{CCAPI_HTTP_HEADERS, "Content-Type: application/json"}});
ord_manager.handle_order_update_out(new_price, new_qty);
me->session->sendRequest(request);
return true;
}
@@ -216,9 +225,10 @@ void send_cancel_generic_private_request(
// std::string query = "cancel?order_id=" + cc_order_key;
std::string query = "order_id=" + cc_order_key;
std::string corr_id = "OrderCxl:" + cc_order_key;
ccapi::Request request(
ccapi::Request::Operation::GENERIC_PRIVATE_REQUEST,
CCAPI_EXCHANGE_NAME_DERIBIT, sec_desc);
CCAPI_EXCHANGE_NAME_DERIBIT, sec_desc, corr_id);
request.appendParam(
{// {CCAPI_HTTP_PROTOCOL, "https"},
{CCAPI_HTTP_METHOD, "GET"},
@@ -236,20 +246,23 @@ bool CCAPIDeribitOrderMux::cancel_order(const CancelOrderMsg &cxl_msg) {
if (it == me->order_map.end()) {
return false;
}
CCAPISingleOrderManager &man = it->second;
const auto &sec_desc = man.order_state().security.deribit().security_desc();
if (man.order_state().security.has_option()) {
CCAPISingleOrderManager &order_man = it->second;
const auto &sec_desc =
order_man.order_state().security.deribit().security_desc();
if (order_man.order_state().security.has_option()) {
send_cancel_generic_private_request(
me->session, man.order_state().cc_order_key, sec_desc);
me->session, order_man.order_state().cc_order_key, sec_desc);
return true;
}
std::string corr_id = "OrderCxl:" + order_man.order_state().cc_order_key;
ccapi::Request request(
ccapi::Request::Operation::CANCEL_ORDER, CCAPI_EXCHANGE_NAME_DERIBIT,
sec_desc);
request.appendParam({{CCAPI_EM_ORDER_ID, man.order_state().cc_order_key}});
request.appendParam(
{{CCAPI_EM_ORDER_ID, order_man.order_state().cc_order_key}});
LOG(OrderMan, Info,
"CCAPIDeribitOrderMux::cancel_order order_key: {} cc_order_key: {}",
man.order_state().order_key, man.order_state().cc_order_key);
order_man.order_state().order_key, order_man.order_state().cc_order_key);
me->session->sendRequest(request);
return true;
}
@@ -266,24 +279,28 @@ OnCCAPIOrderUpdate &CCAPIDeribitOrderMux::order_update_cb() {
// this gets called in the main thread, the events have already been queue'd
// from the CCAPI callback thread
void CCAPIDeribitOrderMux::dispatch_incoming_messages() {
auto n_items = me->incoming_event_queue.read_available();
if (n_items > 0) {
for (auto &kv : me->order_map)
kv.second.dispatch_start();
LOG(OrderMan, Info,
"CCAPIDeribitOrderMux::dispatch_incoming_messages n_items: {}",
n_items);
me->incoming_event_queue.consume_all(
[this](const ccapi::Event &ev) { _handle_event(ev); });
for (auto &kv : me->order_map) {
auto ord_ch = kv.second.dispatch_end();
if (ord_ch) {
me->order_update_cb(kv.second.order_state());
}
if (me->incoming_event_queue.empty())
return;
for (auto &kv : me->order_map)
kv.second.dispatch_start();
LOG(OrderMan, Info,
"CCAPIDeribitOrderMux::dispatch_incoming_messages n_items: {}",
me->incoming_event_queue.read_available());
me->incoming_event_queue.consume_all(
[this](const ccapi::Event &ev) { _handle_event(ev); });
for (auto &kv : me->order_map) {
auto ord_ch = kv.second.dispatch_end();
if (ord_ch) {
me->order_update_cb(kv.second.order_state());
}
// TODO, remove orders in end state
}
// _check_subs();
// TODO: delete terminal state orders
// auto it = std::remove_if(
// me->order_map.begin(), me->order_map.end(),
// [](const std::pair<OrderKey, CCAPISingleOrderManager> &) {
// return true;
// });
// me->order_map.erase(it, me->order_map.end());
}
// Position
@@ -304,6 +321,11 @@ CCAPIDeribitOrderMux::exchange_open_orders_update_cb() {
return me->exchange_open_orders_update_cb;
}
// private functions //////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
// Checks if an orders/trade subscription exists, if not it adds both the subs
// and returns false. If it already exists, returns true
bool CCAPIDeribitOrderMux::_check_and_add_subscription(
const api::Security &sec) {
auto it = me->sub_map.find(sec.id());
@@ -334,10 +356,10 @@ bool CCAPIDeribitOrderMux::_check_and_add_subscription(
}
}
// (Unused) Subscribes to all orders
void CCAPIDeribitOrderMux::_subscribe_to_order_callbacks() {
// if (me->subbed_to_orders)
// return;
// me->subbed_to_orders = true;
// this is not working
bts_assert(false);
std::vector<string> channels = {"user.orders.any.any.raw"};
const nlohmann::json opts_json{
{"method", "private/subscribe"},
@@ -356,21 +378,38 @@ void CCAPIDeribitOrderMux::_subscribe_to_order_callbacks() {
"CCAPIDeribitOrderMux::_subscribe_to_order_callbacks sub: {}", opts_str);
ccapi::Subscription sub(
CCAPI_EXCHANGE_NAME_DERIBIT, "", CCAPI_GENERIC_PUBLIC_SUBSCRIPTION,
opts_str, "AllOrdersSub"
// {
// {"grant_type", "client_credentials"},
// {"scope", "session:bts_spark_deribit1"},
// {"client_id", "ZmIKL6fb"},
// {"client_secret", "_FY2PcXYNc8VdlfkZO5c-pkFJQFhn8P9z4_p8YA2fS0"},
// }
);
opts_str, "AllOrdersSub");
me->session->subscribe(sub);
}
void CCAPIDeribitOrderMux::_handle_event(const ccapi::Event &event) {
ccapi::Event::Type event_type = event.getType();
bts_assert(
event_type == ccapi::Event::Type::RESPONSE || //response to a request
event_type == ccapi::Event::Type::SUBSCRIPTION_DATA ||
event_type == ccapi::Event::Type::SUBSCRIPTION_STATUS);
auto get_corr_id = [&](const ccapi::Message &msg, size_t idx) {
const auto &corr_list = msg.getCorrelationIdList();
// TODO: not sure about this, why isnt corr id a part of Element
return idx < corr_list.size() ? corr_list.at(idx)
: me->missing_corrid_placeholder;
};
for (const auto &msg : event.getMessageList()) {
ccapi::Message::Type msg_type = msg.getType();
size_t i = 0;
for (const auto &elem : msg.getElementList()) {
const std::string &corr_id = get_corr_id(msg, i++);
_handle_element(event.getType(), msg_type, elem, corr_id);
}
}
}
void CCAPIDeribitOrderMux::_handle_element(
ccapi::Event::Type event_type,
ccapi::Message::Type msg_type,
const NVMap &map,
const ccapi::Element &element,
const std::string &corr_id) {
const auto &map = element.getNameValueMap();
switch (msg_type) {
case ccapi::Message::Type::CREATE_ORDER: {
CCAPICreateOrderMsg cr_ord_msg(map, corr_id);
@@ -400,33 +439,17 @@ void CCAPIDeribitOrderMux::_handle_element(
_handle_subscription_status(sub_status_msg);
break;
}
default:
break;
case ccapi::Message::Type::GENERIC_PRIVATE_REQUEST: {
}
}
void CCAPIDeribitOrderMux::_handle_event(const ccapi::Event &event) {
ccapi::Event::Type event_type = event.getType();
bts_assert(
event_type == ccapi::Event::Type::RESPONSE ||
event_type == ccapi::Event::Type::SUBSCRIPTION_DATA ||
event_type == ccapi::Event::Type::SUBSCRIPTION_STATUS);
// switch here to throw out
auto get_corr_id = [&](const ccapi::Message &msg, size_t idx) {
const auto &corr_list = msg.getCorrelationIdList();
return idx < corr_list.size() ? corr_list.at(idx)
: me->missing_corrid_placeholder;
};
for (const auto &msg : event.getMessageList()) {
ccapi::Message::Type msg_type = msg.getType();
size_t i = 0;
for (const auto &elem : msg.getElementList()) {
// TODO: not sure about this, why isnt corr id a part of Element (def
// not right)
const std::string &corr_id = get_corr_id(msg, i);
i++;
_handle_element(event.type, msg_type, corr_id, elem);
}
case ccapi::Message::Type::REQUEST_FAILURE: {
}
default:
LOG(OrderMan, Error,
"CCAPIDeribitOrderMux::_handle_element unhandled message type "
"event_type: {} mgs_type: {} element: {} corr_id: {}",
ccapi::Event::typeToString(event_type),
ccapi::Message::typeToString(msg_type), element.toString(), corr_id);
break;
}
}
@@ -457,17 +480,6 @@ void CCAPIDeribitOrderMux::_handle_subscription_status(
}
}
// void CCAPIDeribitOrderMux::_check_subs() {
// google::protobuf::Timestamp now =
// google::protobuf::util::TimeUtil::GetCurrentTime();
// int diff = now.seconds() - me->last_sec_check_time.seconds();
// if (diff > (int)me->sec_check_period) {
// LOG(OrderMan, Info, "CCAPIDeribitOrderMux::_check_subs diff: {}",
// diff); me->last_sec_check_time = now;
// _add_subscriptions(me->md_oracle.get_securities());
// }
// }
void CCAPIDeribitOrderMux::_send_order(const CreateOrderMsg &cr_msg) {
const auto &sec_desc = cr_msg.security->deribit().security_desc();
ccapi::Request request(
@@ -51,12 +51,12 @@ class CCAPIDeribitOrderMux : public ICCAPIOrderMux {
void _subscribe_to_order_callbacks();
void _handle_event(const ccapi::Event &event);
void _handle_element(
ccapi::Event::Type event_type,
ccapi::Message::Type msg_type,
const std::map<std::string, std::string> &,
const ccapi::Element &element,
const std::string &corr_id);
void _handle_subscription_status(
const CCAPISubscriptionStatusMsg &sub_status);
// void _check_subs();
void _send_order(const CreateOrderMsg &);
void _send_buffered_orders(uint32_t sec_id);
void _query_positions();
+36 -33
View File
@@ -34,26 +34,27 @@ bts::api::orderman::OrderStatus convert_ccapi_status(const std::string &status);
// type = CREATE_ORDER,
// recapType = UNKNOWN,
// time = 1970-01-01T00:00:00.000000000Z,
// timeReceived = 2022-11-03T02:54:53.655916565Z,
// timeReceived = 2023-05-04T20:13:15.425177524Z,
// elementList = [
// Element [
// nameValueMap = {
// CUMULATIVE_FILLED_PRICE_TIMES_QUANTITY = 1224720.000000,
// CUMULATIVE_FILLED_QUANTITY = 60.0,
// INSTRUMENT = BTC-30JUN23,
// LIMIT_PRICE = 20412.0,
// ORDER_ID = 14636988474,
// QUANTITY = 100.0,
// SIDE = SELL,
// CUMULATIVE_FILLED_PRICE_TIMES_QUANTITY = 0.000000,
// CUMULATIVE_FILLED_QUANTITY = 0.0,
// INSTRUMENT = BTC-PERPETUAL,
// LIMIT_PRICE = 28860.0,
// ORDER_ID = 16257763986,
// QUANTITY = 10.0,
// SIDE = BUY,
// STATUS = open
// }
// ]
// ],
// correlationIdList = [ 0 ]
// correlationIdList = [ NewOrder:605003:BTC-PERPETUAL ]
// ]
// ]
// ]
// This is a Response message when a request to send an order is sent
struct CCAPICreateOrderMsg {
std::string order_id;
std::string instrument;
@@ -63,7 +64,6 @@ struct CCAPICreateOrderMsg {
double cumm_filled_price;
double cumm_filled_qty;
bts::api::orderman::OrderStatus status;
// TODO
std::string corr_id;
CCAPICreateOrderMsg(const NVMap &map, const std::string &corr_id)
: order_id(getm(map, CCAPI_EM_ORDER_ID)),
@@ -89,27 +89,27 @@ struct CCAPICreateOrderMsg {
// Message [
// type = EXECUTION_MANAGEMENT_EVENTS_ORDER_UPDATE,
// recapType = UNKNOWN,
// time = 2022-11-03T02:57:34.853000000Z,
// timeReceived = 2022-11-03T02:57:34.906199444Z,
// time = 2023-05-04T20:13:18.559000000Z,
// timeReceived = 2023-05-04T20:13:18.608831966Z,
// elementList = [
// Element [
// nameValueMap = {
// CUMULATIVE_FILLED_PRICE_TIMES_QUANTITY = 1428840.000000,
// CUMULATIVE_FILLED_QUANTITY = 70.0,
// LIMIT_PRICE = 20412.0,
// ORDER_ID = 14636988474,
// QUANTITY = 100.0,
// SIDE = SELL,
// STATUS = open
// CUMULATIVE_FILLED_PRICE_TIMES_QUANTITY = 288635.300000,
// CUMULATIVE_FILLED_QUANTITY = 10.0,
// LIMIT_PRICE = 28865.5,
// ORDER_ID = 16257763986,
// QUANTITY = 10.0,
// SIDE = BUY,
// STATUS = filled
// }
// ]
// ],
// correlationIdList = [ pJFeZrln ]
// correlationIdList = [ OrderSub:BTC-PERPETUAL ]
// ]
// ]
// ]
// at the end, we can receive EXECUTION_MANAGEMENT_EVENTS_ORDER_UPDATE first
// with STATUS = completed, and then the trade(s)
// This is a Subscription message as result of an orders subscription
struct CCAPIOrderUpdateMsg {
std::string order_id;
api::Side side;
@@ -133,34 +133,37 @@ struct CCAPIOrderUpdateMsg {
std::stod(getm(map, CCAPI_EM_ORDER_CUMULATIVE_FILLED_QUANTITY))),
status(convert_ccapi_status(getm(map, CCAPI_EM_ORDER_STATUS))) {}
};
// 3. Event [
// 3.
// Event [
// type = SUBSCRIPTION_DATA,
// messageList = [
// Message [
// type = EXECUTION_MANAGEMENT_EVENTS_PRIVATE_TRADE,
// recapType = UNKNOWN,
// time = 2022-11-03T02:57:34.853000000Z,
// timeReceived = 2022-11-03T02:57:34.907237094Z,
// time = 2023-05-04T20:13:18.559000000Z,
// timeReceived = 2023-05-04T20:13:18.610572197Z,
// elementList = [
// Element [
// nameValueMap = {
// FEE_ASSET = BTC,
// FEE_QUANTITY = 0.0,
// INSTRUMENT = BTC-30JUN23,
// IS_MAKER = 1,
// LAST_EXECUTED_PRICE = 20412.0,
// FEE_QUANTITY = 1.7e-7,
// INSTRUMENT = BTC-PERPETUAL,
// IS_MAKER = 0,
// LAST_EXECUTED_PRICE = 28863.5,
// LAST_EXECUTED_SIZE = 10.0,
// ORDER_ID = 14636988474,
// SEQUENCE_NUMBER = 898679,
// ORDER_ID = 16257763986,
// SEQUENCE_NUMBER = 99252800,
// SIDE = SELL,
// TRADE_ID = 130273180
// TRADE_ID = 161565953
// }
// ]
// ],
// correlationIdList = [ 83Rzbkvs ]
// correlationIdList = [ TradeSub:BTC-PERPETUAL ]
// ]
// ]
// ]
// Trade message
struct CCAPITradeMsg {
std::string order_id;
api::Side side;
+1
View File
@@ -43,6 +43,7 @@ struct CCAPIOrderState {
// same as above
boost::container::small_vector<CCAPISpreadLegFill, 4> cb_leg_trades = {};
size_t n_cbs = 0;
size_t n_outgoing_mods = 0;
CCAPIOrderState(
const api::Security &security,
@@ -244,6 +244,10 @@ bool CCAPISingleOrderManager::dispatch_end() {
return me->prev_state.changed(me->order);
}
void CCAPISingleOrderManager::handle_order_update_out(double, double) {
me->order.n_outgoing_mods++;
}
const CCAPIOrderState &CCAPISingleOrderManager::order_state() const {
return me->order;
}
@@ -35,7 +35,7 @@ class CCAPISingleOrderManager {
// if there is a change return true
bool dispatch_end();
void handle_order_update_out(double new_price, double new_qty);
const CCAPIOrderState &order_state() const;
private: