From 98362c7eb9cf65374f0f9328eb0874be074ec279 Mon Sep 17 00:00:00 2001 From: taha afzal Date: Sat, 26 Feb 2022 23:05:47 -0500 Subject: [PATCH] Fixing the deribit connection drop issue --- src/server/sparkd/cmds/crypto/DeribitDemo.cpp | 39 +--- src/server/sparkd/cmds/crypto/DeribitDemo.h | 4 - .../cmds/crypto/DeribitDemoListener.cpp | 187 ++++++------------ .../sparkd/cmds/crypto/DeribitDemoListener.h | 14 +- .../md/platforms/emumba/EmMDListener.cpp | 1 + .../sparkd/md/platforms/emumba/EmumbaMain.cpp | 10 +- 6 files changed, 82 insertions(+), 173 deletions(-) diff --git a/src/server/sparkd/cmds/crypto/DeribitDemo.cpp b/src/server/sparkd/cmds/crypto/DeribitDemo.cpp index e1ef6177e..42fe6297a 100644 --- a/src/server/sparkd/cmds/crypto/DeribitDemo.cpp +++ b/src/server/sparkd/cmds/crypto/DeribitDemo.cpp @@ -21,42 +21,11 @@ static void signal_handler(int signal_number) { } } -struct DeribitDemo::PImpl { - std::optional conf; - std::optional client_listener; - std::shared_ptr - real_conn; // Shared pointer of deribit connector. This pointer can - // be used to configure/run the connector. - std::shared_ptr - trading_session_interface; // Shared pointer of trading session - // interface. This pointer can be use to send - // order entry messages. -}; - -DeribitDemo::DeribitDemo(const std::string &config_path) - : me_(std::make_unique()) { - // parse config - LOG(md, Info, "DeribitDemo::DeribitDemo constructed config_path:{}", - config_path); - me_->conf = DeribitDemoConfiguration::read_file(config_path); - if (!me_->conf.has_value()) { - return; - } - auto &conf = *me_->conf; - - // connect to deribit +DeribitDemo::DeribitDemo(const std::string &config_path) { std::signal(SIGUSR1, signal_handler); - me_->real_conn = - std::make_shared( - conf.symbol_config_list, conf.trading_config, - conf.logging_configuration, conf.deribit_url, - &me_->client_listener.emplace()); - me_->trading_session_interface = - std::make_shared( - me_->real_conn->create_new_trading_session()); - LOG(md, Info, "DeribitDemo::DeribitDemo starting deribit_handler"); - me_->real_conn->run(); - LOG(md, Info, "DeribitDemo::DeribitDemo deribit_handler exited"); + DeribitDemoListener deribit_listener; + deribit_listener.configure(config_path); + deribit_listener.run(); } DeribitDemo::~DeribitDemo() = default; diff --git a/src/server/sparkd/cmds/crypto/DeribitDemo.h b/src/server/sparkd/cmds/crypto/DeribitDemo.h index 7588d520c..c6094f9d3 100644 --- a/src/server/sparkd/cmds/crypto/DeribitDemo.h +++ b/src/server/sparkd/cmds/crypto/DeribitDemo.h @@ -8,10 +8,6 @@ class DeribitDemo { public: DeribitDemo(const std::string &config_path); ~DeribitDemo(); - - private: - struct PImpl; - std::unique_ptr me_; }; } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/DeribitDemoListener.cpp b/src/server/sparkd/cmds/crypto/DeribitDemoListener.cpp index ed877d81e..8b046eb3c 100644 --- a/src/server/sparkd/cmds/crypto/DeribitDemoListener.cpp +++ b/src/server/sparkd/cmds/crypto/DeribitDemoListener.cpp @@ -5,6 +5,7 @@ #include "rapidjson/ostreamwrapper.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" +#include "server/sparkd/cmds/crypto/DeribitDemoConfiguration.h" #include "server/sparkd/cmds/spartan/utils/FileUtils.h" #include #include @@ -30,8 +31,8 @@ const std::string &extract_security_desc( } bool DeribitDemoListener::configure(const std::string &config_path) { - assert(false); - return true; + // assert(false); + // return true; using namespace rapidjson; auto _main_logger = @@ -39,108 +40,16 @@ bool DeribitDemoListener::configure(const std::string &config_path) { spdlog::set_default_logger(_main_logger); spdlog::set_level(spdlog::level::trace); - std::string deribit_url; - - // Deribit Connector Configuration Structures - std::list - symbol_config_list; - emumba::deribit_connector::configuration::logging_config - logging_configuration; - emumba::deribit_connector::configuration::trading_session_config - trading_config; - - std::ifstream configFile(config_path); - IStreamWrapper isw{configFile}; - - // std::string config_contents((std::istreambuf_iterator(configFile)), - // std::istreambuf_iterator()); std::cout << "config.json is:\n\n" << - // config_contents << "\n***************************\n" << std::endl; - Document config{}; - if (config.ParseStream(isw).HasParseError()) { - SPDLOG_ERROR("Error in parsing config.json"); - return false; - } - - if (config.HasMember("app_logging") == 1) { - logging_configuration.severity = - config["app_logging"]["severity"].GetString(); - logging_configuration.is_console_enabled = - config["app_logging"]["console"].GetBool(); - logging_configuration.is_file_enabled = - config["app_logging"]["file"].GetBool(); - if (config["app_logging"].HasMember("queuesize")) { - logging_configuration.queue_size = - config["app_logging"]["queuesize"].GetInt(); - } - } else { - SPDLOG_INFO("No logging information: Switching to default"); - } - - if (config.HasMember("uri") == 1) { - - deribit_url = config["uri"].GetString(); - if (deribit_url == "") { - deribit_url = "wss://test.deribit.com/ws/api/v2"; - } - } else { - deribit_url = "wss://test.deribit.com/ws/api/v2"; - } - - auto _credentials = config["credentials"].GetObject(); - - trading_config.client_name = _credentials["client_name"].GetString(); - trading_config.client_id = _credentials["id"].GetString(); - trading_config.secret_key = _credentials["secret_key"].GetString(); - trading_config.heartbeat_interval = - config["session"].GetObject()["heartbeat_interval"].GetInt(); - trading_config.core = config["core"].GetInt(); - - auto symbol_config = config["symbol"].GetArray(); - - for (rapidjson::Value::ConstValueIterator _symbol_itr = - symbol_config.Begin(); - _symbol_itr != symbol_config.End(); ++_symbol_itr) { - emumba::deribit_connector::configuration::symbol_config symbol; - emumba::deribit_connector::configuration::subscription_config subs; - auto _symbol = _symbol_itr->GetObject(); - symbol.symbol_name = _symbol["id"].GetString(); - symbol.interested_in_future = _symbol["interested_in_future"].GetBool(); - symbol.interested_in_option = _symbol["interested_in_option"].GetBool(); - symbol.currency_type = _symbol["currency_type"].GetString(); - - for (rapidjson::Value::ConstValueIterator _instrument_itr = - _symbol["instruments"].GetArray().Begin(); - _instrument_itr != _symbol["instruments"].GetArray().End(); - ++_instrument_itr) { - symbol.interested_instruments.insert(_instrument_itr->GetString()); - } - auto channel = _symbol["channel"].GetObject(); - - if (_symbol.HasMember("requests_limit")) { - subs.requests_limit = _symbol["requests_limit"].GetUint(); - } - - subs.book_depth = _symbol["book_depth"].GetInt(); - subs.use_raw_interval = _symbol["raw_enable"].GetBool(); - subs.public_book = channel["PUBLIC_BOOK"].GetBool(); - subs.public_quote = channel["PUBLIC_QUOTE"].GetBool(); - subs.public_trade = channel["PUBLIC_TRADE"].GetBool(); - subs.public_ticker = channel["PUBLIC_TICKER"].GetBool(); - subs.public_markprice = channel["PUBLIC_MARKPRICE"].GetBool(); - subs.public_index = channel["PUBLIC_INDEX"].GetBool(); - subs.private_orders = channel["PRIVATE_ORDERS"].GetBool(); - subs.private_trades = channel["PRIVATE_TRADES"].GetBool(); - subs.private_portfolio = channel["PRIVATE_PORTFOLIO"].GetBool(); - - symbol.interested_subscriptions = subs; - symbol_config_list.push_back(symbol); - } - + std::optional conf = + bts::cmds::DeribitDemoConfiguration::read_file(config_path); // Creating a deribit_handler object using the above configuration structures + // _deribit_conn = _deribit_conn = std::make_shared( - symbol_config_list, trading_config, logging_configuration, deribit_url, - this); + conf->symbol_config_list, conf->trading_config, + conf->logging_configuration, conf->deribit_url, this); + // Creating a trading session interface to send order entry messages + // (removing this line causes connection drops) _trading_session_interface = std::make_shared( _deribit_conn->create_new_trading_session()); @@ -256,7 +165,7 @@ void *DeribitDemoListener::on_instrument_definition( UNUSED_PARAM const std::string &instrument_name, UNUSED_PARAM emumba::deribit_connector::instrument_data_interface *interface) { - SPDLOG_DEBUG( + SPDLOG_INFO( "Market Data API callback | Instrument Definition | Instrument Name: {}", instrument_name); _instrument_data_interface_list[instrument_name] = interface; @@ -273,6 +182,8 @@ void DeribitDemoListener::on_book_state_change( const void *ptr) { UserInstrumentData *instrument_data = (UserInstrumentData *)ptr; UNUSED(instrument_data); + if (_n_other_updates++ % 1000 != 0) + return; SPDLOG_DEBUG( "Market Data API callback | Book State Change | Instrument Name: {}", instrument_data->get_instrument_name()); @@ -306,19 +217,20 @@ void DeribitDemoListener::on_book_state_change( // "name:{},iv:{},liquidation:{},mark_price:{},price:{" // "},tick_direction:{},timestamp:{},trade_id:{},trade_seq:{}", // _message["amount"].GetDouble(), block_trade_id, -// _message["direction"].GetString(), _message["index_price"].GetDouble(), +// _message["direction"].GetString(), +// _message["index_price"].GetDouble(), // _message["instrument_name"].GetString(), iv, liquidation, // _message["mark_price"].GetDouble(), _message["price"].GetDouble(), -// _message["tick_direction"].GetInt(), _message["timestamp"].GetUint64(), -// _message["trade_id"].GetString(), _message["trade_seq"].GetUint64()); +// _message["tick_direction"].GetInt(), +// _message["timestamp"].GetUint64(), _message["trade_id"].GetString(), +// _message["trade_seq"].GetUint64()); // UNUSED(iv); // UserInstrumentData *instrument_data = (UserInstrumentData *)ptr; // UNUSED(instrument_data); // SPDLOG_DEBUG( -// "Market Data API callback | Subscription | Public Trades | Instrument " -// "Name: {}", -// instrument_data->get_instrument_name()); +// "Market Data API callback | Subscription | Public Trades | Instrument +// " "Name: {}", instrument_data->get_instrument_name()); // } void DeribitDemoListener::on_subscription_public_markprice( @@ -404,13 +316,13 @@ void DeribitDemoListener::on_response_get_instruments( void DeribitDemoListener::on_securities( const std::vector &complete_security_list) { - SPDLOG_DEBUG( + SPDLOG_INFO( "On securities callback | total securities: {}", complete_security_list.size()); for (int i = 0; i < (int)complete_security_list.size(); i++) { emumba::market_data::security const *sp = complete_security_list[i]; std::string desc = extract_security_desc(*sp); - SPDLOG_DEBUG("BTS::DeribitDemoListener::on_securities sec:{}", desc); + SPDLOG_INFO("BTS::DeribitDemoListener::on_securities sec:{}", desc); if (complete_security_list[i]->sec_common.sec_type == emumba::market_data::security_type::future) { auto sec_ = std::get<0>(complete_security_list[i]->sec); @@ -504,6 +416,8 @@ void DeribitDemoListener::on_security_statistics( UNUSED_PARAM const void *ptr) { UserInstrumentData *instrument_data = (UserInstrumentData *)ptr; UNUSED(instrument_data); + if (_n_other_updates++ % 1000 != 0) + return; SPDLOG_DEBUG( "Market Data API callback | Security Statistics | SecurityID(using " "ptr): {}, SecurityID: {}, Low: {}, High: {}, Open: " @@ -518,21 +432,32 @@ void DeribitDemoListener::on_security_statistics( void DeribitDemoListener::on_book( UNUSED_PARAM const emumba::market_data::order_book &book, UNUSED_PARAM const void *ptr) { - SPDLOG_DEBUG( - "Market Data API callback | Book Update | SecurityID: {}", - std::get(book.sec_id.value())); - + const auto &id = std::get(book.sec_id.value()); + _book_updates[id] = _book_updates[id] + 1; + if (_n_bk_updates++ % 1000 != 0) + return; + SPDLOG_INFO( + "Market Data API callback | Book Update | bk_update_no: {} " + "sec_bk_update_no: {} SecurityID: " + "{} n_secs: {}", + _n_bk_updates, _book_updates[id], id, _book_updates.size()); + if (_n_bk_updates % 10000 == 0) { + for (const auto &kv : _book_updates) { + SPDLOG_INFO("SecBooks sec: {} n_updates: {}", kv.first, kv.second); + } + } for (int i = 0; i < (int)book.asks.level.size(); i++) { SPDLOG_TRACE( - "Index [{}]: Bid : quantity {}, price {}, Ask : quantity {}, price " + "Index [{}]: Bid : quantity {}, price {}, Ask : quantity {}, " + "price " "{}", i, book.bids.level[i].direct_qty, book.bids.level[i].px.to_double(), book.asks.level[i].direct_qty, book.asks.level[i].px.to_double()); } /* - // Following is an examples of using trading session interface to place a - new order emumba::order_entry::new_order my_order; + // Following is an examples of using trading session interface to place + a new order emumba::order_entry::new_order my_order; my_order.order_def.sec_id.value() = "BTC-PERPETUAL"; my_order.order_def.order_side = emumba::market_data::side::buy; @@ -543,15 +468,16 @@ void DeribitDemoListener::on_book( emumba::order_entry::deribit_order_def_optional deribit_def_optionals; deribit_def.ord_type = - emumba::order_entry::order_type::limit_order; deribit_def_optionals.mmp = - true; deribit_def_optionals.post_only = true; + emumba::order_entry::order_type::limit_order; deribit_def_optionals.mmp + = true; deribit_def_optionals.post_only = true; deribit_def_optionals.reject_post_only = true; deribit_def_optionals.reduce_only = true; deribit_def_optionals.trigger = emumba::order_entry::order_trigger::mark_price; deribit_def_optionals.advanced = - emumba::order_entry::order_advanced::usd; deribit_def_optionals.max_show - = 10.5; deribit_def_optionals.trigger_price = + emumba::order_entry::order_advanced::usd; + deribit_def_optionals.max_show = 10.5; + deribit_def_optionals.trigger_price = emumba::market_data::price((double)15.5); deribit_def.optional_fields = deribit_def_optionals; @@ -563,13 +489,13 @@ void DeribitDemoListener::on_book( emumba::order_entry::order_key order_key = _trading_session_interface->launch_new_order(my_order); - // Following is an examples of using trading session interface to edit an - order emumba::order_entry::order_update my_update; my_update.ord_key = - order_key; my_update.px = 555.5; my_update.qty.value() = + // Following is an examples of using trading session interface to edit + an order emumba::order_entry::order_update my_update; my_update.ord_key + = order_key; my_update.px = 555.5; my_update.qty.value() = (double)666; _trading_session_interface->update_order(my_update); - // Following is an examples of using trading session interface to cancel - an order _trading_session_interface->cancel_order(order_key); + // Following is an examples of using trading session interface to + cancel an order _trading_session_interface->cancel_order(order_key); */ } @@ -579,6 +505,8 @@ void DeribitDemoListener::on_trades( UNUSED_PARAM const void *ptr) { UserInstrumentData *instrument_data = (UserInstrumentData *)ptr; UNUSED(instrument_data); + if (_n_other_updates++ % 1000 != 0) + return; SPDLOG_DEBUG("On trades callback | total trade records: {}", trades.size()); for (int i = 0; i < (int)trades.size(); i++) { auto trade_ts = trades[i]->timestamp; @@ -601,8 +529,11 @@ void DeribitDemoListener::on_quote( { UserInstrumentData *instrument_data = (UserInstrumentData *)ptr; UNUSED(instrument_data); + if (_n_other_updates++ % 1000 != 0) + return; SPDLOG_DEBUG( - "Market Data API callback | Quote Update | SecurityID(using ptr): {}, " + "Market Data API callback | Quote Update | SecurityID(using ptr): " + "{}, " "SecurityID: {}, Best Bid Price: " "{}, Best Bid Amount: {}, Best Ask Price: {}, Best Ask Amount: {}, " "Timestamp: {}", @@ -615,7 +546,7 @@ void DeribitDemoListener::on_quote( void DeribitDemoListener::on_error(__attribute__(( unused)) const emumba::market_data::market_data_error &error) { - SPDLOG_WARN("On error callback | error: {}", error.err_type); + SPDLOG_ERROR("On error callback | error: {}", error.err_type); } } // namespace bts::cmds diff --git a/src/server/sparkd/cmds/crypto/DeribitDemoListener.h b/src/server/sparkd/cmds/crypto/DeribitDemoListener.h index 15ca8a8b0..899d684b7 100644 --- a/src/server/sparkd/cmds/crypto/DeribitDemoListener.h +++ b/src/server/sparkd/cmds/crypto/DeribitDemoListener.h @@ -1,6 +1,7 @@ #pragma once #include "UserInstrumentData.h" #include "rapidjson/document.h" +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include using namespace emumba::deribit_connector; @@ -123,6 +125,9 @@ class DeribitDemoListener : public emumba::deribit_connector::deribit_listener { virtual ~DeribitDemoListener() {} private: + std::unordered_map _book_updates = {}; + int _n_bk_updates = 0; + int _n_other_updates = 0; std::list _instruments; // List of user defined class objects std::map @@ -135,10 +140,11 @@ class DeribitDemoListener : public emumba::deribit_connector::deribit_listener { _deribit_conn; // Shared pointer of deribit connector. This pointer can // be used to configure/run the connector. std::shared_ptr - _trading_session_interface; // Shared pointer of trading session - // interface. This pointer can be use to send - // order entry messages. - rapidjson::Document _message; // RapidJson Object used to parse the JSON + _trading_session_interface; + // // Shared pointer of trading session + // interface. This pointer can be use to send + // order entry messages. + rapidjson::Document _message; // RapidJson Object used to parse the JSON // messages received in some of the callbacks. }; diff --git a/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp b/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp index 4017052de..bb7332ea3 100644 --- a/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmMDListener.cpp @@ -627,6 +627,7 @@ bool is_exp(const ExpSecurity &sec, uint32_t yy, uint32_t mm, uint32_t dd) { } void EmMDListener::_subsribe() { + return; if (me->handler == nullptr) { return; } diff --git a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp index 7d20208b6..547411efa 100644 --- a/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp +++ b/src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp @@ -32,7 +32,10 @@ struct EmumbaMain::PImpl { SecurityIDRecorder recorder; MDProducer producer; EmMDListener listener; - std::unique_ptr real_handler; + std::shared_ptr real_handler; + std::shared_ptr + trading_session_interface; + PImpl() : sec_index(), recorder( {utils::Dispatcher::current(), @@ -57,10 +60,13 @@ struct EmumbaMain::PImpl { // end log bts_assert(conf.has_value()); real_handler = - std::make_unique( + std::make_shared( conf.value().symbol_config_list, conf.value().trading_config, conf.value().logging_configuration, conf.value().deribit_url, &listener); + trading_session_interface = std::make_shared< + emumba::deribit_connector::trading_session_interface>( + real_handler->create_new_trading_session()); listener.set_handler(real_handler.get()); } };