diff --git a/src/lib/io/websocket/websocket_client.cpp b/src/lib/io/websocket/websocket_client.cpp index 64581b2e4..c97051b5e 100644 --- a/src/lib/io/websocket/websocket_client.cpp +++ b/src/lib/io/websocket/websocket_client.cpp @@ -11,7 +11,10 @@ #include "lib/utils/bts_assert.h" #include "lib/utils/formatters/exception_formatter.h" +#include #include +#include +#include #include #include #include @@ -19,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +38,7 @@ namespace bts::websocket { struct WebsocketClientSnapshot { int client_id; WebsocketClientState state; + int n_reconnects; int version; }; @@ -42,9 +47,10 @@ struct WebsocketClientSnapshot { FMT_IMPL( bts::websocket::WebsocketClientSnapshot, t, - "id: {}; state: {}; version: {}", + "id: {}; state: {}; n_reconnects: {}; version: {}", t.client_id, t.state, + t.n_reconnects, t.version); namespace bts::websocket { @@ -62,6 +68,8 @@ using std::size_t; using std::string; using std::thread; using std::uint32_t; +using std::chrono::nanoseconds; +using std::this_thread::sleep_for; using websocketpp::config::asio_tls_client; using WebsocketppClient = @@ -86,9 +94,16 @@ using State = WebsocketClientState; #define ERROR(...) LOG(Http, Error, __VA_ARGS__) #define ERROR_FN(...) LOG(Http, Error, "WebsocketClient::" __VA_ARGS__) +static bts::core::TimeSpan reconnect_interval(std::uint32_t n_attempts) { + std::int64_t secs = 5 * n_attempts; + secs = std::clamp(secs, 5L, 300L); + return bts::core::TimeSpan::from_seconds(secs); +} + struct WebsocketClient::PImpl { WebsocketClientConfig config; WebsocketppClient client{}; + boost::asio::io_context *io_context = nullptr; WebsocketppClient::connection_ptr con; optional open_handler; optional message_handler; @@ -100,6 +115,7 @@ struct WebsocketClient::PImpl { mutex instance_mx; State state = State::IDLE; int version = 0; + int n_reconnects = 0; PImpl() { client.set_access_channels(websocketpp::log::alevel::all); @@ -123,10 +139,17 @@ struct WebsocketClient::PImpl { return true; }); client.set_tls_init_handler([&](auto hdl) { return on_tls_init(hdl); }); - client.set_close_handler([this](auto hdl) { on_close(hdl); }); + client.set_close_handler([=, this](auto hdl) { + on_close(hdl); + if (config.reconnect_on_close) { + sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()}); + connect(); + } + }); client.set_fail_handler([this](auto hdl) { const auto ec = client.get_con_from_hdl(hdl)->get_ec(); ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message()); + on_fail(hdl); }); } @@ -149,7 +172,7 @@ struct WebsocketClient::PImpl { } constexpr WebsocketClientSnapshot snapshot() const { - return {client_id, state, version}; + return {client_id, state, n_reconnects, version}; } void close(websocketpp::close::status::value code, std::string reason) { @@ -193,8 +216,7 @@ struct WebsocketClient::PImpl { /** * Connects to the websocket server */ - void connect( - const map &headers, bool run_in_background = true) { + void connect() { lock_guard lock{instance_mx}; INFO_FN("connect. {}", snapshot()); switch (state.value) { @@ -213,20 +235,11 @@ struct WebsocketClient::PImpl { return; } // Add headers - for (const auto &[k, v] : headers) { + for (const auto &[k, v] : config.headers) { con->append_header(k, v); } // Connect client.connect(con); - if (run_in_background) { - bts_assert_msgf( - !io_thread, "Websocket io_thread already instantiated. {}", - snapshot()); - io_thread = std::thread{[this]() { run(); }}; - if (config.connect_timeout > core::TimeSpan::zero()) { - wait_for_connect(config.connect_timeout); - } - } } catch (const std::exception &ex) { std::string str = ex.what(); ERROR_FN( @@ -238,22 +251,11 @@ struct WebsocketClient::PImpl { } } - void wait_for_connect(const TimeSpan timeout) { - INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout); - const auto start = TimeStamp::now(); - while (state == State::OPENING) { - const auto elapsed = TimeStamp::now() - start; - const auto rem = timeout - elapsed; - if (rem < TimeSpan::zero()) { - throw std::runtime_error{fmt::format( - "Timed out waiting for connection after {} secs. {}", - elapsed.total_seconds(), snapshot())}; - } - WARN_FN( - "wait_for_connect. Still waiting. {}; timeout: {}; rem: {}", - snapshot(), timeout, rem); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } + void reconnect() { + WARN_FN("reconnect. {}", snapshot()); + sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()}); + ++n_reconnects; + connect(); } void run() { @@ -284,10 +286,33 @@ struct WebsocketClient::PImpl { INFO("Websocket 'run' thread end. {}", snapshot()); } + void run_one() { + try { + client.run_one(); + } catch (const websocketpp::exception &ex) { + ERROR_FN( + "io_thread (run_one). Websocket io threw websocketpp::exception. " + "{}; err: {}", + snapshot(), ex); + } catch (const std::exception &ex) { + ERROR_FN( + "io_thread (run_one). Websocket io threw std::exception. {}; err: " + "{}", + snapshot(), ex); + } catch (...) { + bts_assert_msgf( + false, + "io_thread (run_one). Websocket io threw unknown exception. Could " + "this have been thrown by a callback delegate? {}", + snapshot()); + } + } + private: static std::atomic client_id_counter; - auto on_tls_init(ConnectionHdl /*hdl*/) { + std::shared_ptr on_tls_init( + ConnectionHdl /*hdl*/) { INFO_FN("on_tls_init. state: {}; version: {}", state, version); auto result = websocketpp::lib::make_shared(SslContext::sslv23_client); @@ -352,11 +377,37 @@ struct WebsocketClient::PImpl { if (close_handler) { (*close_handler)(ec); } + if (config.reconnect_on_close) { + reconnect(); + } + } + + void on_fail(ConnectionHdl hdl) { + lock_guard lock{instance_mx}; + ERROR_FN("on_fail. {}", snapshot()); + set_state(State::IDLE); + const auto con = client.get_con_from_hdl(hdl); + const auto ec = con->get_ec(); + if (ec) { + ERROR_FN( + "on_fail. {}; local_code: {}; " + "local_reason: {}; remote_code: {}; remote_reason: {}; err: {}", + snapshot(), con->get_local_close_code(), + con->get_local_close_reason(), con->get_remote_close_code(), + con->get_remote_close_reason(), ec.message()); + } + if (config.reconnect_on_fail) { + reconnect(); + } } }; -WebsocketClient::WebsocketClient(WebsocketClientConfig config) - : me_(std::make_unique(std::move(config))) {} +WebsocketClient::WebsocketClient( + WebsocketClientConfig config, boost::asio::io_context *io_context) + : me_(std::make_unique()) { + me_->config = std::move(config); + me_->io_context = io_context; +} WebsocketClient::~WebsocketClient() { me_->close(websocketpp::close::status::going_away, ""); @@ -385,13 +436,16 @@ WebsocketClient::CloseHandler &WebsocketClient::close_handler() { return *me_->close_handler; } -void WebsocketClient::connect( - const std::map &headers, bool run_in_background) { - me_->connect(headers, run_in_background); -} +void WebsocketClient::connect() { me_->connect(); } void WebsocketClient::run() { me_->run(); } +void WebsocketClient::run_one() { me_->run_one(); } + +boost::asio::io_context *WebsocketClient::io_context() { + return me_->io_context; +} + websocketpp::lib::error_code WebsocketClient::send_text(const string &text) { return me_->send_text(text); } @@ -409,4 +463,6 @@ bool WebsocketClient::is_connected() const { void WebsocketClient::ping(const string &payload) { me_->con->ping(payload); } +const string &WebsocketClient::url() const { return me_->config.url; } + } // namespace bts::websocket diff --git a/src/lib/io/websocket/websocket_client.h b/src/lib/io/websocket/websocket_client.h index 629f3df86..4bb793776 100644 --- a/src/lib/io/websocket/websocket_client.h +++ b/src/lib/io/websocket/websocket_client.h @@ -3,6 +3,7 @@ #include "lib/io/websocket/websocket_client_config.h" #include "lib/utils/Delegate.h" +#include #include #include #include @@ -12,28 +13,33 @@ namespace bts::websocket { class WebsocketClient { public: - explicit WebsocketClient(WebsocketClientConfig config); + WebsocketClient( + WebsocketClientConfig config, + boost::asio::io_context *io_context = nullptr); ~WebsocketClient(); using OpenHandler = bts::core::Delegate; using MessageHandler = bts::core::Delegate; using CloseHandler = bts::core::Delegate; - void connect( - const std::map &headers = {}, - bool run_in_background = true); + void connect(); void disconnect(); bool is_connected() const; + boost::asio::io_context *io_context(); void run(); + void run_one(); CloseHandler &open_handler(); MessageHandler &message_handler(); CloseHandler &close_handler(); websocketpp::lib::error_code send_text(const std::string &text); void ping(const std::string &payload = ""); + const std::string &url() const; + private: struct PImpl; std::unique_ptr me_; }; } // namespace bts::websocket + diff --git a/src/lib/io/websocket/websocket_client_config.h b/src/lib/io/websocket/websocket_client_config.h index d78bf2857..2d77ab35a 100644 --- a/src/lib/io/websocket/websocket_client_config.h +++ b/src/lib/io/websocket/websocket_client_config.h @@ -5,9 +5,12 @@ namespace bts::websocket { struct WebsocketClientConfig { - std::string url; - core::TimeSpan connect_timeout; - core::TimeSpan pong_timeout; + std::string url{}; + core::TimeSpan connect_timeout = core::TimeSpan::zero(); + core::TimeSpan pong_timeout = core::TimeSpan::zero(); + bool reconnect_on_close = false; + bool reconnect_on_fail = false; + std::map headers{}; }; } // namespace bts::websocket diff --git a/src/lib/orderman/gateways/OkxExecutionGateway.cpp b/src/lib/orderman/gateways/OkxExecutionGateway.cpp index 453ea078e..3f36dceee 100644 --- a/src/lib/orderman/gateways/OkxExecutionGateway.cpp +++ b/src/lib/orderman/gateways/OkxExecutionGateway.cpp @@ -289,8 +289,33 @@ struct OkxExecutionGateway::PImpl { LOG(OrderMan, Warn, "OkxExecutionAdapter::start Attempting login (Attempt: {})", attempt + 1); - ws_client.connect({}, true); - sleep(2); + if (ws_run_thread) { + // Join this thread to block until the websocket disconnects (then reconnect) + if (ws_run_thread->joinable()) { + ws_run_thread->join(); + } + ws_run_thread.reset(); + } + // Connect + ws_client.connect(); + ws_run_thread = threading::make_thread( + "okx_execution_ws", [&]() { ws_client.run(); }); + // Block until the websocket connects (or times out) + { + const auto start_time = bts::core::TimeStamp::now(); + for (auto elapsed = core::TimeSpan::zero(); + !ws_client.is_connected() && + elapsed < core::TimeSpan::from_seconds(20); + elapsed = core::TimeStamp::now() - start_time) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + if (!ws_client.is_connected()) { + LOG(OrderMan, Warn, + "OkxExecutionAdapter::start. Timed out waiting for " + "websocket to connect"); + continue; + } + } if (ws_ping_thread.has_value()) { if (ws_ping_thread->joinable()) { ws_ping_thread diff --git a/src/lib_binance_execution/binance_execution/usds_futures/user_data_streams/user_data_subscription.cpp b/src/lib_binance_execution/binance_execution/usds_futures/user_data_streams/user_data_subscription.cpp index 4551b5057..6ccaf6ed8 100644 --- a/src/lib_binance_execution/binance_execution/usds_futures/user_data_streams/user_data_subscription.cpp +++ b/src/lib_binance_execution/binance_execution/usds_futures/user_data_streams/user_data_subscription.cpp @@ -58,7 +58,7 @@ struct UserDataSubscription::PImpl { ws_cli.message_handler().bind<&PImpl::on_message>(*this); ws_cli.close_handler().bind<&PImpl::on_close>(*this); INFO_FN("start", "Creating websocket client. url: {}", url); - ws_cli.connect({}, false); + ws_cli.connect(); ws_run_thread.emplace([&]() { INFO_FN("ws_run_thread", "thread starting. url: {}", url); ws_cli.run();