Add reconnect behavior to WebsocketClient

This commit is contained in:
John Colvin committed 2025-03-05 16:00:43 +00:00
1 parent 2c8b3563f1
commit 6ef9bee8e4
5 files changed
+138 -48

No files matched your search

+94 -38
View File
@@ -11,7 +11,10 @@
#include "lib/utils/bts_assert.h"
#include "lib/utils/formatters/exception_formatter.h"
#include <algorithm>
#include <atomic>
#include <boost/asio/io_context.hpp>
#include <chrono>
#include <exception>
#include <fmt/format.h>
#include <mutex>
@@ -19,6 +22,7 @@
#include <optional>
#include <stdexcept>
#include <string>
#include <system_error>
#include <thread>
#include <websocketpp/close.hpp>
#include <websocketpp/common/connection_hdl.hpp>
@@ -34,6 +38,7 @@ namespace bts::websocket {
struct WebsocketClientSnapshot {
int client_id;
WebsocketClientState state;
int n_reconnects;
int version;
};
@@ -42,9 +47,10 @@ struct WebsocketClientSnapshot {
FMT_IMPL(
bts::websocket::WebsocketClientSnapshot,
t,
"id: {}; state: {}; version: {}",
"id: {}; state: {}; n_reconnects: {}; version: {}",
t.client_id,
t.state,
t.n_reconnects,
t.version);
namespace bts::websocket {
@@ -62,6 +68,8 @@ using std::size_t;
using std::string;
using std::thread;
using std::uint32_t;
using std::chrono::nanoseconds;
using std::this_thread::sleep_for;
using websocketpp::config::asio_tls_client;
using WebsocketppClient =
@@ -86,9 +94,16 @@ using State = WebsocketClientState;
#define ERROR(...) LOG(Http, Error, __VA_ARGS__)
#define ERROR_FN(...) LOG(Http, Error, "WebsocketClient::" __VA_ARGS__)
static bts::core::TimeSpan reconnect_interval(std::uint32_t n_attempts) {
std::int64_t secs = 5 * n_attempts;
secs = std::clamp(secs, 5L, 300L);
return bts::core::TimeSpan::from_seconds(secs);
}
struct WebsocketClient::PImpl {
WebsocketClientConfig config;
WebsocketppClient client{};
boost::asio::io_context *io_context = nullptr;
WebsocketppClient::connection_ptr con;
optional<OpenHandler> open_handler;
optional<MessageHandler> message_handler;
@@ -100,6 +115,7 @@ struct WebsocketClient::PImpl {
mutex instance_mx;
State state = State::IDLE;
int version = 0;
int n_reconnects = 0;
PImpl() {
client.set_access_channels(websocketpp::log::alevel::all);
@@ -123,10 +139,17 @@ struct WebsocketClient::PImpl {
return true;
});
client.set_tls_init_handler([&](auto hdl) { return on_tls_init(hdl); });
client.set_close_handler([this](auto hdl) { on_close(hdl); });
client.set_close_handler([=, this](auto hdl) {
on_close(hdl);
if (config.reconnect_on_close) {
sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
connect();
}
});
client.set_fail_handler([this](auto hdl) {
const auto ec = client.get_con_from_hdl(hdl)->get_ec();
ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
on_fail(hdl);
});
}
@@ -149,7 +172,7 @@ struct WebsocketClient::PImpl {
}
constexpr WebsocketClientSnapshot snapshot() const {
return {client_id, state, version};
return {client_id, state, n_reconnects, version};
}
void close(websocketpp::close::status::value code, std::string reason) {
@@ -193,8 +216,7 @@ struct WebsocketClient::PImpl {
/**
* Connects to the websocket server
*/
void connect(
const map<string, string> &headers, bool run_in_background = true) {
void connect() {
lock_guard<mutex> lock{instance_mx};
INFO_FN("connect. {}", snapshot());
switch (state.value) {
@@ -213,20 +235,11 @@ struct WebsocketClient::PImpl {
return;
}
// Add headers
for (const auto &[k, v] : headers) {
for (const auto &[k, v] : config.headers) {
con->append_header(k, v);
}
// Connect
client.connect(con);
if (run_in_background) {
bts_assert_msgf(
!io_thread, "Websocket io_thread already instantiated. {}",
snapshot());
io_thread = std::thread{[this]() { run(); }};
if (config.connect_timeout > core::TimeSpan::zero()) {
wait_for_connect(config.connect_timeout);
}
}
} catch (const std::exception &ex) {
std::string str = ex.what();
ERROR_FN(
@@ -238,22 +251,11 @@ struct WebsocketClient::PImpl {
}
}
void wait_for_connect(const TimeSpan timeout) {
INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout);
const auto start = TimeStamp::now();
while (state == State::OPENING) {
const auto elapsed = TimeStamp::now() - start;
const auto rem = timeout - elapsed;
if (rem < TimeSpan::zero()) {
throw std::runtime_error{fmt::format(
"Timed out waiting for connection after {} secs. {}",
elapsed.total_seconds(), snapshot())};
}
WARN_FN(
"wait_for_connect. Still waiting. {}; timeout: {}; rem: {}",
snapshot(), timeout, rem);
std::this_thread::sleep_for(std::chrono::seconds(1));
}
void reconnect() {
WARN_FN("reconnect. {}", snapshot());
sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
++n_reconnects;
connect();
}
void run() {
@@ -284,10 +286,33 @@ struct WebsocketClient::PImpl {
INFO("Websocket 'run' thread end. {}", snapshot());
}
void run_one() {
try {
client.run_one();
} catch (const websocketpp::exception &ex) {
ERROR_FN(
"io_thread (run_one). Websocket io threw websocketpp::exception. "
"{}; err: {}",
snapshot(), ex);
} catch (const std::exception &ex) {
ERROR_FN(
"io_thread (run_one). Websocket io threw std::exception. {}; err: "
"{}",
snapshot(), ex);
} catch (...) {
bts_assert_msgf(
false,
"io_thread (run_one). Websocket io threw unknown exception. Could "
"this have been thrown by a callback delegate? {}",
snapshot());
}
}
private:
static std::atomic<int> client_id_counter;
auto on_tls_init(ConnectionHdl /*hdl*/) {
std::shared_ptr<boost::asio::ssl::context> on_tls_init(
ConnectionHdl /*hdl*/) {
INFO_FN("on_tls_init. state: {}; version: {}", state, version);
auto result =
websocketpp::lib::make_shared<SslContext>(SslContext::sslv23_client);
@@ -352,11 +377,37 @@ struct WebsocketClient::PImpl {
if (close_handler) {
(*close_handler)(ec);
}
if (config.reconnect_on_close) {
reconnect();
}
}
void on_fail(ConnectionHdl hdl) {
lock_guard<mutex> lock{instance_mx};
ERROR_FN("on_fail. {}", snapshot());
set_state(State::IDLE);
const auto con = client.get_con_from_hdl(hdl);
const auto ec = con->get_ec();
if (ec) {
ERROR_FN(
"on_fail. {}; local_code: {}; "
"local_reason: {}; remote_code: {}; remote_reason: {}; err: {}",
snapshot(), con->get_local_close_code(),
con->get_local_close_reason(), con->get_remote_close_code(),
con->get_remote_close_reason(), ec.message());
}
if (config.reconnect_on_fail) {
reconnect();
}
}
};
WebsocketClient::WebsocketClient(WebsocketClientConfig config)
: me_(std::make_unique<PImpl>(std::move(config))) {}
WebsocketClient::WebsocketClient(
WebsocketClientConfig config, boost::asio::io_context *io_context)
: me_(std::make_unique<PImpl>()) {
me_->config = std::move(config);
me_->io_context = io_context;
}
WebsocketClient::~WebsocketClient() {
me_->close(websocketpp::close::status::going_away, "");
@@ -385,13 +436,16 @@ WebsocketClient::CloseHandler &WebsocketClient::close_handler() {
return *me_->close_handler;
}
void WebsocketClient::connect(
const std::map<std::string, std::string> &headers, bool run_in_background) {
me_->connect(headers, run_in_background);
}
void WebsocketClient::connect() { me_->connect(); }
void WebsocketClient::run() { me_->run(); }
void WebsocketClient::run_one() { me_->run_one(); }
boost::asio::io_context *WebsocketClient::io_context() {
return me_->io_context;
}
websocketpp::lib::error_code WebsocketClient::send_text(const string &text) {
return me_->send_text(text);
}
@@ -409,4 +463,6 @@ bool WebsocketClient::is_connected() const {
void WebsocketClient::ping(const string &payload) { me_->con->ping(payload); }
const string &WebsocketClient::url() const { return me_->config.url; }
} // namespace bts::websocket
+10 -4
View File
@@ -3,6 +3,7 @@
#include "lib/io/websocket/websocket_client_config.h"
#include "lib/utils/Delegate.h"
#include <boost/asio/io_context.hpp>
#include <memory>
#include <string>
#include <websocketpp/config/core_client.hpp>
@@ -12,28 +13,33 @@ namespace bts::websocket {
class WebsocketClient {
public:
explicit WebsocketClient(WebsocketClientConfig config);
WebsocketClient(
WebsocketClientConfig config,
boost::asio::io_context *io_context = nullptr);
~WebsocketClient();
using OpenHandler = bts::core::Delegate<void(const std::error_code &)>;
using MessageHandler = bts::core::Delegate<void(const std::string &)>;
using CloseHandler = bts::core::Delegate<void(const std::error_code &)>;
void connect(
const std::map<std::string, std::string> &headers = {},
bool run_in_background = true);
void connect();
void disconnect();
bool is_connected() const;
boost::asio::io_context *io_context();
void run();
void run_one();
CloseHandler &open_handler();
MessageHandler &message_handler();
CloseHandler &close_handler();
websocketpp::lib::error_code send_text(const std::string &text);
void ping(const std::string &payload = "");
const std::string &url() const;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::websocket
@@ -5,9 +5,12 @@
namespace bts::websocket {
struct WebsocketClientConfig {
std::string url;
core::TimeSpan connect_timeout;
core::TimeSpan pong_timeout;
std::string url{};
core::TimeSpan connect_timeout = core::TimeSpan::zero();
core::TimeSpan pong_timeout = core::TimeSpan::zero();
bool reconnect_on_close = false;
bool reconnect_on_fail = false;
std::map<std::string, std::string> headers{};
};
} // namespace bts::websocket
@@ -289,8 +289,33 @@ struct OkxExecutionGateway::PImpl {
LOG(OrderMan, Warn,
"OkxExecutionAdapter::start Attempting login (Attempt: {})",
attempt + 1);
ws_client.connect({}, true);
sleep(2);
if (ws_run_thread) {
// Join this thread to block until the websocket disconnects (then reconnect)
if (ws_run_thread->joinable()) {
ws_run_thread->join();
}
ws_run_thread.reset();
}
// Connect
ws_client.connect();
ws_run_thread = threading::make_thread(
"okx_execution_ws", [&]() { ws_client.run(); });
// Block until the websocket connects (or times out)
{
const auto start_time = bts::core::TimeStamp::now();
for (auto elapsed = core::TimeSpan::zero();
!ws_client.is_connected() &&
elapsed < core::TimeSpan::from_seconds(20);
elapsed = core::TimeStamp::now() - start_time) {
std::this_thread::sleep_for(std::chrono::milliseconds(500));
}
if (!ws_client.is_connected()) {
LOG(OrderMan, Warn,
"OkxExecutionAdapter::start. Timed out waiting for "
"websocket to connect");
continue;
}
}
if (ws_ping_thread.has_value()) {
if (ws_ping_thread->joinable()) {
ws_ping_thread
@@ -58,7 +58,7 @@ struct UserDataSubscription::PImpl {
ws_cli.message_handler().bind<&PImpl::on_message>(*this);
ws_cli.close_handler().bind<&PImpl::on_close>(*this);
INFO_FN("start", "Creating websocket client. url: {}", url);
ws_cli.connect({}, false);
ws_cli.connect();
ws_run_thread.emplace([&]() {
INFO_FN("ws_run_thread", "thread starting. url: {}", url);
ws_cli.run();