feat: Automatically reconnect in bybit exe gw plugin

Previously, the plugin would just throw an exception and assert crashed
when this happened (the exception would bubble up to the c++ bridge).
This commit is contained in:
John Colvin committed 2025-08-13 21:39:21 +00:00
1 parent e2fdb569c6
commit fc94ec4d46
5 files changed
+409 -53

No files matched your search

@@ -1,8 +1,12 @@
import datetime
import enum
import json
import time
from typing import Optional
from typing import Callable, Optional
import uuid
from pybit_unified_trading import WebSocket, WebSocketTrading, HTTP
from pybit_websocket_stream import WebSocketError
from spark_core import (
JsonElement,
as_bool,
@@ -30,6 +34,15 @@ from spark_enums import OrderStatus, ExecType, OrderType, SecurityType, TimeInFo
import threading
class BybitGatewayStatus(enum.Enum):
IDLE = 0
CONNECTING = 1
SUBSCRIBING = 2
READY = 3
CANCELING_STALE_ORDERS = 4
SYNCING_POSITIONS = 5
class OrderData:
def __init__(
self,
@@ -169,12 +182,31 @@ class BybitExecutionGateway(ExecutionGateway):
self.use_testnet: bool = cfg.get("use_testnet", True)
self.use_trace_logging: bool = cfg.get("use_trace_logging", False)
self.stop_event = threading.Event()
self.stream_cli_disconnect_time: float = 0
self.trade_cli_disconnect_time: float = 0
self.stream_cli: Optional[WebSocket] = None
self.trade_cli: Optional[WebSocketTrading] = None
self.trade_http_cli: Optional[HTTP] = None
self.trade_cli_connected: bool = False
self.stream_cli_connected: bool = False
self.stream_subscriptions: set[str] = set()
# bts id -> exch id
self.orders: dict[int, OrderData] = {}
self.securities: dict[str, PySecurity] = {}
# set of exch ids for orders that being cancelled due to disconnection. We propagate them to
# sparkd with a special flag. Some algos may keep working through cancels due to disconnect
# (instead of the usual 'Cancel on unexpected child cancel' behavior).
self.pending_reconnect_cxls: set[str] = set()
self.status: BybitGatewayStatus = BybitGatewayStatus.IDLE
self.lock = threading.RLock()
@property
def is_connected(self):
return self.trade_cli_connected and self.stream_cli_connected
@property
def is_subscribed(self):
return "order" in self.stream_subscriptions
def _get_order_data(self, exch_id: str):
for _, v in self.orders.items():
@@ -241,6 +273,223 @@ class BybitExecutionGateway(ExecutionGateway):
% (key, req_type, rsn, time.time_ns())
)
def compute_internal_state(
self, positions_synced: bool = False, memo: Optional[str] = None
):
"""
Parameters
----------
positions_sync: bool
True indicates our positions are synchronized with the exchange. It is only
relevant if the current status is SYNCING_POSITIONS, in which case a value
of True signals our new status is READY (barring an unexpected disconnect).
memo: Optional[str]
A brief memo to include in the log message (if the status changes)
"""
# Note: this is NOT a pure function (mutates self.status).
# Computes status as a function of
# 1) the combined stream/trade cli connection status,
# 2) the presence of pending_reconnect_cxls, and
# 3) the current status.
# The general path is
# IDLE -> CONNECTING -> SUBSCRIBING -> CANCELING_STALE_ORDERS -> SYNCING_POSITIONS -> READY
old_status = self.status
new_status = old_status
match self.status:
case BybitGatewayStatus.IDLE:
if self.is_connected:
# Initial Path
new_status = (
BybitGatewayStatus.SYNCING_POSITIONS
if self.is_subscribed
else BybitGatewayStatus.SUBSCRIBING
)
elif self.stream_cli or self.trade_cli:
new_status = BybitGatewayStatus.CONNECTING
case BybitGatewayStatus.CONNECTING:
if self.is_connected:
new_status = (
BybitGatewayStatus.CANCELING_STALE_ORDERS
if self.is_subscribed
else BybitGatewayStatus.SUBSCRIBING
)
case BybitGatewayStatus.CANCELING_STALE_ORDERS:
if not self.is_connected:
self.pending_reconnect_cxls.clear()
new_status = BybitGatewayStatus.CONNECTING
elif not self.pending_reconnect_cxls:
new_status = BybitGatewayStatus.SYNCING_POSITIONS
case BybitGatewayStatus.SUBSCRIBING:
if self.is_subscribed:
new_status = BybitGatewayStatus.CANCELING_STALE_ORDERS
case BybitGatewayStatus.SYNCING_POSITIONS:
if self.pending_reconnect_cxls:
raise ValueError(
f"Invalid internal status '{self.status.name}' when there are pending_reconnect_cxls"
)
if not self.is_connected:
new_status = BybitGatewayStatus.CONNECTING
elif not self.is_subscribed:
new_status = BybitGatewayStatus.SUBSCRIBING
elif positions_synced:
new_status = BybitGatewayStatus.READY
case BybitGatewayStatus.READY:
if not self.is_connected:
new_status = BybitGatewayStatus.CONNECTING
elif not self.is_subscribed:
new_status = BybitGatewayStatus.SUBSCRIBING
self.status = new_status
if new_status != old_status:
self.logger.info(
"[bybit_gw_status_change.json]%s"
% SLog(prev=old_status.name, curr=new_status.name, memo=memo)
)
return True
return False
def on_stream_open(self, ws):
should_cxl_all = False
with self.lock:
self.logger.info("[bybit_gw_stream_open.json]%s" % SLog())
self.stream_cli_connected = True
self.stream_cli_disconnect_time = 0
should_cxl_all = self.compute_internal_state(memo="stream_open") and (
self.status == BybitGatewayStatus.CANCELING_STALE_ORDERS
)
if should_cxl_all:
if not self.cancel_all():
# If cancel_all returns False, then there are no orders to cancel.
# In that case we recompute our state immediately since there
# won't be a cancel ack to trigger it later.
self.compute_internal_state(memo="cxl_all_noop")
def on_stream_subscription_ack(self, subs: list[str], ok: bool):
self.logger.info("[bybit_gw_sub_ack.json]%s" % SLog(subs=subs, ok=ok))
should_cxl_all = False
with self.lock:
for s in subs:
if ok:
self.stream_subscriptions.add(s)
else:
self.stream_subscriptions.remove(s)
should_cxl_all = self.compute_internal_state(memo="subscription_ack") and (
self.status == BybitGatewayStatus.CANCELING_STALE_ORDERS
)
if should_cxl_all:
if not self.cancel_all():
# If cancel_all returns False, then there are no orders to cancel.
# In that case we recompute our state immediately since there
# won't be a cancel ack to trigger it later.
self.compute_internal_state(memo="cxl_all_noop")
def on_stream_error(self, ws, err: WebSocketError):
self.logger.error("[bybit_gw_stream_err.json]%s" % SLog(err=str(err)))
with self.lock:
self.stream_cli_connected = False
self.stream_cli_disconnect_time = time.time()
self.stream_subscriptions.clear()
self.compute_internal_state(memo="stream_error")
return True
def on_stream_close(self, ws, code: Optional[int], rsn: Optional[str]):
self.logger.error("[bybit_gw_stream_close.json]%s" % SLog(code=code, rsn=rsn))
with self.lock:
self.stream_cli_connected = False
self.stream_cli_disconnect_time = time.time()
self.stream_subscriptions.clear()
self.compute_internal_state(memo="stream_close")
return True
def on_trade_cli_open(self, ws):
self.logger.info("[bybit_gw_trd_cli_open.json]%s" % SLog())
should_cxl_all = False
with self.lock:
self.trade_cli_connected = True
self.trade_cli_disconnect_time = 0
should_cxl_all = self.compute_internal_state(memo="trd_cli_open") and (
self.status == BybitGatewayStatus.CANCELING_STALE_ORDERS
)
if should_cxl_all:
if not self.cancel_all():
# If cancel_all returns False, then there are no orders to cancel.
# In that case we recompute our state immediately since there
# won't be a cancel ack to trigger it later.
self.compute_internal_state(memo="cxl_all_noop")
def on_trade_cli_error(self, ws, err: WebSocketError):
self.logger.error("[bybit_gw_trd_cli_err.json]%s" % SLog(err=str(err)))
with self.lock:
self.trade_cli_connected = False
self.trade_cli_disconnect_time = time.time()
self.compute_internal_state(memo="trd_cli_close")
return True
def on_trade_cli_close(self, ws, code: Optional[int], rsn: Optional[str]):
self.logger.error("[bybit_gw_trd_cli_close.json]%s" % SLog(code=code, rsn=rsn))
with self.lock:
self.trade_cli_connected = False
self.trade_cli_disconnect_time = time.time()
self.compute_internal_state(memo="trd_cli_error")
return True
def check_reconnect_timeout(self):
RECONNECT_TIMEOUT = 10
if self.is_connected:
return
with self.lock:
# Check is_connected again now that we're in the lock
if self.is_connected:
return
disconnect_times = []
if not self.stream_cli_connected and self.stream_cli_disconnect_time != 0:
disconnect_times.append(self.stream_cli_disconnect_time)
if not self.trade_cli_connected and self.trade_cli_disconnect_time != 0:
disconnect_times.append(self.trade_cli_disconnect_time)
disconnect_times.sort()
if disconnect_times:
now = time.time()
reconnect_elapsed = now - disconnect_times[-1]
if reconnect_elapsed > RECONNECT_TIMEOUT:
self.logger.error(
"[bybit_gw_reconnect_timeout.json]%s"
% SLog(
stream_connected=self.stream_cli_connected,
stream_disconnect_ts=self.stream_cli_disconnect_time,
trd_cli_connected=self.trade_cli_connected,
trd_cli_disconnect_ts=self.trade_cli_disconnect_time,
err=f"Timed out waiting for reconnect after {RECONNECT_TIMEOUT}s",
)
)
for handler in self.logger.handlers:
handler.flush()
raise ValueError(
"BybitExecutionGateway timed out waiting for reconnect"
)
def cancel_all(self):
self.logger.info(
"[bybit_gw_cxl_all.json]%s"
% SLog(curr_orders=[o for o in self.orders.values()])
)
base_coins = set()
with self.lock:
for o in self.orders.values():
for sec in self.securities.values():
if sec.exchange_id() == o.exch_sec_id:
base_coins.add(sec.asset())
self.pending_reconnect_cxls.add(o.exch_id)
break
if not self.pending_reconnect_cxls:
return False
for coin in base_coins:
self.logger.info("[bybit_gw_cxl_coin.json]%s" % SLog(base_coin=coin))
res = self.trade_http_cli.cancel_all_orders(
category="linear", baseCoin=coin
)
self.logger.info("[bybit_gw_cxl_coin_res.json]%s" % SLog(res=res))
return True
def get_positions(self):
try:
next_pg = None
@@ -317,6 +566,11 @@ class BybitExecutionGateway(ExecutionGateway):
if not ord_data:
on_error("Did not resolve exch order id %s" % exch_id)
return
is_reconnecting_cxl = False
with self.lock:
is_reconnecting_cxl = exch_id in self.pending_reconnect_cxls
if is_reconnecting_cxl:
self.pending_reconnect_cxls.remove(exch_id)
total_filled_qty = parse_float(d["cumExecQty"])
avg_filled_price = parse_float(d["avgPrice"])
status: str = d["orderStatus"]
@@ -348,11 +602,28 @@ class BybitExecutionGateway(ExecutionGateway):
transact_id=str(transact_time),
transact_time=transact_time,
)
if is_reconnecting_cxl:
if ord_data.status == OrderStatus.CANCELED:
# The "RECONNECTING" error msg is a sentinel (don't change it). When
# an order is canceled upon reconnection, some algos may decide to keep
# working rather than terminate due to an "unexpected child cancel."
er.set_error_msg("RECONNECTING")
else:
self.logger.error(
"[bybit_gw_ord_upd_error.json]%s"
% SLog(
err=f"pending reconnect cancel order status is {ord_data.status.name}"
)
)
with self.lock:
if not self.pending_reconnect_cxls:
self.compute_internal_state()
self.logger.info("[bybit_gw_exe_rep_out.json]%s" % er)
bridge.on_execution_report(er)
except Exception as e:
self.logger.error(
'[bybit_gw_ord_upd_err.json]{ "err": %s }' % e, exc_info=True
'[bybit_gw_ord_upd_err.json]{ "err": %s }' % str(e),
exc_info=True,
)
def on_exe_report(msg):
@@ -376,7 +647,13 @@ class BybitExecutionGateway(ExecutionGateway):
"trace_logging": False,
}
self.logger.info("[bybit_gw_stream_sub.json]%s" % JsonElement(params))
self.stream_cli = WebSocket(**params)
self.stream_cli = WebSocket(
**params,
on_open=self.on_stream_open,
on_error=self.on_stream_error,
on_close=self.on_stream_close,
subscription_ack_callback=self.on_stream_subscription_ack,
)
self.stream_cli.order_stream(on_order_update)
self.stream_cli.execution_stream(on_exe_report)
# Trade client (sends orders)
@@ -387,33 +664,31 @@ class BybitExecutionGateway(ExecutionGateway):
"private_auth_expire": 5,
}
self.logger.info("[bybit_gw_start_trd_cli.json]%s" % JsonElement(params))
self.trade_cli = WebSocketTrading(**params)
self.bridge.on_is_ready(True)
self.trade_cli = WebSocketTrading(
**params,
on_open=self.on_trade_cli_open,
on_error=self.on_trade_cli_error,
on_close=self.on_trade_cli_close,
)
has_been_initialized: bool = False
POS_SNAPSHOT_INTERVAL = 10
next_pos_snapshot_time = time.time() + POS_SNAPSHOT_INTERVAL
# Sleep loop to keep thread alive and detect connection status changes
conn_statuses = {
"stream": self.stream_cli.is_connected(),
"trade": self.trade_cli.is_connected(),
}
while not self.stop_event.is_set():
time.sleep(1)
new_conn_statuses = {
"stream": self.stream_cli.is_connected(),
"trade": self.trade_cli.is_connected(),
}
if conn_statuses != new_conn_statuses:
conn_statuses = new_conn_statuses
all_connected = all(map(lambda kv: kv[1], conn_statuses.items()))
self.logger.log(
logging.INFO if all_connected else logging.WARNING,
"[bybit_gw_connection_status.json]%s" % SLog(**conn_statuses),
)
self.bridge.on_is_ready(all_connected)
now = time.time()
if now >= next_pos_snapshot_time:
self.get_positions()
next_pos_snapshot_time = now + POS_SNAPSHOT_INTERVAL
while not self.stop_event.wait(1):
self.check_reconnect_timeout()
match self.status:
case BybitGatewayStatus.READY:
if not has_been_initialized:
self.logger.info("[bybit_gw_is_ready.json]%s" % SLog())
self.bridge.on_is_ready(True)
has_been_initialized = True
now = time.time()
if now >= next_pos_snapshot_time:
self.get_positions()
next_pos_snapshot_time = now + POS_SNAPSHOT_INTERVAL
case BybitGatewayStatus.SYNCING_POSITIONS:
if self.get_positions():
self.compute_internal_state(positions_synced=True)
def start(self, bridge: ExecutionGatewayBridge):
def start_wrapper():
@@ -451,6 +726,7 @@ class BybitExecutionGateway(ExecutionGateway):
px = req.price()
qty = req.quantity()
sec_name = req.exchange_security_id()
self.logger.info("[bybit_gw_crt_ord_step.json]%s" % SLog(msg="Indexing order"))
self.orders[req.key()] = OrderData(
account=self.account,
bts_id=bts_id,
@@ -463,9 +739,16 @@ class BybitExecutionGateway(ExecutionGateway):
price=px,
quantity=qty,
)
self.logger.info(
"[bybit_gw_crt_ord_step.json]%s" % SLog(msg=f"Finding security {sec_name}")
)
sec = self.securities.get(sec_name)
if not sec:
raise ValueError(f"Unknown security '{sec_name}'")
self.logger.error(
"[bybit_gw_crt_ord_err.json]%s"
% SLog(err=f"Unknown security '{sec_name}'")
)
return
params = {
"orderLinkId": exch_ord_id,
"category": "linear",
@@ -0,0 +1,22 @@
import logging
from typing import Optional
from spark_core import StructuredLog as SLog
from python_interop_module import PyExecutionReport, PyPositionInfo
class FakeExecutionGatewayBridge:
def __init__(self, logger: Optional[logging.Logger] = logging.getLogger()):
self.logger = logger
def on_execution_report(self, er: PyExecutionReport) -> None:
if self.logger:
self.logger.info("[exe_rep.json]%s" % er)
def on_is_ready(self, is_ready: bool) -> None:
if self.logger:
self.logger.info("[is_ready.json]%s" % SLog(is_ready=is_ready))
def on_position_info(self, pi: PyPositionInfo) -> None:
if self.logger:
self.logger.info("[pos_info.json]%s" % pi)
@@ -106,6 +106,12 @@ class WebSocket(_V5WebSocketManager):
# Private topics
def reconnect(self):
if not self.WS_URL:
raise ValueError(f"WS_URL is empty")
self._reset()
self._connect(self.WS_URL)
def position_stream(self, callback):
"""Subscribe to the position stream to see changes to your position data in real-time.
@@ -1,3 +1,4 @@
from typing import Callable, Optional
import websocket
import threading
import time
@@ -21,6 +22,8 @@ DOMAIN_ALT = "bytick"
TLD_MAIN = "com"
WebSocketError = websocket.WebSocketConnectionClosedException | ConnectionRefusedError | KeyboardInterrupt | SystemExit | Exception
class _WebSocketManager:
def __init__(
self,
@@ -39,7 +42,26 @@ class _WebSocketManager:
restart_on_error=True,
trace_logging=False,
private_auth_expire=1,
on_open: Optional[Callable[[websocket.WebSocketApp], None]] = None,
on_error : Optional[Callable[[websocket.WebSocketApp, WebSocketError], Optional[bool]]] = None,
on_close : Optional[Callable[[websocket.WebSocketApp, Optional[int], Optional[str]], bool]] = None,
):
"""
Parameters
----------
restart_on_error: bool
Reconnect when the websocket has an error.
on_open: Optional[Callable[[websocket.WebSocketApp], None]]
Optional websocket open handler.
on_error: Optional[Callable[[websocket.WebSocketApp, WebSocketError], Optional[bool]]]
Optional websocket error handler. Return a bool to override the 'restart_on_error'
param (True forces reconnect, False forces no reconnect).
on_close: Optional[Callable[[websocket.WebSocketApp, Optional[int], Optional[str]], bool]]
Optional websocket close handler. Return True to reconnect.
"""
self.testnet = testnet
self.domain = domain
self.tld = tld
@@ -50,6 +72,9 @@ class _WebSocketManager:
self.api_secret = api_secret
self.callback = _callback_function
self.open_callback = on_open
self.close_callback = on_close
self.error_callback = on_error
self.ws_name = ws_name
if api_key:
self.ws_name += " (Auth)"
@@ -86,11 +111,13 @@ class _WebSocketManager:
self._reset()
self.attempting_connection = False
def _on_open(self):
def _on_open(self, ws: websocket.WebSocketApp):
"""
Log WS open.
Fires the open callback if it exists.
"""
logger.debug(f"WebSocket {self.ws_name} opened.")
logger.info(f"WebSocket {self.ws_name} opened. is_connected: {self.is_connected()}")
if self.open_callback:
self.open_callback(ws)
def _on_message(self, message):
"""
@@ -154,9 +181,9 @@ class _WebSocketManager:
self.ws = websocket.WebSocketApp(
url=url,
on_message=lambda ws, msg: self._on_message(msg),
on_close=lambda ws, *args: self._on_close(),
on_open=lambda ws, *args: self._on_open(),
on_error=lambda ws, err: self._on_error(err),
on_close=lambda ws, *args: self._on_close(ws, *args), # type: ignore
on_open=lambda ws: self._on_open(ws), # type: ignore
on_error=lambda ws, err: self._on_error(ws, err), # type: ignore
on_pong=lambda ws, *args: self._on_pong(),
)
@@ -217,10 +244,13 @@ class _WebSocketManager:
)
)
def _on_error(self, error):
def _on_error(self, ws: websocket.WebSocketApp, error: WebSocketError):
"""
Exit on errors and raise exception, or attempt reconnect.
"""
reconnect = (self.error_callback(ws, error) if self.error_callback else None)
if reconnect is None:
reconnect = self.handle_error
if type(error).__name__ not in [
"WebSocketConnectionClosedException",
"ConnectionResetError",
@@ -238,15 +268,16 @@ class _WebSocketManager:
self.exit()
# Reconnect.
if self.handle_error and not self.attempting_connection:
self._reset()
self._connect(self.endpoint)
if reconnect and not self.attempting_connection:
self._reconnect()
def _on_close(self):
def _on_close(self, ws: websocket.WebSocketApp, code: Optional[int], rsn: Optional[str]):
"""
Log WS close.
Fires the close callback if it exists.
"""
logger.debug(f"WebSocket {self.ws_name} closed.")
logger.warning(f"WebSocket {self.ws_name} closed.")
if self.close_callback and self.close_callback(ws, code, rsn):
self._reconnect()
def _on_pong(self):
"""
@@ -260,6 +291,11 @@ class _WebSocketManager:
"""
self._send_custom_ping()
def _reconnect(self):
if not self.attempting_connection:
self._reset()
self._connect(self.endpoint)
def _send_custom_ping(self):
self.ws.send(self.custom_ping_message)
@@ -308,6 +344,7 @@ class _V5WebSocketManager(_WebSocketManager):
if kwargs.get("callback_function")
else self._handle_incoming_message
)
self.subscription_ack_callback: Optional[Callable[[list[str], bool], None]] = kwargs.pop("subscription_ack_callback") if "subscription_ack_callback" in kwargs else None
super().__init__(callback_function, ws_name, **kwargs)
self.subscriptions = {}
@@ -443,21 +480,25 @@ class _V5WebSocketManager(_WebSocketManager):
)
def _process_subscription_message(self, message):
if message.get("req_id"):
sub = self.subscriptions[message["req_id"]]
req_id = message.get("req_id")
if req_id:
sub = self.subscriptions[req_id]
else:
# if req_id is not supported, guess that the last subscription
# sent was successful
sub = json.loads(list(self.subscriptions.items())[0][1])["args"][0]
raise ValueError("Subscription msg has no 'req_id' field")
success = message.get("success")
if success is None:
raise ValueError("Subscription msg has no 'success' field")
# If we get successful futures subscription, notify user
if message.get("success") is True:
if success:
logger.debug(f"Subscription to {sub} successful.")
# Futures subscription fail
elif message.get("success") is False:
else:
# Futures subscription fail
response = message["ret_msg"]
logger.error("Couldn't subscribe to topic." f"Error: {response}.")
self._pop_callback(sub[0])
if self.subscription_ack_callback:
self.subscription_ack_callback(json.loads(sub).get("args"), success)
def _process_normal_message(self, message):
topic = message["topic"]
@@ -1,5 +1,6 @@
from dataclasses import dataclass, field
import json
from typing import Optional
import uuid
import logging
from pybit_websocket_stream import _WebSocketManager
@@ -20,6 +21,9 @@ class _V5TradeWebSocketManager(_WebSocketManager):
self.referral_id = referral_id
self._connect(TRADE_WSS)
def reconnect(self):
self._connect(TRADE_WSS)
def _process_auth_message(self, message):
# If we get successful auth, notify user
if message.get("retCode") == 0:
@@ -75,9 +79,9 @@ class _V5TradeWebSocketManager(_WebSocketManager):
def _pop_error_callback(self, topic):
return self.error_callback_directory.pop(topic)
def _send_order_operation(self, operation, callback, request, error_callback=None):
request_id = str(uuid.uuid4())
def _send_order_operation(self, operation, callback, request, error_callback=None, request_id:Optional[str] = None):
if not request_id:
request_id = str(uuid.uuid4())
message = {
"reqId": request_id,
"header": {