diff --git a/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py b/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py index ae386033a..67c6d31d6 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_execution_gateway.py @@ -41,8 +41,12 @@ class BybitGatewayStatus(enum.Enum): CONNECTING = 1 SUBSCRIBING = 2 READY = 3 - CANCELING_STALE_ORDERS = 4 - SYNCING_POSITIONS = 5 + STOPPING = 4 + + +PyOrderRequest = ( + PyCreateOrderRequest | PyCancelReplaceOrderRequest | PyCancelReplaceOrderRequest +) class OrderData: @@ -75,6 +79,19 @@ class OrderData: self.last_filled_price: float = 0 self.exec_type: ExecType = ExecType.PENDING_NEW self.status: OrderStatus = OrderStatus.PENDING_NEW + # nanos + self.timestamp: int = 0 + + @property + def is_active(self): + match self.status: + case OrderStatus.PENDING_NEW: + return True + case OrderStatus.NEW: + return True + case OrderStatus.PARTIALLY_FILLED: + return True + return False def to_json(self): return { @@ -119,7 +136,7 @@ class OrderData: avg_filled_price=self.avg_filled_price, # There is no transact_id in Bybit, so we use updatedTime transact_id=str(transact_time), - transact_time=transact_time, + transact_time=self.timestamp, ) return er @@ -166,7 +183,87 @@ class OrderData: self.status = OrderStatus.REJECTED case _: raise ValueError(f"Unexpected order status '{status}'") - return self.to_execution_report() + + def on_update_data(self, data): + transact_time = int(data["updatedTime"]) * 1_000_000 + if self.timestamp > transact_time: + return False + self.timestamp = transact_time + total_filled_qty = parse_float(data["cumExecQty"]) + avg_filled_price = parse_float(data["avgPrice"]) + status: str = data["orderStatus"] + px = parse_float(data["price"]) + qty = parse_float(data["qty"]) + self.on_update(px, qty, total_filled_qty, avg_filled_price, status) + return True + + def to_exe_report(self, o: dict, security_type: str = "FUTURE"): + er = PyExecutionReport() + er.set_data( + key=self.bts_id, + exchange_order_id=o["orderLinkId"], + venue="BYBIT", + security_type=security_type, + exchange_security_id=o["symbol"], + account=self.account, + order_type=o["orderType"], + is_buy=o["side"] == "Buy", + price=self.price, + quantity=self.quantity, + time_in_force=o["timeInForce"], + exec_type=self.exec_type.name, + order_status=self.status.name, + last_filled_price=self.last_filled_price, + last_filled_quantity=self.last_filled_quantity, + filled_quantity=self.filled_quantity, + avg_filled_price=self.avg_filled_price, + # There is no transact_id in Bybit, so we use updatedTime + transact_id=str(time.time_ns()), + transact_time=self.timestamp, + ) + return er + + +def merge_order_req( + old_req: PyOrderRequest, new_req: PyOrderRequest +) -> Optional[PyOrderRequest]: + if old_req.key() != new_req.key(): + raise ValueError("Cannot merge PyOrderRequests with different keys") + if isinstance(old_req, PyCreateOrderRequest): + if isinstance(new_req, PyCancelOrderRequest): + # This is a no op + return None + elif isinstance(new_req, PyCancelReplaceOrderRequest): + res = PyCreateOrderRequest() + res.set_exchange_security_id(new_req.exchange_security_id()) + res.set_is_buy(new_req.is_buy()) + res.set_key(new_req.key()) + res.set_order_type(old_req.order_type()) + res.set_price(new_req.price()) + res.set_quantity(new_req.quantity()) + res.set_security_type(new_req.security_type()) + res.set_time_in_force(old_req.time_in_force()) + res.set_venue(new_req.venue()) + return res + elif isinstance(old_req, PyCancelOrderRequest): + if isinstance(new_req, PyCreateOrderRequest): + res = PyCancelReplaceOrderRequest() + res.set_exchange_order_id(old_req.exchange_order_id()) + res.set_exchange_security_id(old_req.exchange_security_id()) + res.set_is_buy(new_req.is_buy()) + res.set_key(old_req.key()) + res.set_price(new_req.price()) + res.set_quantity(new_req.quantity()) + res.set_security_type(new_req.security_type()) + res.set_venue(new_req.venue()) + elif isinstance(new_req, PyCancelOrderRequest): + return old_req + elif isinstance(old_req, PyCancelReplaceOrderRequest): + if isinstance(new_req, PyCancelOrderRequest | PyCancelReplaceOrderRequest): + return new_req + raise ValueError( + f"Cannot merge PyOrderRequest of type {type(old_req)} (old) with {type(new_req)} (new)" + ) class BybitExecutionGateway(ExecutionGateway): @@ -194,18 +291,16 @@ class BybitExecutionGateway(ExecutionGateway): self.trade_cli_connected: bool = False self.stream_cli_connected: bool = False self.stream_subscriptions: set[str] = set() - # bts id -> exch id - self.orders: dict[int, OrderData] = {} + # exch id -> OrderData + self.orders: dict[str, OrderData] = {} self.securities: dict[str, PySecurity] = {} - # set of exch ids for orders that being cancelled due to disconnection. We propagate them to - # sparkd with a special flag. Some algos may keep working through cancels due to disconnect - # (instead of the usual 'Cancel on unexpected child cancel' behavior). - self.pending_reconnect_cxls: set[str] = set() self.order_ack_callback: Optional[ Callable[[dict, Optional[OrderData]], None] ] = None self.status: BybitGatewayStatus = BybitGatewayStatus.IDLE self.lock = threading.RLock() + self.buffered_requests: dict[int, PyOrderRequest] = {} + self.pos_poll_thread: Optional[threading.Thread] = None @property def is_connected(self): @@ -215,11 +310,13 @@ class BybitExecutionGateway(ExecutionGateway): def is_subscribed(self): return "order" in self.stream_subscriptions - def _get_order_data(self, exch_id: str): - for _, v in self.orders.items(): - if v.exch_id == exch_id: - return v - return None + def _get_order_data(self, exch_id=None, bts_id=None, require_exists=False): + with self.lock: + if bts_id is not None: + for o in self.orders.values(): + if o.bts_id == bts_id: + return o + return self.orders[exch_id] if require_exists else self.orders.get(exch_id) def _order_handlers(self, typ: str, exch_id: str): def get_error_desc(msg: dict): @@ -243,7 +340,13 @@ class BybitExecutionGateway(ExecutionGateway): '[bybit_gw_ord_ack_err.json]{ "typ": "%s", "exch_id": "%s", "err": "%s" }' % (typ, exch_id, msg) ) - ord_data = self._get_order_data(exch_id) + if not self.bridge: + self.logger.error( + "[bybit_gw_ord_ack_err_err.json]%s" + % SLog(typ=typ, exch_id=exch_id, err="bridge is None") + ) + return + ord_data = self._get_order_data(exch_id=exch_id) if ord_data: er = ord_data.to_execution_report() match er.order_status(): @@ -282,115 +385,30 @@ class BybitExecutionGateway(ExecutionGateway): % (key, req_type, rsn, time.time_ns()) ) - def compute_internal_state( - self, positions_synced: bool = False, memo: Optional[str] = None - ): - """ - Parameters - ---------- - positions_sync: bool - True indicates our positions are synchronized with the exchange. It is only - relevant if the current status is SYNCING_POSITIONS, in which case a value - of True signals our new status is READY (barring an unexpected disconnect). - - memo: Optional[str] - A brief memo to include in the log message (if the status changes) - """ - # Note: this is NOT a pure function (mutates self.status). - # Computes status as a function of - # 1) the combined stream/trade cli connection status, - # 2) the presence of pending_reconnect_cxls, and - # 3) the current status. - # The general path is - # IDLE -> CONNECTING -> SUBSCRIBING -> CANCELING_STALE_ORDERS -> SYNCING_POSITIONS -> READY - old_status = self.status - new_status = old_status - match self.status: - case BybitGatewayStatus.IDLE: - if self.is_connected: - # Initial Path - new_status = ( - BybitGatewayStatus.SYNCING_POSITIONS - if self.is_subscribed - else BybitGatewayStatus.SUBSCRIBING - ) - elif self.stream_cli or self.trade_cli: - new_status = BybitGatewayStatus.CONNECTING - case BybitGatewayStatus.CONNECTING: - if self.is_connected: - new_status = ( - BybitGatewayStatus.CANCELING_STALE_ORDERS - if self.is_subscribed - else BybitGatewayStatus.SUBSCRIBING - ) - case BybitGatewayStatus.CANCELING_STALE_ORDERS: - if not self.is_connected: - self.pending_reconnect_cxls.clear() - new_status = BybitGatewayStatus.CONNECTING - elif not self.pending_reconnect_cxls: - new_status = BybitGatewayStatus.SYNCING_POSITIONS - case BybitGatewayStatus.SUBSCRIBING: - if self.is_subscribed: - new_status = BybitGatewayStatus.CANCELING_STALE_ORDERS - case BybitGatewayStatus.SYNCING_POSITIONS: - if self.pending_reconnect_cxls: - raise ValueError( - f"Invalid internal status '{self.status.name}' when there are pending_reconnect_cxls" - ) - if not self.is_connected: - new_status = BybitGatewayStatus.CONNECTING - elif not self.is_subscribed: - new_status = BybitGatewayStatus.SUBSCRIBING - elif positions_synced: - new_status = BybitGatewayStatus.READY - case BybitGatewayStatus.READY: - if not self.is_connected: - new_status = BybitGatewayStatus.CONNECTING - elif not self.is_subscribed: - new_status = BybitGatewayStatus.SUBSCRIBING - self.status = new_status - if new_status != old_status: - self.logger.info( - "[bybit_gw_status_change.json]%s" - % SLog(prev=old_status.name, curr=new_status.name, memo=memo) - ) - return True - return False + def get_order_base_coins(self): + # TODO: get base coins for working orders + rv = set() + rv.add("USDT") + return rv def on_stream_open(self, ws): - should_cxl_all = False with self.lock: self.logger.info("[bybit_gw_stream_open.json]%s" % SLog()) self.stream_cli_connected = True self.stream_cli_disconnect_time = 0 - should_cxl_all = self.compute_internal_state(memo="stream_open") and ( - self.status == BybitGatewayStatus.CANCELING_STALE_ORDERS - ) - if should_cxl_all: - if not self.cancel_all(): - # If cancel_all returns False, then there are no orders to cancel. - # In that case we recompute our state immediately since there - # won't be a cancel ack to trigger it later. - self.compute_internal_state(memo="cxl_all_noop") def on_stream_subscription_ack(self, subs: list[str], ok: bool): self.logger.info("[bybit_gw_sub_ack.json]%s" % SLog(subs=subs, ok=ok)) - should_cxl_all = False with self.lock: for s in subs: if ok: self.stream_subscriptions.add(s) else: self.stream_subscriptions.remove(s) - should_cxl_all = self.compute_internal_state(memo="subscription_ack") and ( - self.status == BybitGatewayStatus.CANCELING_STALE_ORDERS - ) - if should_cxl_all: - if not self.cancel_all(): - # If cancel_all returns False, then there are no orders to cancel. - # In that case we recompute our state immediately since there - # won't be a cancel ack to trigger it later. - self.compute_internal_state(memo="cxl_all_noop") + if self.is_subscribed: + self.sync_orders() + if self.is_connected: + self.status = BybitGatewayStatus.READY def on_stream_error(self, ws, err: WebSocketError): self.logger.error("[bybit_gw_stream_err.json]%s" % SLog(err=str(err))) @@ -398,7 +416,7 @@ class BybitExecutionGateway(ExecutionGateway): self.stream_cli_connected = False self.stream_cli_disconnect_time = time.time() self.stream_subscriptions.clear() - self.compute_internal_state(memo="stream_error") + self.status = BybitGatewayStatus.CONNECTING return True def on_stream_close(self, ws, code: Optional[int], rsn: Optional[str]): @@ -407,31 +425,22 @@ class BybitExecutionGateway(ExecutionGateway): self.stream_cli_connected = False self.stream_cli_disconnect_time = time.time() self.stream_subscriptions.clear() - self.compute_internal_state(memo="stream_close") + self.status = BybitGatewayStatus.CONNECTING return True def on_trade_cli_open(self, ws): self.logger.info("[bybit_gw_trd_cli_open.json]%s" % SLog()) - should_cxl_all = False with self.lock: self.trade_cli_connected = True self.trade_cli_disconnect_time = 0 - should_cxl_all = self.compute_internal_state(memo="trd_cli_open") and ( - self.status == BybitGatewayStatus.CANCELING_STALE_ORDERS - ) - if should_cxl_all: - if not self.cancel_all(): - # If cancel_all returns False, then there are no orders to cancel. - # In that case we recompute our state immediately since there - # won't be a cancel ack to trigger it later. - self.compute_internal_state(memo="cxl_all_noop") + if self.is_subscribed: + self.status = BybitGatewayStatus.READY def on_trade_cli_error(self, ws, err: WebSocketError): self.logger.error("[bybit_gw_trd_cli_err.json]%s" % SLog(err=str(err))) with self.lock: self.trade_cli_connected = False self.trade_cli_disconnect_time = time.time() - self.compute_internal_state(memo="trd_cli_close") return True def on_trade_cli_close(self, ws, code: Optional[int], rsn: Optional[str]): @@ -439,67 +448,52 @@ class BybitExecutionGateway(ExecutionGateway): with self.lock: self.trade_cli_connected = False self.trade_cli_disconnect_time = time.time() - self.compute_internal_state(memo="trd_cli_error") return True - def check_reconnect_timeout(self): - RECONNECT_TIMEOUT = 10 - if self.is_connected: - return - with self.lock: - # Check is_connected again now that we're in the lock - if self.is_connected: - return - disconnect_times = [] - if not self.stream_cli_connected and self.stream_cli_disconnect_time != 0: - disconnect_times.append(self.stream_cli_disconnect_time) - if not self.trade_cli_connected and self.trade_cli_disconnect_time != 0: - disconnect_times.append(self.trade_cli_disconnect_time) - disconnect_times.sort() - if disconnect_times: - now = time.time() - reconnect_elapsed = now - disconnect_times[-1] - if reconnect_elapsed > RECONNECT_TIMEOUT: - self.logger.error( - "[bybit_gw_reconnect_timeout.json]%s" - % SLog( - stream_connected=self.stream_cli_connected, - stream_disconnect_ts=self.stream_cli_disconnect_time, - trd_cli_connected=self.trade_cli_connected, - trd_cli_disconnect_ts=self.trade_cli_disconnect_time, - err=f"Timed out waiting for reconnect after {RECONNECT_TIMEOUT}s", - ) - ) - for handler in self.logger.handlers: - handler.flush() + def sync_orders(self, category: str = "linear", settle_coin: str = "USDT"): + self.logger.info("[bybit_gw_sync_ords.json]%s" % SLog()) + next_pg = None + MAX_REQUESTS = 10 + request_count = 0 + params = {"category": category, "settleCoin": settle_coin} + try: + while True: + if request_count == MAX_REQUESTS: raise ValueError( - "BybitExecutionGateway timed out waiting for reconnect" + f"BybitExecutionGateway.sync_orders performed too many requests in loop ({MAX_REQUESTS})" ) + request_count += 1 + if next_pg: + params["cursor"] = next_pg + http_res = self.trade_http_cli.get_open_orders(**params) + self.logger.info( + "[bybit_gw_open_ords_res.json]%s" % JsonElement(http_res) + ) + if type(http_res) != dict or http_res["retCode"] != 0: + break + for o in http_res["result"]["list"]: + od = self.orders.get(o.get("orderLinkId")) + if not od: + continue + if (not od.is_active) and (od.bts_id in self.buffered_requests): + del self.buffered_requests[od.bts_id] + if od.on_update_data(o) and self.exe_report_callback: + self.exe_report_callback(od.to_exe_report(od)) + next_pg = http_res["result"].get("nextPageCursor") + if not next_pg: + break + except Exception as e: + self.logger.error("[bybit_gw_open_ords_err.json]%s" % SLog(err=str(e))) + # Fire buffered requests + for req in self.buffered_requests.values(): + if isinstance(req, PyCreateOrderRequest): + self.create_order(req) + elif isinstance(req, PyCancelReplaceOrderRequest): + self.cancel_replace_order(req) + elif isinstance(req, PyCancelOrderRequest): + self.cancel_order(req) - def cancel_all(self): - self.logger.info( - "[bybit_gw_cxl_all.json]%s" - % SLog(curr_orders=[o for o in self.orders.values()]) - ) - base_coins = set() - with self.lock: - for o in self.orders.values(): - for sec in self.securities.values(): - if sec.exchange_id() == o.exch_sec_id: - base_coins.add(sec.asset()) - self.pending_reconnect_cxls.add(o.exch_id) - break - if not self.pending_reconnect_cxls: - return False - for coin in base_coins: - self.logger.info("[bybit_gw_cxl_coin.json]%s" % SLog(base_coin=coin)) - res = self.trade_http_cli.cancel_all_orders( - category="linear", baseCoin=coin - ) - self.logger.info("[bybit_gw_cxl_coin_res.json]%s" % SLog(res=res)) - return True - - def get_positions(self): + def get_positions(self) -> Optional[PyPositionInfoList]: try: next_pg = None pos_dict: dict[str, PyPositionInfo] = {} @@ -536,9 +530,10 @@ class BybitExecutionGateway(ExecutionGateway): pi.set_security_type(SecurityType.FUTURE.name) pi.set_account(self.account) pi.set_transact_time(ts) + elif pi.transact_time() < ts: + pi.set_transact_time(ts) else: - if pi.transact_time() < ts: - pi.set_transact_time(ts) + continue if p["side"] == "Buy": pi.set_long_avg_cost(float(p["avgPrice"])) pi.set_long_quantity(float(p["size"])) @@ -551,93 +546,42 @@ class BybitExecutionGateway(ExecutionGateway): pos_infos = PyPositionInfoList() for p in pos_dict.values(): pos_infos.append(p) - self.bridge.on_position_info(pos_infos) + return pos_infos except Exception as e: - self.logger.error("[bybit_gw_get_pos_err.json]%s" % SLog(err=str(e))) - return False - return True + self.logger.error("[bybit_gw_pos_err.json]%s" % SLog(err=str(e))) + return None def start_impl(self, bridge: ExecutionGatewayBridge): def on_order_update(msg: dict): - def on_error(err: str): - self.logger.error( - '[bybit_gw_ord_upd_err.json]{ "err": "%s", "ts": %d }' - % (err, time.time_ns()) - ) - self.logger.info("[bybit_gw_ord_upd.json]%s" % msg) try: upds: list[dict] = msg["data"] for d in upds: - er = PyExecutionReport() exch_id: str = d["orderLinkId"] ord_data: Optional[OrderData] = self._get_order_data(exch_id) if not ord_data: - on_error("Did not resolve exch order id %s" % exch_id) + self.logger.error( + "[bybit_gw_ord_upd_err.json]%s" + % SLog(err=f"Unresolved exch order id {exch_id}") + ) return - is_reconnecting_cxl = False - with self.lock: - is_reconnecting_cxl = exch_id in self.pending_reconnect_cxls - if is_reconnecting_cxl: - self.pending_reconnect_cxls.remove(exch_id) - total_filled_qty = parse_float(d["cumExecQty"]) - avg_filled_price = parse_float(d["avgPrice"]) - status: str = d["orderStatus"] - px = parse_float(d["price"]) - qty = parse_float(d["qty"]) - ord_data.on_update( - px, qty, total_filled_qty, avg_filled_price, status - ) - transact_time = int(d["updatedTime"]) * 1_000_000 - er.set_data( - key=ord_data.bts_id, - exchange_order_id=d["orderLinkId"], - venue="BYBIT", - security_type="FUTURE", - exchange_security_id=d["symbol"], - account=self.account, - order_type=d["orderType"], - is_buy=d["side"] == "Buy", - price=px, - quantity=qty, - time_in_force=d["timeInForce"], - exec_type=ord_data.exec_type.name, - order_status=ord_data.status.name, - last_filled_price=ord_data.last_filled_price, - last_filled_quantity=ord_data.last_filled_quantity, - filled_quantity=total_filled_qty, - avg_filled_price=avg_filled_price, - # There is no transact_id in Bybit, so we use updatedTime - transact_id=str(transact_time), - transact_time=transact_time, - ) - if is_reconnecting_cxl: - if ord_data.status == OrderStatus.CANCELED: - # The "RECONNECTING" error msg is a sentinel (don't change it). When - # an order is canceled upon reconnection, some algos may decide to keep - # working rather than terminate due to an "unexpected child cancel." - er.set_error_msg("RECONNECTING") - else: - self.logger.error( - "[bybit_gw_ord_upd_error.json]%s" - % SLog( - err=f"pending reconnect cancel order status is {ord_data.status.name}" - ) - ) - with self.lock: - if not self.pending_reconnect_cxls: - self.compute_internal_state() + if not ord_data.on_update_data(d): + continue + if not ord_data.is_active: + del self.orders[ord_data.exch_id] + er = ord_data.to_exe_report(d) self.logger.info("[bybit_gw_exe_rep_out.json]%s" % er) - bridge.on_execution_report(er) + if self.bridge: + self.bridge.on_execution_report(er) except Exception as e: - self.logger.error( - '[bybit_gw_ord_upd_err.json]{ "err": %s }' % str(e), - exc_info=True, - ) + self.logger.error("[bybit_gw_ord_upd_err.json]%s" % SLog(err=str(e))) def on_exe_report(msg): self.logger.info("[bybit_gw_exe_rep.json]%s" % msg) + with self.lock: + self.status = BybitGatewayStatus.CONNECTING + self.bridge = bridge # Http client (gets position snapshots) params = { "api_key": self.api_key, @@ -679,25 +623,23 @@ class BybitExecutionGateway(ExecutionGateway): on_error=self.on_trade_cli_error, on_close=self.on_trade_cli_close, ) + bridge.on_is_ready(True) - has_been_initialized: bool = False POS_SNAPSHOT_INTERVAL = 10 next_pos_snapshot_time = time.time() + POS_SNAPSHOT_INTERVAL while not self.stop_event.wait(1): - self.check_reconnect_timeout() - match self.status: - case BybitGatewayStatus.READY: - if not has_been_initialized: - self.logger.info("[bybit_gw_is_ready.json]%s" % SLog()) - self.bridge.on_is_ready(True) - has_been_initialized = True - now = time.time() - if now >= next_pos_snapshot_time: - self.get_positions() - next_pos_snapshot_time = now + POS_SNAPSHOT_INTERVAL - case BybitGatewayStatus.SYNCING_POSITIONS: - if self.get_positions(): - self.compute_internal_state(positions_synced=True) + if time.time() > next_pos_snapshot_time: + positions = self.get_positions() + next_pos_snapshot_time = time.time() + POS_SNAPSHOT_INTERVAL + if positions: + self.logger.info("[bybit_gw_pos_infos.json]%s" % positions) + bridge.on_position_info(positions) + else: + self.logger.warning( + "[bybit_gw_pos_infos_err.json]%s" + % SLog(err="Failed to get positions") + ) + self.logger.info("[bybit_gw_poll_pos_done.json]%s" % SLog()) def start(self, bridge: ExecutionGatewayBridge): def start_wrapper(): @@ -719,52 +661,68 @@ class BybitExecutionGateway(ExecutionGateway): def stop(self): self.logger.info("[bybit_gw_stop.json]%s" % SLog()) self.stop_event.set() + if self.pos_poll_thread: + self.pos_poll_thread.join() + self.stop_event.clear() - def _create_order(self, req: PyCreateOrderRequest): - self.logger.info("[bybit_gw_crt_ord.json]%s" % req) - if self.status != BybitGatewayStatus.READY: - self._on_abort(req.key(), "crt", "Status is %s" % self.status.name) - return - if not self.stream_cli or not self.trade_cli or not self.bridge: - self._on_abort( - req.key(), - "crt", - "WebSocket not initialized (stream_cli: %s, trade_cli: %s, bridge: %s)" - % as_bool(self.stream_cli, self.trade_cli, self.bridge), + def _buffer_request(self, req: PyOrderRequest): + bts_id = req.key() + old_req = self.buffered_requests.get(bts_id) + self.logger.warning( + "[bybit_gw_buffer_req.json]%s" + % SLog( + bts_id=bts_id, + old_req_type=str(type(old_req)), + new_req_type=str(type(req)), ) + ) + if old_req: + new_req = merge_order_req(old_req, new_req) + if new_req: + self.buffered_requests[bts_id] = new_req + else: + del self.buffered_requests[bts_id] + else: + self.buffered_requests[bts_id] = req + + def create_order(self, req: PyCreateOrderRequest): + self.logger.info("[bybit_gw_crt.json]%s" % req) + if self.status != BybitGatewayStatus.READY: + self.logger.warning( + "[bybit_gw_crt_err.json]%s", SLog(err=f"Status is {self.status.name}") + ) + self._buffer_request(req) return exch_ord_id = uuid.uuid1().hex bts_id = req.key() px = req.price() qty = req.quantity() - sec_name = req.exchange_security_id() - self.logger.info("[bybit_gw_crt_ord_step.json]%s" % SLog(msg="Indexing order")) - self.orders[req.key()] = OrderData( - account=self.account, - bts_id=bts_id, - exch_id=exch_ord_id, - exch_sec_id=req.exchange_security_id(), - sec_type=SecurityType.parse(req.security_type()), - is_buy=req.is_buy(), - order_type=OrderType.parse(req.order_type()), - time_in_force=TimeInForce.parse(req.time_in_force()), - price=px, - quantity=qty, - ) - self.logger.info( - "[bybit_gw_crt_ord_step.json]%s" % SLog(msg=f"Finding security {sec_name}") - ) - sec = self.securities.get(sec_name) + exch_sec_id = req.exchange_security_id() + with self.lock: + self.orders[exch_ord_id] = OrderData( + account=self.account, + bts_id=bts_id, + exch_id=exch_ord_id, + exch_sec_id=exch_sec_id, + sec_type=SecurityType.parse(req.security_type()), + is_buy=req.is_buy(), + order_type=OrderType.parse(req.order_type()), + time_in_force=TimeInForce.parse(req.time_in_force()), + price=px, + quantity=qty, + ) + sec = self.securities.get(exch_sec_id) if not sec: self.logger.error( - "[bybit_gw_crt_ord_err.json]%s" - % SLog(err=f"Unknown security '{sec_name}'") + "[bybit_gw_crt_err.json]%s" + % SLog(err=f"Unknown security '{exch_sec_id}'") ) + self.buffered_requests[req.key()] = req return params = { "orderLinkId": exch_ord_id, "category": "linear", - "symbol": sec_name, + "symbol": exch_sec_id, "side": "Buy" if req.is_buy() else "Sell", "orderType": req.order_type().title(), "timeInForce": req.time_in_force().upper(), @@ -772,25 +730,56 @@ class BybitExecutionGateway(ExecutionGateway): "qty": format_quantity(sec, qty), } on_ack, on_err = self._order_handlers("crt", exch_ord_id) - self.logger.info("[bybit_gw_crt_ord_send.json]%s" % SLog(params=params)) - self.trade_cli.place_order(on_ack, **params, error_callback=on_err) + self.logger.info("[bybit_gw_crt_send.json]%s" % SLog(params=params)) + try: + self.trade_cli.place_order(on_ack, **params, error_callback=on_err) + return + except websocket.WebSocketConnectionClosedException as e: + self.logger.error("[bybit_gw_crt_err.json]%s" % SLog(err=str(e))) + # Handle websocket closed, signal to algos we are reconnecting + if self.bridge: + er = PyExecutionReport() + ts = time.time_ns() + er.set_data( + key=req.key(), + account=self.account, + exchange_order_id="", + exchange_security_id=req.exchange_security_id(), + venue=req.venue(), + security_type=req.security_type(), + order_type=req.order_type(), + order_status=OrderStatus.REJECTED.name, + time_in_force=req.time_in_force(), + exec_type=ExecType.REJECTED.name, + is_buy=req.is_buy(), + price=req.price(), + quantity=req.quantity(), + filled_quantity=0.0, + last_filled_quantity=0.0, + last_filled_price=0.0, + avg_filled_price=0.0, + transact_id=ts, + transact_time=ts, + ) + er.set_error_msg("RECONNECTING") + self.bridge.on_execution_report(er) - def _cancel_replace_order(self, req: PyCancelReplaceOrderRequest): + def cancel_replace_order(self, req: PyCancelReplaceOrderRequest): self.logger.info("[bybit_gw_cxlrep_ord.json]%s" % req) if self.status != BybitGatewayStatus.READY: - self._on_abort(req.key(), "cxlrep", "Status is %s" % self.status.name) - return - if not self.stream_cli or not self.trade_cli: - self._on_abort(req.key(), "cxlrep", "WebSocket client not created.") + self._buffer_request(req) return bts_id = req.key() - ord_data = self.orders.get(bts_id) + ord_data = self._get_order_data(bts_id=bts_id) if not ord_data: self._on_abort(req.key(), "cxlrep", "Did not resolve exch order id") return sec_name = req.exchange_security_id() sec = self.securities.get(sec_name) if not sec: + self._on_abort( + req.key(), "cxlrep", f"Did not resolve id for sec '{sec_name}'" + ) return params = { "orderLinkId": ord_data.exch_id, @@ -800,21 +789,42 @@ class BybitExecutionGateway(ExecutionGateway): "qty": format_quantity(sec, req.quantity()), } on_ack, on_err = self._order_handlers("cxlrep", ord_data.exch_id) - self.logger.info("[bybit_gw_cxlrep_ord_send.json]%s" % SLog(params=params)) - self.trade_cli.amend_order(on_ack, **params, error_callback=on_err) - - def _cancel_order(self, req: PyCancelOrderRequest): - self.logger.info("[bybit_gw_cxl_ord.json]%s" % req) if self.status != BybitGatewayStatus.READY: - self._on_abort(req.key(), "cxl", "Status is %s" % self.status.name) + self.buffered_requests[req.key()] = req + return + self.logger.info("[bybit_gw_cxlrep_ord_send.json]%s" % SLog(params=params)) + try: + self.trade_cli.amend_order(on_ack, **params, error_callback=on_err) + return + except websocket.WebSocketConnectionClosedException as e: + self.logger.error("[bybit_cxlrep_err.json]%s" % SLog(err=str(e))) + # Handle websocket closed, signal to algos we are reconnecting + ord_data = self.orders.get(bts_id=bts_id) + if not ord_data: + self.logger.error( + "[bybit_cxlrep_err.json]%s" + % SLog(err="Order not found with id %s" % req.exchange_order_id()) + ) + return + ord_data.status = OrderStatus.REJECTED + if self.exe_report_callback: + er = ord_data.to_execution_report() + er.set_error_msg("RECONNECTING") + self.exe_report_callback(er) + + def cancel_order(self, req: PyCancelOrderRequest): + self.logger.info("[bybit_gw_cxl_ord.json]%s" % req) + bts_id = req.key() + if self.status != BybitGatewayStatus.READY: + self.buffered_requests[req.key()] = req + self._on_abort(bts_id, "cxl", "Status is %s" % self.status.name) return if not self.stream_cli or not self.trade_cli: - self._on_abort(req.key(), "cxl", "WebSocket client not created.") + self._on_abort(bts_id, "cxl", "WebSocket client not created.") return - bts_id = req.key() - ord_data = self.orders.get(bts_id) + ord_data = self._get_order_data(bts_id=bts_id) if not ord_data: - self._on_abort(req.key(), "cxl", "Did not resolve exch order id") + self._on_abort(bts_id, "cxl", "Did not resolve exch order id") return params = { "orderLinkId": ord_data.exch_id, @@ -822,76 +832,12 @@ class BybitExecutionGateway(ExecutionGateway): "symbol": req.exchange_security_id(), } on_ack, on_err = self._order_handlers("cxl", ord_data.exch_id) + if self.status != BybitGatewayStatus.READY: + self.buffered_requests[req.key()] = req + return self.logger.info("[bybit_gw_cxl_ord_send.json]%s" % SLog(params=params)) self.trade_cli.cancel_order(on_ack, **params, error_callback=on_err) - def create_order(self, req: PyCreateOrderRequest): - try: - with self.lock: - self._create_order(req) - return - except websocket.WebSocketConnectionClosedException as e: - self.logger.error("[bybit_crt_ord_err.json]%s" % SLog(err=str(e))) - er = PyExecutionReport() - ts = time.time_ns() - er.set_data( - key=req.key(), - account=self.account, - exchange_order_id="", - exchange_security_id=req.exchange_security_id(), - venue=req.venue(), - security_type=req.security_type(), - order_type=req.order_type(), - order_status=OrderStatus.REJECTED.name, - time_in_force=req.time_in_force(), - exec_type=ExecType.REJECTED.name, - is_buy=req.is_buy(), - price=req.price(), - quantity=req.quantity(), - filled_quantity=0.0, - last_filled_quantity=0.0, - last_filled_price=0.0, - avg_filled_price=0.0, - transact_id=ts, - transact_time=ts, - ) - er.set_error_msg("RECONNECTING") - self.bridge.on_execution_report(er) - - def cancel_replace_order(self, req: PyCancelReplaceOrderRequest): - try: - with self.lock: - self._cancel_replace_order(req) - return - except websocket.WebSocketConnectionClosedException as e: - self.logger.error("[bybit_cxlrep_ord_err.json]%s" % SLog(err=str(e))) - od = None - with self.lock: - od = self._get_order_data(req.exchange_order_id()) - if not od: - return - od.status = OrderStatus.REJECTED - er = od.to_execution_report() - er.set_error_msg("RECONNECTING") - self.bridge.on_execution_report(er) - - def cancel_order(self, req: PyCancelOrderRequest): - try: - with self.lock: - self._cancel_order(req) - return - except websocket.WebSocketConnectionClosedException as e: - self.logger.error("[bybit_cxl_ord_err.json]%s" % SLog(err=str(e))) - od = None - with self.lock: - od = self._get_order_data(req.exchange_order_id()) - if not od: - return - od.status = OrderStatus.REJECTED - er = od.to_execution_report() - er.set_error_msg("RECONNECTING") - self.bridge.on_execution_report(er) - def on_securities(self, securities: PySecurityList): self.logger.info("[bybit_gw_sec.json]%s" % securities) self.securities = {sec.name(): sec for sec in securities}