diff --git a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index 3cc2b616a..0ee156e09 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -14,29 +14,38 @@ from python_interop_module import ( PySecurityList, ) from spark_core import ( - JsonElement, configure_logger, get_contract_spec, get_price_info, get_quantity_info, get_security, - get_tick_factors, load_config, ) from py_md_utils import PERP_EXP_STRING import requests -import numpy as np -logger = logging.getLogger("bybit_md_provider") +logger = logging.getLogger("bybit_md_provider") # Global logger restored def normalize_contract_name(contract: str) -> str: + # SPOT: Log normalization for both futures and spot contracts + logger.info(f"Normalizing contract: input={contract}") if len(contract) > 4: - return contract[:-4] + "_" + contract[-4:] + result = contract[:-4] + "_" + contract[-4:] + logger.info(f"Normalizing contract: output={result}") + return result + logger.info(f"Normalizing contract: output={contract} (unchanged)") return contract def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> dict: # Fetch specs for futures only, explicitly, handling pagination - specs_dict = {} + default_specs = { + "tick_unit": 1, + "tick_base": 10, + "cash_per_point": 1, + "qty_tick_unit": 1, + "qty_tick_base": 10000 + } + specs_dict = {contract: default_specs for contract in contracts} try: host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500" @@ -65,37 +74,37 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> d if contract in contracts: try: price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1)) - lot_size = item.get("lotSizeFilter", {}) - qty_step = float(lot_size.get("qtyStep", 0.001)) + lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001)) - if price_tick == 0 or qty_step == 0: - logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={qty_step}") + if price_tick == 0 or lot_size == 0: + logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}") continue - tick_unit, tick_base = get_tick_factors(price_tick) - qty_tick_unit, qty_tick_base = get_tick_factors(qty_step) + tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 + qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 specs_dict[contract] = { - "tick_unit": tick_unit, + "tick_unit": 1, "tick_base": tick_base, "cash_per_point": 1, - "qty_tick_unit": qty_tick_unit, - "qty_tick_base": qty_tick_base, - "min_notional_value": float( - lot_size.get("minNotionalValue", tick_unit / tick_base) - ), + "qty_tick_unit": 1, + "qty_tick_base": qty_tick_base } found_contracts.add(contract) logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]}") except (ValueError, TypeError) as e: - logger.error(f"Failed to parse futures specs for {contract}: {e}") + logger.error(f"Failed to parse futures specs for {contract}): {e}") continue cursor = result.get("nextPageCursor", "") if not cursor: logger.info("No more pages to fetch; pagination complete") break - + + for contract in contracts: + if contract not in found_contracts: + logger.warning(f"Futures contract {contract} not found in API response, using default specs") + logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") logger.debug(f"Final specs: {specs_dict}") return specs_dict @@ -104,11 +113,26 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> d logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True) return specs_dict +# def get_spot_specs(contracts: Iterable[str]) -> dict: +# # SPOT: Use default specs for spot contracts, no API call +# default_specs = { +# "tick_unit": 1, +# "tick_base": 10, +# "cash_per_point": 0.1, +# "qty_tick_unit": 1, +# "qty_tick_base": 10000 +# } +# specs_dict = {contract: default_specs for contract in contracts} +# logger.info(f"Using default specs for spot contracts: {default_specs}") +# return specs_dict + class BybitMdProvider(MdProvider): def __init__(self): super().__init__() self._config: Optional[dict] = None - self.ws = None + # SPOT: Separate WebSockets for futures and spot + self.ws_futures = None + # self.ws_spot = None self.ws_thread: Optional[threading.Thread] = None self.bridge: Optional[MdBridge] = None self.stop_event = threading.Event() @@ -131,60 +155,66 @@ class BybitMdProvider(MdProvider): def futures(self): return self.config.get("futures", []) - @property - def spots(self): - return self.config.get("spots", []) + # @property + # def spots(self): + # return self.config.get("spots", []) def start_impl(self, bridge: MdBridge): configure_logger(logger, "md", **self.config.get("logging", {})) self.bridge = bridge logger.info("Starting BybitMdProvider") - contracts = list( + # SPOT: Generate futures and spot contracts + futures_contracts = list( chain(*[map(lambda f: f"{f}{c}", self.futures) for c in self.currencies]) ) - contract_specs = fetch_futures_specs(contracts, use_testnet=self.use_testnet) - sec_list = PySecurityList() - missing_contracts = list(filter(lambda c: c not in contract_specs, contracts)) - if missing_contracts: - logger.error( - "Did not find instrument info for %s" % JsonElement(missing_contracts) - ) - contracts = list(filter(lambda c: c in contract_specs, contracts)) - for contract in contracts: + # spot_contracts = list( + # chain(*[map(lambda s: s + c, self.spots) for c in self.currencies]) + # ) + + # FIX: Fetch futures specs independently + futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet) + logger.info(f"Fetched futures specs for {len(futures_specs)} contracts") + + # SPOT: Use default specs for spots + # spot_specs = get_spot_specs(spot_contracts) + + # FIX: Create separate security lists for futures and spots + sec_list_futures = PySecurityList() + # sec_list_spots = PySecurityList() + + # FIX: Create securities for futures + for contract in futures_contracts: normalized_contract = normalize_contract_name(contract) - asset, quote = normalized_contract.split("_") - specs = contract_specs.get(contract, {}) - + try: + asset, quote = normalized_contract.split("_") + except ValueError as e: + logger.error(f"Failed to split normalized futures contract {normalized_contract}: {e}") + continue + specs = futures_specs.get(contract, {}) + try: qty_info = get_quantity_info( tick_unit=specs.get("qty_tick_unit", 1), tick_base=specs.get("qty_tick_base", 10000) ) except Exception as e: - logger.error(f"Failed to get quantity info for contract: {contract}, error: {e}") + logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}") continue - + try: - price_tick_unit = specs.get("tick_unit", 1) - price_tick_base=specs.get("tick_base", 10) spec = get_contract_spec( price_info=get_price_info( - tick_unit=price_tick_unit, - tick_base=price_tick_base, - cash_per_point=specs.get("cash_per_point", 0.1), - ), - qty_info=qty_info, - notional_qty_info=get_quantity_info( - tick_unit=price_tick_unit, - tick_base=price_tick_base, - min_value=specs.get("min_notional_value"), + tick_unit=specs.get("tick_unit", 1), + tick_base=specs.get("tick_base", 10), + cash_per_point=specs.get("cash_per_point", 0.1) ), + qty_info=qty_info ) except Exception as e: - logger.error(f"Failed to get contract spec for contract: {contract}, error: {e}") + logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}") continue - + try: sec = get_security( venue="Bybit", @@ -197,42 +227,118 @@ class BybitMdProvider(MdProvider): strike=None, spec=spec, ) - sec_list.append(sec) - logger.info(f"Created security: {contract} with specs: {specs}") + sec_list_futures.append(sec) + logger.info(f"Created FUTURE security: {contract} (market_type=FUTURE, expiration=PERP) with specs: {specs}") except Exception as e: - logger.error(f"Failed to create security for contract: {contract}, error: {e}") + logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}") continue - - bridge.on_securities(sec_list) - logger.info("Sent securities list--bridge.on_securities called.") - + + # SPOT: Create securities for spots + # for contract in spot_contracts: + # normalized_contract = normalize_contract_name(contract) + # try: + # asset, quote = normalized_contract.split("_") + # except ValueError as e: + # logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}") + # continue + # specs = spot_specs.get(contract, {}) + # + # try: + # qty_info = get_quantity_info( + # tick_unit=specs.get("qty_tick_unit", 1), + # tick_base=specs.get("qty_tick_base", 10000) + # ) + # except Exception as e: + # logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}") + # continue + # + # try: + # spec = get_contract_spec( + # price_info=get_price_info( + # tick_unit=specs.get("tick_unit", 1), + # tick_base=specs.get("tick_base", 10), + # cash_per_point=specs.get("cash_per_point", 0.1) + # ), + # qty_info=qty_info + # ) + # except Exception as e: + # logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}") + # continue + # + # try: + # sec = get_security( + # venue="Bybit", + # name=contract, + # asset=asset, + # group=asset, + # currency=quote, + # expiration=None, + # option_kind=None, + # strike=None, + # spec=spec, + # ) + # sec_list_spots.append(sec) + # logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}") + # except Exception as e: + # logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}") + # continue + + # FIX: Combine and send securities + combined_sec_list = PySecurityList() + for sec in sec_list_futures: + combined_sec_list.append(sec) + # for sec in sec_list_spots: + # combined_sec_list.append(sec) + bridge.on_securities(combined_sec_list) + # logger.info(f"Sent securities list: {len(sec_list_futures)} futures, {len(sec_list_spots)} spots") + try: - self.ws = WebSocket( + # SPOT: Initialize WebSocket for futures + self.ws_futures = WebSocket( testnet=self.use_testnet, channel_type="linear", ping_interval=20, ) - logger.info("Connecting to Bybit WebSocket (mainnet, linear)") - - for contract in contracts: + logger.info("Connecting to Bybit WebSocket (futures, linear)") + + for contract in futures_contracts: try: - self.ws.orderbook_stream( + self.ws_futures.orderbook_stream( symbol=contract, depth=50, - callback=self._on_book_update + callback=lambda data: self._on_book_update(data, source="futures") ) - logger.info(f"Subscribed to orderbook.50.{contract}") + logger.info(f"Subscribed to orderbook.50.{contract} (futures)") except Exception as e: - logger.error(f"Failed to subscribe to orderbook for {contract}, error: {e}") - - logger.info("Subscribed to orderbook channels on Bybit unified trading WS.") - + logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}") + + # SPOT: Initialize WebSocket for spot + # self.ws_spot = WebSocket( + # testnet=self.use_testnet, + # channel_type="spot", + # ping_interval=20, + # ) + # logger.info("Connecting to Bybit WebSocket (spot)") + # + # for contract in spot_contracts: + # try: + # self.ws_spot.orderbook_stream( + # symbol=contract, + # depth=50, + # callback=lambda data: self._on_book_update(data, source="spot") + # ) + # logger.info(f"Subscribed to orderbook.50.{contract} (spot)") + # except Exception as e: + # logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}") + + logger.info("Subscribed to orderbook channels on Bybit futures and spot WS.") + for t in threading.enumerate(): logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}") - + while not self.stop_event.is_set(): time.sleep(1) - + except KeyboardInterrupt: logger.info("KeyboardInterrupt received — shutting down.") self.stop() @@ -246,77 +352,99 @@ class BybitMdProvider(MdProvider): self.start_impl(bridge) except Exception as e: logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True) - + self.ws_thread = threading.Thread(target=start_wrapper) self.ws_thread.start() - def _on_book_update(self, data: dict): + def _on_book_update(self, data: dict, source: str): try: + # FIX: Log raw WebSocket data for debugging + logger.debug(f"Raw WebSocket data (source={source}): {data}") + topic = data.get("topic", "") if not topic.startswith("orderbook.50."): + logger.debug(f"Ignoring non-orderbook topic: {topic}") return - + contract = topic.split(".")[-1] payload = data.get("data", {}) - raw_bids = [(float(x[0]), float(x[1])) for x in payload.get("b", [])] - raw_asks = [(float(x[0]), float(x[1])) for x in payload.get("a", [])] - - # Sort and take top 5, padding with [float('nan'), float('nan')] if needed + + # FIX: Validate bids and asks + raw_bids = payload.get("b", []) + raw_asks = payload.get("a", []) + if not raw_bids or not raw_asks: + logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") + + raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids] + raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks] + sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] - sorted_bids += [(np.float64('nan'), np.float64('nan'))] + sorted_bids += [(float('nan'), float('nan'))] sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] - sorted_asks += [(np.float64('nan'), np.float64('nan'))] - - logger.debug(f"📘 BOOK DATA for {contract}: bids={sorted_bids} asks={sorted_asks}") - + sorted_asks += [(float('nan'), float('nan'))] + + # FIX: Log processed orderbook data + logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") + book = PyBook() + # FIX: Determine market type from WebSocket source + market_type = "SPOT" if source == "spot" else "FUTURE" + # FIX: Log market_type assignment + logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") + book.set_data( contract, "Bybit", - "FUTURE", - np.array(sorted_bids, dtype=np.float64), - np.array(sorted_asks, dtype=np.float64), + market_type, + sorted_bids, + sorted_asks, ) book_list = PyBookList() book_list.append(book) if self.bridge: + # FIX: Log book submission + logger.debug(f"✔ on_books called for {contract} (market_type={market_type})") self.bridge.on_books(book_list) - logger.debug(f"✔ on_books called for {contract}") else: logger.warning("Bridge is None, cannot call on_books") except Exception as e: - logger.error(f"Error processing book update for {contract}: {e}", exc_info=True) + logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) def stop(self): logger.info("Stopping BybitMdProvider...") self.stop_event.set() - + logger.debug("Threads before stopping WebSocket:") for t in threading.enumerate(): logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") - - if self.ws: - try: - self.ws.exit() - logger.info("WebSocket exit called") - # Clean up pybit threads with longer timeout - for t in threading.enumerate(): - if t.name in ['Thread-1 ()', 'Thread-2 (_send_ping)'] and t.is_alive(): - logger.info(f"Joining pybit thread: {t.name}") - t.join(timeout=1.0) - except Exception as e: - logger.error(f"Error while closing WebSocket: {e}", exc_info=True) - self.ws = None - + + # SPOT: Close both WebSockets + for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket + # , (self.ws_spot, "spot") + if ws: + try: + ws.exit() + logger.info(f"WebSocket ({ws_type}) exit called") + for t in threading.enumerate(): + if t.name in ['Thread-1 ()', 'Thread-2 (_send_ping)'] and t.is_alive(): + logger.info(f"Joining pybit thread: {t.name} ({ws_type})") + t.join(timeout=1.0) + except Exception as e: + logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True) + ws = None + + self.ws_futures = None + # self.ws_spot = None + if self.ws_thread: self.ws_thread.join() self.bridge = None logger.info("✔ Stopped BybitMdProvider.") - + logger.debug("Threads after stop:") for t in threading.enumerate(): logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") - + import os os._exit(0) @@ -362,4 +490,4 @@ if __name__ == "__main__": md_bridge = FakeMdBridge(logger) md_pro.start(md_bridge) # type: ignore while True: - time.sleep(1) + time.sleep(1) \ No newline at end of file