From 36db69dc6771f9e939de75fb72cedd9c8344a4e5 Mon Sep 17 00:00:00 2001 From: Adeel Khan Date: Tue, 8 Jul 2025 14:06:16 +0000 Subject: [PATCH] FIX: bybit_md_provider.py works with default symbol format in py_bybit_md_provider.json.--MD Freezes on InverseFutures. --- .../bybit_md_provider.py | 581 +++++++++++++++++- 1 file changed, 573 insertions(+), 8 deletions(-) diff --git a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index 8c1483287..ac4d16afe 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -1,3 +1,544 @@ +# import json +# import numpy as np +# import time +# import logging +# import sys +# import threading +# from itertools import chain +# from typing import Iterable, Optional, Dict +# from pybit.unified_trading import WebSocket +# from python_interop_module import ( +# MdBridge, +# MdProvider, +# PyBook, +# PyBookList, +# PySecurityList, +# ) +# from spark_core import ( +# configure_logger, +# get_contract_spec, +# get_price_info, +# get_quantity_info, +# get_security, +# load_config, +# ) +# from py_md_utils import PERP_EXP_STRING +# import requests + +# logger = logging.getLogger("bybit_md_provider") + +# def normalize_contract_name(contract: str) -> str: +# logger.info(f"Normalizing contract: input={contract}") +# if "-" in contract: # Preserve delivery futures (e.g., BTC-26SEP25) +# logger.info(f"Normalizing contract: output={contract} (unchanged, delivery future)") +# return contract +# if contract.endswith("PERP"): +# result = contract[:-4] + "_PERP" +# logger.info(f"Normalizing contract: output={result}") +# return result +# if len(contract) > 4: +# result = contract[:-4] + "_" + contract[-4:] +# logger.info(f"Normalizing contract: output={result}") +# return result +# logger.info(f"Normalizing contract: output={contract} (unchanged)") +# return contract + +# def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: +# """Fetch metadata for all linear contracts and filter for specified contracts.""" +# metadata_dict = {} +# try: +# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" +# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000" +# logger.info(f"Querying Bybit API for metadata: {url}") +# response = requests.get(url, timeout=5) +# response.raise_for_status() +# data = response.json() + +# if data.get("retCode") == 0: +# contracts_data = data.get("result", {}).get("list", []) +# logger.info(f"Retrieved metadata for {len(contracts_data)} linear contracts") +# for item in contracts_data: +# symbol = item.get("symbol") +# if symbol in contracts: +# metadata_dict[symbol] = item +# logger.info(f"Stored metadata for {symbol}") +# else: +# logger.error(f"Metadata API error: {data.get('retMsg', 'Unknown error')}") +# except Exception as e: +# logger.error(f"Failed to fetch metadata: {e}", exc_info=True) +# return metadata_dict + +# def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: +# default_specs = { +# "tick_unit": 1, +# "tick_base": 10, +# "cash_per_point": 1, +# "qty_tick_unit": 1, +# "qty_tick_base": 10000 +# } +# specs_dict = {contract: {"specs": default_specs} for contract in contracts} +# try: +# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" +# base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500" +# logger.info(f"Querying Bybit API for futures specs: {base_url}") + +# found_contracts = set() +# cursor = "" + +# while True: +# url = f"{base_url}&cursor={cursor}" if cursor else base_url +# logger.info(f"Fetching page with cursor: {cursor or 'initial'}") +# response = requests.get(url, timeout=5) +# response.raise_for_status() +# data = response.json() + +# if data.get("retCode") != 0: +# logger.error(f"Futures API error: {data.get('retMsg', 'Unknown error')}") +# return specs_dict + +# result = data.get("result", {}) +# symbols = result.get("list", []) +# logger.info(f"Loaded page with cursor: {cursor or 'initial'}, symbols: {len(symbols)}") + +# for item in symbols: +# contract = item.get("symbol") +# if contract in contracts: +# try: +# price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1)) +# lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001)) + +# if price_tick == 0 or lot_size == 0: +# logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}") +# continue + +# tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 +# if lot_size >= 1: +# qty_tick_unit = lot_size +# qty_tick_base = 1 +# else: +# qty_tick_unit = 1 +# qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 + +# specs_dict[contract] = { +# "specs": { +# "tick_unit": 1, +# "tick_base": tick_base, +# "cash_per_point": 1, +# "qty_tick_unit": qty_tick_unit, +# "qty_tick_base": qty_tick_base +# } +# } +# found_contracts.add(contract) +# logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}") +# except (ValueError, TypeError) as e: +# logger.error(f"Failed to parse futures specs for {contract}: {e}") +# continue + +# cursor = result.get("nextPageCursor", "") +# if not cursor: +# logger.info("No more pages to fetch; pagination complete") +# break + +# for contract in contracts: +# if contract not in found_contracts: +# logger.warning(f"Futures contract {contract} not found in API response, using default specs") + +# logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") +# logger.debug(f"Final specs: {specs_dict}") +# return specs_dict + +# except Exception as e: +# logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True) +# return specs_dict + +# def get_spot_specs(contracts: Iterable[str]) -> Dict[str, Dict]: +# default_specs = { +# "tick_unit": 1, +# "tick_base": 10, +# "cash_per_point": 0.1, +# "qty_tick_unit": 1, +# "qty_tick_base": 10000 +# } +# specs_dict = {contract: {"specs": default_specs} for contract in contracts} +# logger.info(f"Using default specs for spot contracts: {default_specs}") +# return specs_dict + +# def get_expiration(contract: Dict) -> str: +# """Extract expiration from metadata or use default if none.""" +# if contract.get("status") == "Expired": +# return "0" # 8061 format for expired +# delivery_date = contract.get("deliveryDate") # Check if metadata provides this +# if delivery_date: +# from datetime import datetime +# dt = datetime.utcfromtimestamp(delivery_date / 1000) +# return dt.strftime("%Y-%m-%dT%H:%M") +# return PERP_EXP_STRING # Default for perpetuals + +# class BybitMdProvider(MdProvider): +# def __init__(self): +# super().__init__() +# self._config: Optional[dict] = None +# self.ws_futures = None +# # self.ws_spot = None # Commented out spot WebSocket +# self.ws_thread: Optional[threading.Thread] = None +# self.bridge: Optional[MdBridge] = None +# self.stop_event = threading.Event() + +# @property +# def config(self): +# if self._config is None: +# self._config = load_config(["py_bybit_md_provider", "bybit_overrides"]) +# return self._config + +# @property +# def use_testnet(self): +# return self.config.get("use_testnet", True) + +# @property +# def currencies(self): +# return self.config.get("currencies") or ["USDT"] + +# @property +# def futures(self): +# return self.config.get("futures", []) + +# @property +# def spots(self): +# return self.config.get("spots", []) + +# def start_impl(self, bridge: MdBridge): +# configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level +# self.bridge = bridge +# logger.info("Starting BybitMdProvider") + +# # Load futures contracts directly from JSON +# futures_contracts = self.futures +# logger.info(f"futures_contracts: {futures_contracts}") +# # spot_contracts = list(map(lambda s: s + "USDT", self.spots)) # Commented out spot contracts + +# # Fetch metadata for all specified futures contracts +# metadata = get_metadata(futures_contracts, self.use_testnet) +# logger.info(f"Fetched metadata for {len(metadata)} futures contracts") +# logger.info(f"metadata: {metadata}") + +# # Fetch specs +# futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet) +# logger.info(f"Fetched futures specs for {len(futures_specs)} contracts") +# logger.info(f"futures_specs: {futures_specs}") +# # spot_specs = get_spot_specs(spot_contracts) # Commented out spot specs + +# sec_list_futures = PySecurityList() +# # sec_list_spots = PySecurityList() # Commented out spot security list + +# for contract in futures_contracts: +# meta = metadata.get(contract, {}) +# logger.info(f"Metadata for {contract}: {meta}") +# if not meta: +# logger.warning(f"No metadata found for contract: {contract}, using defaults") +# continue + +# try: +# symbol = meta.get("symbol", contract) +# logger.info(f"Security creation for {contract}: symbol={symbol}") +# base_asset = meta.get("baseCoin") +# logger.info(f"Security creation for {contract}: base_asset from metadata={base_asset}") +# quote_currency = meta.get("quoteCoin") +# logger.info(f"Security creation for {contract}: quote_currency from metadata={quote_currency}") +# expiration = get_expiration(meta) +# logger.info(f"Security creation for {contract}: expiration={expiration}") + +# asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) +# logger.info(f"Security creation for {contract}: using asset={asset}") +# group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) +# logger.info(f"Security creation for {contract}: using group={group}") +# currency = quote_currency or "USDT" +# logger.info(f"Security creation for {contract}: using currency={currency}") + +# specs = futures_specs.get(contract, {}).get("specs", {}) +# if not specs: +# logger.warning(f"No specs found for contract: {contract}, using defaults") +# specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 1, "qty_tick_base": 10000} + +# try: +# qty_info = get_quantity_info( +# tick_unit=specs.get("qty_tick_unit", 1), +# tick_base=specs.get("qty_tick_base", 10000) +# ) +# except Exception as e: +# logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}") +# continue + +# try: +# spec = get_contract_spec( +# price_info=get_price_info( +# tick_unit=specs.get("tick_unit", 1), +# tick_base=specs.get("tick_base", 10), +# cash_per_point=specs.get("cash_per_point", 0.1) +# ), +# qty_info=qty_info +# ) +# except Exception as e: +# logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}") +# continue + +# try: +# sec = get_security( +# venue="Bybit", +# name=symbol, +# asset=asset, +# group=group, +# currency=currency, +# expiration=expiration, +# option_kind=None, +# strike=None, +# spec=spec, +# ) +# sec_list_futures.append(sec) +# logger.info(f"Created FUTURE security: name={symbol}, venue=Bybit, asset={asset}, group={group}, currency={currency}, expiration={expiration}, option_kind={None}, strike={None}, spec={specs}") +# except Exception as e: +# logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}", exc_info=True) +# continue + +# except Exception as e: +# logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True) +# continue + +# # for contract in spot_contracts: # Commented out spot security creation +# # normalized_contract = normalize_contract_name(contract) +# # try: +# # asset, quote = normalized_contract.split("_") +# # quote = "USDT" +# # except ValueError as e: +# # logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}") +# # continue +# # specs = spot_specs.get(contract, {}).get("specs", {}) +# # +# # try: +# # qty_info = get_quantity_info( +# # tick_unit=specs.get("qty_tick_unit", 1), +# # tick_base=specs.get("qty_tick_base", 10000) +# # ) +# # except Exception as e: +# # logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}") +# # continue +# # +# # try: +# # spec = get_contract_spec( +# # price_info=get_price_info( +# # tick_unit=specs.get("tick_unit", 1), +# # tick_base=specs.get("tick_base", 10), +# # cash_per_point=specs.get("cash_per_point", 0.1) +# # ), +# # qty_info=qty_info +# # ) +# # except Exception as e: +# # logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}") +# # continue +# # +# # try: +# # sec = get_security( +# # venue="Bybit", +# # name=contract, +# # asset=asset, +# # group=asset, +# # currency=quote, +# # expiration=None, +# # option_kind=None, +# # strike=None, +# # spec=spec, +# # ) +# # sec_list_spots.append(sec) +# # logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}") +# # except Exception as e: +# # logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}", exc_info=True) +# # continue + +# combined_sec_list = PySecurityList() +# for sec in sec_list_futures: +# combined_sec_list.append(sec) +# # for sec in sec_list_spots: # Commented out spot securities +# # combined_sec_list.append(sec) +# bridge.on_securities(combined_sec_list) +# logger.info(f"Sent securities list: {len(sec_list_futures)} futures, 0 spots") + +# try: +# self.ws_futures = WebSocket( +# testnet=self.use_testnet, +# channel_type="linear", +# ping_interval=20, +# ) +# logger.info("Connecting to Bybit WebSocket (futures, linear)") +# for contract in futures_contracts: +# try: +# self.ws_futures.orderbook_stream( +# symbol=contract, +# depth=50, +# callback=lambda data: self._on_book_update(data, source="futures") +# ) +# logger.info(f"Subscribed to orderbook.50.{contract} (futures)") +# except Exception as e: +# logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}") +# # self.ws_spot = WebSocket( # Commented out spot WebSocket +# # testnet=self.use_testnet, +# # channel_type="spot", +# # ping_interval=20, +# # ) +# # logger.info("Connecting to Bybit WebSocket (spot)") +# # +# # for contract in spot_contracts: +# # try: +# # self.ws_spot.orderbook_stream( +# # symbol=contract, +# # depth=50, +# # callback=lambda data: self._on_book_update(data, source="spot") +# # ) +# # logger.info(f"Subscribed to orderbook.50.{contract} (spot)") +# # except Exception as e: +# # logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}") +# logger.info("Subscribed to orderbook channels on Bybit futures WS.") + +# for t in threading.enumerate(): +# logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}") + +# while not self.stop_event.is_set(): +# time.sleep(1) + +# except KeyboardInterrupt: +# logger.info("KeyboardInterrupt received — shutting down.") +# self.stop() +# except Exception as e: +# logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True) +# self.stop() + +# def start(self, bridge: MdBridge): +# def start_wrapper(): +# try: +# self.start_impl(bridge) +# except Exception as e: +# logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True) +# self.ws_thread = threading.Thread(target=start_wrapper) +# self.ws_thread.start() + +# def _on_book_update(self, data: dict, source: str): +# try: +# logger.debug(f"Raw WebSocket data (source={source}): {data}") +# topic = data.get("topic", "") +# if not topic.startswith("orderbook.50."): +# logger.debug(f"Ignoring non-orderbook topic: {topic}") +# return +# contract = topic.split(".")[-1] +# payload = data.get("data", {}) +# raw_bids = payload.get("b", []) +# raw_asks = payload.get("a", []) +# if not raw_bids or not raw_asks: +# logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") +# raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids] +# raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks] +# sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] +# sorted_bids += [(np.float64('nan'), np.float64('nan'))] +# sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] +# sorted_asks += [(np.float64('nan'), np.float64('nan'))] +# logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") +# book = PyBook() +# market_type = "SPOT" if source == "spot" else "FUTURE" +# logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") +# book.set_data( +# contract, +# "Bybit", +# market_type, +# np.array(sorted_bids, dtype=np.float64), +# np.array(sorted_asks, dtype=np.float64), +# ) +# book_list = PyBookList() +# book_list.append(book) +# if self.bridge: +# logger.debug(f"✔ on_books called for {contract} (market_type={market_type})") +# self.bridge.on_books(book_list) +# else: +# logger.warning("Bridge is None, cannot call on_books") +# except Exception as e: +# logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) + +# def stop(self): +# logger.info("Stopping BybitMdProvider...") +# self.stop_event.set() + +# logger.debug("Threads before stopping WebSocket:") +# for t in threading.enumerate(): +# logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") + +# for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket +# # , (self.ws_spot, "spot") +# if ws: +# try: +# ws.exit() +# logger.info(f"WebSocket ({ws_type}) exit called") +# for t in threading.enumerate(): +# if t.name in ['Thread-1 ()', 'Thread-2 (_send_ping)'] and t.is_alive(): +# logger.info(f"Joining pybit thread: {t.name} ({ws_type})") +# t.join(timeout=1.0) +# except Exception as e: +# logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True) +# ws = None + +# self.ws_futures = None +# # self.ws_spot = None # Commented out spot WebSocket + +# if self.ws_thread: +# self.ws_thread.join() +# self.bridge = None +# logger.info("✔ Stopped BybitMdProvider.") + +# logger.debug("Threads after stop:") +# for t in threading.enumerate(): +# logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") + +# import os +# os._exit(0) + +# if __name__ == "__main__": +# import argparse +# from fake_md_bridge import FakeMdBridge + +# ap = argparse.ArgumentParser() +# ap.add_argument("--currencies", nargs="+") +# ap.add_argument("--futures", nargs="+") +# log_levels = list(logging._nameToLevel) +# ap.add_argument("--console-level", choices=log_levels) +# ap.add_argument("--file-level", choices=log_levels) +# ap.add_argument("--dev", action="store_true", dest="use_testnet") +# ap.add_argument("--prod", action="store_false", dest="use_testnet") +# ap.set_defaults( +# currencies=None, +# futures=None, +# console_level=None, +# file_level=None, +# use_testnet=None, +# ) +# args = ap.parse_args() +# print(args) +# logger = logging.getLogger("bybit_md_provider_app") +# md_pro = BybitMdProvider() +# config = md_pro.config +# if args.currencies: +# config["currencies"] = args.currencies +# if args.futures: +# config["futures"] = args.futures +# logging_overrides = {} +# if args.console-level: +# logging_overrides["console_level"] = args.console_level +# if args.file-level: +# logging_overrides["file_level"] = args.file_level +# if logging_overrides: +# config["logging"].update(logging_overrides) +# if args.use_testnet is not None: +# config["use_testnet"] = args.use_testnet +# md_bridge = FakeMdBridge(logger) +# md_pro.start(md_bridge) +# while True: +# time.sleep(1) + + import json import numpy as np import time @@ -174,6 +715,31 @@ def get_expiration(contract: Dict) -> str: return dt.strftime("%Y-%m-%dT%H:%M") return PERP_EXP_STRING # Default for perpetuals +def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: + """Query Bybit API for linear instruments and return full security names for given base coins.""" + futures_contracts = [] + host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" + url = f"https://{host}/v5/market/instruments-info" + + logger.info(f"Querying futures contracts for base symbols: {base_symbols}") + for symbol in base_symbols: + params = { + "category": "linear", + "baseCoin": symbol + } + logger.info(f"Querying for baseCoin={symbol}: {url}") + response = requests.get(url, params=params, timeout=5) + response.raise_for_status() + data = response.json() + if data.get("retCode") == 0 and data.get("result", {}).get("list"): + for item in data.get("result", {}).get("list", []): + futures_contracts.append(item.get("symbol")) + logger.info(f"Found {len(data['result']['list'])} instruments for {symbol}") + else: + logger.warning(f"No instruments found for baseCoin={symbol}: {data.get('retMsg', 'Unknown error')}") + + return sorted(list(set(futures_contracts))) # Remove duplicates and sort + class BybitMdProvider(MdProvider): def __init__(self): super().__init__() @@ -211,8 +777,8 @@ class BybitMdProvider(MdProvider): self.bridge = bridge logger.info("Starting BybitMdProvider") - # Load futures contracts directly from JSON - futures_contracts = self.futures + # Load futures contracts dynamically from API based on base symbols + futures_contracts = get_futures_contracts(self.futures, self.use_testnet) logger.info(f"futures_contracts: {futures_contracts}") # spot_contracts = list(map(lambda s: s + "USDT", self.spots)) # Commented out spot contracts @@ -232,12 +798,13 @@ class BybitMdProvider(MdProvider): for contract in futures_contracts: meta = metadata.get(contract, {}) - logger.info(f"Metadata for {contract}: {meta}") + logger.info(f"Metadata for {contract}: {meta}") # Log metadata to verify contents if not meta: logger.warning(f"No metadata found for contract: {contract}, using defaults") continue try: + # Extract details from metadata symbol = meta.get("symbol", contract) logger.info(f"Security creation for {contract}: symbol={symbol}") base_asset = meta.get("baseCoin") @@ -247,6 +814,7 @@ class BybitMdProvider(MdProvider): expiration = get_expiration(meta) logger.info(f"Security creation for {contract}: expiration={expiration}") + # Use fallbacks with logging asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) logger.info(f"Security creation for {contract}: using asset={asset}") group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) @@ -396,13 +964,10 @@ class BybitMdProvider(MdProvider): # except Exception as e: # logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}") logger.info("Subscribed to orderbook channels on Bybit futures WS.") - for t in threading.enumerate(): logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}") - while not self.stop_event.is_set(): time.sleep(1) - except KeyboardInterrupt: logger.info("KeyboardInterrupt received — shutting down.") self.stop() @@ -525,9 +1090,9 @@ if __name__ == "__main__": if args.futures: config["futures"] = args.futures logging_overrides = {} - if args.console-level: + if args.console_level: logging_overrides["console_level"] = args.console_level - if args.file-level: + if args.file_level: logging_overrides["file_level"] = args.file_level if logging_overrides: config["logging"].update(logging_overrides)