From 0cb3cadc113058e04f480936fa4f77d480dbef42 Mon Sep 17 00:00:00 2001 From: Adeel Khan Date: Tue, 8 Jul 2025 14:16:15 +0000 Subject: [PATCH] FIX: Added filter to exclude InverseFutures--MD not freezing. --- .../bybit_md_provider.py | 549 +----------------- 1 file changed, 5 insertions(+), 544 deletions(-) diff --git a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index ac4d16afe..546df1792 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -1,544 +1,3 @@ -# import json -# import numpy as np -# import time -# import logging -# import sys -# import threading -# from itertools import chain -# from typing import Iterable, Optional, Dict -# from pybit.unified_trading import WebSocket -# from python_interop_module import ( -# MdBridge, -# MdProvider, -# PyBook, -# PyBookList, -# PySecurityList, -# ) -# from spark_core import ( -# configure_logger, -# get_contract_spec, -# get_price_info, -# get_quantity_info, -# get_security, -# load_config, -# ) -# from py_md_utils import PERP_EXP_STRING -# import requests - -# logger = logging.getLogger("bybit_md_provider") - -# def normalize_contract_name(contract: str) -> str: -# logger.info(f"Normalizing contract: input={contract}") -# if "-" in contract: # Preserve delivery futures (e.g., BTC-26SEP25) -# logger.info(f"Normalizing contract: output={contract} (unchanged, delivery future)") -# return contract -# if contract.endswith("PERP"): -# result = contract[:-4] + "_PERP" -# logger.info(f"Normalizing contract: output={result}") -# return result -# if len(contract) > 4: -# result = contract[:-4] + "_" + contract[-4:] -# logger.info(f"Normalizing contract: output={result}") -# return result -# logger.info(f"Normalizing contract: output={contract} (unchanged)") -# return contract - -# def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: -# """Fetch metadata for all linear contracts and filter for specified contracts.""" -# metadata_dict = {} -# try: -# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" -# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000" -# logger.info(f"Querying Bybit API for metadata: {url}") -# response = requests.get(url, timeout=5) -# response.raise_for_status() -# data = response.json() - -# if data.get("retCode") == 0: -# contracts_data = data.get("result", {}).get("list", []) -# logger.info(f"Retrieved metadata for {len(contracts_data)} linear contracts") -# for item in contracts_data: -# symbol = item.get("symbol") -# if symbol in contracts: -# metadata_dict[symbol] = item -# logger.info(f"Stored metadata for {symbol}") -# else: -# logger.error(f"Metadata API error: {data.get('retMsg', 'Unknown error')}") -# except Exception as e: -# logger.error(f"Failed to fetch metadata: {e}", exc_info=True) -# return metadata_dict - -# def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: -# default_specs = { -# "tick_unit": 1, -# "tick_base": 10, -# "cash_per_point": 1, -# "qty_tick_unit": 1, -# "qty_tick_base": 10000 -# } -# specs_dict = {contract: {"specs": default_specs} for contract in contracts} -# try: -# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" -# base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500" -# logger.info(f"Querying Bybit API for futures specs: {base_url}") - -# found_contracts = set() -# cursor = "" - -# while True: -# url = f"{base_url}&cursor={cursor}" if cursor else base_url -# logger.info(f"Fetching page with cursor: {cursor or 'initial'}") -# response = requests.get(url, timeout=5) -# response.raise_for_status() -# data = response.json() - -# if data.get("retCode") != 0: -# logger.error(f"Futures API error: {data.get('retMsg', 'Unknown error')}") -# return specs_dict - -# result = data.get("result", {}) -# symbols = result.get("list", []) -# logger.info(f"Loaded page with cursor: {cursor or 'initial'}, symbols: {len(symbols)}") - -# for item in symbols: -# contract = item.get("symbol") -# if contract in contracts: -# try: -# price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1)) -# lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001)) - -# if price_tick == 0 or lot_size == 0: -# logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}") -# continue - -# tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 -# if lot_size >= 1: -# qty_tick_unit = lot_size -# qty_tick_base = 1 -# else: -# qty_tick_unit = 1 -# qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 - -# specs_dict[contract] = { -# "specs": { -# "tick_unit": 1, -# "tick_base": tick_base, -# "cash_per_point": 1, -# "qty_tick_unit": qty_tick_unit, -# "qty_tick_base": qty_tick_base -# } -# } -# found_contracts.add(contract) -# logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}") -# except (ValueError, TypeError) as e: -# logger.error(f"Failed to parse futures specs for {contract}: {e}") -# continue - -# cursor = result.get("nextPageCursor", "") -# if not cursor: -# logger.info("No more pages to fetch; pagination complete") -# break - -# for contract in contracts: -# if contract not in found_contracts: -# logger.warning(f"Futures contract {contract} not found in API response, using default specs") - -# logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") -# logger.debug(f"Final specs: {specs_dict}") -# return specs_dict - -# except Exception as e: -# logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True) -# return specs_dict - -# def get_spot_specs(contracts: Iterable[str]) -> Dict[str, Dict]: -# default_specs = { -# "tick_unit": 1, -# "tick_base": 10, -# "cash_per_point": 0.1, -# "qty_tick_unit": 1, -# "qty_tick_base": 10000 -# } -# specs_dict = {contract: {"specs": default_specs} for contract in contracts} -# logger.info(f"Using default specs for spot contracts: {default_specs}") -# return specs_dict - -# def get_expiration(contract: Dict) -> str: -# """Extract expiration from metadata or use default if none.""" -# if contract.get("status") == "Expired": -# return "0" # 8061 format for expired -# delivery_date = contract.get("deliveryDate") # Check if metadata provides this -# if delivery_date: -# from datetime import datetime -# dt = datetime.utcfromtimestamp(delivery_date / 1000) -# return dt.strftime("%Y-%m-%dT%H:%M") -# return PERP_EXP_STRING # Default for perpetuals - -# class BybitMdProvider(MdProvider): -# def __init__(self): -# super().__init__() -# self._config: Optional[dict] = None -# self.ws_futures = None -# # self.ws_spot = None # Commented out spot WebSocket -# self.ws_thread: Optional[threading.Thread] = None -# self.bridge: Optional[MdBridge] = None -# self.stop_event = threading.Event() - -# @property -# def config(self): -# if self._config is None: -# self._config = load_config(["py_bybit_md_provider", "bybit_overrides"]) -# return self._config - -# @property -# def use_testnet(self): -# return self.config.get("use_testnet", True) - -# @property -# def currencies(self): -# return self.config.get("currencies") or ["USDT"] - -# @property -# def futures(self): -# return self.config.get("futures", []) - -# @property -# def spots(self): -# return self.config.get("spots", []) - -# def start_impl(self, bridge: MdBridge): -# configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level -# self.bridge = bridge -# logger.info("Starting BybitMdProvider") - -# # Load futures contracts directly from JSON -# futures_contracts = self.futures -# logger.info(f"futures_contracts: {futures_contracts}") -# # spot_contracts = list(map(lambda s: s + "USDT", self.spots)) # Commented out spot contracts - -# # Fetch metadata for all specified futures contracts -# metadata = get_metadata(futures_contracts, self.use_testnet) -# logger.info(f"Fetched metadata for {len(metadata)} futures contracts") -# logger.info(f"metadata: {metadata}") - -# # Fetch specs -# futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet) -# logger.info(f"Fetched futures specs for {len(futures_specs)} contracts") -# logger.info(f"futures_specs: {futures_specs}") -# # spot_specs = get_spot_specs(spot_contracts) # Commented out spot specs - -# sec_list_futures = PySecurityList() -# # sec_list_spots = PySecurityList() # Commented out spot security list - -# for contract in futures_contracts: -# meta = metadata.get(contract, {}) -# logger.info(f"Metadata for {contract}: {meta}") -# if not meta: -# logger.warning(f"No metadata found for contract: {contract}, using defaults") -# continue - -# try: -# symbol = meta.get("symbol", contract) -# logger.info(f"Security creation for {contract}: symbol={symbol}") -# base_asset = meta.get("baseCoin") -# logger.info(f"Security creation for {contract}: base_asset from metadata={base_asset}") -# quote_currency = meta.get("quoteCoin") -# logger.info(f"Security creation for {contract}: quote_currency from metadata={quote_currency}") -# expiration = get_expiration(meta) -# logger.info(f"Security creation for {contract}: expiration={expiration}") - -# asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) -# logger.info(f"Security creation for {contract}: using asset={asset}") -# group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) -# logger.info(f"Security creation for {contract}: using group={group}") -# currency = quote_currency or "USDT" -# logger.info(f"Security creation for {contract}: using currency={currency}") - -# specs = futures_specs.get(contract, {}).get("specs", {}) -# if not specs: -# logger.warning(f"No specs found for contract: {contract}, using defaults") -# specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 1, "qty_tick_base": 10000} - -# try: -# qty_info = get_quantity_info( -# tick_unit=specs.get("qty_tick_unit", 1), -# tick_base=specs.get("qty_tick_base", 10000) -# ) -# except Exception as e: -# logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}") -# continue - -# try: -# spec = get_contract_spec( -# price_info=get_price_info( -# tick_unit=specs.get("tick_unit", 1), -# tick_base=specs.get("tick_base", 10), -# cash_per_point=specs.get("cash_per_point", 0.1) -# ), -# qty_info=qty_info -# ) -# except Exception as e: -# logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}") -# continue - -# try: -# sec = get_security( -# venue="Bybit", -# name=symbol, -# asset=asset, -# group=group, -# currency=currency, -# expiration=expiration, -# option_kind=None, -# strike=None, -# spec=spec, -# ) -# sec_list_futures.append(sec) -# logger.info(f"Created FUTURE security: name={symbol}, venue=Bybit, asset={asset}, group={group}, currency={currency}, expiration={expiration}, option_kind={None}, strike={None}, spec={specs}") -# except Exception as e: -# logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}", exc_info=True) -# continue - -# except Exception as e: -# logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True) -# continue - -# # for contract in spot_contracts: # Commented out spot security creation -# # normalized_contract = normalize_contract_name(contract) -# # try: -# # asset, quote = normalized_contract.split("_") -# # quote = "USDT" -# # except ValueError as e: -# # logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}") -# # continue -# # specs = spot_specs.get(contract, {}).get("specs", {}) -# # -# # try: -# # qty_info = get_quantity_info( -# # tick_unit=specs.get("qty_tick_unit", 1), -# # tick_base=specs.get("qty_tick_base", 10000) -# # ) -# # except Exception as e: -# # logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}") -# # continue -# # -# # try: -# # spec = get_contract_spec( -# # price_info=get_price_info( -# # tick_unit=specs.get("tick_unit", 1), -# # tick_base=specs.get("tick_base", 10), -# # cash_per_point=specs.get("cash_per_point", 0.1) -# # ), -# # qty_info=qty_info -# # ) -# # except Exception as e: -# # logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}") -# # continue -# # -# # try: -# # sec = get_security( -# # venue="Bybit", -# # name=contract, -# # asset=asset, -# # group=asset, -# # currency=quote, -# # expiration=None, -# # option_kind=None, -# # strike=None, -# # spec=spec, -# # ) -# # sec_list_spots.append(sec) -# # logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}") -# # except Exception as e: -# # logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}", exc_info=True) -# # continue - -# combined_sec_list = PySecurityList() -# for sec in sec_list_futures: -# combined_sec_list.append(sec) -# # for sec in sec_list_spots: # Commented out spot securities -# # combined_sec_list.append(sec) -# bridge.on_securities(combined_sec_list) -# logger.info(f"Sent securities list: {len(sec_list_futures)} futures, 0 spots") - -# try: -# self.ws_futures = WebSocket( -# testnet=self.use_testnet, -# channel_type="linear", -# ping_interval=20, -# ) -# logger.info("Connecting to Bybit WebSocket (futures, linear)") -# for contract in futures_contracts: -# try: -# self.ws_futures.orderbook_stream( -# symbol=contract, -# depth=50, -# callback=lambda data: self._on_book_update(data, source="futures") -# ) -# logger.info(f"Subscribed to orderbook.50.{contract} (futures)") -# except Exception as e: -# logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}") -# # self.ws_spot = WebSocket( # Commented out spot WebSocket -# # testnet=self.use_testnet, -# # channel_type="spot", -# # ping_interval=20, -# # ) -# # logger.info("Connecting to Bybit WebSocket (spot)") -# # -# # for contract in spot_contracts: -# # try: -# # self.ws_spot.orderbook_stream( -# # symbol=contract, -# # depth=50, -# # callback=lambda data: self._on_book_update(data, source="spot") -# # ) -# # logger.info(f"Subscribed to orderbook.50.{contract} (spot)") -# # except Exception as e: -# # logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}") -# logger.info("Subscribed to orderbook channels on Bybit futures WS.") - -# for t in threading.enumerate(): -# logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}") - -# while not self.stop_event.is_set(): -# time.sleep(1) - -# except KeyboardInterrupt: -# logger.info("KeyboardInterrupt received — shutting down.") -# self.stop() -# except Exception as e: -# logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True) -# self.stop() - -# def start(self, bridge: MdBridge): -# def start_wrapper(): -# try: -# self.start_impl(bridge) -# except Exception as e: -# logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True) -# self.ws_thread = threading.Thread(target=start_wrapper) -# self.ws_thread.start() - -# def _on_book_update(self, data: dict, source: str): -# try: -# logger.debug(f"Raw WebSocket data (source={source}): {data}") -# topic = data.get("topic", "") -# if not topic.startswith("orderbook.50."): -# logger.debug(f"Ignoring non-orderbook topic: {topic}") -# return -# contract = topic.split(".")[-1] -# payload = data.get("data", {}) -# raw_bids = payload.get("b", []) -# raw_asks = payload.get("a", []) -# if not raw_bids or not raw_asks: -# logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") -# raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids] -# raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks] -# sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] -# sorted_bids += [(np.float64('nan'), np.float64('nan'))] -# sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] -# sorted_asks += [(np.float64('nan'), np.float64('nan'))] -# logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") -# book = PyBook() -# market_type = "SPOT" if source == "spot" else "FUTURE" -# logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") -# book.set_data( -# contract, -# "Bybit", -# market_type, -# np.array(sorted_bids, dtype=np.float64), -# np.array(sorted_asks, dtype=np.float64), -# ) -# book_list = PyBookList() -# book_list.append(book) -# if self.bridge: -# logger.debug(f"✔ on_books called for {contract} (market_type={market_type})") -# self.bridge.on_books(book_list) -# else: -# logger.warning("Bridge is None, cannot call on_books") -# except Exception as e: -# logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) - -# def stop(self): -# logger.info("Stopping BybitMdProvider...") -# self.stop_event.set() - -# logger.debug("Threads before stopping WebSocket:") -# for t in threading.enumerate(): -# logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") - -# for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket -# # , (self.ws_spot, "spot") -# if ws: -# try: -# ws.exit() -# logger.info(f"WebSocket ({ws_type}) exit called") -# for t in threading.enumerate(): -# if t.name in ['Thread-1 ()', 'Thread-2 (_send_ping)'] and t.is_alive(): -# logger.info(f"Joining pybit thread: {t.name} ({ws_type})") -# t.join(timeout=1.0) -# except Exception as e: -# logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True) -# ws = None - -# self.ws_futures = None -# # self.ws_spot = None # Commented out spot WebSocket - -# if self.ws_thread: -# self.ws_thread.join() -# self.bridge = None -# logger.info("✔ Stopped BybitMdProvider.") - -# logger.debug("Threads after stop:") -# for t in threading.enumerate(): -# logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") - -# import os -# os._exit(0) - -# if __name__ == "__main__": -# import argparse -# from fake_md_bridge import FakeMdBridge - -# ap = argparse.ArgumentParser() -# ap.add_argument("--currencies", nargs="+") -# ap.add_argument("--futures", nargs="+") -# log_levels = list(logging._nameToLevel) -# ap.add_argument("--console-level", choices=log_levels) -# ap.add_argument("--file-level", choices=log_levels) -# ap.add_argument("--dev", action="store_true", dest="use_testnet") -# ap.add_argument("--prod", action="store_false", dest="use_testnet") -# ap.set_defaults( -# currencies=None, -# futures=None, -# console_level=None, -# file_level=None, -# use_testnet=None, -# ) -# args = ap.parse_args() -# print(args) -# logger = logging.getLogger("bybit_md_provider_app") -# md_pro = BybitMdProvider() -# config = md_pro.config -# if args.currencies: -# config["currencies"] = args.currencies -# if args.futures: -# config["futures"] = args.futures -# logging_overrides = {} -# if args.console-level: -# logging_overrides["console_level"] = args.console_level -# if args.file-level: -# logging_overrides["file_level"] = args.file_level -# if logging_overrides: -# config["logging"].update(logging_overrides) -# if args.use_testnet is not None: -# config["use_testnet"] = args.use_testnet -# md_bridge = FakeMdBridge(logger) -# md_pro.start(md_bridge) -# while True: -# time.sleep(1) - - import json import numpy as np import time @@ -716,7 +175,7 @@ def get_expiration(contract: Dict) -> str: return PERP_EXP_STRING # Default for perpetuals def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: - """Query Bybit API for linear instruments and return full security names for given base coins.""" + """Query Bybit API for linear instruments and return full security names for given base coins, filtering for LinearFutures and LinearPerpetual.""" futures_contracts = [] host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" url = f"https://{host}/v5/market/instruments-info" @@ -733,8 +192,10 @@ def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: data = response.json() if data.get("retCode") == 0 and data.get("result", {}).get("list"): for item in data.get("result", {}).get("list", []): - futures_contracts.append(item.get("symbol")) - logger.info(f"Found {len(data['result']['list'])} instruments for {symbol}") + contract_type = item.get("contractType") + if contract_type in ["LinearFutures", "LinearPerpetual"]: + futures_contracts.append(item.get("symbol")) + logger.info(f"Found {len(futures_contracts)} compatible instruments for {symbol}") else: logger.warning(f"No instruments found for baseCoin={symbol}: {data.get('retMsg', 'Unknown error')}")