diff --git a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index 7b3126057..a6d6f9295 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -1,450 +1,3 @@ -# import json -# import numpy as np -# import time -# import logging -# import sys -# import threading -# from itertools import chain -# from typing import Iterable, Optional, Dict -# from pybit.unified_trading import WebSocket -# from python_interop_module import ( -# MdBridge, -# MdProvider, -# PyBook, -# PyBookList, -# PySecurityList, -# ) -# from spark_core import ( -# configure_logger, -# get_contract_spec, -# get_price_info, -# get_quantity_info, -# get_security, -# load_config, -# ) -# from py_md_utils import PERP_EXP_STRING -# import requests - -# logger = logging.getLogger("bybit_md_provider") - -# def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: -# """Fetch metadata for all linear contracts and filter for specified contracts.""" -# metadata_dict = {} -# try: -# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" -# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000" -# logger.info(f"Querying Bybit API for metadata: {url}") -# response = requests.get(url, timeout=5) -# response.raise_for_status() -# data = response.json() - -# if data.get("retCode") == 0: -# contracts_data = data.get("result", {}).get("list", []) -# logger.info(f"Retrieved metadata for {len(contracts_data)} linear contracts") -# for item in contracts_data: -# symbol = item.get("symbol") -# if symbol in contracts: -# metadata_dict[symbol] = item -# logger.info(f"Stored metadata for {symbol}") -# else: -# logger.error(f"Metadata API error: {data.get('retMsg', 'Unknown error')}") -# except Exception as e: -# logger.error(f"Failed to fetch metadata: {e}", exc_info=True) -# return metadata_dict - -# def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: -# default_specs = { -# "tick_unit": 1, -# "tick_base": 10, -# "cash_per_point": 1, -# "qty_tick_unit": 1, -# "qty_tick_base": 10000 -# } -# specs_dict = {contract: {"specs": default_specs} for contract in contracts} -# try: -# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" -# base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500" -# logger.info(f"Querying Bybit API for futures specs: {base_url}") - -# found_contracts = set() -# cursor = "" - -# while True: -# url = f"{base_url}&cursor={cursor}" if cursor else base_url -# logger.info(f"Fetching page with cursor: {cursor or 'initial'}") -# response = requests.get(url, timeout=5) -# response.raise_for_status() -# data = response.json() - -# if data.get("retCode") != 0: -# logger.error(f"Futures API error: {data.get('retMsg', 'Unknown error')}") -# return specs_dict - -# result = data.get("result", {}) -# symbols = result.get("list", []) -# logger.info(f"Loaded page with cursor: {cursor or 'initial'}, symbols: {len(symbols)}") - -# for item in symbols: -# contract = item.get("symbol") -# if contract in contracts: -# try: -# price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1)) -# lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001)) - -# if price_tick == 0 or lot_size == 0: -# logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}") -# continue - -# tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 -# if lot_size >= 1: -# qty_tick_unit = lot_size -# qty_tick_base = 1 -# else: -# qty_tick_unit = 1 -# qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 - -# specs_dict[contract] = { -# "specs": { -# "tick_unit": 1, -# "tick_base": tick_base, -# "cash_per_point": 1, -# "qty_tick_unit": qty_tick_unit, -# "qty_tick_base": qty_tick_base -# } -# } -# found_contracts.add(contract) -# logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}") -# except (ValueError, TypeError) as e: -# logger.error(f"Failed to parse futures specs for {contract}: {e}") -# continue - -# cursor = result.get("nextPageCursor", "") -# if not cursor: -# logger.info("No more pages to fetch; pagination complete") -# break - -# for contract in contracts: -# if contract not in found_contracts: -# logger.warning(f"Futures contract {contract} not found in API response, using default specs") - -# logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") -# logger.debug(f"Final specs: {specs_dict}") -# return specs_dict - -# except Exception as e: -# logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True) -# return specs_dict - -# def get_expiration(contract: Dict) -> str: -# """Extract expiration from metadata or use default if none.""" -# if contract.get("status") == "Expired": -# return "0" # 8061 format for expired -# delivery_date = contract.get("deliveryDate") # Check if metadata provides this -# if delivery_date: -# from datetime import datetime -# dt = datetime.utcfromtimestamp(delivery_date / 1000) -# return dt.strftime("%Y-%m-%dT%H:%M") -# return PERP_EXP_STRING # Default for perpetuals - -# def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: -# """Query Bybit API for all linear instruments and filter by base symbols and contract type.""" -# futures_contracts = [] -# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" -# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000" - -# logger.info(f"Querying all linear instruments: {url}") -# response = requests.get(url, timeout=5) -# response.raise_for_status() -# data = response.json() -# if data.get("retCode") == 0 and data.get("result", {}).get("list"): -# logger.info(f"Retrieved {len(data['result']['list'])} linear instruments") -# for item in data.get("result", {}).get("list", []): -# base_coin = item.get("baseCoin") -# contract_type = item.get("contractType") -# if base_coin in base_symbols and contract_type in ["LinearFutures", "LinearPerpetual"]: -# futures_contracts.append(item.get("symbol")) -# logger.info(f"Filtered to {len(futures_contracts)} compatible instruments for base symbols {base_symbols}") -# else: -# logger.warning(f"Failed to retrieve instruments: {data.get('retMsg', 'Unknown error')}") - -# return sorted(list(set(futures_contracts))) # Remove duplicates and sort - -# class BybitMdProvider(MdProvider): -# def __init__(self): -# super().__init__() -# self._config: Optional[dict] = None -# self.ws_futures = None -# self.ws_thread: Optional[threading.Thread] = None -# self.bridge: Optional[MdBridge] = None -# self.stop_event = threading.Event() - -# @property -# def config(self): -# if self._config is None: -# self._config = load_config(["py_bybit_md_provider", "bybit_overrides"]) -# return self._config - -# @property -# def use_testnet(self): -# return self.config.get("use_testnet", True) - -# @property -# def currencies(self): -# return self.config.get("currencies") or ["USDT"] - -# @property -# def futures(self): -# return self.config.get("futures", []) - -# def start_impl(self, bridge: MdBridge): -# configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level -# self.bridge = bridge -# logger.info("Starting BybitMdProvider") - -# # Load futures contracts dynamically from API based on base symbols -# futures_contracts = get_futures_contracts(self.futures, self.use_testnet) -# logger.info(f"futures_contracts: {futures_contracts}") - -# # Fetch metadata for all specified futures contracts -# metadata = get_metadata(futures_contracts, self.use_testnet) -# logger.info(f"Fetched metadata for {len(metadata)} futures contracts") -# logger.info(f"metadata: {metadata}") - -# # Fetch specs -# futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet) -# logger.info(f"Fetched futures specs for {len(futures_specs)} contracts") -# logger.info(f"futures_specs: {futures_specs}") - -# sec_list_futures = PySecurityList() - -# for contract in futures_contracts: -# meta = metadata.get(contract, {}) -# logger.info(f"Metadata for {contract}: {meta}") # Log metadata to verify contents -# if not meta: -# logger.warning(f"No metadata found for contract: {contract}, using defaults") -# continue - -# try: -# # Extract details from metadata -# symbol = meta.get("symbol", contract) -# logger.info(f"Security creation for {contract}: symbol={symbol}") -# base_asset = meta.get("baseCoin") -# logger.info(f"Security creation for {contract}: base_asset from metadata={base_asset}") -# quote_currency = meta.get("quoteCoin") -# logger.info(f"Security creation for {contract}: quote_currency from metadata={quote_currency}") -# expiration = get_expiration(meta) -# logger.info(f"Security creation for {contract}: expiration={expiration}") - -# # Use fallbacks with logging -# asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) -# logger.info(f"Security creation for {contract}: using asset={asset}") -# group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) -# logger.info(f"Security creation for {contract}: using group={group}") -# currency = quote_currency or "USDT" -# logger.info(f"Security creation for {contract}: using currency={currency}") - -# specs = futures_specs.get(contract, {}).get("specs", {}) -# if not specs: -# logger.warning(f"No specs found for contract: {contract}, using defaults") -# specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 1, "qty_tick_unit": 1, "qty_tick_base": 10000} - -# try: -# qty_info = get_quantity_info( -# tick_unit=specs.get("qty_tick_unit", 1), -# tick_base=specs.get("qty_tick_base", 10000) -# ) -# except Exception as e: -# logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}") -# continue - -# try: -# spec = get_contract_spec( -# price_info=get_price_info( -# tick_unit=specs.get("tick_unit", 1), -# tick_base=specs.get("tick_base", 10), -# cash_per_point=specs.get("cash_per_point", 1) -# ), -# qty_info=qty_info -# ) -# except Exception as e: -# logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}") -# continue - -# try: -# sec = get_security( -# venue="Bybit", -# name=symbol, -# asset=asset, -# group=group, -# currency=currency, -# expiration=expiration, -# option_kind=None, -# strike=None, -# spec=spec, -# ) -# sec_list_futures.append(sec) -# logger.info(f"Created FUTURE security: name={symbol}, venue=Bybit, asset={asset}, group={group}, currency={currency}, expiration={expiration}, option_kind={None}, strike={None}, spec={specs}") -# except Exception as e: -# logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}", exc_info=True) -# continue - -# except Exception as e: -# logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True) -# continue - -# combined_sec_list = PySecurityList() -# for sec in sec_list_futures: -# combined_sec_list.append(sec) -# bridge.on_securities(combined_sec_list) -# logger.info(f"Sent securities list: {len(sec_list_futures)} futures") - -# try: -# self.ws_futures = WebSocket( -# testnet=self.use_testnet, -# channel_type="linear", -# ping_interval=20, -# ) -# logger.info("Connecting to Bybit WebSocket (futures, linear)") -# for contract in futures_contracts: -# try: -# self.ws_futures.orderbook_stream( -# symbol=contract, -# depth=50, -# callback=lambda data: self._on_book_update(data, source="futures") -# ) -# logger.info(f"Subscribed to orderbook.50.{contract} (futures)") -# except Exception as e: -# logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}") -# logger.info("Subscribed to orderbook channels on Bybit futures WS.") -# for t in threading.enumerate(): -# logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}") -# while not self.stop_event.is_set(): -# time.sleep(1) -# except KeyboardInterrupt: -# logger.info("KeyboardInterrupt received — shutting down.") -# self.stop() -# except Exception as e: -# logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True) -# self.stop() - -# def start(self, bridge: MdBridge): -# def start_wrapper(): -# try: -# self.start_impl(bridge) -# except Exception as e: -# logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True) -# self.ws_thread = threading.Thread(target=start_wrapper) -# self.ws_thread.start() - -# def _on_book_update(self, data: dict, source: str): -# try: -# logger.debug(f"Raw WebSocket data (source={source}): {data}") -# topic = data.get("topic", "") -# if not topic.startswith("orderbook.50."): -# logger.debug(f"Ignoring non-orderbook topic: {topic}") -# return -# contract = topic.split(".")[-1] -# payload = data.get("data", {}) -# raw_bids = payload.get("b", []) -# raw_asks = payload.get("a", []) -# if not raw_bids or not raw_asks: -# logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") -# raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids] -# raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks] -# sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] -# sorted_bids += [(np.float64('nan'), np.float64('nan'))] -# sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] -# sorted_asks += [(np.float64('nan'), np.float64('nan'))] -# logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") -# book = PyBook() -# market_type = "FUTURE" # Only futures now -# logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") -# book.set_data( -# contract, -# "Bybit", -# market_type, -# np.array(sorted_bids, dtype=np.float64), -# np.array(sorted_asks, dtype=np.float64), -# ) -# book_list = PyBookList() -# book_list.append(book) -# if self.bridge: -# logger.debug(f"✔ on_books called for {contract} (market_type={market_type})") -# self.bridge.on_books(book_list) -# else: -# logger.warning("Bridge is None, cannot call on_books") -# except Exception as e: -# logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) - -# def stop(self): -# logger.info("Stopping BybitMdProvider...") -# self.stop_event.set() -# logger.debug("Threads before stopping WebSocket:") -# for t in threading.enumerate(): -# logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") -# if self.ws_futures: -# try: -# self.ws_futures.exit() -# logger.info("WebSocket (futures) exit called") -# for t in threading.enumerate(): -# if t.name in ['Thread-1 ()', 'Thread-2 (_send_ping)'] and t.is_alive(): -# logger.info(f"Joining pybit thread: {t.name} (futures)") -# t.join(timeout=1.0) -# except Exception as e: -# logger.error(f"Error while closing WebSocket (futures): {e}", exc_info=True) -# self.ws_futures = None -# if self.ws_thread: -# self.ws_thread.join() -# self.bridge = None -# logger.info("✔ Stopped BybitMdProvider.") -# logger.debug("Threads after stop:") -# for t in threading.enumerate(): -# logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") -# import os -# os._exit(0) - -# if __name__ == "__main__": -# import argparse -# from fake_md_bridge import FakeMdBridge - -# ap = argparse.ArgumentParser() -# ap.add_argument("--currencies", nargs="+") -# ap.add_argument("--futures", nargs="+") -# log_levels = list(logging._nameToLevel) -# ap.add_argument("--console-level", choices=log_levels) -# ap.add_argument("--file-level", choices=log_levels) -# ap.add_argument("--dev", action="store_true", dest="use_testnet") -# ap.add_argument("--prod", action="store_false", dest="use_testnet") -# ap.set_defaults( -# currencies=None, -# futures=None, -# console_level=None, -# file_level=None, -# use_testnet=None, -# ) -# args = ap.parse_args() -# print(args) -# logger = logging.getLogger("bybit_md_provider_app") -# md_pro = BybitMdProvider() -# config = md_pro.config -# if args.currencies: -# config["currencies"] = args.currencies -# if args.futures: -# config["futures"] = args.futures -# logging_overrides = {} -# if args.console_level: -# logging_overrides["console_level"] = args.console_level -# if args.file_level: -# logging_overrides["file_level"] = args.file_level -# if logging_overrides: -# config["logging"].update(logging_overrides) -# if args.use_testnet is not None: -# config["use_testnet"] = args.use_testnet -# md_bridge = FakeMdBridge(logger) -# md_pro.start(md_bridge) -# while True: -# time.sleep(1) - import json import numpy as np import time