diff --git a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index 750854cb3..7b3126057 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -1,3 +1,450 @@ +# import json +# import numpy as np +# import time +# import logging +# import sys +# import threading +# from itertools import chain +# from typing import Iterable, Optional, Dict +# from pybit.unified_trading import WebSocket +# from python_interop_module import ( +# MdBridge, +# MdProvider, +# PyBook, +# PyBookList, +# PySecurityList, +# ) +# from spark_core import ( +# configure_logger, +# get_contract_spec, +# get_price_info, +# get_quantity_info, +# get_security, +# load_config, +# ) +# from py_md_utils import PERP_EXP_STRING +# import requests + +# logger = logging.getLogger("bybit_md_provider") + +# def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: +# """Fetch metadata for all linear contracts and filter for specified contracts.""" +# metadata_dict = {} +# try: +# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" +# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000" +# logger.info(f"Querying Bybit API for metadata: {url}") +# response = requests.get(url, timeout=5) +# response.raise_for_status() +# data = response.json() + +# if data.get("retCode") == 0: +# contracts_data = data.get("result", {}).get("list", []) +# logger.info(f"Retrieved metadata for {len(contracts_data)} linear contracts") +# for item in contracts_data: +# symbol = item.get("symbol") +# if symbol in contracts: +# metadata_dict[symbol] = item +# logger.info(f"Stored metadata for {symbol}") +# else: +# logger.error(f"Metadata API error: {data.get('retMsg', 'Unknown error')}") +# except Exception as e: +# logger.error(f"Failed to fetch metadata: {e}", exc_info=True) +# return metadata_dict + +# def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: +# default_specs = { +# "tick_unit": 1, +# "tick_base": 10, +# "cash_per_point": 1, +# "qty_tick_unit": 1, +# "qty_tick_base": 10000 +# } +# specs_dict = {contract: {"specs": default_specs} for contract in contracts} +# try: +# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" +# base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500" +# logger.info(f"Querying Bybit API for futures specs: {base_url}") + +# found_contracts = set() +# cursor = "" + +# while True: +# url = f"{base_url}&cursor={cursor}" if cursor else base_url +# logger.info(f"Fetching page with cursor: {cursor or 'initial'}") +# response = requests.get(url, timeout=5) +# response.raise_for_status() +# data = response.json() + +# if data.get("retCode") != 0: +# logger.error(f"Futures API error: {data.get('retMsg', 'Unknown error')}") +# return specs_dict + +# result = data.get("result", {}) +# symbols = result.get("list", []) +# logger.info(f"Loaded page with cursor: {cursor or 'initial'}, symbols: {len(symbols)}") + +# for item in symbols: +# contract = item.get("symbol") +# if contract in contracts: +# try: +# price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1)) +# lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001)) + +# if price_tick == 0 or lot_size == 0: +# logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}") +# continue + +# tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 +# if lot_size >= 1: +# qty_tick_unit = lot_size +# qty_tick_base = 1 +# else: +# qty_tick_unit = 1 +# qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 + +# specs_dict[contract] = { +# "specs": { +# "tick_unit": 1, +# "tick_base": tick_base, +# "cash_per_point": 1, +# "qty_tick_unit": qty_tick_unit, +# "qty_tick_base": qty_tick_base +# } +# } +# found_contracts.add(contract) +# logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}") +# except (ValueError, TypeError) as e: +# logger.error(f"Failed to parse futures specs for {contract}: {e}") +# continue + +# cursor = result.get("nextPageCursor", "") +# if not cursor: +# logger.info("No more pages to fetch; pagination complete") +# break + +# for contract in contracts: +# if contract not in found_contracts: +# logger.warning(f"Futures contract {contract} not found in API response, using default specs") + +# logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") +# logger.debug(f"Final specs: {specs_dict}") +# return specs_dict + +# except Exception as e: +# logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True) +# return specs_dict + +# def get_expiration(contract: Dict) -> str: +# """Extract expiration from metadata or use default if none.""" +# if contract.get("status") == "Expired": +# return "0" # 8061 format for expired +# delivery_date = contract.get("deliveryDate") # Check if metadata provides this +# if delivery_date: +# from datetime import datetime +# dt = datetime.utcfromtimestamp(delivery_date / 1000) +# return dt.strftime("%Y-%m-%dT%H:%M") +# return PERP_EXP_STRING # Default for perpetuals + +# def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: +# """Query Bybit API for all linear instruments and filter by base symbols and contract type.""" +# futures_contracts = [] +# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" +# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000" + +# logger.info(f"Querying all linear instruments: {url}") +# response = requests.get(url, timeout=5) +# response.raise_for_status() +# data = response.json() +# if data.get("retCode") == 0 and data.get("result", {}).get("list"): +# logger.info(f"Retrieved {len(data['result']['list'])} linear instruments") +# for item in data.get("result", {}).get("list", []): +# base_coin = item.get("baseCoin") +# contract_type = item.get("contractType") +# if base_coin in base_symbols and contract_type in ["LinearFutures", "LinearPerpetual"]: +# futures_contracts.append(item.get("symbol")) +# logger.info(f"Filtered to {len(futures_contracts)} compatible instruments for base symbols {base_symbols}") +# else: +# logger.warning(f"Failed to retrieve instruments: {data.get('retMsg', 'Unknown error')}") + +# return sorted(list(set(futures_contracts))) # Remove duplicates and sort + +# class BybitMdProvider(MdProvider): +# def __init__(self): +# super().__init__() +# self._config: Optional[dict] = None +# self.ws_futures = None +# self.ws_thread: Optional[threading.Thread] = None +# self.bridge: Optional[MdBridge] = None +# self.stop_event = threading.Event() + +# @property +# def config(self): +# if self._config is None: +# self._config = load_config(["py_bybit_md_provider", "bybit_overrides"]) +# return self._config + +# @property +# def use_testnet(self): +# return self.config.get("use_testnet", True) + +# @property +# def currencies(self): +# return self.config.get("currencies") or ["USDT"] + +# @property +# def futures(self): +# return self.config.get("futures", []) + +# def start_impl(self, bridge: MdBridge): +# configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level +# self.bridge = bridge +# logger.info("Starting BybitMdProvider") + +# # Load futures contracts dynamically from API based on base symbols +# futures_contracts = get_futures_contracts(self.futures, self.use_testnet) +# logger.info(f"futures_contracts: {futures_contracts}") + +# # Fetch metadata for all specified futures contracts +# metadata = get_metadata(futures_contracts, self.use_testnet) +# logger.info(f"Fetched metadata for {len(metadata)} futures contracts") +# logger.info(f"metadata: {metadata}") + +# # Fetch specs +# futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet) +# logger.info(f"Fetched futures specs for {len(futures_specs)} contracts") +# logger.info(f"futures_specs: {futures_specs}") + +# sec_list_futures = PySecurityList() + +# for contract in futures_contracts: +# meta = metadata.get(contract, {}) +# logger.info(f"Metadata for {contract}: {meta}") # Log metadata to verify contents +# if not meta: +# logger.warning(f"No metadata found for contract: {contract}, using defaults") +# continue + +# try: +# # Extract details from metadata +# symbol = meta.get("symbol", contract) +# logger.info(f"Security creation for {contract}: symbol={symbol}") +# base_asset = meta.get("baseCoin") +# logger.info(f"Security creation for {contract}: base_asset from metadata={base_asset}") +# quote_currency = meta.get("quoteCoin") +# logger.info(f"Security creation for {contract}: quote_currency from metadata={quote_currency}") +# expiration = get_expiration(meta) +# logger.info(f"Security creation for {contract}: expiration={expiration}") + +# # Use fallbacks with logging +# asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) +# logger.info(f"Security creation for {contract}: using asset={asset}") +# group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) +# logger.info(f"Security creation for {contract}: using group={group}") +# currency = quote_currency or "USDT" +# logger.info(f"Security creation for {contract}: using currency={currency}") + +# specs = futures_specs.get(contract, {}).get("specs", {}) +# if not specs: +# logger.warning(f"No specs found for contract: {contract}, using defaults") +# specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 1, "qty_tick_unit": 1, "qty_tick_base": 10000} + +# try: +# qty_info = get_quantity_info( +# tick_unit=specs.get("qty_tick_unit", 1), +# tick_base=specs.get("qty_tick_base", 10000) +# ) +# except Exception as e: +# logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}") +# continue + +# try: +# spec = get_contract_spec( +# price_info=get_price_info( +# tick_unit=specs.get("tick_unit", 1), +# tick_base=specs.get("tick_base", 10), +# cash_per_point=specs.get("cash_per_point", 1) +# ), +# qty_info=qty_info +# ) +# except Exception as e: +# logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}") +# continue + +# try: +# sec = get_security( +# venue="Bybit", +# name=symbol, +# asset=asset, +# group=group, +# currency=currency, +# expiration=expiration, +# option_kind=None, +# strike=None, +# spec=spec, +# ) +# sec_list_futures.append(sec) +# logger.info(f"Created FUTURE security: name={symbol}, venue=Bybit, asset={asset}, group={group}, currency={currency}, expiration={expiration}, option_kind={None}, strike={None}, spec={specs}") +# except Exception as e: +# logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}", exc_info=True) +# continue + +# except Exception as e: +# logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True) +# continue + +# combined_sec_list = PySecurityList() +# for sec in sec_list_futures: +# combined_sec_list.append(sec) +# bridge.on_securities(combined_sec_list) +# logger.info(f"Sent securities list: {len(sec_list_futures)} futures") + +# try: +# self.ws_futures = WebSocket( +# testnet=self.use_testnet, +# channel_type="linear", +# ping_interval=20, +# ) +# logger.info("Connecting to Bybit WebSocket (futures, linear)") +# for contract in futures_contracts: +# try: +# self.ws_futures.orderbook_stream( +# symbol=contract, +# depth=50, +# callback=lambda data: self._on_book_update(data, source="futures") +# ) +# logger.info(f"Subscribed to orderbook.50.{contract} (futures)") +# except Exception as e: +# logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}") +# logger.info("Subscribed to orderbook channels on Bybit futures WS.") +# for t in threading.enumerate(): +# logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}") +# while not self.stop_event.is_set(): +# time.sleep(1) +# except KeyboardInterrupt: +# logger.info("KeyboardInterrupt received — shutting down.") +# self.stop() +# except Exception as e: +# logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True) +# self.stop() + +# def start(self, bridge: MdBridge): +# def start_wrapper(): +# try: +# self.start_impl(bridge) +# except Exception as e: +# logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True) +# self.ws_thread = threading.Thread(target=start_wrapper) +# self.ws_thread.start() + +# def _on_book_update(self, data: dict, source: str): +# try: +# logger.debug(f"Raw WebSocket data (source={source}): {data}") +# topic = data.get("topic", "") +# if not topic.startswith("orderbook.50."): +# logger.debug(f"Ignoring non-orderbook topic: {topic}") +# return +# contract = topic.split(".")[-1] +# payload = data.get("data", {}) +# raw_bids = payload.get("b", []) +# raw_asks = payload.get("a", []) +# if not raw_bids or not raw_asks: +# logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") +# raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids] +# raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks] +# sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] +# sorted_bids += [(np.float64('nan'), np.float64('nan'))] +# sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] +# sorted_asks += [(np.float64('nan'), np.float64('nan'))] +# logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") +# book = PyBook() +# market_type = "FUTURE" # Only futures now +# logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") +# book.set_data( +# contract, +# "Bybit", +# market_type, +# np.array(sorted_bids, dtype=np.float64), +# np.array(sorted_asks, dtype=np.float64), +# ) +# book_list = PyBookList() +# book_list.append(book) +# if self.bridge: +# logger.debug(f"✔ on_books called for {contract} (market_type={market_type})") +# self.bridge.on_books(book_list) +# else: +# logger.warning("Bridge is None, cannot call on_books") +# except Exception as e: +# logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) + +# def stop(self): +# logger.info("Stopping BybitMdProvider...") +# self.stop_event.set() +# logger.debug("Threads before stopping WebSocket:") +# for t in threading.enumerate(): +# logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") +# if self.ws_futures: +# try: +# self.ws_futures.exit() +# logger.info("WebSocket (futures) exit called") +# for t in threading.enumerate(): +# if t.name in ['Thread-1 ()', 'Thread-2 (_send_ping)'] and t.is_alive(): +# logger.info(f"Joining pybit thread: {t.name} (futures)") +# t.join(timeout=1.0) +# except Exception as e: +# logger.error(f"Error while closing WebSocket (futures): {e}", exc_info=True) +# self.ws_futures = None +# if self.ws_thread: +# self.ws_thread.join() +# self.bridge = None +# logger.info("✔ Stopped BybitMdProvider.") +# logger.debug("Threads after stop:") +# for t in threading.enumerate(): +# logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") +# import os +# os._exit(0) + +# if __name__ == "__main__": +# import argparse +# from fake_md_bridge import FakeMdBridge + +# ap = argparse.ArgumentParser() +# ap.add_argument("--currencies", nargs="+") +# ap.add_argument("--futures", nargs="+") +# log_levels = list(logging._nameToLevel) +# ap.add_argument("--console-level", choices=log_levels) +# ap.add_argument("--file-level", choices=log_levels) +# ap.add_argument("--dev", action="store_true", dest="use_testnet") +# ap.add_argument("--prod", action="store_false", dest="use_testnet") +# ap.set_defaults( +# currencies=None, +# futures=None, +# console_level=None, +# file_level=None, +# use_testnet=None, +# ) +# args = ap.parse_args() +# print(args) +# logger = logging.getLogger("bybit_md_provider_app") +# md_pro = BybitMdProvider() +# config = md_pro.config +# if args.currencies: +# config["currencies"] = args.currencies +# if args.futures: +# config["futures"] = args.futures +# logging_overrides = {} +# if args.console_level: +# logging_overrides["console_level"] = args.console_level +# if args.file_level: +# logging_overrides["file_level"] = args.file_level +# if logging_overrides: +# config["logging"].update(logging_overrides) +# if args.use_testnet is not None: +# config["use_testnet"] = args.use_testnet +# md_bridge = FakeMdBridge(logger) +# md_pro.start(md_bridge) +# while True: +# time.sleep(1) + import json import numpy as np import time @@ -201,7 +648,7 @@ class BybitMdProvider(MdProvider): self.bridge = bridge logger.info("Starting BybitMdProvider") - # Load futures contracts dynamically from API based on base symbols + # Load futures contracts from API based on base symbols futures_contracts = get_futures_contracts(self.futures, self.use_testnet) logger.info(f"futures_contracts: {futures_contracts}") @@ -235,18 +682,17 @@ class BybitMdProvider(MdProvider): expiration = get_expiration(meta) logger.info(f"Security creation for {contract}: expiration={expiration}") - # Use fallbacks with logging - asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) + asset = base_asset logger.info(f"Security creation for {contract}: using asset={asset}") - group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0]) + group = base_asset logger.info(f"Security creation for {contract}: using group={group}") - currency = quote_currency or "USDT" + currency = quote_currency logger.info(f"Security creation for {contract}: using currency={currency}") specs = futures_specs.get(contract, {}).get("specs", {}) if not specs: logger.warning(f"No specs found for contract: {contract}, using defaults") - specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 1, "qty_tick_unit": 1, "qty_tick_base": 10000} + specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 1, "qty_tick_base": 10000} try: qty_info = get_quantity_info( @@ -262,7 +708,7 @@ class BybitMdProvider(MdProvider): price_info=get_price_info( tick_unit=specs.get("tick_unit", 1), tick_base=specs.get("tick_base", 10), - cash_per_point=specs.get("cash_per_point", 1) + cash_per_point=specs.get("cash_per_point", 0.1) ), qty_info=qty_info ) @@ -357,7 +803,7 @@ class BybitMdProvider(MdProvider): sorted_asks += [(np.float64('nan'), np.float64('nan'))] logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") book = PyBook() - market_type = "FUTURE" # Only futures now + market_type = "FUTURE" logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") book.set_data( contract,