FIX: bybit_md_provider.py works with default symbol format in py_bybit_md_provider.json.--MD Freezes on InverseFutures.

This commit is contained in:
Adeel Khan committed 2025-07-14 22:35:36 +00:00
1 parent c7c755a964
commit 36db69dc67
1 file changed
+573 -8
@@ -1,3 +1,544 @@
# import json
# import numpy as np
# import time
# import logging
# import sys
# import threading
# from itertools import chain
# from typing import Iterable, Optional, Dict
# from pybit.unified_trading import WebSocket
# from python_interop_module import (
# MdBridge,
# MdProvider,
# PyBook,
# PyBookList,
# PySecurityList,
# )
# from spark_core import (
# configure_logger,
# get_contract_spec,
# get_price_info,
# get_quantity_info,
# get_security,
# load_config,
# )
# from py_md_utils import PERP_EXP_STRING
# import requests
# logger = logging.getLogger("bybit_md_provider")
# def normalize_contract_name(contract: str) -> str:
# logger.info(f"Normalizing contract: input={contract}")
# if "-" in contract: # Preserve delivery futures (e.g., BTC-26SEP25)
# logger.info(f"Normalizing contract: output={contract} (unchanged, delivery future)")
# return contract
# if contract.endswith("PERP"):
# result = contract[:-4] + "_PERP"
# logger.info(f"Normalizing contract: output={result}")
# return result
# if len(contract) > 4:
# result = contract[:-4] + "_" + contract[-4:]
# logger.info(f"Normalizing contract: output={result}")
# return result
# logger.info(f"Normalizing contract: output={contract} (unchanged)")
# return contract
# def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]:
# """Fetch metadata for all linear contracts and filter for specified contracts."""
# metadata_dict = {}
# try:
# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000"
# logger.info(f"Querying Bybit API for metadata: {url}")
# response = requests.get(url, timeout=5)
# response.raise_for_status()
# data = response.json()
# if data.get("retCode") == 0:
# contracts_data = data.get("result", {}).get("list", [])
# logger.info(f"Retrieved metadata for {len(contracts_data)} linear contracts")
# for item in contracts_data:
# symbol = item.get("symbol")
# if symbol in contracts:
# metadata_dict[symbol] = item
# logger.info(f"Stored metadata for {symbol}")
# else:
# logger.error(f"Metadata API error: {data.get('retMsg', 'Unknown error')}")
# except Exception as e:
# logger.error(f"Failed to fetch metadata: {e}", exc_info=True)
# return metadata_dict
# def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]:
# default_specs = {
# "tick_unit": 1,
# "tick_base": 10,
# "cash_per_point": 1,
# "qty_tick_unit": 1,
# "qty_tick_base": 10000
# }
# specs_dict = {contract: {"specs": default_specs} for contract in contracts}
# try:
# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
# base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500"
# logger.info(f"Querying Bybit API for futures specs: {base_url}")
# found_contracts = set()
# cursor = ""
# while True:
# url = f"{base_url}&cursor={cursor}" if cursor else base_url
# logger.info(f"Fetching page with cursor: {cursor or 'initial'}")
# response = requests.get(url, timeout=5)
# response.raise_for_status()
# data = response.json()
# if data.get("retCode") != 0:
# logger.error(f"Futures API error: {data.get('retMsg', 'Unknown error')}")
# return specs_dict
# result = data.get("result", {})
# symbols = result.get("list", [])
# logger.info(f"Loaded page with cursor: {cursor or 'initial'}, symbols: {len(symbols)}")
# for item in symbols:
# contract = item.get("symbol")
# if contract in contracts:
# try:
# price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1))
# lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001))
# if price_tick == 0 or lot_size == 0:
# logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}")
# continue
# tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10
# if lot_size >= 1:
# qty_tick_unit = lot_size
# qty_tick_base = 1
# else:
# qty_tick_unit = 1
# qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000
# specs_dict[contract] = {
# "specs": {
# "tick_unit": 1,
# "tick_base": tick_base,
# "cash_per_point": 1,
# "qty_tick_unit": qty_tick_unit,
# "qty_tick_base": qty_tick_base
# }
# }
# found_contracts.add(contract)
# logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}")
# except (ValueError, TypeError) as e:
# logger.error(f"Failed to parse futures specs for {contract}: {e}")
# continue
# cursor = result.get("nextPageCursor", "")
# if not cursor:
# logger.info("No more pages to fetch; pagination complete")
# break
# for contract in contracts:
# if contract not in found_contracts:
# logger.warning(f"Futures contract {contract} not found in API response, using default specs")
# logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}")
# logger.debug(f"Final specs: {specs_dict}")
# return specs_dict
# except Exception as e:
# logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True)
# return specs_dict
# def get_spot_specs(contracts: Iterable[str]) -> Dict[str, Dict]:
# default_specs = {
# "tick_unit": 1,
# "tick_base": 10,
# "cash_per_point": 0.1,
# "qty_tick_unit": 1,
# "qty_tick_base": 10000
# }
# specs_dict = {contract: {"specs": default_specs} for contract in contracts}
# logger.info(f"Using default specs for spot contracts: {default_specs}")
# return specs_dict
# def get_expiration(contract: Dict) -> str:
# """Extract expiration from metadata or use default if none."""
# if contract.get("status") == "Expired":
# return "0" # 8061 format for expired
# delivery_date = contract.get("deliveryDate") # Check if metadata provides this
# if delivery_date:
# from datetime import datetime
# dt = datetime.utcfromtimestamp(delivery_date / 1000)
# return dt.strftime("%Y-%m-%dT%H:%M")
# return PERP_EXP_STRING # Default for perpetuals
# class BybitMdProvider(MdProvider):
# def __init__(self):
# super().__init__()
# self._config: Optional[dict] = None
# self.ws_futures = None
# # self.ws_spot = None # Commented out spot WebSocket
# self.ws_thread: Optional[threading.Thread] = None
# self.bridge: Optional[MdBridge] = None
# self.stop_event = threading.Event()
# @property
# def config(self):
# if self._config is None:
# self._config = load_config(["py_bybit_md_provider", "bybit_overrides"])
# return self._config
# @property
# def use_testnet(self):
# return self.config.get("use_testnet", True)
# @property
# def currencies(self):
# return self.config.get("currencies") or ["USDT"]
# @property
# def futures(self):
# return self.config.get("futures", [])
# @property
# def spots(self):
# return self.config.get("spots", [])
# def start_impl(self, bridge: MdBridge):
# configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level
# self.bridge = bridge
# logger.info("Starting BybitMdProvider")
# # Load futures contracts directly from JSON
# futures_contracts = self.futures
# logger.info(f"futures_contracts: {futures_contracts}")
# # spot_contracts = list(map(lambda s: s + "USDT", self.spots)) # Commented out spot contracts
# # Fetch metadata for all specified futures contracts
# metadata = get_metadata(futures_contracts, self.use_testnet)
# logger.info(f"Fetched metadata for {len(metadata)} futures contracts")
# logger.info(f"metadata: {metadata}")
# # Fetch specs
# futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet)
# logger.info(f"Fetched futures specs for {len(futures_specs)} contracts")
# logger.info(f"futures_specs: {futures_specs}")
# # spot_specs = get_spot_specs(spot_contracts) # Commented out spot specs
# sec_list_futures = PySecurityList()
# # sec_list_spots = PySecurityList() # Commented out spot security list
# for contract in futures_contracts:
# meta = metadata.get(contract, {})
# logger.info(f"Metadata for {contract}: {meta}")
# if not meta:
# logger.warning(f"No metadata found for contract: {contract}, using defaults")
# continue
# try:
# symbol = meta.get("symbol", contract)
# logger.info(f"Security creation for {contract}: symbol={symbol}")
# base_asset = meta.get("baseCoin")
# logger.info(f"Security creation for {contract}: base_asset from metadata={base_asset}")
# quote_currency = meta.get("quoteCoin")
# logger.info(f"Security creation for {contract}: quote_currency from metadata={quote_currency}")
# expiration = get_expiration(meta)
# logger.info(f"Security creation for {contract}: expiration={expiration}")
# asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0])
# logger.info(f"Security creation for {contract}: using asset={asset}")
# group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0])
# logger.info(f"Security creation for {contract}: using group={group}")
# currency = quote_currency or "USDT"
# logger.info(f"Security creation for {contract}: using currency={currency}")
# specs = futures_specs.get(contract, {}).get("specs", {})
# if not specs:
# logger.warning(f"No specs found for contract: {contract}, using defaults")
# specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 1, "qty_tick_base": 10000}
# try:
# qty_info = get_quantity_info(
# tick_unit=specs.get("qty_tick_unit", 1),
# tick_base=specs.get("qty_tick_base", 10000)
# )
# except Exception as e:
# logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}")
# continue
# try:
# spec = get_contract_spec(
# price_info=get_price_info(
# tick_unit=specs.get("tick_unit", 1),
# tick_base=specs.get("tick_base", 10),
# cash_per_point=specs.get("cash_per_point", 0.1)
# ),
# qty_info=qty_info
# )
# except Exception as e:
# logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}")
# continue
# try:
# sec = get_security(
# venue="Bybit",
# name=symbol,
# asset=asset,
# group=group,
# currency=currency,
# expiration=expiration,
# option_kind=None,
# strike=None,
# spec=spec,
# )
# sec_list_futures.append(sec)
# logger.info(f"Created FUTURE security: name={symbol}, venue=Bybit, asset={asset}, group={group}, currency={currency}, expiration={expiration}, option_kind={None}, strike={None}, spec={specs}")
# except Exception as e:
# logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}", exc_info=True)
# continue
# except Exception as e:
# logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True)
# continue
# # for contract in spot_contracts: # Commented out spot security creation
# # normalized_contract = normalize_contract_name(contract)
# # try:
# # asset, quote = normalized_contract.split("_")
# # quote = "USDT"
# # except ValueError as e:
# # logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}")
# # continue
# # specs = spot_specs.get(contract, {}).get("specs", {})
# #
# # try:
# # qty_info = get_quantity_info(
# # tick_unit=specs.get("qty_tick_unit", 1),
# # tick_base=specs.get("qty_tick_base", 10000)
# # )
# # except Exception as e:
# # logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}")
# # continue
# #
# # try:
# # spec = get_contract_spec(
# # price_info=get_price_info(
# # tick_unit=specs.get("tick_unit", 1),
# # tick_base=specs.get("tick_base", 10),
# # cash_per_point=specs.get("cash_per_point", 0.1)
# # ),
# # qty_info=qty_info
# # )
# # except Exception as e:
# # logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}")
# # continue
# #
# # try:
# # sec = get_security(
# # venue="Bybit",
# # name=contract,
# # asset=asset,
# # group=asset,
# # currency=quote,
# # expiration=None,
# # option_kind=None,
# # strike=None,
# # spec=spec,
# # )
# # sec_list_spots.append(sec)
# # logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}")
# # except Exception as e:
# # logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}", exc_info=True)
# # continue
# combined_sec_list = PySecurityList()
# for sec in sec_list_futures:
# combined_sec_list.append(sec)
# # for sec in sec_list_spots: # Commented out spot securities
# # combined_sec_list.append(sec)
# bridge.on_securities(combined_sec_list)
# logger.info(f"Sent securities list: {len(sec_list_futures)} futures, 0 spots")
# try:
# self.ws_futures = WebSocket(
# testnet=self.use_testnet,
# channel_type="linear",
# ping_interval=20,
# )
# logger.info("Connecting to Bybit WebSocket (futures, linear)")
# for contract in futures_contracts:
# try:
# self.ws_futures.orderbook_stream(
# symbol=contract,
# depth=50,
# callback=lambda data: self._on_book_update(data, source="futures")
# )
# logger.info(f"Subscribed to orderbook.50.{contract} (futures)")
# except Exception as e:
# logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}")
# # self.ws_spot = WebSocket( # Commented out spot WebSocket
# # testnet=self.use_testnet,
# # channel_type="spot",
# # ping_interval=20,
# # )
# # logger.info("Connecting to Bybit WebSocket (spot)")
# #
# # for contract in spot_contracts:
# # try:
# # self.ws_spot.orderbook_stream(
# # symbol=contract,
# # depth=50,
# # callback=lambda data: self._on_book_update(data, source="spot")
# # )
# # logger.info(f"Subscribed to orderbook.50.{contract} (spot)")
# # except Exception as e:
# # logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}")
# logger.info("Subscribed to orderbook channels on Bybit futures WS.")
# for t in threading.enumerate():
# logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}")
# while not self.stop_event.is_set():
# time.sleep(1)
# except KeyboardInterrupt:
# logger.info("KeyboardInterrupt received — shutting down.")
# self.stop()
# except Exception as e:
# logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True)
# self.stop()
# def start(self, bridge: MdBridge):
# def start_wrapper():
# try:
# self.start_impl(bridge)
# except Exception as e:
# logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True)
# self.ws_thread = threading.Thread(target=start_wrapper)
# self.ws_thread.start()
# def _on_book_update(self, data: dict, source: str):
# try:
# logger.debug(f"Raw WebSocket data (source={source}): {data}")
# topic = data.get("topic", "")
# if not topic.startswith("orderbook.50."):
# logger.debug(f"Ignoring non-orderbook topic: {topic}")
# return
# contract = topic.split(".")[-1]
# payload = data.get("data", {})
# raw_bids = payload.get("b", [])
# raw_asks = payload.get("a", [])
# if not raw_bids or not raw_asks:
# logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}")
# raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids]
# raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks]
# sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
# sorted_bids += [(np.float64('nan'), np.float64('nan'))]
# sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
# sorted_asks += [(np.float64('nan'), np.float64('nan'))]
# logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
# book = PyBook()
# market_type = "SPOT" if source == "spot" else "FUTURE"
# logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})")
# book.set_data(
# contract,
# "Bybit",
# market_type,
# np.array(sorted_bids, dtype=np.float64),
# np.array(sorted_asks, dtype=np.float64),
# )
# book_list = PyBookList()
# book_list.append(book)
# if self.bridge:
# logger.debug(f"✔ on_books called for {contract} (market_type={market_type})")
# self.bridge.on_books(book_list)
# else:
# logger.warning("Bridge is None, cannot call on_books")
# except Exception as e:
# logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True)
# def stop(self):
# logger.info("Stopping BybitMdProvider...")
# self.stop_event.set()
# logger.debug("Threads before stopping WebSocket:")
# for t in threading.enumerate():
# logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
# for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket
# # , (self.ws_spot, "spot")
# if ws:
# try:
# ws.exit()
# logger.info(f"WebSocket ({ws_type}) exit called")
# for t in threading.enumerate():
# if t.name in ['Thread-1 (<lambda>)', 'Thread-2 (_send_ping)'] and t.is_alive():
# logger.info(f"Joining pybit thread: {t.name} ({ws_type})")
# t.join(timeout=1.0)
# except Exception as e:
# logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True)
# ws = None
# self.ws_futures = None
# # self.ws_spot = None # Commented out spot WebSocket
# if self.ws_thread:
# self.ws_thread.join()
# self.bridge = None
# logger.info("✔ Stopped BybitMdProvider.")
# logger.debug("Threads after stop:")
# for t in threading.enumerate():
# logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
# import os
# os._exit(0)
# if __name__ == "__main__":
# import argparse
# from fake_md_bridge import FakeMdBridge
# ap = argparse.ArgumentParser()
# ap.add_argument("--currencies", nargs="+")
# ap.add_argument("--futures", nargs="+")
# log_levels = list(logging._nameToLevel)
# ap.add_argument("--console-level", choices=log_levels)
# ap.add_argument("--file-level", choices=log_levels)
# ap.add_argument("--dev", action="store_true", dest="use_testnet")
# ap.add_argument("--prod", action="store_false", dest="use_testnet")
# ap.set_defaults(
# currencies=None,
# futures=None,
# console_level=None,
# file_level=None,
# use_testnet=None,
# )
# args = ap.parse_args()
# print(args)
# logger = logging.getLogger("bybit_md_provider_app")
# md_pro = BybitMdProvider()
# config = md_pro.config
# if args.currencies:
# config["currencies"] = args.currencies
# if args.futures:
# config["futures"] = args.futures
# logging_overrides = {}
# if args.console-level:
# logging_overrides["console_level"] = args.console_level
# if args.file-level:
# logging_overrides["file_level"] = args.file_level
# if logging_overrides:
# config["logging"].update(logging_overrides)
# if args.use_testnet is not None:
# config["use_testnet"] = args.use_testnet
# md_bridge = FakeMdBridge(logger)
# md_pro.start(md_bridge)
# while True:
# time.sleep(1)
import json
import numpy as np
import time
@@ -174,6 +715,31 @@ def get_expiration(contract: Dict) -> str:
return dt.strftime("%Y-%m-%dT%H:%M")
return PERP_EXP_STRING # Default for perpetuals
def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list:
"""Query Bybit API for linear instruments and return full security names for given base coins."""
futures_contracts = []
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
url = f"https://{host}/v5/market/instruments-info"
logger.info(f"Querying futures contracts for base symbols: {base_symbols}")
for symbol in base_symbols:
params = {
"category": "linear",
"baseCoin": symbol
}
logger.info(f"Querying for baseCoin={symbol}: {url}")
response = requests.get(url, params=params, timeout=5)
response.raise_for_status()
data = response.json()
if data.get("retCode") == 0 and data.get("result", {}).get("list"):
for item in data.get("result", {}).get("list", []):
futures_contracts.append(item.get("symbol"))
logger.info(f"Found {len(data['result']['list'])} instruments for {symbol}")
else:
logger.warning(f"No instruments found for baseCoin={symbol}: {data.get('retMsg', 'Unknown error')}")
return sorted(list(set(futures_contracts))) # Remove duplicates and sort
class BybitMdProvider(MdProvider):
def __init__(self):
super().__init__()
@@ -211,8 +777,8 @@ class BybitMdProvider(MdProvider):
self.bridge = bridge
logger.info("Starting BybitMdProvider")
# Load futures contracts directly from JSON
futures_contracts = self.futures
# Load futures contracts dynamically from API based on base symbols
futures_contracts = get_futures_contracts(self.futures, self.use_testnet)
logger.info(f"futures_contracts: {futures_contracts}")
# spot_contracts = list(map(lambda s: s + "USDT", self.spots)) # Commented out spot contracts
@@ -232,12 +798,13 @@ class BybitMdProvider(MdProvider):
for contract in futures_contracts:
meta = metadata.get(contract, {})
logger.info(f"Metadata for {contract}: {meta}")
logger.info(f"Metadata for {contract}: {meta}") # Log metadata to verify contents
if not meta:
logger.warning(f"No metadata found for contract: {contract}, using defaults")
continue
try:
# Extract details from metadata
symbol = meta.get("symbol", contract)
logger.info(f"Security creation for {contract}: symbol={symbol}")
base_asset = meta.get("baseCoin")
@@ -247,6 +814,7 @@ class BybitMdProvider(MdProvider):
expiration = get_expiration(meta)
logger.info(f"Security creation for {contract}: expiration={expiration}")
# Use fallbacks with logging
asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0])
logger.info(f"Security creation for {contract}: using asset={asset}")
group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0])
@@ -396,13 +964,10 @@ class BybitMdProvider(MdProvider):
# except Exception as e:
# logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}")
logger.info("Subscribed to orderbook channels on Bybit futures WS.")
for t in threading.enumerate():
logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}")
while not self.stop_event.is_set():
time.sleep(1)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt received — shutting down.")
self.stop()
@@ -525,9 +1090,9 @@ if __name__ == "__main__":
if args.futures:
config["futures"] = args.futures
logging_overrides = {}
if args.console-level:
if args.console_level:
logging_overrides["console_level"] = args.console_level
if args.file-level:
if args.file_level:
logging_overrides["file_level"] = args.file_level
if logging_overrides:
config["logging"].update(logging_overrides)