FIX: Added filter to exclude InverseFutures--MD not freezing.

This commit is contained in:
Adeel Khan committed 2025-07-14 22:35:36 +00:00
1 parent 36db69dc67
commit 0cb3cadc11
1 file changed
+5 -544
@@ -1,544 +1,3 @@
# import json
# import numpy as np
# import time
# import logging
# import sys
# import threading
# from itertools import chain
# from typing import Iterable, Optional, Dict
# from pybit.unified_trading import WebSocket
# from python_interop_module import (
# MdBridge,
# MdProvider,
# PyBook,
# PyBookList,
# PySecurityList,
# )
# from spark_core import (
# configure_logger,
# get_contract_spec,
# get_price_info,
# get_quantity_info,
# get_security,
# load_config,
# )
# from py_md_utils import PERP_EXP_STRING
# import requests
# logger = logging.getLogger("bybit_md_provider")
# def normalize_contract_name(contract: str) -> str:
# logger.info(f"Normalizing contract: input={contract}")
# if "-" in contract: # Preserve delivery futures (e.g., BTC-26SEP25)
# logger.info(f"Normalizing contract: output={contract} (unchanged, delivery future)")
# return contract
# if contract.endswith("PERP"):
# result = contract[:-4] + "_PERP"
# logger.info(f"Normalizing contract: output={result}")
# return result
# if len(contract) > 4:
# result = contract[:-4] + "_" + contract[-4:]
# logger.info(f"Normalizing contract: output={result}")
# return result
# logger.info(f"Normalizing contract: output={contract} (unchanged)")
# return contract
# def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]:
# """Fetch metadata for all linear contracts and filter for specified contracts."""
# metadata_dict = {}
# try:
# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
# url = f"https://{host}/v5/market/instruments-info?category=linear&limit=1000"
# logger.info(f"Querying Bybit API for metadata: {url}")
# response = requests.get(url, timeout=5)
# response.raise_for_status()
# data = response.json()
# if data.get("retCode") == 0:
# contracts_data = data.get("result", {}).get("list", [])
# logger.info(f"Retrieved metadata for {len(contracts_data)} linear contracts")
# for item in contracts_data:
# symbol = item.get("symbol")
# if symbol in contracts:
# metadata_dict[symbol] = item
# logger.info(f"Stored metadata for {symbol}")
# else:
# logger.error(f"Metadata API error: {data.get('retMsg', 'Unknown error')}")
# except Exception as e:
# logger.error(f"Failed to fetch metadata: {e}", exc_info=True)
# return metadata_dict
# def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]:
# default_specs = {
# "tick_unit": 1,
# "tick_base": 10,
# "cash_per_point": 1,
# "qty_tick_unit": 1,
# "qty_tick_base": 10000
# }
# specs_dict = {contract: {"specs": default_specs} for contract in contracts}
# try:
# host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
# base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500"
# logger.info(f"Querying Bybit API for futures specs: {base_url}")
# found_contracts = set()
# cursor = ""
# while True:
# url = f"{base_url}&cursor={cursor}" if cursor else base_url
# logger.info(f"Fetching page with cursor: {cursor or 'initial'}")
# response = requests.get(url, timeout=5)
# response.raise_for_status()
# data = response.json()
# if data.get("retCode") != 0:
# logger.error(f"Futures API error: {data.get('retMsg', 'Unknown error')}")
# return specs_dict
# result = data.get("result", {})
# symbols = result.get("list", [])
# logger.info(f"Loaded page with cursor: {cursor or 'initial'}, symbols: {len(symbols)}")
# for item in symbols:
# contract = item.get("symbol")
# if contract in contracts:
# try:
# price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1))
# lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001))
# if price_tick == 0 or lot_size == 0:
# logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}")
# continue
# tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10
# if lot_size >= 1:
# qty_tick_unit = lot_size
# qty_tick_base = 1
# else:
# qty_tick_unit = 1
# qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000
# specs_dict[contract] = {
# "specs": {
# "tick_unit": 1,
# "tick_base": tick_base,
# "cash_per_point": 1,
# "qty_tick_unit": qty_tick_unit,
# "qty_tick_base": qty_tick_base
# }
# }
# found_contracts.add(contract)
# logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}")
# except (ValueError, TypeError) as e:
# logger.error(f"Failed to parse futures specs for {contract}: {e}")
# continue
# cursor = result.get("nextPageCursor", "")
# if not cursor:
# logger.info("No more pages to fetch; pagination complete")
# break
# for contract in contracts:
# if contract not in found_contracts:
# logger.warning(f"Futures contract {contract} not found in API response, using default specs")
# logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}")
# logger.debug(f"Final specs: {specs_dict}")
# return specs_dict
# except Exception as e:
# logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True)
# return specs_dict
# def get_spot_specs(contracts: Iterable[str]) -> Dict[str, Dict]:
# default_specs = {
# "tick_unit": 1,
# "tick_base": 10,
# "cash_per_point": 0.1,
# "qty_tick_unit": 1,
# "qty_tick_base": 10000
# }
# specs_dict = {contract: {"specs": default_specs} for contract in contracts}
# logger.info(f"Using default specs for spot contracts: {default_specs}")
# return specs_dict
# def get_expiration(contract: Dict) -> str:
# """Extract expiration from metadata or use default if none."""
# if contract.get("status") == "Expired":
# return "0" # 8061 format for expired
# delivery_date = contract.get("deliveryDate") # Check if metadata provides this
# if delivery_date:
# from datetime import datetime
# dt = datetime.utcfromtimestamp(delivery_date / 1000)
# return dt.strftime("%Y-%m-%dT%H:%M")
# return PERP_EXP_STRING # Default for perpetuals
# class BybitMdProvider(MdProvider):
# def __init__(self):
# super().__init__()
# self._config: Optional[dict] = None
# self.ws_futures = None
# # self.ws_spot = None # Commented out spot WebSocket
# self.ws_thread: Optional[threading.Thread] = None
# self.bridge: Optional[MdBridge] = None
# self.stop_event = threading.Event()
# @property
# def config(self):
# if self._config is None:
# self._config = load_config(["py_bybit_md_provider", "bybit_overrides"])
# return self._config
# @property
# def use_testnet(self):
# return self.config.get("use_testnet", True)
# @property
# def currencies(self):
# return self.config.get("currencies") or ["USDT"]
# @property
# def futures(self):
# return self.config.get("futures", [])
# @property
# def spots(self):
# return self.config.get("spots", [])
# def start_impl(self, bridge: MdBridge):
# configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level
# self.bridge = bridge
# logger.info("Starting BybitMdProvider")
# # Load futures contracts directly from JSON
# futures_contracts = self.futures
# logger.info(f"futures_contracts: {futures_contracts}")
# # spot_contracts = list(map(lambda s: s + "USDT", self.spots)) # Commented out spot contracts
# # Fetch metadata for all specified futures contracts
# metadata = get_metadata(futures_contracts, self.use_testnet)
# logger.info(f"Fetched metadata for {len(metadata)} futures contracts")
# logger.info(f"metadata: {metadata}")
# # Fetch specs
# futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet)
# logger.info(f"Fetched futures specs for {len(futures_specs)} contracts")
# logger.info(f"futures_specs: {futures_specs}")
# # spot_specs = get_spot_specs(spot_contracts) # Commented out spot specs
# sec_list_futures = PySecurityList()
# # sec_list_spots = PySecurityList() # Commented out spot security list
# for contract in futures_contracts:
# meta = metadata.get(contract, {})
# logger.info(f"Metadata for {contract}: {meta}")
# if not meta:
# logger.warning(f"No metadata found for contract: {contract}, using defaults")
# continue
# try:
# symbol = meta.get("symbol", contract)
# logger.info(f"Security creation for {contract}: symbol={symbol}")
# base_asset = meta.get("baseCoin")
# logger.info(f"Security creation for {contract}: base_asset from metadata={base_asset}")
# quote_currency = meta.get("quoteCoin")
# logger.info(f"Security creation for {contract}: quote_currency from metadata={quote_currency}")
# expiration = get_expiration(meta)
# logger.info(f"Security creation for {contract}: expiration={expiration}")
# asset = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0])
# logger.info(f"Security creation for {contract}: using asset={asset}")
# group = base_asset or (contract.split("-")[0] if "-" in contract else contract.split("_")[0])
# logger.info(f"Security creation for {contract}: using group={group}")
# currency = quote_currency or "USDT"
# logger.info(f"Security creation for {contract}: using currency={currency}")
# specs = futures_specs.get(contract, {}).get("specs", {})
# if not specs:
# logger.warning(f"No specs found for contract: {contract}, using defaults")
# specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 1, "qty_tick_base": 10000}
# try:
# qty_info = get_quantity_info(
# tick_unit=specs.get("qty_tick_unit", 1),
# tick_base=specs.get("qty_tick_base", 10000)
# )
# except Exception as e:
# logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}")
# continue
# try:
# spec = get_contract_spec(
# price_info=get_price_info(
# tick_unit=specs.get("tick_unit", 1),
# tick_base=specs.get("tick_base", 10),
# cash_per_point=specs.get("cash_per_point", 0.1)
# ),
# qty_info=qty_info
# )
# except Exception as e:
# logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}")
# continue
# try:
# sec = get_security(
# venue="Bybit",
# name=symbol,
# asset=asset,
# group=group,
# currency=currency,
# expiration=expiration,
# option_kind=None,
# strike=None,
# spec=spec,
# )
# sec_list_futures.append(sec)
# logger.info(f"Created FUTURE security: name={symbol}, venue=Bybit, asset={asset}, group={group}, currency={currency}, expiration={expiration}, option_kind={None}, strike={None}, spec={specs}")
# except Exception as e:
# logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}", exc_info=True)
# continue
# except Exception as e:
# logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True)
# continue
# # for contract in spot_contracts: # Commented out spot security creation
# # normalized_contract = normalize_contract_name(contract)
# # try:
# # asset, quote = normalized_contract.split("_")
# # quote = "USDT"
# # except ValueError as e:
# # logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}")
# # continue
# # specs = spot_specs.get(contract, {}).get("specs", {})
# #
# # try:
# # qty_info = get_quantity_info(
# # tick_unit=specs.get("qty_tick_unit", 1),
# # tick_base=specs.get("qty_tick_base", 10000)
# # )
# # except Exception as e:
# # logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}")
# # continue
# #
# # try:
# # spec = get_contract_spec(
# # price_info=get_price_info(
# # tick_unit=specs.get("tick_unit", 1),
# # tick_base=specs.get("tick_base", 10),
# # cash_per_point=specs.get("cash_per_point", 0.1)
# # ),
# # qty_info=qty_info
# # )
# # except Exception as e:
# # logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}")
# # continue
# #
# # try:
# # sec = get_security(
# # venue="Bybit",
# # name=contract,
# # asset=asset,
# # group=asset,
# # currency=quote,
# # expiration=None,
# # option_kind=None,
# # strike=None,
# # spec=spec,
# # )
# # sec_list_spots.append(sec)
# # logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}")
# # except Exception as e:
# # logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}", exc_info=True)
# # continue
# combined_sec_list = PySecurityList()
# for sec in sec_list_futures:
# combined_sec_list.append(sec)
# # for sec in sec_list_spots: # Commented out spot securities
# # combined_sec_list.append(sec)
# bridge.on_securities(combined_sec_list)
# logger.info(f"Sent securities list: {len(sec_list_futures)} futures, 0 spots")
# try:
# self.ws_futures = WebSocket(
# testnet=self.use_testnet,
# channel_type="linear",
# ping_interval=20,
# )
# logger.info("Connecting to Bybit WebSocket (futures, linear)")
# for contract in futures_contracts:
# try:
# self.ws_futures.orderbook_stream(
# symbol=contract,
# depth=50,
# callback=lambda data: self._on_book_update(data, source="futures")
# )
# logger.info(f"Subscribed to orderbook.50.{contract} (futures)")
# except Exception as e:
# logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}")
# # self.ws_spot = WebSocket( # Commented out spot WebSocket
# # testnet=self.use_testnet,
# # channel_type="spot",
# # ping_interval=20,
# # )
# # logger.info("Connecting to Bybit WebSocket (spot)")
# #
# # for contract in spot_contracts:
# # try:
# # self.ws_spot.orderbook_stream(
# # symbol=contract,
# # depth=50,
# # callback=lambda data: self._on_book_update(data, source="spot")
# # )
# # logger.info(f"Subscribed to orderbook.50.{contract} (spot)")
# # except Exception as e:
# # logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}")
# logger.info("Subscribed to orderbook channels on Bybit futures WS.")
# for t in threading.enumerate():
# logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}")
# while not self.stop_event.is_set():
# time.sleep(1)
# except KeyboardInterrupt:
# logger.info("KeyboardInterrupt received — shutting down.")
# self.stop()
# except Exception as e:
# logger.error(f"Bybit WebSocket fatal error: {e}", exc_info=True)
# self.stop()
# def start(self, bridge: MdBridge):
# def start_wrapper():
# try:
# self.start_impl(bridge)
# except Exception as e:
# logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True)
# self.ws_thread = threading.Thread(target=start_wrapper)
# self.ws_thread.start()
# def _on_book_update(self, data: dict, source: str):
# try:
# logger.debug(f"Raw WebSocket data (source={source}): {data}")
# topic = data.get("topic", "")
# if not topic.startswith("orderbook.50."):
# logger.debug(f"Ignoring non-orderbook topic: {topic}")
# return
# contract = topic.split(".")[-1]
# payload = data.get("data", {})
# raw_bids = payload.get("b", [])
# raw_asks = payload.get("a", [])
# if not raw_bids or not raw_asks:
# logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}")
# raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids]
# raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks]
# sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
# sorted_bids += [(np.float64('nan'), np.float64('nan'))]
# sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
# sorted_asks += [(np.float64('nan'), np.float64('nan'))]
# logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
# book = PyBook()
# market_type = "SPOT" if source == "spot" else "FUTURE"
# logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})")
# book.set_data(
# contract,
# "Bybit",
# market_type,
# np.array(sorted_bids, dtype=np.float64),
# np.array(sorted_asks, dtype=np.float64),
# )
# book_list = PyBookList()
# book_list.append(book)
# if self.bridge:
# logger.debug(f"✔ on_books called for {contract} (market_type={market_type})")
# self.bridge.on_books(book_list)
# else:
# logger.warning("Bridge is None, cannot call on_books")
# except Exception as e:
# logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True)
# def stop(self):
# logger.info("Stopping BybitMdProvider...")
# self.stop_event.set()
# logger.debug("Threads before stopping WebSocket:")
# for t in threading.enumerate():
# logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
# for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket
# # , (self.ws_spot, "spot")
# if ws:
# try:
# ws.exit()
# logger.info(f"WebSocket ({ws_type}) exit called")
# for t in threading.enumerate():
# if t.name in ['Thread-1 (<lambda>)', 'Thread-2 (_send_ping)'] and t.is_alive():
# logger.info(f"Joining pybit thread: {t.name} ({ws_type})")
# t.join(timeout=1.0)
# except Exception as e:
# logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True)
# ws = None
# self.ws_futures = None
# # self.ws_spot = None # Commented out spot WebSocket
# if self.ws_thread:
# self.ws_thread.join()
# self.bridge = None
# logger.info("✔ Stopped BybitMdProvider.")
# logger.debug("Threads after stop:")
# for t in threading.enumerate():
# logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
# import os
# os._exit(0)
# if __name__ == "__main__":
# import argparse
# from fake_md_bridge import FakeMdBridge
# ap = argparse.ArgumentParser()
# ap.add_argument("--currencies", nargs="+")
# ap.add_argument("--futures", nargs="+")
# log_levels = list(logging._nameToLevel)
# ap.add_argument("--console-level", choices=log_levels)
# ap.add_argument("--file-level", choices=log_levels)
# ap.add_argument("--dev", action="store_true", dest="use_testnet")
# ap.add_argument("--prod", action="store_false", dest="use_testnet")
# ap.set_defaults(
# currencies=None,
# futures=None,
# console_level=None,
# file_level=None,
# use_testnet=None,
# )
# args = ap.parse_args()
# print(args)
# logger = logging.getLogger("bybit_md_provider_app")
# md_pro = BybitMdProvider()
# config = md_pro.config
# if args.currencies:
# config["currencies"] = args.currencies
# if args.futures:
# config["futures"] = args.futures
# logging_overrides = {}
# if args.console-level:
# logging_overrides["console_level"] = args.console_level
# if args.file-level:
# logging_overrides["file_level"] = args.file_level
# if logging_overrides:
# config["logging"].update(logging_overrides)
# if args.use_testnet is not None:
# config["use_testnet"] = args.use_testnet
# md_bridge = FakeMdBridge(logger)
# md_pro.start(md_bridge)
# while True:
# time.sleep(1)
import json
import numpy as np
import time
@@ -716,7 +175,7 @@ def get_expiration(contract: Dict) -> str:
return PERP_EXP_STRING # Default for perpetuals
def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list:
"""Query Bybit API for linear instruments and return full security names for given base coins."""
"""Query Bybit API for linear instruments and return full security names for given base coins, filtering for LinearFutures and LinearPerpetual."""
futures_contracts = []
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
url = f"https://{host}/v5/market/instruments-info"
@@ -733,8 +192,10 @@ def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list:
data = response.json()
if data.get("retCode") == 0 and data.get("result", {}).get("list"):
for item in data.get("result", {}).get("list", []):
futures_contracts.append(item.get("symbol"))
logger.info(f"Found {len(data['result']['list'])} instruments for {symbol}")
contract_type = item.get("contractType")
if contract_type in ["LinearFutures", "LinearPerpetual"]:
futures_contracts.append(item.get("symbol"))
logger.info(f"Found {len(futures_contracts)} compatible instruments for {symbol}")
else:
logger.warning(f"No instruments found for baseCoin={symbol}: {data.get('retMsg', 'Unknown error')}")