ADD: BYBIT MD support for Spots and Linear USDT Margined Perps/Futures.

This commit is contained in:
Adeel Khan committed 2025-07-14 22:35:36 +00:00
1 parent 767fa25d61
commit 5198236b45
1 file changed
+235 -107
@@ -14,29 +14,38 @@ from python_interop_module import (
PySecurityList,
)
from spark_core import (
JsonElement,
configure_logger,
get_contract_spec,
get_price_info,
get_quantity_info,
get_security,
get_tick_factors,
load_config,
)
from py_md_utils import PERP_EXP_STRING
import requests
import numpy as np
logger = logging.getLogger("bybit_md_provider")
logger = logging.getLogger("bybit_md_provider") # Global logger restored
def normalize_contract_name(contract: str) -> str:
# SPOT: Log normalization for both futures and spot contracts
logger.info(f"Normalizing contract: input={contract}")
if len(contract) > 4:
return contract[:-4] + "_" + contract[-4:]
result = contract[:-4] + "_" + contract[-4:]
logger.info(f"Normalizing contract: output={result}")
return result
logger.info(f"Normalizing contract: output={contract} (unchanged)")
return contract
def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> dict:
# Fetch specs for futures only, explicitly, handling pagination
specs_dict = {}
default_specs = {
"tick_unit": 1,
"tick_base": 10,
"cash_per_point": 1,
"qty_tick_unit": 1,
"qty_tick_base": 10000
}
specs_dict = {contract: default_specs for contract in contracts}
try:
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500"
@@ -65,37 +74,37 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> d
if contract in contracts:
try:
price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1))
lot_size = item.get("lotSizeFilter", {})
qty_step = float(lot_size.get("qtyStep", 0.001))
lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001))
if price_tick == 0 or qty_step == 0:
logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={qty_step}")
if price_tick == 0 or lot_size == 0:
logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}")
continue
tick_unit, tick_base = get_tick_factors(price_tick)
qty_tick_unit, qty_tick_base = get_tick_factors(qty_step)
tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10
qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000
specs_dict[contract] = {
"tick_unit": tick_unit,
"tick_unit": 1,
"tick_base": tick_base,
"cash_per_point": 1,
"qty_tick_unit": qty_tick_unit,
"qty_tick_base": qty_tick_base,
"min_notional_value": float(
lot_size.get("minNotionalValue", tick_unit / tick_base)
),
"qty_tick_unit": 1,
"qty_tick_base": qty_tick_base
}
found_contracts.add(contract)
logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]}")
except (ValueError, TypeError) as e:
logger.error(f"Failed to parse futures specs for {contract}: {e}")
logger.error(f"Failed to parse futures specs for {contract}): {e}")
continue
cursor = result.get("nextPageCursor", "")
if not cursor:
logger.info("No more pages to fetch; pagination complete")
break
for contract in contracts:
if contract not in found_contracts:
logger.warning(f"Futures contract {contract} not found in API response, using default specs")
logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}")
logger.debug(f"Final specs: {specs_dict}")
return specs_dict
@@ -104,11 +113,26 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> d
logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True)
return specs_dict
# def get_spot_specs(contracts: Iterable[str]) -> dict:
# # SPOT: Use default specs for spot contracts, no API call
# default_specs = {
# "tick_unit": 1,
# "tick_base": 10,
# "cash_per_point": 0.1,
# "qty_tick_unit": 1,
# "qty_tick_base": 10000
# }
# specs_dict = {contract: default_specs for contract in contracts}
# logger.info(f"Using default specs for spot contracts: {default_specs}")
# return specs_dict
class BybitMdProvider(MdProvider):
def __init__(self):
super().__init__()
self._config: Optional[dict] = None
self.ws = None
# SPOT: Separate WebSockets for futures and spot
self.ws_futures = None
# self.ws_spot = None
self.ws_thread: Optional[threading.Thread] = None
self.bridge: Optional[MdBridge] = None
self.stop_event = threading.Event()
@@ -131,60 +155,66 @@ class BybitMdProvider(MdProvider):
def futures(self):
return self.config.get("futures", [])
@property
def spots(self):
return self.config.get("spots", [])
# @property
# def spots(self):
# return self.config.get("spots", [])
def start_impl(self, bridge: MdBridge):
configure_logger(logger, "md", **self.config.get("logging", {}))
self.bridge = bridge
logger.info("Starting BybitMdProvider")
contracts = list(
# SPOT: Generate futures and spot contracts
futures_contracts = list(
chain(*[map(lambda f: f"{f}{c}", self.futures) for c in self.currencies])
)
contract_specs = fetch_futures_specs(contracts, use_testnet=self.use_testnet)
sec_list = PySecurityList()
missing_contracts = list(filter(lambda c: c not in contract_specs, contracts))
if missing_contracts:
logger.error(
"Did not find instrument info for %s" % JsonElement(missing_contracts)
)
contracts = list(filter(lambda c: c in contract_specs, contracts))
for contract in contracts:
# spot_contracts = list(
# chain(*[map(lambda s: s + c, self.spots) for c in self.currencies])
# )
# FIX: Fetch futures specs independently
futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet)
logger.info(f"Fetched futures specs for {len(futures_specs)} contracts")
# SPOT: Use default specs for spots
# spot_specs = get_spot_specs(spot_contracts)
# FIX: Create separate security lists for futures and spots
sec_list_futures = PySecurityList()
# sec_list_spots = PySecurityList()
# FIX: Create securities for futures
for contract in futures_contracts:
normalized_contract = normalize_contract_name(contract)
asset, quote = normalized_contract.split("_")
specs = contract_specs.get(contract, {})
try:
asset, quote = normalized_contract.split("_")
except ValueError as e:
logger.error(f"Failed to split normalized futures contract {normalized_contract}: {e}")
continue
specs = futures_specs.get(contract, {})
try:
qty_info = get_quantity_info(
tick_unit=specs.get("qty_tick_unit", 1),
tick_base=specs.get("qty_tick_base", 10000)
)
except Exception as e:
logger.error(f"Failed to get quantity info for contract: {contract}, error: {e}")
logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}")
continue
try:
price_tick_unit = specs.get("tick_unit", 1)
price_tick_base=specs.get("tick_base", 10)
spec = get_contract_spec(
price_info=get_price_info(
tick_unit=price_tick_unit,
tick_base=price_tick_base,
cash_per_point=specs.get("cash_per_point", 0.1),
),
qty_info=qty_info,
notional_qty_info=get_quantity_info(
tick_unit=price_tick_unit,
tick_base=price_tick_base,
min_value=specs.get("min_notional_value"),
tick_unit=specs.get("tick_unit", 1),
tick_base=specs.get("tick_base", 10),
cash_per_point=specs.get("cash_per_point", 0.1)
),
qty_info=qty_info
)
except Exception as e:
logger.error(f"Failed to get contract spec for contract: {contract}, error: {e}")
logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}")
continue
try:
sec = get_security(
venue="Bybit",
@@ -197,42 +227,118 @@ class BybitMdProvider(MdProvider):
strike=None,
spec=spec,
)
sec_list.append(sec)
logger.info(f"Created security: {contract} with specs: {specs}")
sec_list_futures.append(sec)
logger.info(f"Created FUTURE security: {contract} (market_type=FUTURE, expiration=PERP) with specs: {specs}")
except Exception as e:
logger.error(f"Failed to create security for contract: {contract}, error: {e}")
logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}")
continue
bridge.on_securities(sec_list)
logger.info("Sent securities list--bridge.on_securities called.")
# SPOT: Create securities for spots
# for contract in spot_contracts:
# normalized_contract = normalize_contract_name(contract)
# try:
# asset, quote = normalized_contract.split("_")
# except ValueError as e:
# logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}")
# continue
# specs = spot_specs.get(contract, {})
#
# try:
# qty_info = get_quantity_info(
# tick_unit=specs.get("qty_tick_unit", 1),
# tick_base=specs.get("qty_tick_base", 10000)
# )
# except Exception as e:
# logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}")
# continue
#
# try:
# spec = get_contract_spec(
# price_info=get_price_info(
# tick_unit=specs.get("tick_unit", 1),
# tick_base=specs.get("tick_base", 10),
# cash_per_point=specs.get("cash_per_point", 0.1)
# ),
# qty_info=qty_info
# )
# except Exception as e:
# logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}")
# continue
#
# try:
# sec = get_security(
# venue="Bybit",
# name=contract,
# asset=asset,
# group=asset,
# currency=quote,
# expiration=None,
# option_kind=None,
# strike=None,
# spec=spec,
# )
# sec_list_spots.append(sec)
# logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}")
# except Exception as e:
# logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}")
# continue
# FIX: Combine and send securities
combined_sec_list = PySecurityList()
for sec in sec_list_futures:
combined_sec_list.append(sec)
# for sec in sec_list_spots:
# combined_sec_list.append(sec)
bridge.on_securities(combined_sec_list)
# logger.info(f"Sent securities list: {len(sec_list_futures)} futures, {len(sec_list_spots)} spots")
try:
self.ws = WebSocket(
# SPOT: Initialize WebSocket for futures
self.ws_futures = WebSocket(
testnet=self.use_testnet,
channel_type="linear",
ping_interval=20,
)
logger.info("Connecting to Bybit WebSocket (mainnet, linear)")
for contract in contracts:
logger.info("Connecting to Bybit WebSocket (futures, linear)")
for contract in futures_contracts:
try:
self.ws.orderbook_stream(
self.ws_futures.orderbook_stream(
symbol=contract,
depth=50,
callback=self._on_book_update
callback=lambda data: self._on_book_update(data, source="futures")
)
logger.info(f"Subscribed to orderbook.50.{contract}")
logger.info(f"Subscribed to orderbook.50.{contract} (futures)")
except Exception as e:
logger.error(f"Failed to subscribe to orderbook for {contract}, error: {e}")
logger.info("Subscribed to orderbook channels on Bybit unified trading WS.")
logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}")
# SPOT: Initialize WebSocket for spot
# self.ws_spot = WebSocket(
# testnet=self.use_testnet,
# channel_type="spot",
# ping_interval=20,
# )
# logger.info("Connecting to Bybit WebSocket (spot)")
#
# for contract in spot_contracts:
# try:
# self.ws_spot.orderbook_stream(
# symbol=contract,
# depth=50,
# callback=lambda data: self._on_book_update(data, source="spot")
# )
# logger.info(f"Subscribed to orderbook.50.{contract} (spot)")
# except Exception as e:
# logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}")
logger.info("Subscribed to orderbook channels on Bybit futures and spot WS.")
for t in threading.enumerate():
logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}")
while not self.stop_event.is_set():
time.sleep(1)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt received — shutting down.")
self.stop()
@@ -246,77 +352,99 @@ class BybitMdProvider(MdProvider):
self.start_impl(bridge)
except Exception as e:
logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True)
self.ws_thread = threading.Thread(target=start_wrapper)
self.ws_thread.start()
def _on_book_update(self, data: dict):
def _on_book_update(self, data: dict, source: str):
try:
# FIX: Log raw WebSocket data for debugging
logger.debug(f"Raw WebSocket data (source={source}): {data}")
topic = data.get("topic", "")
if not topic.startswith("orderbook.50."):
logger.debug(f"Ignoring non-orderbook topic: {topic}")
return
contract = topic.split(".")[-1]
payload = data.get("data", {})
raw_bids = [(float(x[0]), float(x[1])) for x in payload.get("b", [])]
raw_asks = [(float(x[0]), float(x[1])) for x in payload.get("a", [])]
# Sort and take top 5, padding with [float('nan'), float('nan')] if needed
# FIX: Validate bids and asks
raw_bids = payload.get("b", [])
raw_asks = payload.get("a", [])
if not raw_bids or not raw_asks:
logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}")
raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids]
raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks]
sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
sorted_bids += [(np.float64('nan'), np.float64('nan'))]
sorted_bids += [(float('nan'), float('nan'))]
sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
sorted_asks += [(np.float64('nan'), np.float64('nan'))]
logger.debug(f"📘 BOOK DATA for {contract}: bids={sorted_bids} asks={sorted_asks}")
sorted_asks += [(float('nan'), float('nan'))]
# FIX: Log processed orderbook data
logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
book = PyBook()
# FIX: Determine market type from WebSocket source
market_type = "SPOT" if source == "spot" else "FUTURE"
# FIX: Log market_type assignment
logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})")
book.set_data(
contract,
"Bybit",
"FUTURE",
np.array(sorted_bids, dtype=np.float64),
np.array(sorted_asks, dtype=np.float64),
market_type,
sorted_bids,
sorted_asks,
)
book_list = PyBookList()
book_list.append(book)
if self.bridge:
# FIX: Log book submission
logger.debug(f"✔ on_books called for {contract} (market_type={market_type})")
self.bridge.on_books(book_list)
logger.debug(f"✔ on_books called for {contract}")
else:
logger.warning("Bridge is None, cannot call on_books")
except Exception as e:
logger.error(f"Error processing book update for {contract}: {e}", exc_info=True)
logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True)
def stop(self):
logger.info("Stopping BybitMdProvider...")
self.stop_event.set()
logger.debug("Threads before stopping WebSocket:")
for t in threading.enumerate():
logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
if self.ws:
try:
self.ws.exit()
logger.info("WebSocket exit called")
# Clean up pybit threads with longer timeout
for t in threading.enumerate():
if t.name in ['Thread-1 (<lambda>)', 'Thread-2 (_send_ping)'] and t.is_alive():
logger.info(f"Joining pybit thread: {t.name}")
t.join(timeout=1.0)
except Exception as e:
logger.error(f"Error while closing WebSocket: {e}", exc_info=True)
self.ws = None
# SPOT: Close both WebSockets
for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket
# , (self.ws_spot, "spot")
if ws:
try:
ws.exit()
logger.info(f"WebSocket ({ws_type}) exit called")
for t in threading.enumerate():
if t.name in ['Thread-1 (<lambda>)', 'Thread-2 (_send_ping)'] and t.is_alive():
logger.info(f"Joining pybit thread: {t.name} ({ws_type})")
t.join(timeout=1.0)
except Exception as e:
logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True)
ws = None
self.ws_futures = None
# self.ws_spot = None
if self.ws_thread:
self.ws_thread.join()
self.bridge = None
logger.info("✔ Stopped BybitMdProvider.")
logger.debug("Threads after stop:")
for t in threading.enumerate():
logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
import os
os._exit(0)
@@ -362,4 +490,4 @@ if __name__ == "__main__":
md_bridge = FakeMdBridge(logger)
md_pro.start(md_bridge) # type: ignore
while True:
time.sleep(1)
time.sleep(1)