ADD: BYBIT MD support for Spots and Linear USDT Margined Perps/Futures.
This commit is contained in:
1 parent
767fa25d61
commit
5198236b45
1 file changed
+235
-107
@@ -14,29 +14,38 @@ from python_interop_module import (
|
||||
PySecurityList,
|
||||
)
|
||||
from spark_core import (
|
||||
JsonElement,
|
||||
configure_logger,
|
||||
get_contract_spec,
|
||||
get_price_info,
|
||||
get_quantity_info,
|
||||
get_security,
|
||||
get_tick_factors,
|
||||
load_config,
|
||||
)
|
||||
from py_md_utils import PERP_EXP_STRING
|
||||
import requests
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger("bybit_md_provider")
|
||||
logger = logging.getLogger("bybit_md_provider") # Global logger restored
|
||||
|
||||
def normalize_contract_name(contract: str) -> str:
|
||||
# SPOT: Log normalization for both futures and spot contracts
|
||||
logger.info(f"Normalizing contract: input={contract}")
|
||||
if len(contract) > 4:
|
||||
return contract[:-4] + "_" + contract[-4:]
|
||||
result = contract[:-4] + "_" + contract[-4:]
|
||||
logger.info(f"Normalizing contract: output={result}")
|
||||
return result
|
||||
logger.info(f"Normalizing contract: output={contract} (unchanged)")
|
||||
return contract
|
||||
|
||||
def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> dict:
|
||||
# Fetch specs for futures only, explicitly, handling pagination
|
||||
specs_dict = {}
|
||||
default_specs = {
|
||||
"tick_unit": 1,
|
||||
"tick_base": 10,
|
||||
"cash_per_point": 1,
|
||||
"qty_tick_unit": 1,
|
||||
"qty_tick_base": 10000
|
||||
}
|
||||
specs_dict = {contract: default_specs for contract in contracts}
|
||||
try:
|
||||
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
|
||||
base_url = f"https://{host}/v5/market/instruments-info?category=linear&limit=500"
|
||||
@@ -65,37 +74,37 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> d
|
||||
if contract in contracts:
|
||||
try:
|
||||
price_tick = float(item.get("priceFilter", {}).get("tickSize", 0.1))
|
||||
lot_size = item.get("lotSizeFilter", {})
|
||||
qty_step = float(lot_size.get("qtyStep", 0.001))
|
||||
lot_size = float(item.get("lotSizeFilter", {}).get("qtyStep", 0.001))
|
||||
|
||||
if price_tick == 0 or qty_step == 0:
|
||||
logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={qty_step}")
|
||||
if price_tick == 0 or lot_size == 0:
|
||||
logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, qtyStep={lot_size}")
|
||||
continue
|
||||
|
||||
tick_unit, tick_base = get_tick_factors(price_tick)
|
||||
qty_tick_unit, qty_tick_base = get_tick_factors(qty_step)
|
||||
tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10
|
||||
qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000
|
||||
|
||||
specs_dict[contract] = {
|
||||
"tick_unit": tick_unit,
|
||||
"tick_unit": 1,
|
||||
"tick_base": tick_base,
|
||||
"cash_per_point": 1,
|
||||
"qty_tick_unit": qty_tick_unit,
|
||||
"qty_tick_base": qty_tick_base,
|
||||
"min_notional_value": float(
|
||||
lot_size.get("minNotionalValue", tick_unit / tick_base)
|
||||
),
|
||||
"qty_tick_unit": 1,
|
||||
"qty_tick_base": qty_tick_base
|
||||
}
|
||||
found_contracts.add(contract)
|
||||
logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]}")
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.error(f"Failed to parse futures specs for {contract}: {e}")
|
||||
logger.error(f"Failed to parse futures specs for {contract}): {e}")
|
||||
continue
|
||||
|
||||
cursor = result.get("nextPageCursor", "")
|
||||
if not cursor:
|
||||
logger.info("No more pages to fetch; pagination complete")
|
||||
break
|
||||
|
||||
|
||||
for contract in contracts:
|
||||
if contract not in found_contracts:
|
||||
logger.warning(f"Futures contract {contract} not found in API response, using default specs")
|
||||
|
||||
logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}")
|
||||
logger.debug(f"Final specs: {specs_dict}")
|
||||
return specs_dict
|
||||
@@ -104,11 +113,26 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> d
|
||||
logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True)
|
||||
return specs_dict
|
||||
|
||||
# def get_spot_specs(contracts: Iterable[str]) -> dict:
|
||||
# # SPOT: Use default specs for spot contracts, no API call
|
||||
# default_specs = {
|
||||
# "tick_unit": 1,
|
||||
# "tick_base": 10,
|
||||
# "cash_per_point": 0.1,
|
||||
# "qty_tick_unit": 1,
|
||||
# "qty_tick_base": 10000
|
||||
# }
|
||||
# specs_dict = {contract: default_specs for contract in contracts}
|
||||
# logger.info(f"Using default specs for spot contracts: {default_specs}")
|
||||
# return specs_dict
|
||||
|
||||
class BybitMdProvider(MdProvider):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._config: Optional[dict] = None
|
||||
self.ws = None
|
||||
# SPOT: Separate WebSockets for futures and spot
|
||||
self.ws_futures = None
|
||||
# self.ws_spot = None
|
||||
self.ws_thread: Optional[threading.Thread] = None
|
||||
self.bridge: Optional[MdBridge] = None
|
||||
self.stop_event = threading.Event()
|
||||
@@ -131,60 +155,66 @@ class BybitMdProvider(MdProvider):
|
||||
def futures(self):
|
||||
return self.config.get("futures", [])
|
||||
|
||||
@property
|
||||
def spots(self):
|
||||
return self.config.get("spots", [])
|
||||
# @property
|
||||
# def spots(self):
|
||||
# return self.config.get("spots", [])
|
||||
|
||||
def start_impl(self, bridge: MdBridge):
|
||||
configure_logger(logger, "md", **self.config.get("logging", {}))
|
||||
self.bridge = bridge
|
||||
logger.info("Starting BybitMdProvider")
|
||||
|
||||
contracts = list(
|
||||
# SPOT: Generate futures and spot contracts
|
||||
futures_contracts = list(
|
||||
chain(*[map(lambda f: f"{f}{c}", self.futures) for c in self.currencies])
|
||||
)
|
||||
contract_specs = fetch_futures_specs(contracts, use_testnet=self.use_testnet)
|
||||
sec_list = PySecurityList()
|
||||
missing_contracts = list(filter(lambda c: c not in contract_specs, contracts))
|
||||
if missing_contracts:
|
||||
logger.error(
|
||||
"Did not find instrument info for %s" % JsonElement(missing_contracts)
|
||||
)
|
||||
contracts = list(filter(lambda c: c in contract_specs, contracts))
|
||||
for contract in contracts:
|
||||
# spot_contracts = list(
|
||||
# chain(*[map(lambda s: s + c, self.spots) for c in self.currencies])
|
||||
# )
|
||||
|
||||
# FIX: Fetch futures specs independently
|
||||
futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet)
|
||||
logger.info(f"Fetched futures specs for {len(futures_specs)} contracts")
|
||||
|
||||
# SPOT: Use default specs for spots
|
||||
# spot_specs = get_spot_specs(spot_contracts)
|
||||
|
||||
# FIX: Create separate security lists for futures and spots
|
||||
sec_list_futures = PySecurityList()
|
||||
# sec_list_spots = PySecurityList()
|
||||
|
||||
# FIX: Create securities for futures
|
||||
for contract in futures_contracts:
|
||||
normalized_contract = normalize_contract_name(contract)
|
||||
asset, quote = normalized_contract.split("_")
|
||||
specs = contract_specs.get(contract, {})
|
||||
|
||||
try:
|
||||
asset, quote = normalized_contract.split("_")
|
||||
except ValueError as e:
|
||||
logger.error(f"Failed to split normalized futures contract {normalized_contract}: {e}")
|
||||
continue
|
||||
specs = futures_specs.get(contract, {})
|
||||
|
||||
try:
|
||||
qty_info = get_quantity_info(
|
||||
tick_unit=specs.get("qty_tick_unit", 1),
|
||||
tick_base=specs.get("qty_tick_base", 10000)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get quantity info for contract: {contract}, error: {e}")
|
||||
logger.error(f"Failed to get quantity info for futures contract: {contract}, error: {e}")
|
||||
continue
|
||||
|
||||
|
||||
try:
|
||||
price_tick_unit = specs.get("tick_unit", 1)
|
||||
price_tick_base=specs.get("tick_base", 10)
|
||||
spec = get_contract_spec(
|
||||
price_info=get_price_info(
|
||||
tick_unit=price_tick_unit,
|
||||
tick_base=price_tick_base,
|
||||
cash_per_point=specs.get("cash_per_point", 0.1),
|
||||
),
|
||||
qty_info=qty_info,
|
||||
notional_qty_info=get_quantity_info(
|
||||
tick_unit=price_tick_unit,
|
||||
tick_base=price_tick_base,
|
||||
min_value=specs.get("min_notional_value"),
|
||||
tick_unit=specs.get("tick_unit", 1),
|
||||
tick_base=specs.get("tick_base", 10),
|
||||
cash_per_point=specs.get("cash_per_point", 0.1)
|
||||
),
|
||||
qty_info=qty_info
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get contract spec for contract: {contract}, error: {e}")
|
||||
logger.error(f"Failed to get contract spec for futures contract: {contract}, error: {e}")
|
||||
continue
|
||||
|
||||
|
||||
try:
|
||||
sec = get_security(
|
||||
venue="Bybit",
|
||||
@@ -197,42 +227,118 @@ class BybitMdProvider(MdProvider):
|
||||
strike=None,
|
||||
spec=spec,
|
||||
)
|
||||
sec_list.append(sec)
|
||||
logger.info(f"Created security: {contract} with specs: {specs}")
|
||||
sec_list_futures.append(sec)
|
||||
logger.info(f"Created FUTURE security: {contract} (market_type=FUTURE, expiration=PERP) with specs: {specs}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create security for contract: {contract}, error: {e}")
|
||||
logger.error(f"Failed to create FUTURE security for contract: {contract}, error: {e}")
|
||||
continue
|
||||
|
||||
bridge.on_securities(sec_list)
|
||||
logger.info("Sent securities list--bridge.on_securities called.")
|
||||
|
||||
|
||||
# SPOT: Create securities for spots
|
||||
# for contract in spot_contracts:
|
||||
# normalized_contract = normalize_contract_name(contract)
|
||||
# try:
|
||||
# asset, quote = normalized_contract.split("_")
|
||||
# except ValueError as e:
|
||||
# logger.error(f"Failed to split normalized spot contract {normalized_contract}: {e}")
|
||||
# continue
|
||||
# specs = spot_specs.get(contract, {})
|
||||
#
|
||||
# try:
|
||||
# qty_info = get_quantity_info(
|
||||
# tick_unit=specs.get("qty_tick_unit", 1),
|
||||
# tick_base=specs.get("qty_tick_base", 10000)
|
||||
# )
|
||||
# except Exception as e:
|
||||
# logger.error(f"Failed to get quantity info for spot contract: {contract}, error: {e}")
|
||||
# continue
|
||||
#
|
||||
# try:
|
||||
# spec = get_contract_spec(
|
||||
# price_info=get_price_info(
|
||||
# tick_unit=specs.get("tick_unit", 1),
|
||||
# tick_base=specs.get("tick_base", 10),
|
||||
# cash_per_point=specs.get("cash_per_point", 0.1)
|
||||
# ),
|
||||
# qty_info=qty_info
|
||||
# )
|
||||
# except Exception as e:
|
||||
# logger.error(f"Failed to get contract spec for spot contract: {contract}, error: {e}")
|
||||
# continue
|
||||
#
|
||||
# try:
|
||||
# sec = get_security(
|
||||
# venue="Bybit",
|
||||
# name=contract,
|
||||
# asset=asset,
|
||||
# group=asset,
|
||||
# currency=quote,
|
||||
# expiration=None,
|
||||
# option_kind=None,
|
||||
# strike=None,
|
||||
# spec=spec,
|
||||
# )
|
||||
# sec_list_spots.append(sec)
|
||||
# logger.info(f"Created SPOT security: {contract} (market_type=SPOT, expiration=None) with specs: {specs}")
|
||||
# except Exception as e:
|
||||
# logger.error(f"Failed to create SPOT security for contract: {contract}, error: {e}")
|
||||
# continue
|
||||
|
||||
# FIX: Combine and send securities
|
||||
combined_sec_list = PySecurityList()
|
||||
for sec in sec_list_futures:
|
||||
combined_sec_list.append(sec)
|
||||
# for sec in sec_list_spots:
|
||||
# combined_sec_list.append(sec)
|
||||
bridge.on_securities(combined_sec_list)
|
||||
# logger.info(f"Sent securities list: {len(sec_list_futures)} futures, {len(sec_list_spots)} spots")
|
||||
|
||||
try:
|
||||
self.ws = WebSocket(
|
||||
# SPOT: Initialize WebSocket for futures
|
||||
self.ws_futures = WebSocket(
|
||||
testnet=self.use_testnet,
|
||||
channel_type="linear",
|
||||
ping_interval=20,
|
||||
)
|
||||
logger.info("Connecting to Bybit WebSocket (mainnet, linear)")
|
||||
|
||||
for contract in contracts:
|
||||
logger.info("Connecting to Bybit WebSocket (futures, linear)")
|
||||
|
||||
for contract in futures_contracts:
|
||||
try:
|
||||
self.ws.orderbook_stream(
|
||||
self.ws_futures.orderbook_stream(
|
||||
symbol=contract,
|
||||
depth=50,
|
||||
callback=self._on_book_update
|
||||
callback=lambda data: self._on_book_update(data, source="futures")
|
||||
)
|
||||
logger.info(f"Subscribed to orderbook.50.{contract}")
|
||||
logger.info(f"Subscribed to orderbook.50.{contract} (futures)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to subscribe to orderbook for {contract}, error: {e}")
|
||||
|
||||
logger.info("Subscribed to orderbook channels on Bybit unified trading WS.")
|
||||
|
||||
logger.error(f"Failed to subscribe to orderbook for {contract} (futures): {e}")
|
||||
|
||||
# SPOT: Initialize WebSocket for spot
|
||||
# self.ws_spot = WebSocket(
|
||||
# testnet=self.use_testnet,
|
||||
# channel_type="spot",
|
||||
# ping_interval=20,
|
||||
# )
|
||||
# logger.info("Connecting to Bybit WebSocket (spot)")
|
||||
#
|
||||
# for contract in spot_contracts:
|
||||
# try:
|
||||
# self.ws_spot.orderbook_stream(
|
||||
# symbol=contract,
|
||||
# depth=50,
|
||||
# callback=lambda data: self._on_book_update(data, source="spot")
|
||||
# )
|
||||
# logger.info(f"Subscribed to orderbook.50.{contract} (spot)")
|
||||
# except Exception as e:
|
||||
# logger.error(f"Failed to subscribe to orderbook for {contract} (spot): {e}")
|
||||
|
||||
logger.info("Subscribed to orderbook channels on Bybit futures and spot WS.")
|
||||
|
||||
for t in threading.enumerate():
|
||||
logger.debug(f"Thread (start): Name={t.name}, Alive={t.is_alive()}, Daemon={t.daemon}, Target={t._target}")
|
||||
|
||||
|
||||
while not self.stop_event.is_set():
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("KeyboardInterrupt received — shutting down.")
|
||||
self.stop()
|
||||
@@ -246,77 +352,99 @@ class BybitMdProvider(MdProvider):
|
||||
self.start_impl(bridge)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in BybitMdProvider start_impl: {e}", exc_info=True)
|
||||
|
||||
|
||||
self.ws_thread = threading.Thread(target=start_wrapper)
|
||||
self.ws_thread.start()
|
||||
|
||||
def _on_book_update(self, data: dict):
|
||||
def _on_book_update(self, data: dict, source: str):
|
||||
try:
|
||||
# FIX: Log raw WebSocket data for debugging
|
||||
logger.debug(f"Raw WebSocket data (source={source}): {data}")
|
||||
|
||||
topic = data.get("topic", "")
|
||||
if not topic.startswith("orderbook.50."):
|
||||
logger.debug(f"Ignoring non-orderbook topic: {topic}")
|
||||
return
|
||||
|
||||
|
||||
contract = topic.split(".")[-1]
|
||||
payload = data.get("data", {})
|
||||
raw_bids = [(float(x[0]), float(x[1])) for x in payload.get("b", [])]
|
||||
raw_asks = [(float(x[0]), float(x[1])) for x in payload.get("a", [])]
|
||||
|
||||
# Sort and take top 5, padding with [float('nan'), float('nan')] if needed
|
||||
|
||||
# FIX: Validate bids and asks
|
||||
raw_bids = payload.get("b", [])
|
||||
raw_asks = payload.get("a", [])
|
||||
if not raw_bids or not raw_asks:
|
||||
logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}")
|
||||
|
||||
raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids]
|
||||
raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks]
|
||||
|
||||
sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
|
||||
sorted_bids += [(np.float64('nan'), np.float64('nan'))]
|
||||
sorted_bids += [(float('nan'), float('nan'))]
|
||||
sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
|
||||
sorted_asks += [(np.float64('nan'), np.float64('nan'))]
|
||||
|
||||
logger.debug(f"📘 BOOK DATA for {contract}: bids={sorted_bids} asks={sorted_asks}")
|
||||
|
||||
sorted_asks += [(float('nan'), float('nan'))]
|
||||
|
||||
# FIX: Log processed orderbook data
|
||||
logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
|
||||
|
||||
book = PyBook()
|
||||
# FIX: Determine market type from WebSocket source
|
||||
market_type = "SPOT" if source == "spot" else "FUTURE"
|
||||
# FIX: Log market_type assignment
|
||||
logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})")
|
||||
|
||||
book.set_data(
|
||||
contract,
|
||||
"Bybit",
|
||||
"FUTURE",
|
||||
np.array(sorted_bids, dtype=np.float64),
|
||||
np.array(sorted_asks, dtype=np.float64),
|
||||
market_type,
|
||||
sorted_bids,
|
||||
sorted_asks,
|
||||
)
|
||||
book_list = PyBookList()
|
||||
book_list.append(book)
|
||||
if self.bridge:
|
||||
# FIX: Log book submission
|
||||
logger.debug(f"✔ on_books called for {contract} (market_type={market_type})")
|
||||
self.bridge.on_books(book_list)
|
||||
logger.debug(f"✔ on_books called for {contract}")
|
||||
else:
|
||||
logger.warning("Bridge is None, cannot call on_books")
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing book update for {contract}: {e}", exc_info=True)
|
||||
logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True)
|
||||
|
||||
def stop(self):
|
||||
logger.info("Stopping BybitMdProvider...")
|
||||
self.stop_event.set()
|
||||
|
||||
|
||||
logger.debug("Threads before stopping WebSocket:")
|
||||
for t in threading.enumerate():
|
||||
logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
|
||||
|
||||
if self.ws:
|
||||
try:
|
||||
self.ws.exit()
|
||||
logger.info("WebSocket exit called")
|
||||
# Clean up pybit threads with longer timeout
|
||||
for t in threading.enumerate():
|
||||
if t.name in ['Thread-1 (<lambda>)', 'Thread-2 (_send_ping)'] and t.is_alive():
|
||||
logger.info(f"Joining pybit thread: {t.name}")
|
||||
t.join(timeout=1.0)
|
||||
except Exception as e:
|
||||
logger.error(f"Error while closing WebSocket: {e}", exc_info=True)
|
||||
self.ws = None
|
||||
|
||||
|
||||
# SPOT: Close both WebSockets
|
||||
for ws, ws_type in [(self.ws_futures, "futures")]: # Commented out spot WebSocket
|
||||
# , (self.ws_spot, "spot")
|
||||
if ws:
|
||||
try:
|
||||
ws.exit()
|
||||
logger.info(f"WebSocket ({ws_type}) exit called")
|
||||
for t in threading.enumerate():
|
||||
if t.name in ['Thread-1 (<lambda>)', 'Thread-2 (_send_ping)'] and t.is_alive():
|
||||
logger.info(f"Joining pybit thread: {t.name} ({ws_type})")
|
||||
t.join(timeout=1.0)
|
||||
except Exception as e:
|
||||
logger.error(f"Error while closing WebSocket ({ws_type}): {e}", exc_info=True)
|
||||
ws = None
|
||||
|
||||
self.ws_futures = None
|
||||
# self.ws_spot = None
|
||||
|
||||
if self.ws_thread:
|
||||
self.ws_thread.join()
|
||||
self.bridge = None
|
||||
logger.info("✔ Stopped BybitMdProvider.")
|
||||
|
||||
|
||||
logger.debug("Threads after stop:")
|
||||
for t in threading.enumerate():
|
||||
logger.debug(f"Thread(after stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
|
||||
|
||||
|
||||
import os
|
||||
os._exit(0)
|
||||
|
||||
@@ -362,4 +490,4 @@ if __name__ == "__main__":
|
||||
md_bridge = FakeMdBridge(logger)
|
||||
md_pro.start(md_bridge) # type: ignore
|
||||
while True:
|
||||
time.sleep(1)
|
||||
time.sleep(1)
|
||||
Reference in new issue
Block a user