ADD: Support for Spreads on Bybit. Gateio fixes.

This commit is contained in:
Adeel Khan committed 2025-08-21 22:12:03 +00:00
1 parent 1b5f1bfb27
commit ca27903f63
7 files changed
+1192 -191

No files matched your search

@@ -30,6 +30,12 @@ void to_json(nlohmann::json &j, const PySecurity &sec) {
j["k"] = opt.strike();
j["cp"] = opt.is_call() ? "C" : "P";
},
[&](const PySpread &spr) {
j["typ"] = "SPR";
j["exp"] = spr.expiration();
j["spr_type"] = spr.spread_type();
j["legs"] = spr.legs();
},
},
sec.sec_);
}
@@ -78,6 +84,11 @@ void from_json(const nlohmann::json &j, PySecurity &sec) {
opt.set_expiration(j.at("exp").get<string>());
opt.set_is_call(j.at("cp").get<string>().starts_with('C'));
opt.set_strike(j.at("k").get<double>());
} else if (typ.starts_with('S') && typ != "SPOT") {
auto &spr = sec.sec_.emplace<PySpread>();
spr.set_expiration(j.at("exp").get<string>());
spr.set_spread_type(j.at("spr_type").get<string>());
spr.set_legs(j.at("legs").get<std::vector<string>>());
}
}
@@ -4,6 +4,7 @@
#include "python_interop/py_future.h"
#include "python_interop/py_option.h"
#include "python_interop/py_spot.h"
#include "python_interop/py_spread.h"
#include "spark/io/json_formatter.h"
#include <cstdint>
@@ -50,6 +51,10 @@ class PySecurity {
void set_option(PyOption);
constexpr bool is_option() const;
const PySpread &spread() const;
void set_spread(PySpread);
constexpr bool is_spread() const;
const PyContractSpecification &contract_spec() const;
void set_contract_spec(PyContractSpecification value);
@@ -65,7 +70,7 @@ class PySecurity {
std::string name_;
std::string currency_;
PyContractSpecification contract_spec_;
std::variant<PySpot, PyFuture, PyOption> sec_;
std::variant<PySpot, PyFuture, PyOption, PySpread> sec_;
friend struct fmt::formatter<PySecurity>;
friend void to_json(nlohmann::json &j, const PySecurity &sec);
@@ -180,6 +185,18 @@ constexpr bool PySecurity::is_option() const {
return std::holds_alternative<PyOption>(sec_);
}
inline const PySpread &PySecurity::spread() const {
if (std::holds_alternative<PySpread>(sec_)) {
return std::get<PySpread>(sec_);
}
throw std::runtime_error{"PySecurity is not a spread"};
}
inline void PySecurity::set_spread(PySpread value) { sec_ = std::move(value); }
constexpr bool PySecurity::is_spread() const {
return std::holds_alternative<PySpread>(sec_);
}
inline const PyContractSpecification &PySecurity::contract_spec() const {
return contract_spec_;
}
@@ -16,6 +16,8 @@
#include "python_interop/py_contract_quantity_info.h"
#include "python_interop/py_contract_specification.h"
#include "python_interop/py_security.h"
#include "python_interop/py_spread.h"
#include <memory>
#include <pybind11/detail/common.h>
@@ -35,6 +37,7 @@ using bts::md::py_interop::PyMdProvider;
using bts::md::py_interop::PyOption;
using bts::md::py_interop::PySecurity;
using bts::md::py_interop::PySpot;
using bts::md::py_interop::PySpread;
using bts::md::py_interop::PyTrade;
using python_execution::ExecutionGateway;
using python_execution::ExecutionGatewayBridge;
@@ -129,6 +132,19 @@ PYBIND11_MODULE(python_interop_module, m) {
.PROP(PyOption, is_call)
.PROP(PyOption, strike)
.def("__str__", &PyOption::to_string);
py::class_<PySpread>(m, "PySpread")
.def(py::init<>())
.PROP(PySpread, spread_type)
.def("legs", &PySpread::legs) // ← Changed this line
.def("set_legs", [](PySpread& self, py::list legs_list) { // ← Custom binding
std::vector<std::string> legs;
for (auto item : legs_list) {
legs.push_back(py::cast<std::string>(item));
}
self.set_legs(legs);
})
.PROP(PySpread, expiration)
.def("__str__", &PySpread::to_string);
py::class_<PySecurity, shared_ptr<PySecurity>>(m, "PySecurity")
.def(py::init<>())
.PROP(PySecurity, id)
@@ -142,11 +158,35 @@ PYBIND11_MODULE(python_interop_module, m) {
.PROP(PySecurity, spot)
.PROP(PySecurity, future)
.PROP(PySecurity, option)
.PROP(PySecurity, spread)
.def("set_spread_type", [](PySecurity& self, const std::string& spread_type) {
if (self.is_spread()) {
const_cast<PySpread&>(self.spread()).set_spread_type(spread_type);
}
})
.def("set_legs", [](PySecurity& self, py::list legs_list) {
if (self.is_spread()) {
std::vector<std::string> legs;
for (auto item : legs_list) {
legs.push_back(py::cast<std::string>(item));
}
const_cast<PySpread&>(self.spread()).set_legs(legs);
}
})
.def("set_expiration", [](PySecurity& self, const std::string& expiration) {
if (self.is_spread()) {
const_cast<PySpread&>(self.spread()).set_expiration(expiration);
}
})
.def("is_spread", &PySecurity::is_spread)
.def("__str__", &PySecurity::to_string)
.def("__format__", &PySecurity::format);
py::bind_vector<std::vector<shared_ptr<PySecurity>>>(m, "PySecurityList")
.PY_LIST_TO_STRING(PySecurity);
py::bind_vector<std::vector<std::string>>(m, "StringVector");
//////////////////////////////////////////////////
// Md
//////////////////////////////////////////////////
@@ -4,15 +4,20 @@ import time
import logging
import sys
import threading
import ctypes
from itertools import chain
from typing import Iterable, Optional, Dict
from pybit.unified_trading import WebSocket
import websockets
import asyncio
from python_interop_module import (
MdBridge,
MdProvider,
PyBook,
PyBookList,
PySecurityList,
PySecurity,
PySpread
)
from spark_core import (
configure_logger,
@@ -25,6 +30,7 @@ from spark_core import (
from py_md_utils import PERP_EXP_STRING
import requests
logger = logging.getLogger("bybit_md_provider")
@@ -128,7 +134,7 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> D
if contract not in found_contracts:
logger.warning(f"Futures contract {contract} not found in API response, using default specs")
logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}")
logger.debug(f"Returning specs for contracts: {list(specs_dict.keys())}")
logger.debug(f"Final specs: {specs_dict}")
return specs_dict
@@ -170,9 +176,6 @@ def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list:
return sorted(list(set(futures_contracts))) # Remove duplicates and sort
# Spot support starts here (updated)
def get_spot_metadata(base_symbols: list, currencies: list, logger, use_testnet: bool = True) -> dict:
"""Fetch metadata for spot contracts filtered by base symbols and supported currencies."""
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
@@ -194,14 +197,72 @@ def get_spot_metadata(base_symbols: list, currencies: list, logger, use_testnet:
else:
logger.error(f"Spot metadata API error: {data.get('retMsg', 'Unknown error')}")
return spots_meta
# Spot support ends here
def get_spread_contracts(base_symbols: list, use_testnet: bool = True) -> list:
"""Query Bybit API for all spread instruments and filter by base symbols."""
spread_contracts = []
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
url = f"https://{host}/v5/spread/instrument?limit=500"
logger.info(f"Querying all spread instruments: {url}")
response = requests.get(url, timeout=5)
response.raise_for_status()
data = response.json()
if data.get("retCode") == 0 and data.get("result", {}).get("list"):
logger.info(f"Retrieved {len(data['result']['list'])} spread instruments")
for item in data.get("result", {}).get("list", []):
base_coin = item.get("baseCoin")
if base_coin in base_symbols:
spread_contracts.append(item.get("symbol"))
logger.info(f"Filtered to {len(spread_contracts)} compatible spread instruments for base symbols {base_symbols}")
else:
logger.warning(f"Failed to retrieve instruments: {data.get('retMsg', 'Unknown error')}")
return sorted(list(set(spread_contracts))) # Remove duplicates and sort
def get_spread_metadata(base_symbols: list, logger, use_testnet: bool = True) -> dict:
"""Fetch metadata for spread contracts filtered by base symbols."""
host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com"
url = f"https://{host}/v5/spread/instrument?limit=500"
logger.info(f"Querying Bybit API for spread metadata: {url}")
try:
response = requests.get(url, timeout=5)
response.raise_for_status()
data = response.json()
logger.info(f"Spread API response retCode: {data.get('retCode')}")
logger.info(f"Spread API response retMsg: {data.get('retMsg')}")
spreads_meta = {}
if data.get("retCode") == 0:
spread_list = data.get("result", {}).get("list", [])
logger.info(f"Retrieved {len(spread_list)} spread contracts from API")
for item in spread_list:
base_coin = item.get("baseCoin")
if base_coin in base_symbols:
symbol = item.get("symbol")
spreads_meta[symbol] = item
logger.info(f"Stored metadata for spread {symbol} with base {base_coin}")
logger.info(f"Total spreads stored: {len(spreads_meta)}")
else:
logger.error(f"Spread metadata API error: {data.get('retMsg', 'Unknown error')}")
except Exception as e:
logger.error(f"Failed to fetch spread metadata: {e}", exc_info=True)
spreads_meta = {}
return spreads_meta
class BybitMdProvider(MdProvider):
def __init__(self):
super().__init__()
self._config: Optional[dict] = None
self.ws_futures = None
self.ws_spots = None # Add WebSocket for spots
self.ws_spots = None
self.ws_thread: Optional[threading.Thread] = None
self.bridge: Optional[MdBridge] = None
self.stop_event = threading.Event()
@@ -224,11 +285,14 @@ class BybitMdProvider(MdProvider):
def futures(self):
return self.config.get("futures", [])
# Spot support starts here
@property
def spots(self):
return self.config.get("spots", [])
# Spot support ends here
@property
def spreads(self):
return self.config.get("spreads", [])
def start_impl(self, bridge: MdBridge):
configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level
@@ -250,7 +314,7 @@ class BybitMdProvider(MdProvider):
sec_list_futures = PySecurityList()
for contract in futures_contracts:
meta = metadata.get(contract, {})
logger.info(f"Metadata for {contract}: {meta}")
logger.debug(f"Metadata for {contract}: {meta}")
if not meta:
logger.warning(f"No metadata found for contract: {contract}, using defaults")
continue
@@ -300,7 +364,6 @@ class BybitMdProvider(MdProvider):
bridge.on_securities(combined_sec_list)
logger.info(f"Sent securities list: {len(sec_list_futures)} futures")
# Spot support starts here
# Get spot metadata
spots_meta = get_spot_metadata(self.spots, self.currencies, logger, self.use_testnet)
logger.info(f"Fetched metadata for {len(spots_meta)} spot contracts")
@@ -403,8 +466,297 @@ class BybitMdProvider(MdProvider):
combined_sec_list.append(sec)
bridge.on_securities(combined_sec_list)
logger.info(f"Sent securities list: {len(sec_list_spots)} spots + {len(sec_list_futures)} futures")
# Spot support ends here
# Get spread metadata
spreads_meta = get_spread_metadata(self.spreads, logger, self.use_testnet)
logger.info(f"Fetched metadata for {len(spreads_meta)} spread contracts")
# Extract all unique leg symbols from spread metadata
all_leg_symbols = set()
leg_metadata = {} # Store metadata for each leg symbol
for symbol, meta in spreads_meta.items():
legs = meta.get("legs", [])
for leg in legs:
leg_symbol = leg.get("symbol")
leg_contract_type = leg.get("contractType")
if leg_symbol and leg_contract_type:
all_leg_symbols.add(leg_symbol)
leg_metadata[leg_symbol] = {
"contractType": leg_contract_type,
"baseCoin": meta.get("baseCoin"),
"quoteCoin": meta.get("quoteCoin")
}
logger.info(f"Found {len(all_leg_symbols)} unique leg symbols: {sorted(all_leg_symbols)}")
logger.debug(f"Leg metadata summary: {leg_metadata}")
# Create leg securities (before spread securities)
sec_list_legs = PySecurityList()
# Log what metadata we have for legs
for leg_symbol in sorted(all_leg_symbols):
if leg_symbol in metadata:
meta = metadata.get(leg_symbol, {})
delivery_date = meta.get("deliveryDate")
status = meta.get("status")
logger.info(f"Leg {leg_symbol} metadata: deliveryDate={delivery_date}, status={status}")
else:
logger.warning(f"No metadata found for leg {leg_symbol} in existing futures metadata")
# Use existing futures specs for legs that are futures
futures_leg_symbols = [symbol for symbol in all_leg_symbols
if leg_metadata.get(symbol, {}).get("contractType") in ["LinearFutures", "LinearPerpetual"]]
logger.info(f"Found {len(futures_leg_symbols)} futures legs that can use existing specs")
# Also check what spot specs we have available for legs
spot_leg_symbols = [symbol for symbol in all_leg_symbols
if leg_metadata.get(symbol, {}).get("contractType") == "Spot"]
logger.info(f"Found {len(spot_leg_symbols)} spot legs that can use existing specs")
# Log what specs are available for legs
for leg_symbol in sorted(all_leg_symbols):
leg_meta = leg_metadata.get(leg_symbol, {})
contract_type = leg_meta.get("contractType", "Unknown")
if contract_type in ["LinearFutures", "LinearPerpetual"]:
if leg_symbol in futures_specs:
logger.info(f"Leg {leg_symbol} ({contract_type}): futures specs available")
else:
logger.warning(f"Leg {leg_symbol} ({contract_type}): no futures specs found")
elif contract_type == "Spot":
if leg_symbol in spot_specs:
logger.info(f"Leg {leg_symbol} ({contract_type}): spot specs available")
else:
logger.warning(f"Leg {leg_symbol} ({contract_type}): no spot specs found")
else:
logger.warning(f"Leg {leg_symbol} ({contract_type}): unknown contract type")
for leg_symbol in sorted(all_leg_symbols):
leg_meta = leg_metadata.get(leg_symbol, {})
contract_type = leg_meta.get("contractType", "Unknown")
base_asset = leg_meta.get("baseCoin", "BTC")
quote_currency = leg_meta.get("quoteCoin", "USDT")
logger.info(f"Creating leg security for {leg_symbol}: type={contract_type}, asset={base_asset}, currency={quote_currency}")
try:
# Get the full metadata for this leg symbol from existing futures metadata
full_leg_meta = metadata.get(leg_symbol, {})
# Determine expiration based on contract type (same logic as futures/spots)
if contract_type == "Spot":
expiration = None # Spots have no expiration
elif contract_type in ["LinearPerpetual", "LinearFutures"]:
# Use the same get_expiration logic as regular futures
if full_leg_meta:
expiration = get_expiration(full_leg_meta)
logger.info(f"Using existing metadata expiration for {leg_symbol}: {expiration}")
else:
# Fallback: try to parse from symbol name for futures
if contract_type == "LinearFutures" and "-" in leg_symbol and len(leg_symbol.split("-")) == 2:
try:
date_part = leg_symbol.split("-")[1]
from datetime import datetime
dt = datetime.strptime(date_part, "%d%b%y")
if dt.year < 2000:
dt = dt.replace(year=dt.year + 2000)
expiration = dt.strftime("%Y-%m-%dT%H:%M")
logger.info(f"Parsed expiration from symbol for {leg_symbol}: {date_part} -> {expiration}")
except Exception as e:
logger.warning(f"Failed to parse expiration for {leg_symbol}, using default: {e}")
expiration = PERP_EXP_STRING
else:
expiration = PERP_EXP_STRING
logger.info(f"Using fallback expiration for {leg_symbol}: {expiration}")
else:
logger.warning(f"Unknown contract type {contract_type} for leg {leg_symbol}, using default expiration")
expiration = PERP_EXP_STRING
# Get specs based on contract type - reuse existing futures specs
if contract_type in ["LinearFutures", "LinearPerpetual"] and leg_symbol in futures_specs:
specs = futures_specs.get(leg_symbol, {}).get("specs", {})
logger.info(f"Using existing futures specs for {leg_symbol}: {specs}")
elif contract_type == "Spot" and leg_symbol in spot_specs:
specs = spot_specs.get(leg_symbol, {}).get("specs", {})
logger.info(f"Using existing spot specs for {leg_symbol}: {specs}")
else:
# Use default specs only if no existing specs available
specs = {
"tick_unit": 1,
"tick_base": 100,
"cash_per_point": 1,
"qty_tick_unit": 1,
"qty_tick_base": 10000
}
logger.info(f"Using default specs for {leg_symbol} (no existing specs found)")
qty_info = get_quantity_info(
tick_unit=specs.get("qty_tick_unit", 1),
tick_base=specs.get("qty_tick_base", 10000)
)
spec = get_contract_spec(
price_info=get_price_info(
tick_unit=specs.get("tick_unit", 1),
tick_base=specs.get("tick_base", 10),
cash_per_point=specs.get("cash_per_point", 1)
),
qty_info=qty_info
)
# Create leg security using get_security
sec = get_security(
venue="Bybit",
name=leg_symbol,
asset=base_asset,
group=base_asset,
currency=quote_currency,
expiration=expiration,
option_kind=None,
strike=None,
spec=spec,
)
sec_list_legs.append(sec)
logger.info(f"Created LEG security: name={leg_symbol}, type={contract_type}, asset={base_asset}, currency={quote_currency}, expiration={expiration}")
except Exception as e:
logger.error(f"Failed to create LEG security for {leg_symbol}: {e}", exc_info=True)
# Send leg securities to bridge
if sec_list_legs:
for sec in sec_list_legs:
combined_sec_list.append(sec)
logger.info(f"Added {len(sec_list_legs)} leg securities to combined list")
# create spread securities
spread_contracts = get_spread_contracts(self.spreads, self.use_testnet)
logger.info(f"Available spread contracts: {list(spreads_meta.keys())}")
logger.info(f"Configured base symbols: {self.spreads}")
# Create spread specs using metadata
spread_specs = {}
for symbol in spread_contracts:
meta = spreads_meta.get(symbol, {})
try:
# Extract price and quantity from metadata
price_tick = float(meta.get("tickSize", 0.1))
lot_size = float(meta.get("lotSize", 0.001))
if price_tick == 0 or lot_size == 0:
logger.error(f"Invalid tick or lot size for spread {symbol}: tickSize={price_tick}, lotSize={lot_size}")
continue
# Calculate tick_base and qty values
tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10
if lot_size >= 1:
qty_tick_unit = lot_size
qty_tick_base = 1
else:
qty_tick_unit = 1
qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000
spread_specs[symbol] = {
"tick_unit": 1,
"tick_base": tick_base,
"cash_per_point": 1,
"qty_tick_unit": qty_tick_unit,
"qty_tick_base": qty_tick_base
}
logger.info(f"Calculated spread specs for {symbol}: {spread_specs[symbol]}")
except (ValueError, TypeError) as e:
logger.error(f"Failed to parse spread specs for {symbol}: {e}")
continue
# Log summary of spread specs
logger.info(f"Successfully calculated specs for {len(spread_specs)} spread contracts")
# Create spread securities
sec_list_spreads = PySecurityList()
for symbol in spread_contracts:
meta = spreads_meta.get(symbol, {})
logger.debug(f"Spread security creation for {symbol}: {meta}")
try:
base_asset = meta.get("baseCoin", "BTC")
quote_currency = meta.get("quoteCoin", "USDT")
contract_type = meta.get("contractType", "Unknown")
# Process all spread types (FutureSpread, FundingRateArb, CarryTrade, PerpBasis)
logger.info(f"Processing spread {symbol} with contractType {contract_type}")
# Extract leg information for spread
legs = meta.get("legs", [])
leg_symbols = [leg.get("symbol") for leg in legs if leg.get("symbol")]
logger.info(f"Spread {symbol} has legs: {leg_symbols}")
# Get calculated specs for this spread symbol
specs = spread_specs.get(symbol)
if not specs:
logger.warning(f"No specs found for spread {symbol}, skipping")
continue
qty_info = get_quantity_info(
tick_unit=specs.get("qty_tick_unit", 1),
tick_base=specs.get("qty_tick_base", 10000)
)
spec = get_contract_spec(
price_info=get_price_info(
tick_unit=specs.get("tick_unit", 1),
tick_base=specs.get("tick_base", 10),
cash_per_point=specs.get("cash_per_point", 1)
),
qty_info=qty_info
)
# Fix expiration for spreads
delivery_time = meta.get("deliveryTime")
if delivery_time and delivery_time != "0":
from datetime import datetime
dt = datetime.utcfromtimestamp(int(delivery_time) / 1000)
expiration = dt.strftime("%Y-%m-%dT%H:%M")
else:
expiration = PERP_EXP_STRING # or some other default for spreads
# Create PySecurity directly (not using get_security)
sec = PySecurity()
sec.set_venue("Bybit")
sec.set_name(symbol)
sec.set_asset(base_asset)
sec.set_group(base_asset)
sec.set_currency(quote_currency)
sec.set_contract_spec(spec)
# Create and set the spread:
spread = PySpread()
spread.set_spread_type(contract_type)
spread.set_legs(leg_symbols)
spread.set_expiration(expiration)
# Set the security as a spread:
sec.set_spread(spread)
sec.set_exchange_id(symbol) # This ensures unique SecurityID generation
sec_list_spreads.append(sec)
logger.info(f"Created SPREAD security: name={symbol}, asset={base_asset}, currency={quote_currency}, type={contract_type}, legs={leg_symbols}")
except Exception as e:
logger.error(f"Failed to create SPREAD security for {symbol}: {e}", exc_info=True)
# Spread count
logger.info(f"Total spreads created: {len(sec_list_spreads)}")
# Debug: all created spread securities with their types
for i, sec in enumerate(sec_list_spreads):
spread_info = sec.spread()
logger.info(f"Spread {i+1}: name={sec.name()}, exchange_id={sec.exchange_id()}, asset={sec.asset()}, type={spread_info.spread_type()}, legs={spread_info.legs()}")
logger.info(f"Created {len(sec_list_spreads)} spread securities")
for sec in sec_list_spreads:
combined_sec_list.append(sec)
logger.info(f"Added spread to combined list: {sec.name()}")
bridge.on_securities(combined_sec_list)
logger.info(f"Sent {len(sec_list_spreads)} spreads to bridge")
try:
# WebSocket for futures
@@ -443,7 +795,71 @@ class BybitMdProvider(MdProvider):
except Exception as e:
logger.error(f"Failed to subscribe to orderbook for {contract} (spots): {e}")
logger.info("Subscribed to orderbook channels on Bybit futures and spots WS.")
# WebSocket for spreads
if sec_list_spreads:
try:
# Get spread symbols from metadata (all spread types)
spread_symbols = [meta.get("symbol") for meta in spreads_meta.values()]
logger.info(f"Found {len(spread_symbols)} spread symbols to subscribe to: {spread_symbols}")
if spread_symbols:
# Create spread WebSocket connection
host = "stream-testnet.bybit.com" if self.use_testnet else "stream.bybit.com"
uri = f"wss://{host}/v5/public/spread"
logger.info(f"Connecting to Bybit spread WebSocket: {uri}")
# Run WebSocket in separate thread to avoid blocking
def run_spread_websocket():
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self._spread_websocket_async(uri, spread_symbols))
except Exception as e:
logger.error(f"Spread WebSocket thread error: {e}", exc_info=True)
spread_thread = threading.Thread(target=run_spread_websocket, daemon=True, name="SpreadWebSocket")
spread_thread.start()
logger.info("Spread WebSocket started in separate thread")
# Add thread monitoring
def monitor_spread_thread():
while not self.stop_event.is_set():
time.sleep(10)
if not spread_thread.is_alive():
logger.warning("Spread WebSocket thread died, attempting restart...")
try:
# Restart the thread
new_spread_thread = threading.Thread(target=run_spread_websocket, daemon=True, name="SpreadWebSocket-Restart")
new_spread_thread.start()
logger.info("Spread WebSocket thread restarted")
break
except Exception as e:
logger.error(f"Failed to restart spread WebSocket thread: {e}")
break
monitor_thread = threading.Thread(target=monitor_spread_thread, daemon=True, name="SpreadWebSocketMonitor")
monitor_thread.start()
logger.info("Spread WebSocket monitor thread started")
# Add update monitoring thread
def monitor_spread_updates():
while not self.stop_event.is_set():
time.sleep(60) # Check every minute
if not self.stop_event.is_set() and hasattr(self, 'last_spread_update_time'):
current_time = time.time()
for symbol, last_update in self.last_spread_update_time.items():
if current_time - last_update > 120: # 2 minutes without updates
logger.warning(f"No spread updates for {symbol} in {int(current_time - last_update)} seconds")
update_monitor_thread = threading.Thread(target=monitor_spread_updates, daemon=True, name="SpreadUpdateMonitor")
update_monitor_thread.start()
except Exception as e:
logger.error(f"Failed to setup spread WebSocket: {e}", exc_info=True)
logger.info("Subscribed to orderbook channels on Bybit futures, spots, and spreads WS.")
while not self.stop_event.is_set():
time.sleep(1)
except KeyboardInterrupt:
@@ -475,13 +891,14 @@ class BybitMdProvider(MdProvider):
raw_asks = payload.get("a", [])
if not raw_bids or not raw_asks:
logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}")
return # Skip processing if no valid data
raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids]
raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks]
sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
sorted_bids += [(np.float64('nan'), np.float64('nan'))]
sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
sorted_asks += [(np.float64('nan'), np.float64('nan'))]
logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
logger.debug(f"BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
book = PyBook()
market_type = "FUTURE" if source == "futures" else "SPOT" # Dynamically set market type
logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})")
@@ -495,19 +912,301 @@ class BybitMdProvider(MdProvider):
book_list = PyBookList()
book_list.append(book)
if self.bridge:
logger.debug(f"✔ on_books called for {contract} (market_type={market_type})")
logger.debug(f" on_books called for {contract} (market_type={market_type})")
self.bridge.on_books(book_list)
else:
logger.warning("Bridge is None, cannot call on_books")
except Exception as e:
logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True)
async def _spread_websocket_async(self, uri: str, symbols: list):
"""Async WebSocket connection for spreads with improved connection management."""
max_reconnect_attempts = 5
reconnect_delay = 5 # seconds
for attempt in range(max_reconnect_attempts):
try:
logger.info(f"Attempting to connect to spread WebSocket (attempt {attempt + 1}/{max_reconnect_attempts}): {uri}")
# Disable built-in ping/pong since Bybit spread endpoint doesn't support it
async with websockets.connect(
uri,
ping_interval=None, # Disable built-in ping
ping_timeout=None, # Disable built-in ping timeout
close_timeout=10, # Wait 10 seconds for close frame
max_size=2**20 # 1MB max message size
) as websocket:
logger.info(f"Connected to spread WebSocket: {uri}")
# Subscribe to all spread symbols
subscribe_msg = {
"op": "subscribe",
"args": [f"orderbook.25.{symbol}" for symbol in symbols]
}
await websocket.send(json.dumps(subscribe_msg))
logger.info(f"Subscribed to spread orderbooks: {subscribe_msg['args']}")
# Start custom heartbeat task (in addition to websockets built-in ping)
heartbeat_task = asyncio.create_task(self._spread_heartbeat(websocket))
# Start connection monitor task
monitor_task = asyncio.create_task(self._spread_connection_monitor(websocket))
try:
# Process messages
async for message in websocket:
if self.stop_event.is_set():
break
try:
data = json.loads(message)
logger.debug(f"SPREAD WEBSOCKET MESSAGE: {data}")
# Handle pong responses
if data.get("op") == "pong":
logger.debug("Received pong response from spread WebSocket")
continue
# Check message type for debugging
msg_type = data.get("type", "unknown")
topic = data.get("topic", "unknown")
logger.debug(f"Processing spread message: type={msg_type}, topic={topic}")
# Process as complete orderbook (simple approach)
self._on_spread_book_update(data)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse spread WebSocket message: {e}")
except Exception as e:
logger.error(f"Error processing spread message: {e}")
finally:
# Cancel tasks
heartbeat_task.cancel()
monitor_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
try:
await monitor_task
except asyncio.CancelledError:
pass
except websockets.exceptions.ConnectionClosed as e:
logger.warning(f"Spread WebSocket connection closed (attempt {attempt + 1}): {e}")
if attempt < max_reconnect_attempts - 1:
logger.info(f"Reconnecting in {reconnect_delay} seconds...")
await asyncio.sleep(reconnect_delay)
reconnect_delay = min(reconnect_delay * 2, 60) # Exponential backoff, max 60s
else:
logger.error("Max reconnection attempts reached for spread WebSocket")
break
except Exception as e:
logger.error(f"Spread WebSocket connection error (attempt {attempt + 1}): {e}", exc_info=True)
if attempt < max_reconnect_attempts - 1:
logger.info(f"Reconnecting in {reconnect_delay} seconds...")
await asyncio.sleep(reconnect_delay)
reconnect_delay = min(reconnect_delay * 2, 60)
else:
logger.error("Max reconnection attempts reached for spread WebSocket")
break
async def _spread_connection_monitor(self, websocket):
"""Monitor WebSocket connection health and log status."""
monitor_count = 0
try:
while not self.stop_event.is_set():
await asyncio.sleep(30) # Check every 30 seconds
if not self.stop_event.is_set():
monitor_count += 1
if websocket.open:
logger.debug(f"Spread WebSocket connection is healthy (check #{monitor_count})")
else:
logger.warning(f"Spread WebSocket connection is closed (check #{monitor_count})")
break
except asyncio.CancelledError:
logger.debug("Spread connection monitor task cancelled")
except Exception as e:
logger.error(f"Error in spread connection monitor: {e}")
async def _spread_heartbeat(self, websocket):
"""Send heartbeat ping every 20 seconds to keep connection alive."""
heartbeat_count = 0
try:
while not self.stop_event.is_set():
await asyncio.sleep(20) # Wait 20 seconds
if not self.stop_event.is_set():
try:
# Send custom ping message
heartbeat_count += 1
ping_msg = {"req_id": str(int(time.time())), "op": "ping"}
await websocket.send(json.dumps(ping_msg))
logger.debug(f"Sent spread WebSocket ping heartbeat #{heartbeat_count}")
except websockets.exceptions.ConnectionClosed:
logger.warning("Spread WebSocket connection closed during heartbeat")
break
except Exception as e:
logger.error(f"Error sending spread heartbeat: {e}")
break
except asyncio.CancelledError:
logger.debug("Spread heartbeat task cancelled")
except Exception as e:
logger.error(f"Error in spread heartbeat: {e}")
def _on_spread_book_update(self, data: dict):
"""Handle spread orderbook updates with proper delta/snapshot processing."""
try:
# Track last update time for debugging
if not hasattr(self, 'last_spread_update_time'):
self.last_spread_update_time = {}
topic = data.get("topic", "")
if not topic.startswith("orderbook.25."):
logger.debug(f"Ignoring non-orderbook topic: {topic}")
return
# Extract symbol from topic
symbol = topic.split(".", 2)[-1] # Get everything after "orderbook.25."
self.last_spread_update_time[symbol] = time.time()
logger.debug(f"Processing spread orderbook update for {symbol}")
# Initialize spread orderbook storage if not exists
if not hasattr(self, 'spread_orderbooks'):
self.spread_orderbooks = {}
# Initialize orderbook for this symbol if not exists
if symbol not in self.spread_orderbooks:
self.spread_orderbooks[symbol] = {"b": [], "a": []}
payload = data.get("data", {})
msg_type = data.get("type", "")
# Handle snapshot (initial data or reset)
if msg_type == "snapshot":
logger.debug(f"Received snapshot for spread {symbol}")
self.spread_orderbooks[symbol] = payload.copy()
elif msg_type == "delta":
logger.debug(f"Processing delta for spread {symbol}")
# Process delta updates
book_sides = {"b": payload.get("b", []), "a": payload.get("a", [])}
# Update sequence numbers
if "u" in payload:
self.spread_orderbooks[symbol]["u"] = payload["u"]
if "seq" in payload:
self.spread_orderbooks[symbol]["seq"] = payload["seq"]
# Process each side (bids and asks)
for side, entries in book_sides.items():
for entry in entries:
price, qty = entry[0], float(entry[1])
# Delete if quantity is 0
if qty == 0:
self.spread_orderbooks[symbol][side] = [
x for x in self.spread_orderbooks[symbol][side]
if x[0] != price
]
else:
# Update or insert
existing = next((x for x in self.spread_orderbooks[symbol][side]
if x[0] == price), None)
if existing:
existing[1] = str(qty)
else:
self.spread_orderbooks[symbol][side].append([price, str(qty)])
else:
logger.debug(f"Unknown message type for spread {symbol}: {msg_type}")
return
# Get the processed orderbook
orderbook = self.spread_orderbooks[symbol]
raw_bids = orderbook.get("b", [])
raw_asks = orderbook.get("a", [])
if not raw_bids and not raw_asks:
logger.warning(f"Empty spread orderbook data for {symbol}")
return
# Convert to same format as futures/spots
bids = []
asks = []
# Process bids (price, quantity pairs)
for bid in raw_bids:
if len(bid) >= 2:
price = float(bid[0])
quantity = float(bid[1])
if quantity > 0: # Only include non-zero quantities
bids.append((price, quantity))
# Process asks (price, quantity pairs)
for ask in raw_asks:
if len(ask) >= 2:
price = float(ask[0])
quantity = float(ask[1])
if quantity > 0: # Only include non-zero quantities
asks.append((price, quantity))
# Sort and limit to top 5 levels (maintain bridge format)
sorted_bids = sorted(bids, key=lambda x: -x[0])[:5] # Sort by price descending
sorted_asks = sorted(asks, key=lambda x: x[0])[:5] # Sort by price ascending
# Pad with NaN to ensure exactly 5 levels (spreads might have fewer levels)
while len(sorted_bids) < 5:
sorted_bids.append((np.float64('nan'), np.float64('nan')))
while len(sorted_asks) < 5:
sorted_asks.append((np.float64('nan'), np.float64('nan')))
# Add final NaN row (total: 5 levels + 1 NaN = 6 rows)
sorted_bids.append((np.float64('nan'), np.float64('nan')))
sorted_asks.append((np.float64('nan'), np.float64('nan')))
logger.debug(f"SPREAD BOOK DATA for {symbol}: bids={sorted_bids} asks={sorted_asks}")
# Create PyBook
book = PyBook()
book.set_data(
symbol,
"Bybit",
"SPREAD",
np.array(sorted_bids, dtype=np.float64),
np.array(sorted_asks, dtype=np.float64),
)
book_list = PyBookList()
book_list.append(book)
if self.bridge:
logger.debug(f"on_books called for spread {symbol}")
self.bridge.on_books(book_list)
else:
logger.warning("Bridge is None, cannot call on_books")
except Exception as e:
logger.error(f"Error processing spread book update: {e}", exc_info=True)
def stop(self):
logger.info("Stopping BybitMdProvider...")
self.stop_event.set()
logger.debug("Threads before stopping WebSocket:")
for t in threading.enumerate():
logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}")
# Clean up spread WebSocket threads
for t in threading.enumerate():
if t.name in ['SpreadWebSocket', 'SpreadWebSocket-Restart', 'SpreadWebSocketMonitor', 'SpreadUpdateMonitor'] and t.is_alive():
logger.info(f"Joining spread WebSocket thread: {t.name}")
t.join(timeout=2.0)
if t.is_alive():
logger.warning(f"Spread WebSocket thread {t.name} did not terminate gracefully")
if self.ws_futures:
try:
self.ws_futures.exit()
@@ -584,4 +1283,4 @@ if __name__ == "__main__":
md_bridge = FakeMdBridge(logger)
md_pro.start(md_bridge)
while True:
time.sleep(1)
time.sleep(1)
@@ -4,10 +4,11 @@ import time
import logging
import sys
import threading
from typing import Iterable, Optional, Dict
import websocket
import requests
from itertools import chain
from websocket import create_connection, WebSocketConnectionClosedException
from typing import Iterable, Optional, Any
import ssl
import certifi
from python_interop_module import (
MdBridge,
MdProvider,
@@ -23,214 +24,407 @@ from spark_core import (
get_security,
load_config,
)
# Urls:
# prod perps (ws) - "wss://fx-ws.gateio.ws/v4/ws/usdt"
# testnet perps (ws) - "wss://fx-ws-testnet.gateio.ws/v4/ws/usdt"
from py_md_utils import PERP_EXP_STRING
logger = logging.getLogger("gateio_md_provider")
def fetch_contract_specs(contracts: Iterable[str]) -> dict:
"""Fetch specs for all contracts in one request."""
def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]:
"""Fetch metadata for all USDT-margined futures contracts and filter for specified contracts."""
metadata_dict = {}
try:
host = "api.gateio.ws" if not use_testnet else "api.testnet.gateio.ws"
url = f"https://{host}/api/v4/futures/usdt/contracts"
logger.info(f"Querying Gate.io API for metadata: {url}")
response = requests.get(url, timeout=5)
response.raise_for_status()
contracts_data = response.json()
logger.info(f"Retrieved metadata for {len(contracts_data)} futures contracts")
for item in contracts_data:
symbol = item.get("name")
if symbol in contracts and not item.get("in_delisting", False):
metadata_dict[symbol] = item
logger.info(f"Stored metadata for {symbol}")
except Exception as e:
logger.error(f"Failed to fetch metadata: {e}", exc_info=True)
return metadata_dict
def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]:
default_specs = {
"tick_unit": 1,
"tick_base": 10,
"cash_per_point": 0.1,
"qty_tick_unit": 1,
"cash_per_point": 1,
"qty_tick_unit": 0.001,
"qty_tick_base": 10000
}
specs_dict = {contract: default_specs for contract in contracts}
specs_dict = {contract: {"specs": default_specs} for contract in contracts}
try:
url = "https://api.gateio.ws/api/v4/futures/usdt/contracts"
host = "api.gateio.ws" if not use_testnet else "api.testnet.gateio.ws"
url = f"https://{host}/api/v4/futures/usdt/contracts"
logger.info(f"Querying Gate.io API for futures specs: {url}")
response = requests.get(url, timeout=5)
response.raise_for_status()
data = response.json()
for item in data:
symbols = response.json()
logger.info(f"Loaded {len(symbols)} futures contracts")
for item in symbols:
contract = item.get("name")
if contract in contracts:
price_tick = float(item.get("price_tick", "0.1"))
contract_size = float(item.get("contract_size", "0.0001"))
tick_base = int(1 / price_tick) if price_tick != 0 else 10
qty_tick_base = int(1 / contract_size) if contract_size != 0 else 10000
specs_dict[contract] = {
"tick_unit": 1,
"tick_base": tick_base,
"cash_per_point": price_tick,
"qty_tick_unit": 1,
"qty_tick_base": qty_tick_base
}
return specs_dict
except Exception as e:
logger.error(f"Failed to fetch batch specs: {e}", exc_info=True)
try:
price_tick = float(item.get("order_price_round", 0.1))
lot_size = float(item.get("quanto_multiplier", 0.001))
if price_tick == 0 or lot_size == 0:
logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, lotSize={lot_size}")
continue
tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10
if lot_size >= 1:
qty_tick_unit = lot_size
qty_tick_base = 1
else:
qty_tick_unit = 1
qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000
specs_dict[contract] = {
"specs": {
"tick_unit": 1,
"tick_base": tick_base,
"cash_per_point": 1,
"qty_tick_unit": qty_tick_unit,
"qty_tick_base": qty_tick_base
}
}
logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}")
except (ValueError, TypeError) as e:
logger.error(f"Failed to parse futures specs for {contract}: {e}")
continue
logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}")
logger.debug(f"Final specs: {specs_dict}")
return specs_dict
class GateIoMdProvider(MdProvider):
except Exception as e:
logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True)
return specs_dict
def get_expiration(contract: Dict) -> str:
"""Extract expiration from metadata or use default for perpetuals."""
if contract.get("in_delisting", False):
return "0" # 8061 format for expired
return PERP_EXP_STRING # Default for perpetuals
def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list:
"""Query Gate.io API for all USDT-margined futures and filter by base symbols."""
futures_contracts = []
host = "api.gateio.ws" if not use_testnet else "api.testnet.gateio.ws"
url = f"https://{host}/api/v4/futures/usdt/contracts"
logger.info(f"Querying all futures contracts: {url}")
response = requests.get(url, timeout=5)
response.raise_for_status()
data = response.json()
logger.info(f"Retrieved {len(data)} futures contracts")
base_symbols_upper = [s.upper() for s in base_symbols] # Normalize to uppercase
for item in data:
symbol = item.get("name")
base_coin = symbol.split("_")[0] if symbol and "_" in symbol else None
contract_type = item.get("type", "").lower()
in_delisting = item.get("in_delisting", False)
# logger.debug(f"Checking contract: {symbol}, base: {base_coin}, type: {contract_type}, in_delisting: {in_delisting}")
if base_coin and base_coin.upper() in base_symbols_upper and contract_type == "direct" and not in_delisting:
futures_contracts.append(symbol)
logger.info(f"Matched contract: {symbol} (base: {base_coin}, type: {contract_type}, in_delisting: {in_delisting})")
logger.info(f"Filtered to {len(futures_contracts)} compatible instruments for base symbols {base_symbols}")
return sorted(list(set(futures_contracts)))
class GateioMdProvider(MdProvider):
def __init__(self):
super().__init__()
self.ws = None
self._config: Optional[dict] = None
self.ws_futures = None
self.ws_thread: Optional[threading.Thread] = None
self.bridge: Optional[MdBridge] = None
self.stop_event = threading.Event()
self.ping_thread = None
def send_ping(self):
"""Send ping messages every 15 seconds to keep connection alive."""
while not self.stop_event.is_set():
try:
ping_msg = {
"time": int(time.time() * 1000),
"channel": "futures.ping",
"event": "",
"payload": []
}
self.ws.send(json.dumps(ping_msg))
logger.debug("Sent ping")
except Exception as e:
logger.error(f"Ping error: {e}", exc_info=True)
break
time.sleep(15)
@property
def config(self):
if self._config is None:
self._config = load_config(["py_gateio_md_provider", "gateio_overrides"])
return self._config
@property
def use_testnet(self):
return self.config.get("use_testnet", True)
@property
def currencies(self):
return self.config.get("currencies") or ["USDT"]
@property
def futures(self):
return self.config.get("futures", [])
def start_impl(self, bridge: MdBridge):
cfg = load_config(["gateio_md_provider", "gateio_overrides"])
configure_logger(logger, "md", **cfg.get("logging", {}))
configure_logger(logger, "md", level="INFO", **self.config.get("logging", {}))
self.bridge = bridge
ws_url = cfg["ws_url"]
logger.info(f"Connecting to {ws_url}")
self.ws = create_connection(ws_url)
self.ws.settimeout(5)
logger.info("Connected.")
logger.info("Starting GateioMdProvider")
# Start ping thread
self.ping_thread = threading.Thread(target=self.send_ping, daemon=True)
self.ping_thread.start()
logger.info("Started ping thread")
futures_contracts = get_futures_contracts(self.futures, self.use_testnet)
logger.info(f"futures_contracts: {futures_contracts}")
# Fetch all contract specs at once
spots: list[str] = cfg["spots"]
currencies: list[str] = cfg.get("currencies") or ["USDT"]
contracts = list(chain(*[map(lambda s: f"{s}{c}", spots) for c in currencies]))
contract_specs = fetch_contract_specs(contracts)
metadata = get_metadata(futures_contracts, self.use_testnet)
logger.info(f"Fetched metadata for {len(metadata)} futures contracts")
logger.info(f"metadata: {metadata}")
# 1) Send security info
sec_list = PySecurityList()
for contract in contracts:
asset, quote = contract.split("_")
specs = contract_specs[contract]
spec = get_contract_spec(
price_info=get_price_info(
tick_unit=specs["tick_unit"],
tick_base=specs["tick_base"],
cash_per_point=specs["cash_per_point"]
),
qty_info=get_quantity_info(
tick_unit=specs["qty_tick_unit"],
tick_base=specs["qty_tick_base"]
futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet)
logger.info(f"Fetched futures specs for {len(futures_specs)} contracts")
logger.info(f"futures_specs: {futures_specs}")
sec_list_futures = PySecurityList()
for contract in futures_contracts:
meta = metadata.get(contract, {})
if not meta:
logger.warning(f"No metadata found for contract: {contract}, using defaults")
continue
try:
symbol = meta.get("name", contract)
base_asset = symbol.split("_")[0]
quote_currency = symbol.split("_")[1]
expiration = get_expiration(meta)
asset = base_asset
group = base_asset
currency = quote_currency
specs = futures_specs.get(contract, {}).get("specs", {})
if not specs:
logger.warning(f"No specs found for contract: {contract}, using defaults")
specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 0.001, "qty_tick_base": 10000}
qty_info = get_quantity_info(
tick_unit=specs.get("qty_tick_unit", 0.001),
tick_base=specs.get("qty_tick_base", 10000)
)
)
sec = get_security(
venue="GateIO",
name=contract,
asset=asset,
group=asset,
currency=quote,
expiration=None,
option_kind=None,
strike=None,
spec=spec,
)
sec_list.append(sec)
logger.info(f"Created security: {contract} with specs: {specs}")
bridge.on_securities(sec_list)
logger.info("Sent securities list--sink.on_securities called.")
# 2) Subscribe to order books
for contract in contracts:
sub_msg = {
"time": int(time.time() * 1000),
"channel": "futures.order_book",
"event": "subscribe",
"payload": [contract, "5", "0"]
}
self.ws.send(json.dumps(sub_msg))
logger.info(f"✔ Sent subscribe for {contract}: {sub_msg}")
spec = get_contract_spec(
price_info=get_price_info(
tick_unit=specs.get("tick_unit", 1),
tick_base=specs.get("tick_base", 10),
cash_per_point=specs.get("cash_per_point", 0.1)
),
qty_info=qty_info
)
sec = get_security(
venue="Gateio",
name=symbol,
asset=asset,
group=group,
currency=currency,
expiration=expiration,
option_kind=None,
strike=None,
spec=spec,
)
sec_list_futures.append(sec)
logger.info(f"Created FUTURE security: name={symbol}, venue=Gateio, asset={asset}, group={group}, currency={currency}, expiration={expiration}, spec={specs}")
except Exception as e:
logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True)
continue
combined_sec_list = PySecurityList()
for sec in sec_list_futures:
combined_sec_list.append(sec)
bridge.on_securities(combined_sec_list)
logger.info(f"Sent securities list: {len(sec_list_futures)} futures")
retry_count = 0
max_retries = 3
while not self.stop_event.is_set() and retry_count < max_retries:
try:
ws_url = "wss://fx-ws.gateio.ws/v4/ws/usdt" if not self.use_testnet else "wss://fx-ws.testnet.gateio.ws/v4/ws/usdt"
self.ws_futures = websocket.WebSocketApp(
ws_url,
on_message=self._on_message,
on_error=self._on_error,
on_close=self._on_close,
on_open=self._on_open,
)
logger.info(f"Connecting to Gate.io WebSocket (futures), attempt {retry_count + 1}/{max_retries}")
websocket.enableTrace(True) # Enable tracing for debugging
self.ws_thread = threading.Thread(
target=self.ws_futures.run_forever,
kwargs={"ping_interval": 20, "ping_timeout": 10}
)
self.ws_thread.start()
# Wait for connection to establish
time.sleep(2)
if self.ws_futures and self.ws_futures.sock and self.ws_futures.sock.connected:
for contract in futures_contracts:
subscribe_msg = {
"channel": "futures.order_book",
"event": "subscribe",
"payload": [contract, "5", "0"]
}
self.ws_futures.send(json.dumps(subscribe_msg))
logger.info(f"Subscribed to orderbook for {contract} (futures)")
break
else:
logger.warning("WebSocket connection not established, retrying...")
retry_count += 1
time.sleep(5)
except Exception as e:
logger.error(f"WebSocket connection attempt {retry_count + 1} failed: {e}", exc_info=True)
retry_count += 1
time.sleep(5)
if retry_count >= max_retries:
logger.error("Max WebSocket retries reached, stopping provider")
self.stop()
return
# 3) Listen for incoming messages in main thread
logger.info("▶ Listening for data")
try:
while not self.stop_event.is_set():
try:
msg = self.ws.recv()
except Exception as e:
logger.debug(f"⚠ recv() error or timeout: {e}")
continue
logger.debug(f"⟵ raw WS: {msg}")
try:
data = json.loads(msg)
logger.debug(f"Parsed JSON: {data}")
except Exception as e:
logger.error(f"JSON parse error: {e}", exc_info=True)
continue
event = data.get("event")
channel = data.get("channel")
# Handle subscription acknowledgements
if event == "subscribe":
logger.info(f"🔔 Subscribe ACK: {data}")
continue
# Handle order book updates
if channel == "futures.order_book" and event in ["all", "update"]:
logger.debug(f"Handling '{event}' event")
payload = data.get("result", {})
if isinstance(payload, dict) and ("bids" in payload or "asks" in payload):
contract = payload.get("contract", "<no-contract>")
bids = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("bids", [])]
asks = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("asks", [])]
logger.debug(f"📘 BOOK DATA for {contract}: bids={bids} asks={asks}")
book = PyBook()
book.set_data(
contract,
"GateIO",
"SPOT",
np.array(bids, dtype=np.float64),
np.array(asks, dtype=np.float64),
)
book_list = PyBookList()
book_list.append(book)
self.bridge.on_books(book_list)
logger.debug(f"✔ on_books called for {contract}")
else:
logger.debug("… no bids/asks in payload; skipping")
except WebSocketConnectionClosedException:
logger.info("WebSocket closed.")
time.sleep(1)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt received — shutting down.")
self.stop()
except Exception as e:
logger.error("Fatal error in listen loop:", exc_info=True)
logger.error(f"Gate.io WebSocket fatal error: {e}", exc_info=True)
self.stop()
def _on_open(self, ws):
logger.info("WebSocket connection opened")
def _on_message(self, ws, message):
try:
data = json.loads(message)
logger.debug(f"Raw WebSocket message: {data}")
# Only process order book updates (event: 'all'), skip subscription confirmations
if data.get("channel") == "futures.order_book" and data.get("event") == "all":
contract = data.get("result", {}).get("contract")
if not contract:
logger.warning(f"No contract symbol in order book update: {data}")
return
self._on_book_update(data, source="futures")
elif data.get("event") == "subscribe":
logger.debug("Received subscription confirmation, ignoring for book update")
else:
logger.debug(f"Ignoring non-orderbook message: {data}")
except Exception as e:
logger.error(f"Error processing WebSocket message: {e}", exc_info=True)
def _on_error(self, ws, error):
logger.error(f"WebSocket error: {error}", exc_info=True)
def _on_close(self, ws, close_status_code, close_msg):
logger.info(f"WebSocket closed: status={close_status_code}, msg={close_msg}")
def start(self, bridge: MdBridge):
self.ws_thread = threading.Thread(target=lambda: self.start_impl(bridge))
def start_wrapper():
try:
self.start_impl(bridge)
except Exception as e:
logger.error(f"Error in GateioMdProvider start_impl: {e}", exc_info=True)
self.ws_thread = threading.Thread(target=start_wrapper)
self.ws_thread.start()
def _on_book_update(self, data: dict, source: str):
try:
logger.debug(f"Raw WebSocket data (source={source}): {data}")
contract = data.get("result", {}).get("contract")
if not contract:
logger.warning(f"No contract symbol in book update: {data}")
return
raw_bids = data.get("result", {}).get("bids", [])
raw_asks = data.get("result", {}).get("asks", [])
if not raw_bids or not raw_asks:
logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}")
return
# Convert to (price, size) pairs
raw_bids = [(float(x["p"]), float(x["s"])) for x in raw_bids]
raw_asks = [(float(x["p"]), float(x["s"])) for x in raw_asks]
# Sort and limit to 5 levels
sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5]
sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5]
logger.info(f"BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}")
book = PyBook()
market_type = "FUTURE"
book.set_data(
contract,
"Gateio",
market_type,
np.array(sorted_bids, dtype=np.float64),
np.array(sorted_asks, dtype=np.float64),
)
book_list = PyBookList()
book_list.append(book)
logger.debug(f"Sending book_list to bridge: {book_list}")
if self.bridge:
logger.debug(f"Calling on_books for {contract} (market_type={market_type})")
self.bridge.on_books(book_list)
else:
logger.warning("Bridge is None, cannot call on_books")
except Exception as e:
logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True)
def stop(self):
"""Disconnect and cleanup."""
logger.info("Stopping GateIoMdProvider...")
logger.info("Stopping GateioMdProvider...")
self.stop_event.set()
if self.ping_thread and self.ping_thread.is_alive():
self.ping_thread.join()
logger.info("Ping thread stopped")
if self.ws:
if self.ws_futures:
try:
self.ws.close()
except Exception:
pass
self.ws = None
self.ws_futures.close()
logger.info("WebSocket (futures) closed")
except Exception as e:
logger.error(f"Error while closing WebSocket (futures): {e}", exc_info=True)
self.ws_futures = None
if self.ws_thread:
self.ws_thread.join()
self.bridge = None
logger.info("✔ Stopped GateIoMdProvider.")
logger.info("✔ Stopped GateioMdProvider.")
import os
os._exit(0)
if __name__ == "__main__":
import argparse
from fake_md_bridge import FakeMdBridge
ap = argparse.ArgumentParser()
ap.add_argument("--currencies", nargs="+")
ap.add_argument("--futures", nargs="+")
log_levels = list(logging._nameToLevel)
ap.add_argument("--console-level", choices=log_levels)
ap.add_argument("--file-level", choices=log_levels)
ap.add_argument("--dev", action="store_true", dest="use_testnet")
ap.add_argument("--prod", action="store_false", dest="use_testnet")
ap.set_defaults(
currencies=None,
futures=None,
console_level=None,
file_level=None,
use_testnet=None,
)
args = ap.parse_args()
logger = logging.getLogger("gateio_md_provider_app")
md_pro = GateioMdProvider()
config = md_pro.config
if args.currencies:
config["currencies"] = args.currencies
if args.futures:
config["futures"] = args.futures
logging_overrides = {}
if args.console_level:
logging_overrides["console_level"] = args.console_level
if args.file_level:
logging_overrides["file_level"] = args.file_level
if logging_overrides:
config["logging"].update(logging_overrides)
if args.use_testnet is not None:
config["use_testnet"] = args.use_testnet
md_bridge = FakeMdBridge(logger)
md_pro.start(md_bridge)
while True:
time.sleep(1)