diff --git a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp index 7b04223aa..a5af2b05b 100644 --- a/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp +++ b/src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp @@ -5,6 +5,11 @@ #include "python_interop/plugin_manager.h" #include "python_interop/py_contract_specification.h" #include "python_interop/py_security.h" +#include "python_interop/py_spread.h" +#include "spark/domain/SpreadSecurity.h" +#include "spark/domain/LegDefinition.h" +#include "spark/domain/Quantity.h" +#include "spark/domain/Side.h" #include "python_interop/python_object.h" #include "spark/api/domain/CallPut.pb.h" #include "spark/api/domain/SecurityType.h" @@ -80,6 +85,10 @@ using bts::core::TimeStamp; using bts::domain::AggressorSide; using bts::domain::Book; using bts::domain::FutureSecurity; +using bts::domain::SpreadSecurity; +using bts::domain::LegDefinition; +using bts::domain::Quantity; +using bts::domain::Side; using bts::domain::OptionSecurity; using bts::domain::Price; using bts::domain::Security; @@ -355,10 +364,41 @@ struct PyMarketDataSubscription::PImpl { opt.call_put = py_opt.is_call() ? CallPut::CALL : CallPut::PUT; opt.strike = Price::from_double(py_opt.strike()); sec.security = std::move(opt); - } else { + } else if (py_sec.is_spread()) { + SpreadSecurity spread; + const auto &py_spread = py_sec.spread(); + + // Convert Python spread legs to C++ LegDefinition + for (const auto &leg_name : py_spread.legs()) { + LOG(md, Info, "Processing spread leg: {}", leg_name); + + // Use the existing security resolution mechanism + // Each leg should be resolved using the same venue and appropriate security type + // For now, we'll assume the leg is a future (most common for spreads) + // You may need to adjust this based on your specific leg types + auto leg_sec_id = get_sec_id_(sec_comm.venue, SecurityType::SECURITY_TYPE_FUTURE, leg_name); + auto quantity = bts::domain::Quantity(1); // Default quantity + auto side = bts::domain::Side::Value::Buy; // Default side + + bts::domain::LegDefinition leg_d{ + leg_sec_id, + quantity, + side + }; + spread.legs.push_back(leg_d); + } + + LOG(md, Info, + "Created spread with {} legs. Spread type: {}, expiration: {}", + py_spread.legs().size(), + py_spread.spread_type(), + py_spread.expiration()); + + sec.security = std::move(spread); + } else { LOG(md, Error, - "PySecurity has incomplete definition (not spot, future, or " - "option)"); + "PySecurity has incomplete definition (not spot, future, " + "option or spread)"); return std::nullopt; } sec_comm.id = diff --git a/src/lib_python_interop/python_interop/py_security.cpp b/src/lib_python_interop/python_interop/py_security.cpp index 94da8fdfc..fbd314ca2 100644 --- a/src/lib_python_interop/python_interop/py_security.cpp +++ b/src/lib_python_interop/python_interop/py_security.cpp @@ -30,6 +30,12 @@ void to_json(nlohmann::json &j, const PySecurity &sec) { j["k"] = opt.strike(); j["cp"] = opt.is_call() ? "C" : "P"; }, + [&](const PySpread &spr) { + j["typ"] = "SPR"; + j["exp"] = spr.expiration(); + j["spr_type"] = spr.spread_type(); + j["legs"] = spr.legs(); + }, }, sec.sec_); } @@ -78,6 +84,11 @@ void from_json(const nlohmann::json &j, PySecurity &sec) { opt.set_expiration(j.at("exp").get()); opt.set_is_call(j.at("cp").get().starts_with('C')); opt.set_strike(j.at("k").get()); + } else if (typ.starts_with('S') && typ != "SPOT") { + auto &spr = sec.sec_.emplace(); + spr.set_expiration(j.at("exp").get()); + spr.set_spread_type(j.at("spr_type").get()); + spr.set_legs(j.at("legs").get>()); } } diff --git a/src/lib_python_interop/python_interop/py_security.h b/src/lib_python_interop/python_interop/py_security.h index 0d3d4ba01..ff76df2b4 100644 --- a/src/lib_python_interop/python_interop/py_security.h +++ b/src/lib_python_interop/python_interop/py_security.h @@ -4,6 +4,7 @@ #include "python_interop/py_future.h" #include "python_interop/py_option.h" #include "python_interop/py_spot.h" +#include "python_interop/py_spread.h" #include "spark/io/json_formatter.h" #include @@ -50,6 +51,10 @@ class PySecurity { void set_option(PyOption); constexpr bool is_option() const; + const PySpread &spread() const; + void set_spread(PySpread); + constexpr bool is_spread() const; + const PyContractSpecification &contract_spec() const; void set_contract_spec(PyContractSpecification value); @@ -65,7 +70,7 @@ class PySecurity { std::string name_; std::string currency_; PyContractSpecification contract_spec_; - std::variant sec_; + std::variant sec_; friend struct fmt::formatter; friend void to_json(nlohmann::json &j, const PySecurity &sec); @@ -180,6 +185,18 @@ constexpr bool PySecurity::is_option() const { return std::holds_alternative(sec_); } +inline const PySpread &PySecurity::spread() const { + if (std::holds_alternative(sec_)) { + return std::get(sec_); + } + throw std::runtime_error{"PySecurity is not a spread"}; +} +inline void PySecurity::set_spread(PySpread value) { sec_ = std::move(value); } +constexpr bool PySecurity::is_spread() const { + return std::holds_alternative(sec_); +} + + inline const PyContractSpecification &PySecurity::contract_spec() const { return contract_spec_; } diff --git a/src/lib_python_interop/python_interop/python_interop_module.cpp b/src/lib_python_interop/python_interop/python_interop_module.cpp index 0578e7a0b..6b3d3572a 100644 --- a/src/lib_python_interop/python_interop/python_interop_module.cpp +++ b/src/lib_python_interop/python_interop/python_interop_module.cpp @@ -16,6 +16,8 @@ #include "python_interop/py_contract_quantity_info.h" #include "python_interop/py_contract_specification.h" #include "python_interop/py_security.h" +#include "python_interop/py_spread.h" + #include #include @@ -35,6 +37,7 @@ using bts::md::py_interop::PyMdProvider; using bts::md::py_interop::PyOption; using bts::md::py_interop::PySecurity; using bts::md::py_interop::PySpot; +using bts::md::py_interop::PySpread; using bts::md::py_interop::PyTrade; using python_execution::ExecutionGateway; using python_execution::ExecutionGatewayBridge; @@ -129,6 +132,19 @@ PYBIND11_MODULE(python_interop_module, m) { .PROP(PyOption, is_call) .PROP(PyOption, strike) .def("__str__", &PyOption::to_string); + py::class_(m, "PySpread") + .def(py::init<>()) + .PROP(PySpread, spread_type) + .def("legs", &PySpread::legs) // ← Changed this line + .def("set_legs", [](PySpread& self, py::list legs_list) { // ← Custom binding + std::vector legs; + for (auto item : legs_list) { + legs.push_back(py::cast(item)); + } + self.set_legs(legs); + }) + .PROP(PySpread, expiration) + .def("__str__", &PySpread::to_string); py::class_>(m, "PySecurity") .def(py::init<>()) .PROP(PySecurity, id) @@ -142,11 +158,35 @@ PYBIND11_MODULE(python_interop_module, m) { .PROP(PySecurity, spot) .PROP(PySecurity, future) .PROP(PySecurity, option) + .PROP(PySecurity, spread) + .def("set_spread_type", [](PySecurity& self, const std::string& spread_type) { + if (self.is_spread()) { + const_cast(self.spread()).set_spread_type(spread_type); + } + }) + .def("set_legs", [](PySecurity& self, py::list legs_list) { + if (self.is_spread()) { + std::vector legs; + for (auto item : legs_list) { + legs.push_back(py::cast(item)); + } + const_cast(self.spread()).set_legs(legs); + } + }) + .def("set_expiration", [](PySecurity& self, const std::string& expiration) { + if (self.is_spread()) { + const_cast(self.spread()).set_expiration(expiration); + } + }) + .def("is_spread", &PySecurity::is_spread) .def("__str__", &PySecurity::to_string) .def("__format__", &PySecurity::format); py::bind_vector>>(m, "PySecurityList") .PY_LIST_TO_STRING(PySecurity); + py::bind_vector>(m, "StringVector"); + + ////////////////////////////////////////////////// // Md ////////////////////////////////////////////////// diff --git a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py index 52ac64729..e9088b6db 100644 --- a/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/bybit_md_provider.py @@ -4,15 +4,20 @@ import time import logging import sys import threading +import ctypes from itertools import chain from typing import Iterable, Optional, Dict from pybit.unified_trading import WebSocket +import websockets +import asyncio from python_interop_module import ( MdBridge, MdProvider, PyBook, PyBookList, PySecurityList, + PySecurity, + PySpread ) from spark_core import ( configure_logger, @@ -25,6 +30,7 @@ from spark_core import ( from py_md_utils import PERP_EXP_STRING import requests + logger = logging.getLogger("bybit_md_provider") @@ -128,7 +134,7 @@ def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> D if contract not in found_contracts: logger.warning(f"Futures contract {contract} not found in API response, using default specs") - logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") + logger.debug(f"Returning specs for contracts: {list(specs_dict.keys())}") logger.debug(f"Final specs: {specs_dict}") return specs_dict @@ -170,9 +176,6 @@ def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: return sorted(list(set(futures_contracts))) # Remove duplicates and sort - - -# Spot support starts here (updated) def get_spot_metadata(base_symbols: list, currencies: list, logger, use_testnet: bool = True) -> dict: """Fetch metadata for spot contracts filtered by base symbols and supported currencies.""" host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" @@ -194,14 +197,72 @@ def get_spot_metadata(base_symbols: list, currencies: list, logger, use_testnet: else: logger.error(f"Spot metadata API error: {data.get('retMsg', 'Unknown error')}") return spots_meta -# Spot support ends here + +def get_spread_contracts(base_symbols: list, use_testnet: bool = True) -> list: + """Query Bybit API for all spread instruments and filter by base symbols.""" + spread_contracts = [] + host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" + url = f"https://{host}/v5/spread/instrument?limit=500" + + logger.info(f"Querying all spread instruments: {url}") + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + if data.get("retCode") == 0 and data.get("result", {}).get("list"): + logger.info(f"Retrieved {len(data['result']['list'])} spread instruments") + for item in data.get("result", {}).get("list", []): + base_coin = item.get("baseCoin") + if base_coin in base_symbols: + spread_contracts.append(item.get("symbol")) + logger.info(f"Filtered to {len(spread_contracts)} compatible spread instruments for base symbols {base_symbols}") + else: + logger.warning(f"Failed to retrieve instruments: {data.get('retMsg', 'Unknown error')}") + + return sorted(list(set(spread_contracts))) # Remove duplicates and sort + +def get_spread_metadata(base_symbols: list, logger, use_testnet: bool = True) -> dict: + """Fetch metadata for spread contracts filtered by base symbols.""" + host = "api-testnet.bybit.com" if use_testnet else "api.bybit.com" + url = f"https://{host}/v5/spread/instrument?limit=500" + + logger.info(f"Querying Bybit API for spread metadata: {url}") + + try: + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + + logger.info(f"Spread API response retCode: {data.get('retCode')}") + logger.info(f"Spread API response retMsg: {data.get('retMsg')}") + + spreads_meta = {} + if data.get("retCode") == 0: + spread_list = data.get("result", {}).get("list", []) + logger.info(f"Retrieved {len(spread_list)} spread contracts from API") + + for item in spread_list: + base_coin = item.get("baseCoin") + if base_coin in base_symbols: + symbol = item.get("symbol") + spreads_meta[symbol] = item + logger.info(f"Stored metadata for spread {symbol} with base {base_coin}") + + logger.info(f"Total spreads stored: {len(spreads_meta)}") + else: + logger.error(f"Spread metadata API error: {data.get('retMsg', 'Unknown error')}") + except Exception as e: + logger.error(f"Failed to fetch spread metadata: {e}", exc_info=True) + spreads_meta = {} + + return spreads_meta + class BybitMdProvider(MdProvider): def __init__(self): super().__init__() self._config: Optional[dict] = None self.ws_futures = None - self.ws_spots = None # Add WebSocket for spots + self.ws_spots = None self.ws_thread: Optional[threading.Thread] = None self.bridge: Optional[MdBridge] = None self.stop_event = threading.Event() @@ -224,11 +285,14 @@ class BybitMdProvider(MdProvider): def futures(self): return self.config.get("futures", []) - # Spot support starts here + @property def spots(self): return self.config.get("spots", []) - # Spot support ends here + + @property + def spreads(self): + return self.config.get("spreads", []) def start_impl(self, bridge: MdBridge): configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) # Enforce INFO level @@ -250,7 +314,7 @@ class BybitMdProvider(MdProvider): sec_list_futures = PySecurityList() for contract in futures_contracts: meta = metadata.get(contract, {}) - logger.info(f"Metadata for {contract}: {meta}") + logger.debug(f"Metadata for {contract}: {meta}") if not meta: logger.warning(f"No metadata found for contract: {contract}, using defaults") continue @@ -300,7 +364,6 @@ class BybitMdProvider(MdProvider): bridge.on_securities(combined_sec_list) logger.info(f"Sent securities list: {len(sec_list_futures)} futures") - # Spot support starts here # Get spot metadata spots_meta = get_spot_metadata(self.spots, self.currencies, logger, self.use_testnet) logger.info(f"Fetched metadata for {len(spots_meta)} spot contracts") @@ -403,8 +466,297 @@ class BybitMdProvider(MdProvider): combined_sec_list.append(sec) bridge.on_securities(combined_sec_list) logger.info(f"Sent securities list: {len(sec_list_spots)} spots + {len(sec_list_futures)} futures") - # Spot support ends here + # Get spread metadata + spreads_meta = get_spread_metadata(self.spreads, logger, self.use_testnet) + logger.info(f"Fetched metadata for {len(spreads_meta)} spread contracts") + + # Extract all unique leg symbols from spread metadata + all_leg_symbols = set() + leg_metadata = {} # Store metadata for each leg symbol + + for symbol, meta in spreads_meta.items(): + legs = meta.get("legs", []) + for leg in legs: + leg_symbol = leg.get("symbol") + leg_contract_type = leg.get("contractType") + if leg_symbol and leg_contract_type: + all_leg_symbols.add(leg_symbol) + leg_metadata[leg_symbol] = { + "contractType": leg_contract_type, + "baseCoin": meta.get("baseCoin"), + "quoteCoin": meta.get("quoteCoin") + } + + logger.info(f"Found {len(all_leg_symbols)} unique leg symbols: {sorted(all_leg_symbols)}") + logger.debug(f"Leg metadata summary: {leg_metadata}") + + # Create leg securities (before spread securities) + sec_list_legs = PySecurityList() + + # Log what metadata we have for legs + for leg_symbol in sorted(all_leg_symbols): + if leg_symbol in metadata: + meta = metadata.get(leg_symbol, {}) + delivery_date = meta.get("deliveryDate") + status = meta.get("status") + logger.info(f"Leg {leg_symbol} metadata: deliveryDate={delivery_date}, status={status}") + else: + logger.warning(f"No metadata found for leg {leg_symbol} in existing futures metadata") + + # Use existing futures specs for legs that are futures + futures_leg_symbols = [symbol for symbol in all_leg_symbols + if leg_metadata.get(symbol, {}).get("contractType") in ["LinearFutures", "LinearPerpetual"]] + logger.info(f"Found {len(futures_leg_symbols)} futures legs that can use existing specs") + + # Also check what spot specs we have available for legs + spot_leg_symbols = [symbol for symbol in all_leg_symbols + if leg_metadata.get(symbol, {}).get("contractType") == "Spot"] + logger.info(f"Found {len(spot_leg_symbols)} spot legs that can use existing specs") + + # Log what specs are available for legs + for leg_symbol in sorted(all_leg_symbols): + leg_meta = leg_metadata.get(leg_symbol, {}) + contract_type = leg_meta.get("contractType", "Unknown") + if contract_type in ["LinearFutures", "LinearPerpetual"]: + if leg_symbol in futures_specs: + logger.info(f"Leg {leg_symbol} ({contract_type}): futures specs available") + else: + logger.warning(f"Leg {leg_symbol} ({contract_type}): no futures specs found") + elif contract_type == "Spot": + if leg_symbol in spot_specs: + logger.info(f"Leg {leg_symbol} ({contract_type}): spot specs available") + else: + logger.warning(f"Leg {leg_symbol} ({contract_type}): no spot specs found") + else: + logger.warning(f"Leg {leg_symbol} ({contract_type}): unknown contract type") + + for leg_symbol in sorted(all_leg_symbols): + leg_meta = leg_metadata.get(leg_symbol, {}) + contract_type = leg_meta.get("contractType", "Unknown") + base_asset = leg_meta.get("baseCoin", "BTC") + quote_currency = leg_meta.get("quoteCoin", "USDT") + + logger.info(f"Creating leg security for {leg_symbol}: type={contract_type}, asset={base_asset}, currency={quote_currency}") + + try: + # Get the full metadata for this leg symbol from existing futures metadata + full_leg_meta = metadata.get(leg_symbol, {}) + + # Determine expiration based on contract type (same logic as futures/spots) + if contract_type == "Spot": + expiration = None # Spots have no expiration + elif contract_type in ["LinearPerpetual", "LinearFutures"]: + # Use the same get_expiration logic as regular futures + if full_leg_meta: + expiration = get_expiration(full_leg_meta) + logger.info(f"Using existing metadata expiration for {leg_symbol}: {expiration}") + else: + # Fallback: try to parse from symbol name for futures + if contract_type == "LinearFutures" and "-" in leg_symbol and len(leg_symbol.split("-")) == 2: + try: + date_part = leg_symbol.split("-")[1] + from datetime import datetime + dt = datetime.strptime(date_part, "%d%b%y") + if dt.year < 2000: + dt = dt.replace(year=dt.year + 2000) + expiration = dt.strftime("%Y-%m-%dT%H:%M") + logger.info(f"Parsed expiration from symbol for {leg_symbol}: {date_part} -> {expiration}") + except Exception as e: + logger.warning(f"Failed to parse expiration for {leg_symbol}, using default: {e}") + expiration = PERP_EXP_STRING + else: + expiration = PERP_EXP_STRING + logger.info(f"Using fallback expiration for {leg_symbol}: {expiration}") + else: + logger.warning(f"Unknown contract type {contract_type} for leg {leg_symbol}, using default expiration") + expiration = PERP_EXP_STRING + + # Get specs based on contract type - reuse existing futures specs + if contract_type in ["LinearFutures", "LinearPerpetual"] and leg_symbol in futures_specs: + specs = futures_specs.get(leg_symbol, {}).get("specs", {}) + logger.info(f"Using existing futures specs for {leg_symbol}: {specs}") + elif contract_type == "Spot" and leg_symbol in spot_specs: + specs = spot_specs.get(leg_symbol, {}).get("specs", {}) + logger.info(f"Using existing spot specs for {leg_symbol}: {specs}") + else: + # Use default specs only if no existing specs available + specs = { + "tick_unit": 1, + "tick_base": 100, + "cash_per_point": 1, + "qty_tick_unit": 1, + "qty_tick_base": 10000 + } + logger.info(f"Using default specs for {leg_symbol} (no existing specs found)") + + qty_info = get_quantity_info( + tick_unit=specs.get("qty_tick_unit", 1), + tick_base=specs.get("qty_tick_base", 10000) + ) + spec = get_contract_spec( + price_info=get_price_info( + tick_unit=specs.get("tick_unit", 1), + tick_base=specs.get("tick_base", 10), + cash_per_point=specs.get("cash_per_point", 1) + ), + qty_info=qty_info + ) + + # Create leg security using get_security + sec = get_security( + venue="Bybit", + name=leg_symbol, + asset=base_asset, + group=base_asset, + currency=quote_currency, + expiration=expiration, + option_kind=None, + strike=None, + spec=spec, + ) + + sec_list_legs.append(sec) + logger.info(f"Created LEG security: name={leg_symbol}, type={contract_type}, asset={base_asset}, currency={quote_currency}, expiration={expiration}") + + except Exception as e: + logger.error(f"Failed to create LEG security for {leg_symbol}: {e}", exc_info=True) + + # Send leg securities to bridge + if sec_list_legs: + for sec in sec_list_legs: + combined_sec_list.append(sec) + logger.info(f"Added {len(sec_list_legs)} leg securities to combined list") + + # create spread securities + spread_contracts = get_spread_contracts(self.spreads, self.use_testnet) + logger.info(f"Available spread contracts: {list(spreads_meta.keys())}") + logger.info(f"Configured base symbols: {self.spreads}") + + # Create spread specs using metadata + spread_specs = {} + for symbol in spread_contracts: + meta = spreads_meta.get(symbol, {}) + try: + # Extract price and quantity from metadata + price_tick = float(meta.get("tickSize", 0.1)) + lot_size = float(meta.get("lotSize", 0.001)) + + if price_tick == 0 or lot_size == 0: + logger.error(f"Invalid tick or lot size for spread {symbol}: tickSize={price_tick}, lotSize={lot_size}") + continue + + # Calculate tick_base and qty values + tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 + if lot_size >= 1: + qty_tick_unit = lot_size + qty_tick_base = 1 + else: + qty_tick_unit = 1 + qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 + + spread_specs[symbol] = { + "tick_unit": 1, + "tick_base": tick_base, + "cash_per_point": 1, + "qty_tick_unit": qty_tick_unit, + "qty_tick_base": qty_tick_base + } + logger.info(f"Calculated spread specs for {symbol}: {spread_specs[symbol]}") + except (ValueError, TypeError) as e: + logger.error(f"Failed to parse spread specs for {symbol}: {e}") + continue + + # Log summary of spread specs + logger.info(f"Successfully calculated specs for {len(spread_specs)} spread contracts") + + # Create spread securities + sec_list_spreads = PySecurityList() + for symbol in spread_contracts: + meta = spreads_meta.get(symbol, {}) + logger.debug(f"Spread security creation for {symbol}: {meta}") + try: + base_asset = meta.get("baseCoin", "BTC") + quote_currency = meta.get("quoteCoin", "USDT") + contract_type = meta.get("contractType", "Unknown") + + # Process all spread types (FutureSpread, FundingRateArb, CarryTrade, PerpBasis) + logger.info(f"Processing spread {symbol} with contractType {contract_type}") + + # Extract leg information for spread + legs = meta.get("legs", []) + leg_symbols = [leg.get("symbol") for leg in legs if leg.get("symbol")] + + logger.info(f"Spread {symbol} has legs: {leg_symbols}") + + # Get calculated specs for this spread symbol + specs = spread_specs.get(symbol) + if not specs: + logger.warning(f"No specs found for spread {symbol}, skipping") + continue + + qty_info = get_quantity_info( + tick_unit=specs.get("qty_tick_unit", 1), + tick_base=specs.get("qty_tick_base", 10000) + ) + spec = get_contract_spec( + price_info=get_price_info( + tick_unit=specs.get("tick_unit", 1), + tick_base=specs.get("tick_base", 10), + cash_per_point=specs.get("cash_per_point", 1) + ), + qty_info=qty_info + ) + + # Fix expiration for spreads + delivery_time = meta.get("deliveryTime") + if delivery_time and delivery_time != "0": + from datetime import datetime + dt = datetime.utcfromtimestamp(int(delivery_time) / 1000) + expiration = dt.strftime("%Y-%m-%dT%H:%M") + else: + expiration = PERP_EXP_STRING # or some other default for spreads + + # Create PySecurity directly (not using get_security) + sec = PySecurity() + sec.set_venue("Bybit") + sec.set_name(symbol) + sec.set_asset(base_asset) + sec.set_group(base_asset) + sec.set_currency(quote_currency) + sec.set_contract_spec(spec) + + # Create and set the spread: + spread = PySpread() + spread.set_spread_type(contract_type) + spread.set_legs(leg_symbols) + spread.set_expiration(expiration) + + # Set the security as a spread: + sec.set_spread(spread) + sec.set_exchange_id(symbol) # This ensures unique SecurityID generation + + + sec_list_spreads.append(sec) + logger.info(f"Created SPREAD security: name={symbol}, asset={base_asset}, currency={quote_currency}, type={contract_type}, legs={leg_symbols}") + except Exception as e: + logger.error(f"Failed to create SPREAD security for {symbol}: {e}", exc_info=True) + + # Spread count + logger.info(f"Total spreads created: {len(sec_list_spreads)}") + + # Debug: all created spread securities with their types + for i, sec in enumerate(sec_list_spreads): + spread_info = sec.spread() + logger.info(f"Spread {i+1}: name={sec.name()}, exchange_id={sec.exchange_id()}, asset={sec.asset()}, type={spread_info.spread_type()}, legs={spread_info.legs()}") + + + logger.info(f"Created {len(sec_list_spreads)} spread securities") + for sec in sec_list_spreads: + combined_sec_list.append(sec) + logger.info(f"Added spread to combined list: {sec.name()}") + bridge.on_securities(combined_sec_list) + logger.info(f"Sent {len(sec_list_spreads)} spreads to bridge") try: # WebSocket for futures @@ -443,7 +795,71 @@ class BybitMdProvider(MdProvider): except Exception as e: logger.error(f"Failed to subscribe to orderbook for {contract} (spots): {e}") - logger.info("Subscribed to orderbook channels on Bybit futures and spots WS.") + # WebSocket for spreads + if sec_list_spreads: + try: + # Get spread symbols from metadata (all spread types) + spread_symbols = [meta.get("symbol") for meta in spreads_meta.values()] + + logger.info(f"Found {len(spread_symbols)} spread symbols to subscribe to: {spread_symbols}") + + if spread_symbols: + # Create spread WebSocket connection + host = "stream-testnet.bybit.com" if self.use_testnet else "stream.bybit.com" + uri = f"wss://{host}/v5/public/spread" + + logger.info(f"Connecting to Bybit spread WebSocket: {uri}") + + # Run WebSocket in separate thread to avoid blocking + def run_spread_websocket(): + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._spread_websocket_async(uri, spread_symbols)) + except Exception as e: + logger.error(f"Spread WebSocket thread error: {e}", exc_info=True) + + spread_thread = threading.Thread(target=run_spread_websocket, daemon=True, name="SpreadWebSocket") + spread_thread.start() + logger.info("Spread WebSocket started in separate thread") + + # Add thread monitoring + def monitor_spread_thread(): + while not self.stop_event.is_set(): + time.sleep(10) + if not spread_thread.is_alive(): + logger.warning("Spread WebSocket thread died, attempting restart...") + try: + # Restart the thread + new_spread_thread = threading.Thread(target=run_spread_websocket, daemon=True, name="SpreadWebSocket-Restart") + new_spread_thread.start() + logger.info("Spread WebSocket thread restarted") + break + except Exception as e: + logger.error(f"Failed to restart spread WebSocket thread: {e}") + break + + monitor_thread = threading.Thread(target=monitor_spread_thread, daemon=True, name="SpreadWebSocketMonitor") + monitor_thread.start() + logger.info("Spread WebSocket monitor thread started") + + # Add update monitoring thread + def monitor_spread_updates(): + while not self.stop_event.is_set(): + time.sleep(60) # Check every minute + if not self.stop_event.is_set() and hasattr(self, 'last_spread_update_time'): + current_time = time.time() + for symbol, last_update in self.last_spread_update_time.items(): + if current_time - last_update > 120: # 2 minutes without updates + logger.warning(f"No spread updates for {symbol} in {int(current_time - last_update)} seconds") + + update_monitor_thread = threading.Thread(target=monitor_spread_updates, daemon=True, name="SpreadUpdateMonitor") + update_monitor_thread.start() + + except Exception as e: + logger.error(f"Failed to setup spread WebSocket: {e}", exc_info=True) + + logger.info("Subscribed to orderbook channels on Bybit futures, spots, and spreads WS.") while not self.stop_event.is_set(): time.sleep(1) except KeyboardInterrupt: @@ -475,13 +891,14 @@ class BybitMdProvider(MdProvider): raw_asks = payload.get("a", []) if not raw_bids or not raw_asks: logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") + return # Skip processing if no valid data raw_bids = [(float(x[0]), float(x[1])) for x in raw_bids] raw_asks = [(float(x[0]), float(x[1])) for x in raw_asks] sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] sorted_bids += [(np.float64('nan'), np.float64('nan'))] sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] sorted_asks += [(np.float64('nan'), np.float64('nan'))] - logger.info(f"📘 BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") + logger.debug(f"BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") book = PyBook() market_type = "FUTURE" if source == "futures" else "SPOT" # Dynamically set market type logger.debug(f"Assigned market_type={market_type} for contract={contract} (source={source})") @@ -495,19 +912,301 @@ class BybitMdProvider(MdProvider): book_list = PyBookList() book_list.append(book) if self.bridge: - logger.debug(f"✔ on_books called for {contract} (market_type={market_type})") + logger.debug(f" on_books called for {contract} (market_type={market_type})") self.bridge.on_books(book_list) else: logger.warning("Bridge is None, cannot call on_books") except Exception as e: logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) + async def _spread_websocket_async(self, uri: str, symbols: list): + """Async WebSocket connection for spreads with improved connection management.""" + max_reconnect_attempts = 5 + reconnect_delay = 5 # seconds + + for attempt in range(max_reconnect_attempts): + try: + logger.info(f"Attempting to connect to spread WebSocket (attempt {attempt + 1}/{max_reconnect_attempts}): {uri}") + + # Disable built-in ping/pong since Bybit spread endpoint doesn't support it + async with websockets.connect( + uri, + ping_interval=None, # Disable built-in ping + ping_timeout=None, # Disable built-in ping timeout + close_timeout=10, # Wait 10 seconds for close frame + max_size=2**20 # 1MB max message size + ) as websocket: + logger.info(f"Connected to spread WebSocket: {uri}") + + # Subscribe to all spread symbols + subscribe_msg = { + "op": "subscribe", + "args": [f"orderbook.25.{symbol}" for symbol in symbols] + } + + await websocket.send(json.dumps(subscribe_msg)) + logger.info(f"Subscribed to spread orderbooks: {subscribe_msg['args']}") + + # Start custom heartbeat task (in addition to websockets built-in ping) + heartbeat_task = asyncio.create_task(self._spread_heartbeat(websocket)) + + # Start connection monitor task + monitor_task = asyncio.create_task(self._spread_connection_monitor(websocket)) + + try: + # Process messages + async for message in websocket: + if self.stop_event.is_set(): + break + + try: + data = json.loads(message) + logger.debug(f"SPREAD WEBSOCKET MESSAGE: {data}") + + # Handle pong responses + if data.get("op") == "pong": + logger.debug("Received pong response from spread WebSocket") + continue + + # Check message type for debugging + msg_type = data.get("type", "unknown") + topic = data.get("topic", "unknown") + logger.debug(f"Processing spread message: type={msg_type}, topic={topic}") + + # Process as complete orderbook (simple approach) + self._on_spread_book_update(data) + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse spread WebSocket message: {e}") + except Exception as e: + logger.error(f"Error processing spread message: {e}") + + finally: + # Cancel tasks + heartbeat_task.cancel() + monitor_task.cancel() + try: + await heartbeat_task + except asyncio.CancelledError: + pass + try: + await monitor_task + except asyncio.CancelledError: + pass + + except websockets.exceptions.ConnectionClosed as e: + logger.warning(f"Spread WebSocket connection closed (attempt {attempt + 1}): {e}") + if attempt < max_reconnect_attempts - 1: + logger.info(f"Reconnecting in {reconnect_delay} seconds...") + await asyncio.sleep(reconnect_delay) + reconnect_delay = min(reconnect_delay * 2, 60) # Exponential backoff, max 60s + else: + logger.error("Max reconnection attempts reached for spread WebSocket") + break + except Exception as e: + logger.error(f"Spread WebSocket connection error (attempt {attempt + 1}): {e}", exc_info=True) + if attempt < max_reconnect_attempts - 1: + logger.info(f"Reconnecting in {reconnect_delay} seconds...") + await asyncio.sleep(reconnect_delay) + reconnect_delay = min(reconnect_delay * 2, 60) + else: + logger.error("Max reconnection attempts reached for spread WebSocket") + break + + async def _spread_connection_monitor(self, websocket): + """Monitor WebSocket connection health and log status.""" + monitor_count = 0 + try: + while not self.stop_event.is_set(): + await asyncio.sleep(30) # Check every 30 seconds + if not self.stop_event.is_set(): + monitor_count += 1 + if websocket.open: + logger.debug(f"Spread WebSocket connection is healthy (check #{monitor_count})") + else: + logger.warning(f"Spread WebSocket connection is closed (check #{monitor_count})") + break + except asyncio.CancelledError: + logger.debug("Spread connection monitor task cancelled") + except Exception as e: + logger.error(f"Error in spread connection monitor: {e}") + + async def _spread_heartbeat(self, websocket): + """Send heartbeat ping every 20 seconds to keep connection alive.""" + heartbeat_count = 0 + + try: + while not self.stop_event.is_set(): + await asyncio.sleep(20) # Wait 20 seconds + if not self.stop_event.is_set(): + try: + # Send custom ping message + heartbeat_count += 1 + ping_msg = {"req_id": str(int(time.time())), "op": "ping"} + await websocket.send(json.dumps(ping_msg)) + logger.debug(f"Sent spread WebSocket ping heartbeat #{heartbeat_count}") + + except websockets.exceptions.ConnectionClosed: + logger.warning("Spread WebSocket connection closed during heartbeat") + break + except Exception as e: + logger.error(f"Error sending spread heartbeat: {e}") + break + + except asyncio.CancelledError: + logger.debug("Spread heartbeat task cancelled") + except Exception as e: + logger.error(f"Error in spread heartbeat: {e}") + + def _on_spread_book_update(self, data: dict): + """Handle spread orderbook updates with proper delta/snapshot processing.""" + try: + # Track last update time for debugging + if not hasattr(self, 'last_spread_update_time'): + self.last_spread_update_time = {} + + topic = data.get("topic", "") + if not topic.startswith("orderbook.25."): + logger.debug(f"Ignoring non-orderbook topic: {topic}") + return + + # Extract symbol from topic + symbol = topic.split(".", 2)[-1] # Get everything after "orderbook.25." + self.last_spread_update_time[symbol] = time.time() + + logger.debug(f"Processing spread orderbook update for {symbol}") + + # Initialize spread orderbook storage if not exists + if not hasattr(self, 'spread_orderbooks'): + self.spread_orderbooks = {} + + # Initialize orderbook for this symbol if not exists + if symbol not in self.spread_orderbooks: + self.spread_orderbooks[symbol] = {"b": [], "a": []} + + payload = data.get("data", {}) + msg_type = data.get("type", "") + + # Handle snapshot (initial data or reset) + if msg_type == "snapshot": + logger.debug(f"Received snapshot for spread {symbol}") + self.spread_orderbooks[symbol] = payload.copy() + elif msg_type == "delta": + logger.debug(f"Processing delta for spread {symbol}") + # Process delta updates + book_sides = {"b": payload.get("b", []), "a": payload.get("a", [])} + + # Update sequence numbers + if "u" in payload: + self.spread_orderbooks[symbol]["u"] = payload["u"] + if "seq" in payload: + self.spread_orderbooks[symbol]["seq"] = payload["seq"] + + # Process each side (bids and asks) + for side, entries in book_sides.items(): + for entry in entries: + price, qty = entry[0], float(entry[1]) + + # Delete if quantity is 0 + if qty == 0: + self.spread_orderbooks[symbol][side] = [ + x for x in self.spread_orderbooks[symbol][side] + if x[0] != price + ] + else: + # Update or insert + existing = next((x for x in self.spread_orderbooks[symbol][side] + if x[0] == price), None) + if existing: + existing[1] = str(qty) + else: + self.spread_orderbooks[symbol][side].append([price, str(qty)]) + else: + logger.debug(f"Unknown message type for spread {symbol}: {msg_type}") + return + + # Get the processed orderbook + orderbook = self.spread_orderbooks[symbol] + raw_bids = orderbook.get("b", []) + raw_asks = orderbook.get("a", []) + + if not raw_bids and not raw_asks: + logger.warning(f"Empty spread orderbook data for {symbol}") + return + + # Convert to same format as futures/spots + bids = [] + asks = [] + + # Process bids (price, quantity pairs) + for bid in raw_bids: + if len(bid) >= 2: + price = float(bid[0]) + quantity = float(bid[1]) + if quantity > 0: # Only include non-zero quantities + bids.append((price, quantity)) + + # Process asks (price, quantity pairs) + for ask in raw_asks: + if len(ask) >= 2: + price = float(ask[0]) + quantity = float(ask[1]) + if quantity > 0: # Only include non-zero quantities + asks.append((price, quantity)) + + # Sort and limit to top 5 levels (maintain bridge format) + sorted_bids = sorted(bids, key=lambda x: -x[0])[:5] # Sort by price descending + sorted_asks = sorted(asks, key=lambda x: x[0])[:5] # Sort by price ascending + + # Pad with NaN to ensure exactly 5 levels (spreads might have fewer levels) + while len(sorted_bids) < 5: + sorted_bids.append((np.float64('nan'), np.float64('nan'))) + while len(sorted_asks) < 5: + sorted_asks.append((np.float64('nan'), np.float64('nan'))) + + # Add final NaN row (total: 5 levels + 1 NaN = 6 rows) + sorted_bids.append((np.float64('nan'), np.float64('nan'))) + sorted_asks.append((np.float64('nan'), np.float64('nan'))) + + logger.debug(f"SPREAD BOOK DATA for {symbol}: bids={sorted_bids} asks={sorted_asks}") + + # Create PyBook + book = PyBook() + book.set_data( + symbol, + "Bybit", + "SPREAD", + np.array(sorted_bids, dtype=np.float64), + np.array(sorted_asks, dtype=np.float64), + ) + + book_list = PyBookList() + book_list.append(book) + + if self.bridge: + logger.debug(f"on_books called for spread {symbol}") + self.bridge.on_books(book_list) + else: + logger.warning("Bridge is None, cannot call on_books") + + except Exception as e: + logger.error(f"Error processing spread book update: {e}", exc_info=True) + def stop(self): logger.info("Stopping BybitMdProvider...") self.stop_event.set() logger.debug("Threads before stopping WebSocket:") for t in threading.enumerate(): logger.debug(f"Thread(stop): {t.name}, Alive: {t.is_alive()}, Daemon: {t.daemon}, Target={t._target}") + + # Clean up spread WebSocket threads + for t in threading.enumerate(): + if t.name in ['SpreadWebSocket', 'SpreadWebSocket-Restart', 'SpreadWebSocketMonitor', 'SpreadUpdateMonitor'] and t.is_alive(): + logger.info(f"Joining spread WebSocket thread: {t.name}") + t.join(timeout=2.0) + if t.is_alive(): + logger.warning(f"Spread WebSocket thread {t.name} did not terminate gracefully") + if self.ws_futures: try: self.ws_futures.exit() @@ -584,4 +1283,4 @@ if __name__ == "__main__": md_bridge = FakeMdBridge(logger) md_pro.start(md_bridge) while True: - time.sleep(1) + time.sleep(1) \ No newline at end of file diff --git a/src/lib_python_interop/python_interop_plugins/gateio_md_provider.py b/src/lib_python_interop/python_interop_plugins/gateio_md_provider.py index ad52ec5e0..5985e6d14 100644 --- a/src/lib_python_interop/python_interop_plugins/gateio_md_provider.py +++ b/src/lib_python_interop/python_interop_plugins/gateio_md_provider.py @@ -4,10 +4,11 @@ import time import logging import sys import threading +from typing import Iterable, Optional, Dict +import websocket import requests -from itertools import chain -from websocket import create_connection, WebSocketConnectionClosedException -from typing import Iterable, Optional, Any +import ssl +import certifi from python_interop_module import ( MdBridge, MdProvider, @@ -23,214 +24,407 @@ from spark_core import ( get_security, load_config, ) - -# Urls: -# prod perps (ws) - "wss://fx-ws.gateio.ws/v4/ws/usdt" -# testnet perps (ws) - "wss://fx-ws-testnet.gateio.ws/v4/ws/usdt" +from py_md_utils import PERP_EXP_STRING logger = logging.getLogger("gateio_md_provider") -def fetch_contract_specs(contracts: Iterable[str]) -> dict: - """Fetch specs for all contracts in one request.""" +def get_metadata(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: + """Fetch metadata for all USDT-margined futures contracts and filter for specified contracts.""" + metadata_dict = {} + try: + host = "api.gateio.ws" if not use_testnet else "api.testnet.gateio.ws" + url = f"https://{host}/api/v4/futures/usdt/contracts" + logger.info(f"Querying Gate.io API for metadata: {url}") + response = requests.get(url, timeout=5) + response.raise_for_status() + contracts_data = response.json() + + logger.info(f"Retrieved metadata for {len(contracts_data)} futures contracts") + for item in contracts_data: + symbol = item.get("name") + if symbol in contracts and not item.get("in_delisting", False): + metadata_dict[symbol] = item + logger.info(f"Stored metadata for {symbol}") + except Exception as e: + logger.error(f"Failed to fetch metadata: {e}", exc_info=True) + return metadata_dict + +def fetch_futures_specs(contracts: Iterable[str], use_testnet: bool = True) -> Dict[str, Dict]: default_specs = { "tick_unit": 1, "tick_base": 10, - "cash_per_point": 0.1, - "qty_tick_unit": 1, + "cash_per_point": 1, + "qty_tick_unit": 0.001, "qty_tick_base": 10000 } - specs_dict = {contract: default_specs for contract in contracts} + specs_dict = {contract: {"specs": default_specs} for contract in contracts} try: - url = "https://api.gateio.ws/api/v4/futures/usdt/contracts" + host = "api.gateio.ws" if not use_testnet else "api.testnet.gateio.ws" + url = f"https://{host}/api/v4/futures/usdt/contracts" + logger.info(f"Querying Gate.io API for futures specs: {url}") response = requests.get(url, timeout=5) response.raise_for_status() - data = response.json() - for item in data: + symbols = response.json() + + logger.info(f"Loaded {len(symbols)} futures contracts") + for item in symbols: contract = item.get("name") if contract in contracts: - price_tick = float(item.get("price_tick", "0.1")) - contract_size = float(item.get("contract_size", "0.0001")) - tick_base = int(1 / price_tick) if price_tick != 0 else 10 - qty_tick_base = int(1 / contract_size) if contract_size != 0 else 10000 - specs_dict[contract] = { - "tick_unit": 1, - "tick_base": tick_base, - "cash_per_point": price_tick, - "qty_tick_unit": 1, - "qty_tick_base": qty_tick_base - } - return specs_dict - except Exception as e: - logger.error(f"Failed to fetch batch specs: {e}", exc_info=True) + try: + price_tick = float(item.get("order_price_round", 0.1)) + lot_size = float(item.get("quanto_multiplier", 0.001)) + + if price_tick == 0 or lot_size == 0: + logger.error(f"Invalid tick or lot size for {contract}: tickSize={price_tick}, lotSize={lot_size}") + continue + + tick_base = min(int(1 / price_tick), 1000000) if price_tick != 0 else 10 + if lot_size >= 1: + qty_tick_unit = lot_size + qty_tick_base = 1 + else: + qty_tick_unit = 1 + qty_tick_base = min(int(1 / lot_size), 1000000) if lot_size != 0 else 10000 + + specs_dict[contract] = { + "specs": { + "tick_unit": 1, + "tick_base": tick_base, + "cash_per_point": 1, + "qty_tick_unit": qty_tick_unit, + "qty_tick_base": qty_tick_base + } + } + logger.info(f"Fetched futures specs for {contract}: {specs_dict[contract]['specs']}") + except (ValueError, TypeError) as e: + logger.error(f"Failed to parse futures specs for {contract}: {e}") + continue + + logger.info(f"Returning specs for contracts: {list(specs_dict.keys())}") + logger.debug(f"Final specs: {specs_dict}") return specs_dict -class GateIoMdProvider(MdProvider): + except Exception as e: + logger.error(f"Failed to fetch futures contract specs: {e}", exc_info=True) + return specs_dict + +def get_expiration(contract: Dict) -> str: + """Extract expiration from metadata or use default for perpetuals.""" + if contract.get("in_delisting", False): + return "0" # 8061 format for expired + return PERP_EXP_STRING # Default for perpetuals + +def get_futures_contracts(base_symbols: list, use_testnet: bool = True) -> list: + """Query Gate.io API for all USDT-margined futures and filter by base symbols.""" + futures_contracts = [] + host = "api.gateio.ws" if not use_testnet else "api.testnet.gateio.ws" + url = f"https://{host}/api/v4/futures/usdt/contracts" + + logger.info(f"Querying all futures contracts: {url}") + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + logger.info(f"Retrieved {len(data)} futures contracts") + base_symbols_upper = [s.upper() for s in base_symbols] # Normalize to uppercase + for item in data: + symbol = item.get("name") + base_coin = symbol.split("_")[0] if symbol and "_" in symbol else None + contract_type = item.get("type", "").lower() + in_delisting = item.get("in_delisting", False) + # logger.debug(f"Checking contract: {symbol}, base: {base_coin}, type: {contract_type}, in_delisting: {in_delisting}") + if base_coin and base_coin.upper() in base_symbols_upper and contract_type == "direct" and not in_delisting: + futures_contracts.append(symbol) + logger.info(f"Matched contract: {symbol} (base: {base_coin}, type: {contract_type}, in_delisting: {in_delisting})") + logger.info(f"Filtered to {len(futures_contracts)} compatible instruments for base symbols {base_symbols}") + + return sorted(list(set(futures_contracts))) + +class GateioMdProvider(MdProvider): def __init__(self): super().__init__() - self.ws = None + self._config: Optional[dict] = None + self.ws_futures = None self.ws_thread: Optional[threading.Thread] = None self.bridge: Optional[MdBridge] = None self.stop_event = threading.Event() - self.ping_thread = None - def send_ping(self): - """Send ping messages every 15 seconds to keep connection alive.""" - while not self.stop_event.is_set(): - try: - ping_msg = { - "time": int(time.time() * 1000), - "channel": "futures.ping", - "event": "", - "payload": [] - } - self.ws.send(json.dumps(ping_msg)) - logger.debug("Sent ping") - except Exception as e: - logger.error(f"Ping error: {e}", exc_info=True) - break - time.sleep(15) + @property + def config(self): + if self._config is None: + self._config = load_config(["py_gateio_md_provider", "gateio_overrides"]) + return self._config + + @property + def use_testnet(self): + return self.config.get("use_testnet", True) + + @property + def currencies(self): + return self.config.get("currencies") or ["USDT"] + + @property + def futures(self): + return self.config.get("futures", []) def start_impl(self, bridge: MdBridge): - cfg = load_config(["gateio_md_provider", "gateio_overrides"]) - configure_logger(logger, "md", **cfg.get("logging", {})) + configure_logger(logger, "md", level="INFO", **self.config.get("logging", {})) self.bridge = bridge - ws_url = cfg["ws_url"] - logger.info(f"Connecting to {ws_url}") - self.ws = create_connection(ws_url) - self.ws.settimeout(5) - logger.info("Connected.") + logger.info("Starting GateioMdProvider") - # Start ping thread - self.ping_thread = threading.Thread(target=self.send_ping, daemon=True) - self.ping_thread.start() - logger.info("Started ping thread") + futures_contracts = get_futures_contracts(self.futures, self.use_testnet) + logger.info(f"futures_contracts: {futures_contracts}") - # Fetch all contract specs at once - spots: list[str] = cfg["spots"] - currencies: list[str] = cfg.get("currencies") or ["USDT"] - contracts = list(chain(*[map(lambda s: f"{s}{c}", spots) for c in currencies])) - contract_specs = fetch_contract_specs(contracts) + metadata = get_metadata(futures_contracts, self.use_testnet) + logger.info(f"Fetched metadata for {len(metadata)} futures contracts") + logger.info(f"metadata: {metadata}") - # 1) Send security info - sec_list = PySecurityList() - for contract in contracts: - asset, quote = contract.split("_") - specs = contract_specs[contract] - spec = get_contract_spec( - price_info=get_price_info( - tick_unit=specs["tick_unit"], - tick_base=specs["tick_base"], - cash_per_point=specs["cash_per_point"] - ), - qty_info=get_quantity_info( - tick_unit=specs["qty_tick_unit"], - tick_base=specs["qty_tick_base"] + futures_specs = fetch_futures_specs(futures_contracts, self.use_testnet) + logger.info(f"Fetched futures specs for {len(futures_specs)} contracts") + logger.info(f"futures_specs: {futures_specs}") + + sec_list_futures = PySecurityList() + + for contract in futures_contracts: + meta = metadata.get(contract, {}) + if not meta: + logger.warning(f"No metadata found for contract: {contract}, using defaults") + continue + + try: + symbol = meta.get("name", contract) + base_asset = symbol.split("_")[0] + quote_currency = symbol.split("_")[1] + expiration = get_expiration(meta) + + asset = base_asset + group = base_asset + currency = quote_currency + + specs = futures_specs.get(contract, {}).get("specs", {}) + if not specs: + logger.warning(f"No specs found for contract: {contract}, using defaults") + specs = {"tick_unit": 1, "tick_base": 10, "cash_per_point": 0.1, "qty_tick_unit": 0.001, "qty_tick_base": 10000} + + qty_info = get_quantity_info( + tick_unit=specs.get("qty_tick_unit", 0.001), + tick_base=specs.get("qty_tick_base", 10000) ) - ) - sec = get_security( - venue="GateIO", - name=contract, - asset=asset, - group=asset, - currency=quote, - expiration=None, - option_kind=None, - strike=None, - spec=spec, - ) - sec_list.append(sec) - logger.info(f"Created security: {contract} with specs: {specs}") - bridge.on_securities(sec_list) - logger.info("Sent securities list--sink.on_securities called.") - # 2) Subscribe to order books - for contract in contracts: - sub_msg = { - "time": int(time.time() * 1000), - "channel": "futures.order_book", - "event": "subscribe", - "payload": [contract, "5", "0"] - } - self.ws.send(json.dumps(sub_msg)) - logger.info(f"✔ Sent subscribe for {contract}: {sub_msg}") + spec = get_contract_spec( + price_info=get_price_info( + tick_unit=specs.get("tick_unit", 1), + tick_base=specs.get("tick_base", 10), + cash_per_point=specs.get("cash_per_point", 0.1) + ), + qty_info=qty_info + ) + + sec = get_security( + venue="Gateio", + name=symbol, + asset=asset, + group=group, + currency=currency, + expiration=expiration, + option_kind=None, + strike=None, + spec=spec, + ) + sec_list_futures.append(sec) + logger.info(f"Created FUTURE security: name={symbol}, venue=Gateio, asset={asset}, group={group}, currency={currency}, expiration={expiration}, spec={specs}") + except Exception as e: + logger.error(f"Error processing metadata for {contract}: {e}", exc_info=True) + continue + + combined_sec_list = PySecurityList() + for sec in sec_list_futures: + combined_sec_list.append(sec) + bridge.on_securities(combined_sec_list) + logger.info(f"Sent securities list: {len(sec_list_futures)} futures") + + retry_count = 0 + max_retries = 3 + while not self.stop_event.is_set() and retry_count < max_retries: + try: + ws_url = "wss://fx-ws.gateio.ws/v4/ws/usdt" if not self.use_testnet else "wss://fx-ws.testnet.gateio.ws/v4/ws/usdt" + self.ws_futures = websocket.WebSocketApp( + ws_url, + on_message=self._on_message, + on_error=self._on_error, + on_close=self._on_close, + on_open=self._on_open, + ) + logger.info(f"Connecting to Gate.io WebSocket (futures), attempt {retry_count + 1}/{max_retries}") + websocket.enableTrace(True) # Enable tracing for debugging + self.ws_thread = threading.Thread( + target=self.ws_futures.run_forever, + kwargs={"ping_interval": 20, "ping_timeout": 10} + ) + self.ws_thread.start() + # Wait for connection to establish + time.sleep(2) + if self.ws_futures and self.ws_futures.sock and self.ws_futures.sock.connected: + for contract in futures_contracts: + subscribe_msg = { + "channel": "futures.order_book", + "event": "subscribe", + "payload": [contract, "5", "0"] + } + self.ws_futures.send(json.dumps(subscribe_msg)) + logger.info(f"Subscribed to orderbook for {contract} (futures)") + break + else: + logger.warning("WebSocket connection not established, retrying...") + retry_count += 1 + time.sleep(5) + except Exception as e: + logger.error(f"WebSocket connection attempt {retry_count + 1} failed: {e}", exc_info=True) + retry_count += 1 + time.sleep(5) + if retry_count >= max_retries: + logger.error("Max WebSocket retries reached, stopping provider") + self.stop() + return - # 3) Listen for incoming messages in main thread - logger.info("▶ Listening for data") try: while not self.stop_event.is_set(): - try: - msg = self.ws.recv() - except Exception as e: - logger.debug(f"⚠ recv() error or timeout: {e}") - continue - - logger.debug(f"⟵ raw WS: {msg}") - try: - data = json.loads(msg) - logger.debug(f"Parsed JSON: {data}") - except Exception as e: - logger.error(f"JSON parse error: {e}", exc_info=True) - continue - - event = data.get("event") - channel = data.get("channel") - - # Handle subscription acknowledgements - if event == "subscribe": - logger.info(f"🔔 Subscribe ACK: {data}") - continue - - # Handle order book updates - if channel == "futures.order_book" and event in ["all", "update"]: - logger.debug(f"Handling '{event}' event") - payload = data.get("result", {}) - if isinstance(payload, dict) and ("bids" in payload or "asks" in payload): - contract = payload.get("contract", "") - bids = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("bids", [])] - asks = [(np.float64(l["p"]), np.float64(l["s"])) for l in payload.get("asks", [])] - logger.debug(f"📘 BOOK DATA for {contract}: bids={bids} asks={asks}") - - book = PyBook() - book.set_data( - contract, - "GateIO", - "SPOT", - np.array(bids, dtype=np.float64), - np.array(asks, dtype=np.float64), - ) - book_list = PyBookList() - book_list.append(book) - self.bridge.on_books(book_list) - logger.debug(f"✔ on_books called for {contract}") - else: - logger.debug("… no bids/asks in payload; skipping") - - except WebSocketConnectionClosedException: - logger.info("WebSocket closed.") + time.sleep(1) + except KeyboardInterrupt: + logger.info("KeyboardInterrupt received — shutting down.") + self.stop() except Exception as e: - logger.error("Fatal error in listen loop:", exc_info=True) + logger.error(f"Gate.io WebSocket fatal error: {e}", exc_info=True) + self.stop() + + def _on_open(self, ws): + logger.info("WebSocket connection opened") + + def _on_message(self, ws, message): + try: + data = json.loads(message) + logger.debug(f"Raw WebSocket message: {data}") + # Only process order book updates (event: 'all'), skip subscription confirmations + if data.get("channel") == "futures.order_book" and data.get("event") == "all": + contract = data.get("result", {}).get("contract") + if not contract: + logger.warning(f"No contract symbol in order book update: {data}") + return + self._on_book_update(data, source="futures") + elif data.get("event") == "subscribe": + logger.debug("Received subscription confirmation, ignoring for book update") + else: + logger.debug(f"Ignoring non-orderbook message: {data}") + except Exception as e: + logger.error(f"Error processing WebSocket message: {e}", exc_info=True) + + def _on_error(self, ws, error): + logger.error(f"WebSocket error: {error}", exc_info=True) + + def _on_close(self, ws, close_status_code, close_msg): + logger.info(f"WebSocket closed: status={close_status_code}, msg={close_msg}") def start(self, bridge: MdBridge): - self.ws_thread = threading.Thread(target=lambda: self.start_impl(bridge)) + def start_wrapper(): + try: + self.start_impl(bridge) + except Exception as e: + logger.error(f"Error in GateioMdProvider start_impl: {e}", exc_info=True) + self.ws_thread = threading.Thread(target=start_wrapper) self.ws_thread.start() + def _on_book_update(self, data: dict, source: str): + try: + logger.debug(f"Raw WebSocket data (source={source}): {data}") + contract = data.get("result", {}).get("contract") + if not contract: + logger.warning(f"No contract symbol in book update: {data}") + return + raw_bids = data.get("result", {}).get("bids", []) + raw_asks = data.get("result", {}).get("asks", []) + if not raw_bids or not raw_asks: + logger.warning(f"Empty or invalid orderbook data for {contract} (source={source}): bids={raw_bids}, asks={raw_asks}") + return + # Convert to (price, size) pairs + raw_bids = [(float(x["p"]), float(x["s"])) for x in raw_bids] + raw_asks = [(float(x["p"]), float(x["s"])) for x in raw_asks] + # Sort and limit to 5 levels + sorted_bids = sorted(raw_bids, key=lambda x: -x[0])[:5] + sorted_asks = sorted(raw_asks, key=lambda x: x[0])[:5] + logger.info(f"BOOK DATA for {contract} (source={source}): bids={sorted_bids} asks={sorted_asks}") + book = PyBook() + market_type = "FUTURE" + book.set_data( + contract, + "Gateio", + market_type, + np.array(sorted_bids, dtype=np.float64), + np.array(sorted_asks, dtype=np.float64), + ) + book_list = PyBookList() + book_list.append(book) + logger.debug(f"Sending book_list to bridge: {book_list}") + if self.bridge: + logger.debug(f"Calling on_books for {contract} (market_type={market_type})") + self.bridge.on_books(book_list) + else: + logger.warning("Bridge is None, cannot call on_books") + except Exception as e: + logger.error(f"Error processing book update for {contract} (source={source}): {e}", exc_info=True) + def stop(self): - """Disconnect and cleanup.""" - logger.info("Stopping GateIoMdProvider...") + logger.info("Stopping GateioMdProvider...") self.stop_event.set() - - if self.ping_thread and self.ping_thread.is_alive(): - self.ping_thread.join() - logger.info("Ping thread stopped") - - if self.ws: + if self.ws_futures: try: - self.ws.close() - except Exception: - pass - self.ws = None - + self.ws_futures.close() + logger.info("WebSocket (futures) closed") + except Exception as e: + logger.error(f"Error while closing WebSocket (futures): {e}", exc_info=True) + self.ws_futures = None if self.ws_thread: self.ws_thread.join() self.bridge = None - logger.info("✔ Stopped GateIoMdProvider.") + logger.info("✔ Stopped GateioMdProvider.") + import os + os._exit(0) + +if __name__ == "__main__": + import argparse + from fake_md_bridge import FakeMdBridge + + ap = argparse.ArgumentParser() + ap.add_argument("--currencies", nargs="+") + ap.add_argument("--futures", nargs="+") + log_levels = list(logging._nameToLevel) + ap.add_argument("--console-level", choices=log_levels) + ap.add_argument("--file-level", choices=log_levels) + ap.add_argument("--dev", action="store_true", dest="use_testnet") + ap.add_argument("--prod", action="store_false", dest="use_testnet") + ap.set_defaults( + currencies=None, + futures=None, + console_level=None, + file_level=None, + use_testnet=None, + ) + args = ap.parse_args() + logger = logging.getLogger("gateio_md_provider_app") + md_pro = GateioMdProvider() + config = md_pro.config + if args.currencies: + config["currencies"] = args.currencies + if args.futures: + config["futures"] = args.futures + logging_overrides = {} + if args.console_level: + logging_overrides["console_level"] = args.console_level + if args.file_level: + logging_overrides["file_level"] = args.file_level + if logging_overrides: + config["logging"].update(logging_overrides) + if args.use_testnet is not None: + config["use_testnet"] = args.use_testnet + md_bridge = FakeMdBridge(logger) + md_pro.start(md_bridge) + while True: + time.sleep(1) \ No newline at end of file diff --git a/src/lib_spark/spark/domain/SpreadSecurity.h b/src/lib_spark/spark/domain/SpreadSecurity.h index c3493da64..6e433061f 100644 --- a/src/lib_spark/spark/domain/SpreadSecurity.h +++ b/src/lib_spark/spark/domain/SpreadSecurity.h @@ -1,5 +1,5 @@ #pragma once -#include "LegDefinition.h" +#include "spark/domain/LegDefinition.h" #include "spark/io/JsonReader.h" #include #include