Files
spark/tools/binance/binance_client.py
T
John Colvin c8a89c1e8d cleanup: Rename get_binance_md.py -> binance_client.py
This is a more appropriate name (it's not just md anymore).
2025-05-20 20:08:34 +00:00

387 lines
14 KiB
Python

import argparse
import hmac
import hashlib
import os
import logging
import sys
import time
import websocket
import json
import requests
from string import Template
from threading import Thread
from typing import Any, Callable, Optional, TextIO
from binance_env_helper import (
required_env_var,
expand_vars,
expand_defaults,
load_env,
)
def hmac_hashing(api_secret: str, payload: str):
m = hmac.new(api_secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256)
return m.hexdigest()
def rest_request(base_url: str, api_key: str, api_secret: str, parameters: dict = {}):
parameters["timestamp"] = int(time.time() * 1000)
sorted_keys = list(parameters)
sorted_keys.sort()
query_params = "&".join([f"{k}={parameters[k]}" for k in sorted_keys])
signature = hmac_hashing(api_secret=api_secret, payload=query_params)
parameters["signature"] = signature
query_params += "&signature=" + signature
url = base_url + "/api/v3/userDataStream" + "?" + query_params
res = requests.post(url, headers={"X-MBX-APIKEY": api_key})
return res.text
class CircularBufferWriter:
def __init__(self, buffer_line_count: int, sink: TextIO = sys.stdout):
self.buffer_lines = []
self.buffer_line_count = buffer_line_count
self.sink = sink
def print(self, msg):
if self.buffer_line_count <= 0:
self.sink.write(msg + "\n")
return
for _ in range(0, len(self.buffer_lines)):
self.sink.write("\x1b[1A\x1b[2K") # move up cursor and delete whole line
if len(self.buffer_lines) == self.buffer_line_count:
self.buffer_lines.pop()
self.buffer_lines.insert(0, msg)
lines = ""
num_lines = len(self.buffer_lines)
for i in range(0, num_lines):
lines += self.buffer_lines[num_lines - i - 1] + "\n"
self.sink.write(lines)
def _get_arg_parser():
# Cli Args
arg_parser = argparse.ArgumentParser(description="Request market data at Binance")
arg_parser.add_argument(
"--streams",
help="Streams to subscribe (e.g, btcusdt@depth5@500ms)",
)
arg_parser.add_argument(
"--prod", action="store_const", const="PROD", dest="mode", help="Run at prod"
)
arg_parser.add_argument(
"--dev", action="store_const", const="DEV", dest="mode", help="Run at testnet"
)
arg_parser.add_argument(
"--spot",
action="store_const",
const="BINANCE",
dest="venue",
help="Run at spots",
)
arg_parser.add_argument("--op", help="The op to run")
arg_parser.add_argument(
"--param", help="Parameter(s) formatted as key=val)", nargs="+"
)
arg_parser.add_argument("--create-order", action="store_true", help="Create order")
arg_parser.add_argument(
"--usds-future",
action="store_const",
const="BINANCE_USDS_FUTURES",
dest="venue",
help="Run at usds futures",
)
arg_parser.add_argument("--user-data", action="store_true", help="User data")
arg_parser.add_argument(
"--exch-info",
action="store_const",
const="EXCHANGE_INFO",
dest="op",
help="Get exchangeInfo (REST)",
)
arg_parser.add_argument(
"--list-symbols", action="store_true", help="Get symbols (REST)"
)
arg_parser.add_argument(
"--listen-key",
action="store_const",
const="LISTEN_KEY",
dest="op",
help="Get listen key (needed for streams)",
)
arg_parser.add_argument(
"--list-streams", action="store_true", help="List all streams"
)
arg_parser.add_argument(
"--display-lines",
help="Cap the number of lines to display from incoming messages (applies circular buffer to console output).",
)
arg_parser.add_argument("--url", help="The url to send requests to")
arg_parser.set_defaults(mode="DEV", venue="BINANCE", user_data=False, op=None)
return arg_parser
def _parse_bool(s: Optional[str]):
match s:
case "1":
return True
case "true":
return True
case "on":
return True
case "y":
return True
case "yes":
return True
case _:
return False
class BinanceClient:
def __init__(self, venue="BINANCE", mode="DEV", logger=None):
self.venue = venue
self.mode = mode
self.rest_url = required_env_var(f"{venue}_{mode}_URL")
self.ws_url = required_env_var(f"{venue}_{mode}_WS_URL")
self.logger = logger or logging.getLogger(__name__)
self.api_key = os.environ.get(f"{self.venue}_{self.mode}_API_KEY")
self.secret_key = os.environ.get(f"{self.venue}_{self.mode}_SECRET_KEY")
def _get_base_url(self, op, is_ws: bool = False, vars: Optional[dict] = None):
"""Gets the base url for the request (includes host and path parts, but not query params)."""
url_var_suffix = "WS_URL" if is_ws else "URL"
url = os.environ.get(f"{self.venue}_{self.mode}_{url_var_suffix}") or ""
req_path = Template(os.environ.get(f"{self.venue}_{op}_PATH") or "").substitute(
os.environ
)
if vars:
req_path = Template(req_path).substitute(vars)
if req_path:
n = 1 if req_path.startswith("/") else 0
n += 1 if self.rest_url.endswith("/") else 0
match n:
case 0:
url += "/" + req_path
case 1:
url += req_path
case 2:
url += req_path[1:]
return url
def _get_params_str(self, op, **kwargs):
params = os.environ.get(f"{self.venue}_{op}_PARAMS") or ""
params = expand_vars(params, kwargs)
params = expand_defaults(params)
return params
def _get_params_dict(self, op, **kwargs):
params = self._get_params_str(op=op, **kwargs)
if not params:
return {}
return {p[0]: p[1] for p in map(lambda x: x.split("="), params.split("&"))}
def request(self, op, **kwargs):
is_signed = _parse_bool(os.environ.get(f"{self.venue}_{op}_IS_SIGNED"))
is_keyed = _parse_bool(os.environ.get(f"{self.venue}_{op}_IS_KEYED"))
method = os.environ.get(f"{self.venue}_{op}_METHOD")
params = self._get_params_dict(op, **kwargs)
base_url = self._get_base_url(op, is_ws=False)
res = None
headers = {"Content-Type": "application/json"}
if is_keyed or is_signed:
if not self.api_key:
raise ValueError("Api key not defined")
headers["X-MBX-APIKEY"] = self.api_key
if is_signed:
if not self.secret_key:
raise ValueError("Secret key not defined")
params["timestamp"] = str(int(time.time() * 1000))
base_url = self._get_base_url(op)
sorted_keys = list(params)
sorted_keys.sort()
params_str = "&".join([f"{k}={params[k]}" for k in sorted_keys])
signature = hmac_hashing(api_secret=self.secret_key, payload=params_str)
params_str += "&signature=" + signature
url = base_url + "?" + params_str
if not method:
method = "POST"
del headers["Content-Type"]
res = requests.request(method, url, headers=headers)
else:
params_str = "&".join([f"{k}={v}" for k, v in params.items()])
url = base_url
if params_str:
url += "?" + params_str
if not method:
method = "GET"
res = requests.request(method, url, headers=headers)
if not res.ok:
self.logger.error("Request failed. op=%s, res=\n%s", op, str(res))
return res
def subscribe_user_data(
self,
on_message: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
on_error: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
on_close: Optional[Callable[[websocket.WebSocket, Any, Any], None]] = None,
):
listen_key_res = self.request("LISTEN_KEY")
if not listen_key_res.ok:
raise ValueError("Failed to get listen key")
listen_key = listen_key_res.json().get("listenKey")
url = self._get_base_url(
"USER_DATA", is_ws=True, vars={"LISTEN_KEY": listen_key}
)
ws_app = websocket.WebSocketApp(
url=url,
on_message=on_message,
on_error=on_error,
on_close=on_close,
)
ws_app.run_forever(ping_interval=600, ping_timeout=180)
def subscribe_streams(
self,
streams,
on_message: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
on_error: Optional[Callable[[websocket.WebSocket, Any], None]] = None,
on_close: Optional[Callable[[websocket.WebSocket, Any, Any], None]] = None,
**kwargs,
):
def on_open(ws: websocket.WebSocket):
def run(*args):
try:
req = {
"method": "SUBSCRIBE",
"params": streams,
"id": 1,
}
req_str = json.dumps(req)
self.logger.info("Send: %s", req_str)
ws.send_text(req_str)
except Exception as ex:
self.logger.error("Error: %s", ex)
self.logger.info("Disconnecting websocket")
ws.close()
self.logger.info("Websocket connected")
th = Thread(target=run)
th.start()
th.join()
url = self._get_base_url("STREAMS", is_ws=True, **kwargs)
ws_app = websocket.WebSocketApp(
url=url,
on_open=on_open,
on_message=on_message,
on_error=on_error,
on_close=on_close,
)
self.logger.info("StreamSubscription @ %s", url)
ws_app.run_forever(ping_interval=600, ping_timeout=180)
def _main():
load_env()
arg_parser = _get_arg_parser()
args = arg_parser.parse_args()
if args.list_streams:
stream_descs = [
"{symbol}@aggTrade",
"{symbol}@markPrice",
"{symbol}@markPrice@{interval} (intervals: 1s)",
"!markPrice@arr",
"!markPrice@arr@1s",
"{symbol}@kline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)",
"{pair}_{contractType}@continuousKline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)"
"{symbol}@miniTicker",
"!ticker@arr",
"{symbol}@ticker",
"!miniTicker@arr",
"{symbol}@bookTicker",
"!bookTicker",
"{symbol}@forceOrder",
"!forceOrder@arr",
"{symbol}@depth{levels} (levels: 5, 10, 20)",
"{symbol}@depth{levels}@{interval} (levels: 5, 10, 20)",
" (intervals: 100ms, 500ms)",
"{symbol}@depth",
"{symbol}@depth@{interval} (intervals: 100ms, 500ms)",
"{symbol}@compositeIndex",
"!contractInfo",
"!assetIndex@arr",
"{assetSymbol}@assetIndex",
]
print("\n".join(stream_descs))
return
logger = logging.getLogger()
cli = BinanceClient(args.venue, args.mode, logger=logger)
params = {kv[0]: kv[1] for kv in map(lambda x: x.split("="), args.param or [])}
if args.op:
res = cli.request(args.op, **params)
if res.ok:
print(res.text)
elif args.list_symbols:
# Get symbols
res = cli.request("EXCHANGE_INFO")
if res.status_code < 200 or res.status_code > 300:
sys.stderr.write(
f"Request failed. code: {res.status_code}, reason: {res.reason}\n"
)
return 1
syms = {}
res_j = json.loads(res.text)
for sym in res_j["symbols"]:
syms[sym["symbol"]] = sym
sym_list = list(syms)
sym_list.sort()
for sym in sym_list:
print(sym)
elif args.user_data:
# Subscribe user data
writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
def on_message(ws, msg):
logger.debug("on_message: %s", msg)
writer.print(msg)
def on_error(ws, err):
logger.error("on_error: %s", err)
def on_close(ws, code, msg):
logger.info("on_close (%d): %s", code, msg)
cli.subscribe_user_data(on_message, on_error, on_close)
elif args.streams:
# Subscribe streams
writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
def on_message(ws, msg):
logger.debug("Recv: %s", msg)
writer.print(msg)
def on_error(ws, err):
logger.error("Error: %s", err)
def on_close(ws, code, msg):
logger.info("Closed (%d): %s", code, msg)
cli.subscribe_streams(
streams=args.streams if isinstance(args.streams, list) else [args.streams],
on_message=on_message,
on_error=on_error,
on_close=on_close,
)
else:
sys.stderr.write(
"No data requested (run with --streams <stream1> <stream2> ... or --exch-info or --list-symbols)"
)
return 1
return 0
if __name__ == "__main__":
code = _main()
exit(code)