Files
spark/tools/binance/get_binance_md.py
T

296 lines
10 KiB
Python

import argparse
import hmac
import hashlib
import os
import dotenv
from base64 import b64encode
import logging
import sys
import time
import websocket
import json
import requests
from threading import Thread
from typing import Callable, Optional, TextIO
def hmac_hashing(api_secret: str, payload: str):
m = hmac.new(api_secret.encode("utf-8"), payload.encode("utf-8"), hashlib.sha256)
return m.hexdigest()
def rest_request(base_url: str, api_key: str, api_secret: str, parameters: dict = {}):
parameters["timestamp"] = int(time.time() * 1000)
sorted_keys = list(parameters)
sorted_keys.sort()
query_params="&".join([f"{k}={parameters[k]}" for k in sorted_keys])
signature = hmac_hashing(api_secret=api_secret, payload=query_params)
parameters["signature"] = signature
query_params += "&signature=" + signature
url = base_url + "/api/v3/userDataStream" + "?" + query_params
res = requests.post(url, headers={"X-MBX-APIKEY": api_key})
return res.text
def get_listen_key(base_url: str, api_key: str):
url = base_url + "/api/v3/userDataStream"
res = requests.post(url, headers={"X-MBX-APIKEY": api_key})
return res.text
class CircularBufferWriter:
def __init__(self, buffer_line_count: int, sink: TextIO = sys.stdout):
self.buffer_lines = []
self.buffer_line_count = buffer_line_count
self.sink = sink
def print(self, msg):
if self.buffer_line_count <= 0:
self.sink.write(msg + '\n')
return
for _ in range(0, len(self.buffer_lines)):
self.sink.write("\x1b[1A\x1b[2K") # move up cursor and delete whole line
if len(self.buffer_lines) == self.buffer_line_count:
self.buffer_lines.pop()
self.buffer_lines.insert(0, msg)
lines = ''
num_lines = len(self.buffer_lines)
for i in range(0, num_lines):
lines += self.buffer_lines[num_lines - i - 1] + '\n'
self.sink.write(lines)
def book_ticker_stream():
return "bookTicker"
def depth_stream(symbol: str, num_levels: Optional[int] = 5, interval_ms: Optional[int] = 500):
rv = f"{symbol}@depth"
if num_levels is not None:
rv += str(num_levels)
if interval_ms is not None:
rv += f"{interval_ms}ms"
return rv
class OrderBookSubscription:
def __init__(
self,
msg_handler: Callable,
streams: list[str],
url: str = "wss://stream.binancefuture.com:443/ws",
logger: logging.Logger = logging.getLogger(),
):
self.msg_handler = msg_handler
self.streams = streams
self.url = url
self.logger = logger
def on_open(self, ws: websocket.WebSocketApp):
def run(*args):
try:
req = {
"method": "SUBSCRIBE",
"params": self.streams,
"id": 1,
}
req_str = json.dumps(req)
self.logger.info("Send: %s", req_str)
ws.send_text(req_str)
except Exception as ex:
self.logger.error("Error: %s", ex)
self.logger.info("Disconnecting websocket")
ws.close()
self.logger.info("Websocket connected")
th = Thread(target=run)
th.start()
th.join()
def on_message(self, ws, msg):
self.logger.debug("Recv: %s", msg)
self.msg_handler(msg)
def on_error(self, ws, err):
self.logger.error("Error: %s", err)
def on_close(self, ws, code, msg):
self.logger.info("Closed (%d): %s", code, msg)
def run(self):
ws_app = websocket.WebSocketApp(
url=self.url,
on_open=self.on_open,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close,
)
ws_app.run_forever(ping_interval=600, ping_timeout=180)
class UserDataSubscription:
def __init__(
self,
msg_handler: Callable,
url: str,
logger: logging.Logger = logging.getLogger(),
):
self.msg_handler = msg_handler
self.url = url
self.logger = logger
def on_open(self, ws: websocket.WebSocketApp):
self.logger.info("Websocket connected")
def on_message(self, ws, msg):
self.logger.debug("Recv: %s", msg)
self.msg_handler(msg)
def on_error(self, ws, err):
self.logger.error("Error: %s", err)
def on_close(self, ws, code, msg):
self.logger.info("Closed (%d): %s", code, msg)
def run(self):
ws_app = websocket.WebSocketApp(
url=self.url,
on_open=self.on_open,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close,
)
ws_app.run_forever(ping_interval=600, ping_timeout=180)
def _main():
# Urls
FUTURES_WS_URL_PROD = "wss://fstream.binance.com:443/ws"
FUTURES_WS_URL_DEV = "wss://stream.binancefuture.com:443/ws"
FUTURES_REST_URL_PROD = "https://fapi.binance.com"
FUTURES_REST_URL_DEV = "https://testnet.binancefuture.com"
SPOTS_WS_URL_DEV = "wss://stream.testnet.binance.vision:9443/ws"
SPOTS_WS_URL_PROD = "wss://stream.binance.com:9443/ws"
SPOTS_REST_URL_DEV = "https://testnet.binance.vision"
SPOTS_REST_URL_PROD = "https://api.binance.com"
# Cli Args
arg_parser = argparse.ArgumentParser(
description="Request market data at Binance"
)
arg_parser.add_argument(
"--streams",
help="Streams to subscribe (e.g, btcusdt@depth5@500ms)",
)
arg_parser.add_argument(
"--user-data",
action="store_true",
help="User data",
)
arg_parser.add_argument("--exch-info", action="store_true", help="Get exchangeInfo (REST)")
arg_parser.add_argument("--list-symbols", action="store_true", help="Get symbols (REST)")
arg_parser.add_argument("--list-streams", action="store_true", help="List all streams")
arg_parser.add_argument("--display-lines", help="Cap the number of lines to display from incoming messages (applies circular buffer to console output).")
arg_parser.add_argument("--url", help="The url to send requests to")
args = arg_parser.parse_args()
if args.list_streams:
stream_descs = [
"{symbol}@aggTrade",
"{symbol}@markPrice",
"{symbol}@markPrice@{interval} (intervals: 1s)",
"!markPrice@arr",
"!markPrice@arr@1s",
"{symbol}@kline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)",
"{pair}_{contractType}@continuousKline_{interval} (intervals: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M)"
"{symbol}@miniTicker",
"!ticker@arr",
"{symbol}@ticker",
"!miniTicker@arr",
"{symbol}@bookTicker",
"!bookTicker",
"{symbol}@forceOrder",
"!forceOrder@arr",
"{symbol}@depth{levels} (levels: 5, 10, 20)",
"{symbol}@depth{levels}@{interval} (levels: 5, 10, 20)",
" (intervals: 100ms, 500ms)",
"{symbol}@depth",
"{symbol}@depth@{interval} (intervals: 100ms, 500ms)",
"{symbol}@compositeIndex",
"!contractInfo",
"!assetIndex@arr",
"{assetSymbol}@assetIndex",
]
print("\n".join(stream_descs))
elif args.exch_info:
# Get exch info
url = args.url or (FUTURES_REST_URL_DEV + "/fapi/v1/exchangeInfo")
res = requests.get(url, headers={"Content-Type": "application/json"})
if res.status_code < 200 or res.status_code > 300:
sys.stderr.write(f"Request failed. code: {res.status_code}, reason: {res.reason}\n")
return 1
print(res.text)
elif args.list_symbols:
# Get symbols
url = args.url or (FUTURES_REST_URL_DEV + "/fapi/v1/exchangeInfo")
res = requests.get(url, headers={"Content-Type": "application/json"})
if res.status_code < 200 or res.status_code > 300:
sys.stderr.write(f"Request failed. code: {res.status_code}, reason: {res.reason}\n")
return 1
syms = {}
res_j = json.loads(res.text)
for sym in res_j["symbols"]:
syms[sym["symbol"]] = sym
sym_list = list(syms)
sym_list.sort()
for sym in sym_list:
print(sym)
elif args.user_data:
logger = logging.getLogger()
env_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "binance.env")
dotenv.load_dotenv(env_path)
api_key = os.environ.get("BINANCE_DEV_API_KEY")
secret_key = os.environ.get("BINANCE_DEV_SECRET_KEY")
listen_key_res = get_listen_key(SPOTS_REST_URL_DEV, api_key)
listen_key = json.loads(listen_key_res).get("listenKey")
if not listen_key:
logger.error("Failed to get listen key")
return 1
url = args.url or SPOTS_WS_URL_DEV
url += "/ws/" + listen_key
writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
def on_message(ws, msg):
logger.debug("on_message: %s", msg)
writer.print(msg)
def on_error(ws, err):
logger.error("on_error: %s", err)
def on_close(ws, code, msg):
logger.info("on_close (%d): %s", code, msg)
ws_app = websocket.WebSocketApp(
url=url,
on_message=on_message,
on_error=on_error,
on_close=on_close,
)
ws_app.run_forever(ping_interval=600, ping_timeout=180)
elif args.streams:
# Subscribe streams
url = args.url or FUTURES_WS_URL_DEV
writer = CircularBufferWriter(buffer_line_count=int(args.display_lines or -1))
sub = OrderBookSubscription(
msg_handler=writer.print,
url=url,
streams=args.streams if isinstance(args.streams, list) else [args.streams],
)
sub.run()
else:
sys.stderr.write("No data requested (run with --streams <stream1> <stream2> ... or --exch-info or --list-symbols)")
return 1
return 0
if __name__ == "__main__":
code = _main()
exit(code)