Files
spark/tools/py/databento/databento_client.py
T

281 lines
9.0 KiB
Python

import argparse
import datetime
import json
import logging
from typing import Any, Callable, Literal, Optional
import databento
import databento_dbn
import dotenv
import os
import sys
import string
import pandas as pd
import sqlite3
from pathlib import Path
from core.table_formatter import fmt_table
from core.logging import configure_logger, JsonElement, StructuredLog as SLog
logger = configure_logger(logging.getLogger("databento_client"), "md")
Schemas = ["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
Schema = Literal["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
Venues = ["GLBX", "XNAS"]
Venue = Literal["GLBX", "XNAS"]
FileTypes = ["parquet", "feather", "csv"]
FileType = Literal["parquet", "feather", "csv"]
def _datetime_to_epoch_nanos(dt: datetime.datetime):
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
delta = dt - epoch
ns = (delta.days * 86_400 * 1_000_000_000 + delta.seconds * 1_000_000_000 +
delta.microseconds * 1_000)
return ns
def _epoch_nanos_to_datetime(ns: int):
return datetime.datetime.fromtimestamp(ns / 1e9, tz=datetime.timezone.utc)
def _to_datetime(dt: datetime.datetime | int):
if isinstance(dt, int):
return _epoch_nanos_to_datetime(dt)
return dt
def _fmt_date(dt: datetime.datetime):
return dt.strftime("%Y%m%d-%H%M%S%z")
def _get_data_dir():
if sys.platform == "win32":
return Path(string.Template("${LOCALAPPDATA}").substitute(
os.environ)).joinpath("bts", "databento")
else:
return Path(string.Template("${HOME}").substitute(
os.environ)).joinpath(".local", "share", "bts", "databento")
def _get_dataset(venue: str, data_type: str):
if venue == "GLBX" or venue == "CME":
return f"GLBX.MDP3"
return f"{venue}.ITCH"
# match venue.upper():
# case "NASDAQ":
# venue = "XNAS"
# case "CME":
# venue = "GLBX"
# case _:
# pass
# match venue.upper():
# case "GLBX":
# return f"{venue}.MDP3"
# case _:
# return f"{venue}.ITCH"
def _parse_date(date_str: str) -> datetime.datetime:
formats = ["%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"]
for f in formats:
try:
return datetime.datetime.strptime(date_str, f)
except Exception:
pass
raise ValueError("Failed to parse date")
def _metadata_column_defs():
columns: list[tuple[str, str]] = [
("schema", "TEXT"),
("date", "TEXT"),
("venue", "TEXT"),
("symbol", "TEXT"),
("sec_type", "TEXT"),
("file_type", "TEXT"),
("location", "TEXT"),
]
return columns
def _get_out_path(schema, venue, symbol, typ, date: int | datetime.datetime,
file_type: str):
date_str = _to_datetime(date).strftime("%Y%m%d")
rv = Path(
schema,
venue,
symbol,
f"{schema}_{venue}_{symbol}_{typ}_{date_str}",
)
if file_type:
if not file_type.startswith("."):
file_type = "." + file_type
rv = rv.with_suffix(file_type)
return rv
class DownloadArgs:
def __init__(self, venue, schema, symbol, sec_type: str, api_key,
date: int | datetime.datetime | str, dataset: str,
file_type: str):
self.venue = venue
self.schema = schema
self.symbol = symbol
self.sec_type = sec_type
if isinstance(date, str):
self.date = _parse_date(date)
elif isinstance(date, int):
self.date = _epoch_nanos_to_datetime(date)
else:
self.date = date
self.api_key = string.Template(api_key).substitute(os.environ)
self.file_type = file_type
if not dataset:
dataset = _get_dataset(self.venue, self.schema)
self.dataset = dataset
self.out_path = _get_out_path(self.schema, self.venue, self.symbol,
self.sec_type, self.date, self.file_type)
self.full_out_path = _get_data_dir().joinpath(self.out_path).absolute()
def _store_metadata(req: DownloadArgs):
data_dir = _get_data_dir()
data_dir.mkdir(parents=True, exist_ok=True)
db_file = str(data_dir.joinpath("requests.db"))
table = "store"
logger.info("[store_metadata_init.json]%s" % SLog(db_file=db_file))
with sqlite3.connect(db_file) as conn:
conn = sqlite3.connect(db_file)
cur = conn.cursor()
columns = _metadata_column_defs()
crt_expr = f"""CREATE TABLE IF NOT EXISTS {table} (
{", ".join(map(lambda n: n[0] + " " + n[1], columns))},
UNIQUE(schema, date, venue, symbol, file_type)
);
"""
logger.info("[store_metadata_exec.json]%s" % SLog(query=crt_expr))
cur.execute(crt_expr)
date_str = req.date.strftime("%Y%m%d")
fvals = [
f'"{req.schema}"',
f'"{date_str}"',
f'"{req.venue}"',
f'"{req.symbol}"',
f'"{req.sec_type}"',
f'"{req.file_type}"',
f'"{req.out_path}"',
]
ins_expr = f"INSERT INTO {table} VALUES ({','.join(['?' for _ in fvals])})"
logger.info("[store_metadata_execmany.json]%s" %
SLog(query=ins_expr, vals=fvals))
cur.executemany(ins_expr, [fvals])
conn.commit()
return True
def _download_data(args: DownloadArgs):
if args.full_out_path.exists():
logger.error(
"[download_data_err.json]%s" %
SLog(err=f"Output file already exists '{args.full_out_path}'"))
return
logger.info("[download_data.json]%s" % SLog(
venue=args.venue,
schema=args.schema,
symbol=args.symbol,
api_key=args.api_key,
date=_fmt_date(args.date),
dataset=args.dataset,
out=str(args.full_out_path),
))
if not args.dataset:
raise ValueError(
f"Unable to determine dataset for given venue ({args.venue}) and schema ({args.schema})"
)
# Fetch data
cli = databento.Historical(key=args.api_key)
req_data = {
"dataset": args.dataset,
"schema": args.schema,
"symbols": [f"{sym}.FUT" for sym in [args.symbol]],
"stype_in": "parent",
"start": _fmt_date(args.date),
"end": _fmt_date(args.date + datetime.timedelta(days=1)),
}
logger.info("[download_data_req.json]%s" % SLog(**req_data))
data = cli.timeseries.get_range(**req_data)
# Save metadata
if not _store_metadata(args):
logger.error("[download_data_err.json]%s" %
SLog(err="Failed to store metadata"))
return
# Save data
logger.info("[download_data_save.json]%s" % SLog())
args.full_out_path.parent.mkdir(parents=True, exist_ok=True)
if args.full_out_path.suffix == ".csv":
logger.info("[download_data_to_csv.json]%s" %
SLog(dest=args.full_out_path))
data.to_csv(args.full_out_path)
elif args.full_out_path.suffix == ".parquet":
data.to_parquet(args.full_out_path)
elif args.full_out_path.suffix == ".feather":
df = data.to_df()
df.to_feather(args.full_out_path)
elif args.full_out_path.suffixes[-2:] == [".dbn", ".zst"]:
data.to_file(args.full_out_path)
else:
logger.error(
"[download_data_err.json]%s" %
SLog(dest=args.full_out_path, err="File type not supported"))
def _main():
parser = argparse.ArgumentParser()
parser.add_argument("--venue")
parser.add_argument(
"--schema",
choices=[
"trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m", "definition"
],
)
parser.add_argument("--dataset")
parser.add_argument("--date")
parser.add_argument("--api-key")
parser.add_argument("--symbol")
parser.add_argument("--sec-type")
parser.add_argument("--file-type")
log_levels = list(logging._nameToLevel)
parser.add_argument("--console-level", choices=log_levels)
parser.add_argument("--file-level", choices=log_levels)
parser.add_argument("--dry-run", action="store_true")
parser.set_defaults(
venue=None,
schema=None,
dataset=None,
date=None,
api_key="${DATABENTO_API_KEY}",
symbol=None,
sec_type="FUT",
file_type="dbn.zst",
dry_run=False,
console_level=None,
file_level=None,
)
args = parser.parse_args()
logging_config = {}
if args.console_level:
logging_config["console_level"] = args.console_level
if args.file_level:
logging_config["file_level"] = args.file_level
configure_logger(logger, "md", **logging_config)
this_dir = Path(__file__).parent.absolute()
if this_dir.joinpath(".env.local").exists():
dotenv.load_dotenv(this_dir.joinpath(".env.local"))
date = _parse_date(args.date)
dl_args = DownloadArgs(args.venue, args.schema, args.symbol, args.sec_type,
args.api_key, date, args.dataset, args.file_type)
_download_data(dl_args)
if __name__ == "__main__":
_main()