Files
spark/tools/py/databento/databento_client.py
T
2025-11-15 12:46:23 -05:00

366 lines
12 KiB
Python

import argparse
import datetime
import logging
from typing import Literal, Optional
import databento
import dotenv
import os
import sys
import string
import pandas as pd
import sqlite3
from pathlib import Path
from core.logging import configure_logger, StructuredLog as SLog
from core.table_formatter import fmt_table
logger = configure_logger(logging.getLogger("databento_client"), "md")
Schemas = ["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
Schema = Literal["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
Venues = ["GLBX", "XNAS"]
Venue = Literal["GLBX", "XNAS"]
FileTypes = ["parquet", "feather", "csv"]
FileType = Literal["parquet", "feather", "csv"]
def _datetime_to_epoch_nanos(dt: datetime.datetime):
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
delta = dt - epoch
ns = (delta.days * 86_400 * 1_000_000_000 + delta.seconds * 1_000_000_000 +
delta.microseconds * 1_000)
return ns
def _epoch_nanos_to_datetime(ns: int):
return datetime.datetime.fromtimestamp(ns / 1e9, tz=datetime.timezone.utc)
def _to_datetime(dt: datetime.datetime | int):
if isinstance(dt, int):
return _epoch_nanos_to_datetime(dt)
return dt
def _get_data_dir():
if sys.platform == "win32":
return Path(string.Template("${LOCALAPPDATA}").substitute(
os.environ)).joinpath("bts", "databento")
else:
return Path(string.Template("${HOME}").substitute(
os.environ)).joinpath(".local", "share", "bts", "databento")
def _get_dataset(venue: str):
venue = venue.upper()
if venue == "GLBX" or venue == "CME":
return "GLBX.MDP3"
if venue == "NASDAQ" or venue == "NASQ" or venue == "XNAS":
venue = "XNAS.ITCH"
return f"{venue}"
def _parse_date(date_str: str) -> datetime.datetime:
formats = ["%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"]
for f in formats:
try:
return datetime.datetime.strptime(date_str, f)
except Exception:
pass
raise ValueError("Failed to parse date")
def _metadata_column_defs():
columns: list[tuple[str, str]] = [
("schema", "TEXT"),
("date", "TEXT"),
("venue", "TEXT"),
("symbol", "TEXT"),
("sec_type", "TEXT"),
("file_type", "TEXT"),
("location", "TEXT"),
]
return columns
def _get_out_path(schema, venue, symbol, typ, date: int | datetime.datetime,
file_type: str):
date_str = _to_datetime(date).strftime("%Y%m%d")
name = "_".join(
filter(lambda x: x is not None,
[schema, venue, symbol, typ, date_str]))
rv = Path(
schema,
venue,
symbol,
name,
)
if file_type:
if not file_type.startswith("."):
file_type = "." + file_type
rv = rv.with_suffix(file_type)
return rv
class DownloadArgs:
def __init__(self, venue, schema, symbol, sec_type: str, api_key,
date: int | datetime.datetime | str, dataset: str,
file_type: str):
self.venue = venue
self.schema = schema
self.symbol = symbol
self.sec_type = sec_type
if isinstance(date, str):
self.date = _parse_date(date)
elif isinstance(date, int):
self.date = _epoch_nanos_to_datetime(date)
else:
self.date = date
self.api_key = string.Template(api_key).substitute(os.environ)
self.file_type = file_type
if not dataset:
dataset = _get_dataset(self.venue)
self.dataset = dataset
self.out_path = _get_out_path(self.schema, self.venue, self.symbol,
self.sec_type, self.date, self.file_type)
self.full_out_path = _get_data_dir().joinpath(self.out_path).absolute()
def _store_metadata(req: DownloadArgs):
data_dir = _get_data_dir()
data_dir.mkdir(parents=True, exist_ok=True)
db_file = str(data_dir.joinpath("requests.db"))
table = "store"
logger.info("[store_metadata_init.json]%s" % SLog(db_file=db_file))
with sqlite3.connect(db_file) as conn:
conn = sqlite3.connect(db_file)
cur = conn.cursor()
columns = _metadata_column_defs()
crt_expr = f"""CREATE TABLE IF NOT EXISTS {table} (
{", ".join(map(lambda n: n[0] + " " + n[1], columns))},
UNIQUE(schema, date, venue, symbol, file_type)
);
"""
logger.info("[store_metadata_exec.json]%s" % SLog(query=crt_expr))
cur.execute(crt_expr)
date_str = req.date.strftime("%Y%m%d")
fvals = [
f'{req.schema}',
f'{date_str}',
f'{req.venue}',
f'{req.symbol}',
f'{req.sec_type or ""}',
f'{req.file_type}',
f'{req.out_path}',
]
ins_expr = f"INSERT INTO {table} VALUES ({','.join(['?' for _ in fvals])})"
logger.info("[store_metadata_execmany.json]%s" %
SLog(query=ins_expr, vals=fvals))
cur.executemany(ins_expr, [fvals])
conn.commit()
return True
def _download_data(args: DownloadArgs):
if args.full_out_path.exists():
logger.error(
"[download_data_err.json]%s" %
SLog(err=f"Output file already exists '{args.full_out_path}'"))
return
def fmt_date_ymdhms(dt: datetime.datetime):
return dt.strftime("%Y%m%d-%H%M%S%z")
logger.info("[download_data.json]%s" % SLog(
venue=args.venue,
schema=args.schema,
symbol=args.symbol,
api_key=args.api_key,
date=fmt_date_ymdhms(args.date),
dataset=args.dataset,
out=str(args.full_out_path),
))
if not args.dataset:
raise ValueError(
f"Unable to determine dataset for given venue ({args.venue}) and schema ({args.schema})"
)
# Save metadata
if not _store_metadata(args):
logger.error("[download_data_err.json]%s" %
SLog(err="Failed to store metadata"))
return
# Fetch data
cli = databento.Historical(key=args.api_key)
req_data = {
"dataset": args.dataset,
"schema": args.schema,
"stype_in": "raw_symbol",
"start": fmt_date_ymdhms(args.date),
"end": fmt_date_ymdhms(args.date + datetime.timedelta(days=1)),
}
def is_raw_glbx_fut(sym: str):
return sym and sym[-1].isnumeric()
if args.sec_type == "FUT":
if args.venue == "GLBX" and is_raw_glbx_fut(args.symbol):
req_data["symbols"] = [args.symbol]
else:
req_data["symbols"] = [f"{args.symbol}.{args.sec_type}"]
req_data["stype_in"] = "parent"
logger.info("[download_data_req.json]%s" % SLog(**req_data))
data = cli.timeseries.get_range(**req_data)
# Save data
logger.info("[download_data_save.json]%s" % SLog())
args.full_out_path.parent.mkdir(parents=True, exist_ok=True)
if args.full_out_path.suffix == ".csv":
logger.info("[download_data_to_csv.json]%s" %
SLog(dest=args.full_out_path))
data.to_csv(args.full_out_path)
elif args.full_out_path.suffix == ".parquet":
data.to_parquet(args.full_out_path)
elif args.full_out_path.suffix == ".feather":
df = data.to_df()
df.to_feather(args.full_out_path)
elif args.full_out_path.suffixes[-2:] == [".dbn", ".zst"]:
data.to_file(args.full_out_path)
else:
logger.error(
"[download_data_err.json]%s" %
SLog(dest=args.full_out_path, err="File type not supported"))
def _get_or_convert_cached(args: DownloadArgs):
path = args.full_out_path
if path.exists():
logger.info(
"[get_or_convert_cached.json]%s" %
SLog(msg="Destination path already exists", dest=str(path)))
return True
if path.suffixes[-2:] == [".dbn", ".zst"]:
logger.warning("[get_or_convert_cached_err.json]%s" %
SLog(err="Conversion to .dbn.zst not supported"))
return False
# The with_suffix("").with_suffix("") chain ensures double suffixes,
# like .dbn.zst, are fully removed.
base_path = path.with_suffix("").with_suffix("")
def try_read_as(suffix, read_df):
p = base_path.with_suffix(suffix)
if p.exists():
return read_df(p)
return None
df: Optional[pd.DataFrame] = try_read_as(
".dbn.zst", lambda p: databento.DBNStore.from_file(p).to_df())
if df is None:
df = try_read_as(".parquet", pd.read_parquet)
if df is None:
df = try_read_as(".feather", pd.read_feather)
if df is None:
df = try_read_as(".csv", pd.read_csv)
if df is None:
return False
# Save metadata
if not _store_metadata(args):
logger.error("[get_or_convert_cached_err.json]%s" %
SLog(err="Failed to store metadata"))
return False
if df is None:
return False
if path.suffix == ".parquet":
df.to_parquet(path)
return True
elif path.suffix == ".feather":
df.to_feather(path)
return True
elif path.suffix == ".csv":
df.to_csv(path)
return True
return False
def _main():
parser = argparse.ArgumentParser()
parser.add_argument("--venue")
parser.add_argument(
"--schema",
choices=[
"trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m", "definition"
],
)
parser.add_argument("--dataset")
parser.add_argument("--date")
parser.add_argument("--start-date")
parser.add_argument("--end-date")
parser.add_argument("--api-key")
parser.add_argument("--symbol")
parser.add_argument("--sec-type")
parser.add_argument("--file-type")
parser.add_argument("--list", action="store_true")
log_levels = list(logging._nameToLevel)
parser.add_argument("--console-level", choices=log_levels)
parser.add_argument("--file-level", choices=log_levels)
parser.add_argument("--dry-run", action="store_true")
parser.set_defaults(
venue=None,
schema=None,
dataset=None,
date=None,
start_date=None,
end_date=None,
api_key="${DATABENTO_API_KEY}",
symbol=None,
sec_type=None,
list=False,
file_type="dbn.zst",
dry_run=False,
console_level=None,
file_level=None,
)
args = parser.parse_args()
logging_config = {}
if args.console_level:
logging_config["console_level"] = args.console_level
if args.file_level:
logging_config["file_level"] = args.file_level
configure_logger(logger, "md", **logging_config)
this_dir = Path(__file__).parent.absolute()
if this_dir.joinpath(".env.local").exists():
dotenv.load_dotenv(this_dir.joinpath(".env.local"))
if not args.date and not args.start_date:
sys.stderr.write(
"Please specify date (--date), or date range (--start-date, --end-date)"
)
return 1
start_date = _parse_date(args.date or args.start_date)
end_date = _parse_date(
args.end_date) if args.end_date else (start_date +
datetime.timedelta(days=1))
dates = map(lambda d: start_date + datetime.timedelta(days=d),
range(0, int((end_date - start_date).days)))
dl_argss = map(
lambda d:
DownloadArgs(args.venue, args.schema, args.symbol, args.sec_type, args.
api_key, d, args.dataset, args.file_type), dates)
if args.list:
file_infos = []
for dl_args in dl_argss:
if dl_args.full_out_path.exists():
file_stats = dl_args.full_out_path.stat()
mebibytes = file_stats.st_size / 1048576
mebibytes = round(mebibytes, 2)
mebibytes = f"{mebibytes:.2f}MiB"
file_infos.append((dl_args.full_out_path, mebibytes))
file_table = fmt_table(columns=["Path", "Size"], rows=file_infos)
print(file_table)
else:
for dl_args in dl_argss:
if not _get_or_convert_cached(dl_args):
_download_data(dl_args)
return 0
if __name__ == "__main__":
code = _main()
exit(code)