import argparse import datetime import json import logging from typing import Any, Callable, Literal, Optional import databento import databento_dbn import dotenv import os import sys import string import pandas as pd import sqlite3 from pathlib import Path from core.table_formatter import fmt_table from core.logging import configure_logger, JsonElement, StructuredLog as SLog logger = configure_logger(logging.getLogger("databento_client"), "md") Schemas = ["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"] Schema = Literal["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"] Venues = ["GLBX", "XNAS"] Venue = Literal["GLBX", "XNAS"] FileTypes = ["parquet", "feather", "csv"] FileType = Literal["parquet", "feather", "csv"] def _datetime_to_epoch_nanos(dt: datetime.datetime): epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) delta = dt - epoch ns = (delta.days * 86_400 * 1_000_000_000 + delta.seconds * 1_000_000_000 + delta.microseconds * 1_000) return ns def _epoch_nanos_to_datetime(ns: int): return datetime.datetime.fromtimestamp(ns / 1e9, tz=datetime.timezone.utc) def _to_datetime(dt: datetime.datetime | int): if isinstance(dt, int): return _epoch_nanos_to_datetime(dt) return dt def _fmt_date(dt: datetime.datetime): return dt.strftime("%Y%m%d-%H%M%S%z") def _get_data_dir(): if sys.platform == "win32": return Path(string.Template("${LOCALAPPDATA}").substitute( os.environ)).joinpath("bts", "databento") else: return Path(string.Template("${HOME}").substitute( os.environ)).joinpath(".local", "share", "bts", "databento") def _get_dataset(venue: str, data_type: str): if venue == "GLBX" or venue == "CME": return f"GLBX.MDP3" return f"{venue}.ITCH" # match venue.upper(): # case "NASDAQ": # venue = "XNAS" # case "CME": # venue = "GLBX" # case _: # pass # match venue.upper(): # case "GLBX": # return f"{venue}.MDP3" # case _: # return f"{venue}.ITCH" def _parse_date(date_str: str) -> datetime.datetime: formats = ["%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"] for f in formats: try: return datetime.datetime.strptime(date_str, f) except Exception: pass raise ValueError("Failed to parse date") def _metadata_column_defs(): columns: list[tuple[str, str]] = [ ("schema", "TEXT"), ("date", "TEXT"), ("venue", "TEXT"), ("symbol", "TEXT"), ("sec_type", "TEXT"), ("file_type", "TEXT"), ("location", "TEXT"), ] return columns def _get_out_path(schema, venue, symbol, typ, date: int | datetime.datetime, file_type: str): date_str = _to_datetime(date).strftime("%Y%m%d") rv = Path( schema, venue, symbol, f"{schema}_{venue}_{symbol}_{typ}_{date_str}", ) if file_type: if not file_type.startswith("."): file_type = "." + file_type rv = rv.with_suffix(file_type) return rv class DownloadArgs: def __init__(self, venue, schema, symbol, sec_type: str, api_key, date: int | datetime.datetime | str, dataset: str, file_type: str): self.venue = venue self.schema = schema self.symbol = symbol self.sec_type = sec_type if isinstance(date, str): self.date = _parse_date(date) elif isinstance(date, int): self.date = _epoch_nanos_to_datetime(date) else: self.date = date self.api_key = string.Template(api_key).substitute(os.environ) self.file_type = file_type if not dataset: dataset = _get_dataset(self.venue, self.schema) self.dataset = dataset self.out_path = _get_out_path(self.schema, self.venue, self.symbol, self.sec_type, self.date, self.file_type) self.full_out_path = _get_data_dir().joinpath(self.out_path).absolute() def _store_metadata(req: DownloadArgs): data_dir = _get_data_dir() data_dir.mkdir(parents=True, exist_ok=True) db_file = str(data_dir.joinpath("requests.db")) table = "store" logger.info("[store_metadata_init.json]%s" % SLog(db_file=db_file)) with sqlite3.connect(db_file) as conn: conn = sqlite3.connect(db_file) cur = conn.cursor() columns = _metadata_column_defs() crt_expr = f"""CREATE TABLE IF NOT EXISTS {table} ( {", ".join(map(lambda n: n[0] + " " + n[1], columns))}, UNIQUE(schema, date, venue, symbol, file_type) ); """ logger.info("[store_metadata_exec.json]%s" % SLog(query=crt_expr)) cur.execute(crt_expr) date_str = req.date.strftime("%Y%m%d") fvals = [ f'"{req.schema}"', f'"{date_str}"', f'"{req.venue}"', f'"{req.symbol}"', f'"{req.sec_type}"', f'"{req.file_type}"', f'"{req.out_path}"', ] ins_expr = f"INSERT INTO {table} VALUES ({','.join(['?' for _ in fvals])})" logger.info("[store_metadata_execmany.json]%s" % SLog(query=ins_expr, vals=fvals)) cur.executemany(ins_expr, [fvals]) conn.commit() return True def _download_data(args: DownloadArgs): if args.full_out_path.exists(): logger.error( "[download_data_err.json]%s" % SLog(err=f"Output file already exists '{args.full_out_path}'")) return logger.info("[download_data.json]%s" % SLog( venue=args.venue, schema=args.schema, symbol=args.symbol, api_key=args.api_key, date=_fmt_date(args.date), dataset=args.dataset, out=str(args.full_out_path), )) if not args.dataset: raise ValueError( f"Unable to determine dataset for given venue ({args.venue}) and schema ({args.schema})" ) # Fetch data cli = databento.Historical(key=args.api_key) req_data = { "dataset": args.dataset, "schema": args.schema, "symbols": [f"{sym}.FUT" for sym in [args.symbol]], "stype_in": "parent", "start": _fmt_date(args.date), "end": _fmt_date(args.date + datetime.timedelta(days=1)), } logger.info("[download_data_req.json]%s" % SLog(**req_data)) data = cli.timeseries.get_range(**req_data) # Save metadata if not _store_metadata(args): logger.error("[download_data_err.json]%s" % SLog(err="Failed to store metadata")) return # Save data logger.info("[download_data_save.json]%s" % SLog()) args.full_out_path.parent.mkdir(parents=True, exist_ok=True) if args.full_out_path.suffix == ".csv": logger.info("[download_data_to_csv.json]%s" % SLog(dest=args.full_out_path)) data.to_csv(args.full_out_path) elif args.full_out_path.suffix == ".parquet": data.to_parquet(args.full_out_path) elif args.full_out_path.suffix == ".feather": df = data.to_df() df.to_feather(args.full_out_path) elif args.full_out_path.suffixes[-2:] == [".dbn", ".zst"]: data.to_file(args.full_out_path) else: logger.error( "[download_data_err.json]%s" % SLog(dest=args.full_out_path, err="File type not supported")) def _main(): parser = argparse.ArgumentParser() parser.add_argument("--venue") parser.add_argument( "--schema", choices=[ "trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m", "definition" ], ) parser.add_argument("--dataset") parser.add_argument("--date") parser.add_argument("--api-key") parser.add_argument("--symbol") parser.add_argument("--sec-type") parser.add_argument("--file-type") log_levels = list(logging._nameToLevel) parser.add_argument("--console-level", choices=log_levels) parser.add_argument("--file-level", choices=log_levels) parser.add_argument("--dry-run", action="store_true") parser.set_defaults( venue=None, schema=None, dataset=None, date=None, api_key="${DATABENTO_API_KEY}", symbol=None, sec_type="FUT", file_type="dbn.zst", dry_run=False, console_level=None, file_level=None, ) args = parser.parse_args() logging_config = {} if args.console_level: logging_config["console_level"] = args.console_level if args.file_level: logging_config["file_level"] = args.file_level configure_logger(logger, "md", **logging_config) this_dir = Path(__file__).parent.absolute() if this_dir.joinpath(".env.local").exists(): dotenv.load_dotenv(this_dir.joinpath(".env.local")) date = _parse_date(args.date) dl_args = DownloadArgs(args.venue, args.schema, args.symbol, args.sec_type, args.api_key, date, args.dataset, args.file_type) _download_data(dl_args) if __name__ == "__main__": _main()