import argparse import datetime import logging from typing import Literal, Optional import databento import dotenv import os import sys import string import pandas as pd import sqlite3 from pathlib import Path from core.logging import configure_logger, StructuredLog as SLog from core.table_formatter import fmt_table logger = configure_logger(logging.getLogger("databento_client"), "md") Schemas = ["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"] Schema = Literal["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"] Venues = ["GLBX", "XNAS"] Venue = Literal["GLBX", "XNAS"] FileTypes = ["parquet", "feather", "csv"] FileType = Literal["parquet", "feather", "csv"] def _datetime_to_epoch_nanos(dt: datetime.datetime): epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) delta = dt - epoch ns = (delta.days * 86_400 * 1_000_000_000 + delta.seconds * 1_000_000_000 + delta.microseconds * 1_000) return ns def _epoch_nanos_to_datetime(ns: int): return datetime.datetime.fromtimestamp(ns / 1e9, tz=datetime.timezone.utc) def _to_datetime(dt: datetime.datetime | int): if isinstance(dt, int): return _epoch_nanos_to_datetime(dt) return dt def _get_data_dir(): if sys.platform == "win32": return Path(string.Template("${LOCALAPPDATA}").substitute( os.environ)).joinpath("bts", "databento") else: return Path(string.Template("${HOME}").substitute( os.environ)).joinpath(".local", "share", "bts", "databento") def _get_dataset(venue: str): venue = venue.upper() if venue == "GLBX" or venue == "CME": return "GLBX.MDP3" if venue == "NASDAQ" or venue == "NASQ" or venue == "XNAS": venue = "XNAS.ITCH" return f"{venue}" def _parse_date(date_str: str) -> datetime.datetime: formats = ["%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"] for f in formats: try: return datetime.datetime.strptime(date_str, f) except Exception: pass raise ValueError("Failed to parse date") def _metadata_column_defs(): columns: list[tuple[str, str]] = [ ("schema", "TEXT"), ("date", "TEXT"), ("venue", "TEXT"), ("symbol", "TEXT"), ("sec_type", "TEXT"), ("file_type", "TEXT"), ("location", "TEXT"), ] return columns def _get_out_path(schema, venue, symbol, typ, date: int | datetime.datetime, file_type: str): date_str = _to_datetime(date).strftime("%Y%m%d") name = "_".join( filter(lambda x: x is not None, [schema, venue, symbol, typ, date_str])) rv = Path( schema, venue, symbol, name, ) if file_type: if not file_type.startswith("."): file_type = "." + file_type rv = rv.with_suffix(file_type) return rv class DownloadArgs: def __init__(self, venue, schema, symbol, sec_type: str, api_key, date: int | datetime.datetime | str, dataset: str, file_type: str): self.venue = venue self.schema = schema self.symbol = symbol self.sec_type = sec_type if isinstance(date, str): self.date = _parse_date(date) elif isinstance(date, int): self.date = _epoch_nanos_to_datetime(date) else: self.date = date self.api_key = string.Template(api_key).substitute(os.environ) self.file_type = file_type if not dataset: dataset = _get_dataset(self.venue) self.dataset = dataset self.out_path = _get_out_path(self.schema, self.venue, self.symbol, self.sec_type, self.date, self.file_type) self.full_out_path = _get_data_dir().joinpath(self.out_path).absolute() def _store_metadata(req: DownloadArgs): data_dir = _get_data_dir() data_dir.mkdir(parents=True, exist_ok=True) db_file = str(data_dir.joinpath("requests.db")) table = "store" logger.info("[store_metadata_init.json]%s" % SLog(db_file=db_file)) with sqlite3.connect(db_file) as conn: conn = sqlite3.connect(db_file) cur = conn.cursor() columns = _metadata_column_defs() crt_expr = f"""CREATE TABLE IF NOT EXISTS {table} ( {", ".join(map(lambda n: n[0] + " " + n[1], columns))}, UNIQUE(schema, date, venue, symbol, file_type) ); """ logger.info("[store_metadata_exec.json]%s" % SLog(query=crt_expr)) cur.execute(crt_expr) date_str = req.date.strftime("%Y%m%d") fvals = [ f'{req.schema}', f'{date_str}', f'{req.venue}', f'{req.symbol}', f'{req.sec_type or ""}', f'{req.file_type}', f'{req.out_path}', ] ins_expr = f"INSERT INTO {table} VALUES ({','.join(['?' for _ in fvals])})" logger.info("[store_metadata_execmany.json]%s" % SLog(query=ins_expr, vals=fvals)) cur.executemany(ins_expr, [fvals]) conn.commit() return True def _download_data(args: DownloadArgs): if args.full_out_path.exists(): logger.error( "[download_data_err.json]%s" % SLog(err=f"Output file already exists '{args.full_out_path}'")) return def fmt_date_ymdhms(dt: datetime.datetime): return dt.strftime("%Y%m%d-%H%M%S%z") logger.info("[download_data.json]%s" % SLog( venue=args.venue, schema=args.schema, symbol=args.symbol, api_key=args.api_key, date=fmt_date_ymdhms(args.date), dataset=args.dataset, out=str(args.full_out_path), )) if not args.dataset: raise ValueError( f"Unable to determine dataset for given venue ({args.venue}) and schema ({args.schema})" ) # Save metadata if not _store_metadata(args): logger.error("[download_data_err.json]%s" % SLog(err="Failed to store metadata")) return # Fetch data cli = databento.Historical(key=args.api_key) req_data = { "dataset": args.dataset, "schema": args.schema, "stype_in": "raw_symbol", "start": fmt_date_ymdhms(args.date), "end": fmt_date_ymdhms(args.date + datetime.timedelta(days=1)), } def is_raw_glbx_fut(sym: str): return sym and sym[-1].isnumeric() if args.sec_type == "FUT": if args.venue == "GLBX" and is_raw_glbx_fut(args.symbol): req_data["symbols"] = [args.symbol] else: req_data["symbols"] = [f"{args.symbol}.{args.sec_type}"] req_data["stype_in"] = "parent" logger.info("[download_data_req.json]%s" % SLog(**req_data)) data = cli.timeseries.get_range(**req_data) # Save data logger.info("[download_data_save.json]%s" % SLog()) args.full_out_path.parent.mkdir(parents=True, exist_ok=True) if args.full_out_path.suffix == ".csv": logger.info("[download_data_to_csv.json]%s" % SLog(dest=args.full_out_path)) data.to_csv(args.full_out_path) elif args.full_out_path.suffix == ".parquet": data.to_parquet(args.full_out_path) elif args.full_out_path.suffix == ".feather": df = data.to_df() df.to_feather(args.full_out_path) elif args.full_out_path.suffixes[-2:] == [".dbn", ".zst"]: data.to_file(args.full_out_path) else: logger.error( "[download_data_err.json]%s" % SLog(dest=args.full_out_path, err="File type not supported")) def _get_or_convert_cached(args: DownloadArgs): path = args.full_out_path if path.exists(): logger.info( "[get_or_convert_cached.json]%s" % SLog(msg="Destination path already exists", dest=str(path))) return True if path.suffixes[-2:] == [".dbn", ".zst"]: logger.warning("[get_or_convert_cached_err.json]%s" % SLog(err="Conversion to .dbn.zst not supported")) return False # The with_suffix("").with_suffix("") chain ensures double suffixes, # like .dbn.zst, are fully removed. base_path = path.with_suffix("").with_suffix("") def try_read_as(suffix, read_df): p = base_path.with_suffix(suffix) if p.exists(): return read_df(p) return None df: Optional[pd.DataFrame] = try_read_as( ".dbn.zst", lambda p: databento.DBNStore.from_file(p).to_df()) if df is None: df = try_read_as(".parquet", pd.read_parquet) if df is None: df = try_read_as(".feather", pd.read_feather) if df is None: df = try_read_as(".csv", pd.read_csv) if df is None: return False # Save metadata if not _store_metadata(args): logger.error("[get_or_convert_cached_err.json]%s" % SLog(err="Failed to store metadata")) return False if df is None: return False if path.suffix == ".parquet": df.to_parquet(path) return True elif path.suffix == ".feather": df.to_feather(path) return True elif path.suffix == ".csv": df.to_csv(path) return True return False def _main(): parser = argparse.ArgumentParser() parser.add_argument("--venue") parser.add_argument( "--schema", choices=[ "trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m", "definition" ], ) parser.add_argument("--dataset") parser.add_argument("--date") parser.add_argument("--start-date") parser.add_argument("--end-date") parser.add_argument("--api-key") parser.add_argument("--symbol") parser.add_argument("--sec-type") parser.add_argument("--file-type") parser.add_argument("--list", action="store_true") log_levels = list(logging._nameToLevel) parser.add_argument("--console-level", choices=log_levels) parser.add_argument("--file-level", choices=log_levels) parser.add_argument("--dry-run", action="store_true") parser.set_defaults( venue=None, schema=None, dataset=None, date=None, start_date=None, end_date=None, api_key="${DATABENTO_API_KEY}", symbol=None, sec_type=None, list=False, file_type="dbn.zst", dry_run=False, console_level=None, file_level=None, ) args = parser.parse_args() logging_config = {} if args.console_level: logging_config["console_level"] = args.console_level if args.file_level: logging_config["file_level"] = args.file_level configure_logger(logger, "md", **logging_config) this_dir = Path(__file__).parent.absolute() if this_dir.joinpath(".env.local").exists(): dotenv.load_dotenv(this_dir.joinpath(".env.local")) if not args.date and not args.start_date: sys.stderr.write( "Please specify date (--date), or date range (--start-date, --end-date)" ) return 1 start_date = _parse_date(args.date or args.start_date) end_date = _parse_date( args.end_date) if args.end_date else (start_date + datetime.timedelta(days=1)) dates = map(lambda d: start_date + datetime.timedelta(days=d), range(0, int((end_date - start_date).days))) dl_argss = map( lambda d: DownloadArgs(args.venue, args.schema, args.symbol, args.sec_type, args. api_key, d, args.dataset, args.file_type), dates) if args.list: file_infos = [] for dl_args in dl_argss: if dl_args.full_out_path.exists(): file_stats = dl_args.full_out_path.stat() mebibytes = file_stats.st_size / 1048576 mebibytes = round(mebibytes, 2) mebibytes = f"{mebibytes:.2f}MiB" file_infos.append((dl_args.full_out_path, mebibytes)) file_table = fmt_table(columns=["Path", "Size"], rows=file_infos) print(file_table) else: for dl_args in dl_argss: if not _get_or_convert_cached(dl_args): _download_data(dl_args) return 0 if __name__ == "__main__": code = _main() exit(code)