281 lines
9.0 KiB
Python
281 lines
9.0 KiB
Python
import argparse
|
|
import datetime
|
|
import json
|
|
import logging
|
|
from typing import Any, Callable, Literal, Optional
|
|
import databento
|
|
import databento_dbn
|
|
import dotenv
|
|
import os
|
|
import sys
|
|
import string
|
|
import pandas as pd
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from core.table_formatter import fmt_table
|
|
from core.logging import configure_logger, JsonElement, StructuredLog as SLog
|
|
|
|
logger = configure_logger(logging.getLogger("databento_client"), "md")
|
|
|
|
Schemas = ["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
|
|
Schema = Literal["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
|
|
Venues = ["GLBX", "XNAS"]
|
|
Venue = Literal["GLBX", "XNAS"]
|
|
FileTypes = ["parquet", "feather", "csv"]
|
|
FileType = Literal["parquet", "feather", "csv"]
|
|
|
|
|
|
def _datetime_to_epoch_nanos(dt: datetime.datetime):
|
|
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
|
|
delta = dt - epoch
|
|
ns = (delta.days * 86_400 * 1_000_000_000 + delta.seconds * 1_000_000_000 +
|
|
delta.microseconds * 1_000)
|
|
return ns
|
|
|
|
|
|
def _epoch_nanos_to_datetime(ns: int):
|
|
return datetime.datetime.fromtimestamp(ns / 1e9, tz=datetime.timezone.utc)
|
|
|
|
|
|
def _to_datetime(dt: datetime.datetime | int):
|
|
if isinstance(dt, int):
|
|
return _epoch_nanos_to_datetime(dt)
|
|
return dt
|
|
|
|
|
|
def _fmt_date(dt: datetime.datetime):
|
|
return dt.strftime("%Y%m%d-%H%M%S%z")
|
|
|
|
|
|
def _get_data_dir():
|
|
if sys.platform == "win32":
|
|
return Path(string.Template("${LOCALAPPDATA}").substitute(
|
|
os.environ)).joinpath("bts", "databento")
|
|
else:
|
|
return Path(string.Template("${HOME}").substitute(
|
|
os.environ)).joinpath(".local", "share", "bts", "databento")
|
|
|
|
|
|
def _get_dataset(venue: str, data_type: str):
|
|
if venue == "GLBX" or venue == "CME":
|
|
return f"GLBX.MDP3"
|
|
return f"{venue}.ITCH"
|
|
# match venue.upper():
|
|
# case "NASDAQ":
|
|
# venue = "XNAS"
|
|
# case "CME":
|
|
# venue = "GLBX"
|
|
# case _:
|
|
# pass
|
|
# match venue.upper():
|
|
# case "GLBX":
|
|
# return f"{venue}.MDP3"
|
|
# case _:
|
|
# return f"{venue}.ITCH"
|
|
|
|
|
|
def _parse_date(date_str: str) -> datetime.datetime:
|
|
formats = ["%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"]
|
|
for f in formats:
|
|
try:
|
|
return datetime.datetime.strptime(date_str, f)
|
|
except Exception:
|
|
pass
|
|
raise ValueError("Failed to parse date")
|
|
|
|
|
|
def _metadata_column_defs():
|
|
columns: list[tuple[str, str]] = [
|
|
("schema", "TEXT"),
|
|
("date", "TEXT"),
|
|
("venue", "TEXT"),
|
|
("symbol", "TEXT"),
|
|
("sec_type", "TEXT"),
|
|
("file_type", "TEXT"),
|
|
("location", "TEXT"),
|
|
]
|
|
return columns
|
|
|
|
|
|
def _get_out_path(schema, venue, symbol, typ, date: int | datetime.datetime,
|
|
file_type: str):
|
|
date_str = _to_datetime(date).strftime("%Y%m%d")
|
|
rv = Path(
|
|
schema,
|
|
venue,
|
|
symbol,
|
|
f"{schema}_{venue}_{symbol}_{typ}_{date_str}",
|
|
)
|
|
if file_type:
|
|
if not file_type.startswith("."):
|
|
file_type = "." + file_type
|
|
rv = rv.with_suffix(file_type)
|
|
return rv
|
|
|
|
|
|
class DownloadArgs:
|
|
def __init__(self, venue, schema, symbol, sec_type: str, api_key,
|
|
date: int | datetime.datetime | str, dataset: str,
|
|
file_type: str):
|
|
self.venue = venue
|
|
self.schema = schema
|
|
self.symbol = symbol
|
|
self.sec_type = sec_type
|
|
if isinstance(date, str):
|
|
self.date = _parse_date(date)
|
|
elif isinstance(date, int):
|
|
self.date = _epoch_nanos_to_datetime(date)
|
|
else:
|
|
self.date = date
|
|
self.api_key = string.Template(api_key).substitute(os.environ)
|
|
self.file_type = file_type
|
|
if not dataset:
|
|
dataset = _get_dataset(self.venue, self.schema)
|
|
self.dataset = dataset
|
|
self.out_path = _get_out_path(self.schema, self.venue, self.symbol,
|
|
self.sec_type, self.date, self.file_type)
|
|
self.full_out_path = _get_data_dir().joinpath(self.out_path).absolute()
|
|
|
|
|
|
def _store_metadata(req: DownloadArgs):
|
|
data_dir = _get_data_dir()
|
|
data_dir.mkdir(parents=True, exist_ok=True)
|
|
db_file = str(data_dir.joinpath("requests.db"))
|
|
table = "store"
|
|
logger.info("[store_metadata_init.json]%s" % SLog(db_file=db_file))
|
|
with sqlite3.connect(db_file) as conn:
|
|
conn = sqlite3.connect(db_file)
|
|
cur = conn.cursor()
|
|
columns = _metadata_column_defs()
|
|
crt_expr = f"""CREATE TABLE IF NOT EXISTS {table} (
|
|
{", ".join(map(lambda n: n[0] + " " + n[1], columns))},
|
|
UNIQUE(schema, date, venue, symbol, file_type)
|
|
);
|
|
"""
|
|
logger.info("[store_metadata_exec.json]%s" % SLog(query=crt_expr))
|
|
cur.execute(crt_expr)
|
|
date_str = req.date.strftime("%Y%m%d")
|
|
fvals = [
|
|
f'"{req.schema}"',
|
|
f'"{date_str}"',
|
|
f'"{req.venue}"',
|
|
f'"{req.symbol}"',
|
|
f'"{req.sec_type}"',
|
|
f'"{req.file_type}"',
|
|
f'"{req.out_path}"',
|
|
]
|
|
ins_expr = f"INSERT INTO {table} VALUES ({','.join(['?' for _ in fvals])})"
|
|
logger.info("[store_metadata_execmany.json]%s" %
|
|
SLog(query=ins_expr, vals=fvals))
|
|
cur.executemany(ins_expr, [fvals])
|
|
conn.commit()
|
|
return True
|
|
|
|
|
|
def _download_data(args: DownloadArgs):
|
|
if args.full_out_path.exists():
|
|
logger.error(
|
|
"[download_data_err.json]%s" %
|
|
SLog(err=f"Output file already exists '{args.full_out_path}'"))
|
|
return
|
|
logger.info("[download_data.json]%s" % SLog(
|
|
venue=args.venue,
|
|
schema=args.schema,
|
|
symbol=args.symbol,
|
|
api_key=args.api_key,
|
|
date=_fmt_date(args.date),
|
|
dataset=args.dataset,
|
|
out=str(args.full_out_path),
|
|
))
|
|
if not args.dataset:
|
|
raise ValueError(
|
|
f"Unable to determine dataset for given venue ({args.venue}) and schema ({args.schema})"
|
|
)
|
|
# Fetch data
|
|
cli = databento.Historical(key=args.api_key)
|
|
req_data = {
|
|
"dataset": args.dataset,
|
|
"schema": args.schema,
|
|
"symbols": [f"{sym}.FUT" for sym in [args.symbol]],
|
|
"stype_in": "parent",
|
|
"start": _fmt_date(args.date),
|
|
"end": _fmt_date(args.date + datetime.timedelta(days=1)),
|
|
}
|
|
logger.info("[download_data_req.json]%s" % SLog(**req_data))
|
|
data = cli.timeseries.get_range(**req_data)
|
|
# Save metadata
|
|
if not _store_metadata(args):
|
|
logger.error("[download_data_err.json]%s" %
|
|
SLog(err="Failed to store metadata"))
|
|
return
|
|
# Save data
|
|
logger.info("[download_data_save.json]%s" % SLog())
|
|
args.full_out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if args.full_out_path.suffix == ".csv":
|
|
logger.info("[download_data_to_csv.json]%s" %
|
|
SLog(dest=args.full_out_path))
|
|
data.to_csv(args.full_out_path)
|
|
elif args.full_out_path.suffix == ".parquet":
|
|
data.to_parquet(args.full_out_path)
|
|
elif args.full_out_path.suffix == ".feather":
|
|
df = data.to_df()
|
|
df.to_feather(args.full_out_path)
|
|
elif args.full_out_path.suffixes[-2:] == [".dbn", ".zst"]:
|
|
data.to_file(args.full_out_path)
|
|
else:
|
|
logger.error(
|
|
"[download_data_err.json]%s" %
|
|
SLog(dest=args.full_out_path, err="File type not supported"))
|
|
|
|
|
|
def _main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--venue")
|
|
parser.add_argument(
|
|
"--schema",
|
|
choices=[
|
|
"trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m", "definition"
|
|
],
|
|
)
|
|
parser.add_argument("--dataset")
|
|
parser.add_argument("--date")
|
|
parser.add_argument("--api-key")
|
|
parser.add_argument("--symbol")
|
|
parser.add_argument("--sec-type")
|
|
parser.add_argument("--file-type")
|
|
log_levels = list(logging._nameToLevel)
|
|
parser.add_argument("--console-level", choices=log_levels)
|
|
parser.add_argument("--file-level", choices=log_levels)
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
parser.set_defaults(
|
|
venue=None,
|
|
schema=None,
|
|
dataset=None,
|
|
date=None,
|
|
api_key="${DATABENTO_API_KEY}",
|
|
symbol=None,
|
|
sec_type="FUT",
|
|
file_type="dbn.zst",
|
|
dry_run=False,
|
|
console_level=None,
|
|
file_level=None,
|
|
)
|
|
args = parser.parse_args()
|
|
logging_config = {}
|
|
if args.console_level:
|
|
logging_config["console_level"] = args.console_level
|
|
if args.file_level:
|
|
logging_config["file_level"] = args.file_level
|
|
configure_logger(logger, "md", **logging_config)
|
|
this_dir = Path(__file__).parent.absolute()
|
|
if this_dir.joinpath(".env.local").exists():
|
|
dotenv.load_dotenv(this_dir.joinpath(".env.local"))
|
|
date = _parse_date(args.date)
|
|
dl_args = DownloadArgs(args.venue, args.schema, args.symbol, args.sec_type,
|
|
args.api_key, date, args.dataset, args.file_type)
|
|
_download_data(dl_args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_main()
|