366 lines
12 KiB
Python
366 lines
12 KiB
Python
import argparse
|
|
import datetime
|
|
import logging
|
|
from typing import Literal, Optional
|
|
import databento
|
|
import dotenv
|
|
import os
|
|
import sys
|
|
import string
|
|
import pandas as pd
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from core.logging import configure_logger, StructuredLog as SLog
|
|
from core.table_formatter import fmt_table
|
|
|
|
logger = configure_logger(logging.getLogger("databento_client"), "md")
|
|
|
|
Schemas = ["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
|
|
Schema = Literal["trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m"]
|
|
Venues = ["GLBX", "XNAS"]
|
|
Venue = Literal["GLBX", "XNAS"]
|
|
FileTypes = ["parquet", "feather", "csv"]
|
|
FileType = Literal["parquet", "feather", "csv"]
|
|
|
|
|
|
def _datetime_to_epoch_nanos(dt: datetime.datetime):
|
|
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
|
|
delta = dt - epoch
|
|
ns = (delta.days * 86_400 * 1_000_000_000 + delta.seconds * 1_000_000_000 +
|
|
delta.microseconds * 1_000)
|
|
return ns
|
|
|
|
|
|
def _epoch_nanos_to_datetime(ns: int):
|
|
return datetime.datetime.fromtimestamp(ns / 1e9, tz=datetime.timezone.utc)
|
|
|
|
|
|
def _to_datetime(dt: datetime.datetime | int):
|
|
if isinstance(dt, int):
|
|
return _epoch_nanos_to_datetime(dt)
|
|
return dt
|
|
|
|
|
|
def _get_data_dir():
|
|
if sys.platform == "win32":
|
|
return Path(string.Template("${LOCALAPPDATA}").substitute(
|
|
os.environ)).joinpath("bts", "databento")
|
|
else:
|
|
return Path(string.Template("${HOME}").substitute(
|
|
os.environ)).joinpath(".local", "share", "bts", "databento")
|
|
|
|
|
|
def _get_dataset(venue: str):
|
|
venue = venue.upper()
|
|
if venue == "GLBX" or venue == "CME":
|
|
return "GLBX.MDP3"
|
|
if venue == "NASDAQ" or venue == "NASQ" or venue == "XNAS":
|
|
venue = "XNAS.ITCH"
|
|
return f"{venue}"
|
|
|
|
|
|
def _parse_date(date_str: str) -> datetime.datetime:
|
|
formats = ["%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"]
|
|
for f in formats:
|
|
try:
|
|
return datetime.datetime.strptime(date_str, f)
|
|
except Exception:
|
|
pass
|
|
raise ValueError("Failed to parse date")
|
|
|
|
|
|
def _metadata_column_defs():
|
|
columns: list[tuple[str, str]] = [
|
|
("schema", "TEXT"),
|
|
("date", "TEXT"),
|
|
("venue", "TEXT"),
|
|
("symbol", "TEXT"),
|
|
("sec_type", "TEXT"),
|
|
("file_type", "TEXT"),
|
|
("location", "TEXT"),
|
|
]
|
|
return columns
|
|
|
|
|
|
def _get_out_path(schema, venue, symbol, typ, date: int | datetime.datetime,
|
|
file_type: str):
|
|
date_str = _to_datetime(date).strftime("%Y%m%d")
|
|
name = "_".join(
|
|
filter(lambda x: x is not None,
|
|
[schema, venue, symbol, typ, date_str]))
|
|
rv = Path(
|
|
schema,
|
|
venue,
|
|
symbol,
|
|
name,
|
|
)
|
|
if file_type:
|
|
if not file_type.startswith("."):
|
|
file_type = "." + file_type
|
|
rv = rv.with_suffix(file_type)
|
|
return rv
|
|
|
|
|
|
class DownloadArgs:
|
|
def __init__(self, venue, schema, symbol, sec_type: str, api_key,
|
|
date: int | datetime.datetime | str, dataset: str,
|
|
file_type: str):
|
|
self.venue = venue
|
|
self.schema = schema
|
|
self.symbol = symbol
|
|
self.sec_type = sec_type
|
|
if isinstance(date, str):
|
|
self.date = _parse_date(date)
|
|
elif isinstance(date, int):
|
|
self.date = _epoch_nanos_to_datetime(date)
|
|
else:
|
|
self.date = date
|
|
self.api_key = string.Template(api_key).substitute(os.environ)
|
|
self.file_type = file_type
|
|
if not dataset:
|
|
dataset = _get_dataset(self.venue)
|
|
self.dataset = dataset
|
|
self.out_path = _get_out_path(self.schema, self.venue, self.symbol,
|
|
self.sec_type, self.date, self.file_type)
|
|
self.full_out_path = _get_data_dir().joinpath(self.out_path).absolute()
|
|
|
|
|
|
def _store_metadata(req: DownloadArgs):
|
|
data_dir = _get_data_dir()
|
|
data_dir.mkdir(parents=True, exist_ok=True)
|
|
db_file = str(data_dir.joinpath("requests.db"))
|
|
table = "store"
|
|
logger.info("[store_metadata_init.json]%s" % SLog(db_file=db_file))
|
|
with sqlite3.connect(db_file) as conn:
|
|
conn = sqlite3.connect(db_file)
|
|
cur = conn.cursor()
|
|
columns = _metadata_column_defs()
|
|
crt_expr = f"""CREATE TABLE IF NOT EXISTS {table} (
|
|
{", ".join(map(lambda n: n[0] + " " + n[1], columns))},
|
|
UNIQUE(schema, date, venue, symbol, file_type)
|
|
);
|
|
"""
|
|
logger.info("[store_metadata_exec.json]%s" % SLog(query=crt_expr))
|
|
cur.execute(crt_expr)
|
|
date_str = req.date.strftime("%Y%m%d")
|
|
fvals = [
|
|
f'{req.schema}',
|
|
f'{date_str}',
|
|
f'{req.venue}',
|
|
f'{req.symbol}',
|
|
f'{req.sec_type or ""}',
|
|
f'{req.file_type}',
|
|
f'{req.out_path}',
|
|
]
|
|
ins_expr = f"INSERT INTO {table} VALUES ({','.join(['?' for _ in fvals])})"
|
|
logger.info("[store_metadata_execmany.json]%s" %
|
|
SLog(query=ins_expr, vals=fvals))
|
|
cur.executemany(ins_expr, [fvals])
|
|
conn.commit()
|
|
return True
|
|
|
|
|
|
def _download_data(args: DownloadArgs):
|
|
if args.full_out_path.exists():
|
|
logger.error(
|
|
"[download_data_err.json]%s" %
|
|
SLog(err=f"Output file already exists '{args.full_out_path}'"))
|
|
return
|
|
|
|
def fmt_date_ymdhms(dt: datetime.datetime):
|
|
return dt.strftime("%Y%m%d-%H%M%S%z")
|
|
|
|
logger.info("[download_data.json]%s" % SLog(
|
|
venue=args.venue,
|
|
schema=args.schema,
|
|
symbol=args.symbol,
|
|
api_key=args.api_key,
|
|
date=fmt_date_ymdhms(args.date),
|
|
dataset=args.dataset,
|
|
out=str(args.full_out_path),
|
|
))
|
|
if not args.dataset:
|
|
raise ValueError(
|
|
f"Unable to determine dataset for given venue ({args.venue}) and schema ({args.schema})"
|
|
)
|
|
# Save metadata
|
|
if not _store_metadata(args):
|
|
logger.error("[download_data_err.json]%s" %
|
|
SLog(err="Failed to store metadata"))
|
|
return
|
|
# Fetch data
|
|
cli = databento.Historical(key=args.api_key)
|
|
req_data = {
|
|
"dataset": args.dataset,
|
|
"schema": args.schema,
|
|
"stype_in": "raw_symbol",
|
|
"start": fmt_date_ymdhms(args.date),
|
|
"end": fmt_date_ymdhms(args.date + datetime.timedelta(days=1)),
|
|
}
|
|
|
|
def is_raw_glbx_fut(sym: str):
|
|
return sym and sym[-1].isnumeric()
|
|
|
|
if args.sec_type == "FUT":
|
|
if args.venue == "GLBX" and is_raw_glbx_fut(args.symbol):
|
|
req_data["symbols"] = [args.symbol]
|
|
else:
|
|
req_data["symbols"] = [f"{args.symbol}.{args.sec_type}"]
|
|
req_data["stype_in"] = "parent"
|
|
logger.info("[download_data_req.json]%s" % SLog(**req_data))
|
|
data = cli.timeseries.get_range(**req_data)
|
|
# Save data
|
|
logger.info("[download_data_save.json]%s" % SLog())
|
|
args.full_out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if args.full_out_path.suffix == ".csv":
|
|
logger.info("[download_data_to_csv.json]%s" %
|
|
SLog(dest=args.full_out_path))
|
|
data.to_csv(args.full_out_path)
|
|
elif args.full_out_path.suffix == ".parquet":
|
|
data.to_parquet(args.full_out_path)
|
|
elif args.full_out_path.suffix == ".feather":
|
|
df = data.to_df()
|
|
df.to_feather(args.full_out_path)
|
|
elif args.full_out_path.suffixes[-2:] == [".dbn", ".zst"]:
|
|
data.to_file(args.full_out_path)
|
|
else:
|
|
logger.error(
|
|
"[download_data_err.json]%s" %
|
|
SLog(dest=args.full_out_path, err="File type not supported"))
|
|
|
|
|
|
def _get_or_convert_cached(args: DownloadArgs):
|
|
path = args.full_out_path
|
|
if path.exists():
|
|
logger.info(
|
|
"[get_or_convert_cached.json]%s" %
|
|
SLog(msg="Destination path already exists", dest=str(path)))
|
|
return True
|
|
if path.suffixes[-2:] == [".dbn", ".zst"]:
|
|
logger.warning("[get_or_convert_cached_err.json]%s" %
|
|
SLog(err="Conversion to .dbn.zst not supported"))
|
|
return False
|
|
# The with_suffix("").with_suffix("") chain ensures double suffixes,
|
|
# like .dbn.zst, are fully removed.
|
|
base_path = path.with_suffix("").with_suffix("")
|
|
|
|
def try_read_as(suffix, read_df):
|
|
p = base_path.with_suffix(suffix)
|
|
if p.exists():
|
|
return read_df(p)
|
|
return None
|
|
|
|
df: Optional[pd.DataFrame] = try_read_as(
|
|
".dbn.zst", lambda p: databento.DBNStore.from_file(p).to_df())
|
|
if df is None:
|
|
df = try_read_as(".parquet", pd.read_parquet)
|
|
if df is None:
|
|
df = try_read_as(".feather", pd.read_feather)
|
|
if df is None:
|
|
df = try_read_as(".csv", pd.read_csv)
|
|
if df is None:
|
|
return False
|
|
# Save metadata
|
|
if not _store_metadata(args):
|
|
logger.error("[get_or_convert_cached_err.json]%s" %
|
|
SLog(err="Failed to store metadata"))
|
|
return False
|
|
if df is None:
|
|
return False
|
|
if path.suffix == ".parquet":
|
|
df.to_parquet(path)
|
|
return True
|
|
elif path.suffix == ".feather":
|
|
df.to_feather(path)
|
|
return True
|
|
elif path.suffix == ".csv":
|
|
df.to_csv(path)
|
|
return True
|
|
return False
|
|
|
|
|
|
def _main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--venue")
|
|
parser.add_argument(
|
|
"--schema",
|
|
choices=[
|
|
"trades", "mbp-1", "mbp-10", "ohlcv-1s", "ohlcv-1m", "definition"
|
|
],
|
|
)
|
|
parser.add_argument("--dataset")
|
|
parser.add_argument("--date")
|
|
parser.add_argument("--start-date")
|
|
parser.add_argument("--end-date")
|
|
parser.add_argument("--api-key")
|
|
parser.add_argument("--symbol")
|
|
parser.add_argument("--sec-type")
|
|
parser.add_argument("--file-type")
|
|
parser.add_argument("--list", action="store_true")
|
|
log_levels = list(logging._nameToLevel)
|
|
parser.add_argument("--console-level", choices=log_levels)
|
|
parser.add_argument("--file-level", choices=log_levels)
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
parser.set_defaults(
|
|
venue=None,
|
|
schema=None,
|
|
dataset=None,
|
|
date=None,
|
|
start_date=None,
|
|
end_date=None,
|
|
api_key="${DATABENTO_API_KEY}",
|
|
symbol=None,
|
|
sec_type=None,
|
|
list=False,
|
|
file_type="dbn.zst",
|
|
dry_run=False,
|
|
console_level=None,
|
|
file_level=None,
|
|
)
|
|
args = parser.parse_args()
|
|
logging_config = {}
|
|
if args.console_level:
|
|
logging_config["console_level"] = args.console_level
|
|
if args.file_level:
|
|
logging_config["file_level"] = args.file_level
|
|
configure_logger(logger, "md", **logging_config)
|
|
this_dir = Path(__file__).parent.absolute()
|
|
if this_dir.joinpath(".env.local").exists():
|
|
dotenv.load_dotenv(this_dir.joinpath(".env.local"))
|
|
if not args.date and not args.start_date:
|
|
sys.stderr.write(
|
|
"Please specify date (--date), or date range (--start-date, --end-date)"
|
|
)
|
|
return 1
|
|
start_date = _parse_date(args.date or args.start_date)
|
|
end_date = _parse_date(
|
|
args.end_date) if args.end_date else (start_date +
|
|
datetime.timedelta(days=1))
|
|
dates = map(lambda d: start_date + datetime.timedelta(days=d),
|
|
range(0, int((end_date - start_date).days)))
|
|
dl_argss = map(
|
|
lambda d:
|
|
DownloadArgs(args.venue, args.schema, args.symbol, args.sec_type, args.
|
|
api_key, d, args.dataset, args.file_type), dates)
|
|
if args.list:
|
|
file_infos = []
|
|
for dl_args in dl_argss:
|
|
if dl_args.full_out_path.exists():
|
|
file_stats = dl_args.full_out_path.stat()
|
|
mebibytes = file_stats.st_size / 1048576
|
|
mebibytes = round(mebibytes, 2)
|
|
mebibytes = f"{mebibytes:.2f}MiB"
|
|
file_infos.append((dl_args.full_out_path, mebibytes))
|
|
file_table = fmt_table(columns=["Path", "Size"], rows=file_infos)
|
|
print(file_table)
|
|
else:
|
|
for dl_args in dl_argss:
|
|
if not _get_or_convert_cached(dl_args):
|
|
_download_data(dl_args)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
code = _main()
|
|
exit(code)
|