Files
spark/tools/py/trade_latency_stats.py
2026-07-02 16:21:08 +00:00

107 lines
3.0 KiB
Python

#!/usr/bin/env python3
import argparse
import csv
import math
from collections import defaultdict
from pathlib import Path
PAIR_KEY_FIELDS = (
"sec",
"key",
"tradeTsUnixNs",
"price",
"qty",
"agg",
"totalVolume",
"fromSpread",
)
PERCENTILES = (50, 90, 95, 99)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Print simple stats for trade latency CSV output."
)
parser.add_argument("csv_file", help="CSV file created by trade_latency_to_csv.py")
return parser.parse_args()
def to_int(row: dict, key: str) -> int:
value = row.get(key, "")
return int(value) if value not in ("", None) else 0
def percentile(values: list[float], pct: int) -> float:
if not values:
return float("nan")
if len(values) == 1:
return values[0]
rank = (len(values) - 1) * (pct / 100.0)
lower = math.floor(rank)
upper = math.ceil(rank)
if lower == upper:
return values[lower]
weight = rank - lower
return values[lower] * (1 - weight) + values[upper] * weight
def print_stats(title: str, values: list[float], unit: str) -> None:
values = sorted(values)
if not values:
print(f"{title}: no data")
return
print(f"{title}:")
print(f" count={len(values)}")
print(f" avg={sum(values) / len(values):.3f} {unit}")
print(f" min={values[0]:.3f} {unit}")
print(f" max={values[-1]:.3f} {unit}")
for pct in PERCENTILES:
print(f" p{pct}={percentile(values, pct):.3f} {unit}")
def build_pair_key(row: dict) -> tuple[str, ...]:
return tuple(str(row.get(field, "")) for field in PAIR_KEY_FIELDS)
def main() -> int:
args = parse_args()
csv_path = Path(args.csv_file)
rows = []
with csv_path.open("r", encoding="utf-8", newline="") as infile:
reader = csv.DictReader(infile)
for row in reader:
rows.append(row)
by_stage = defaultdict(list)
grouped = defaultdict(lambda: defaultdict(list))
for row in rows:
exchange_to_log_us = float(row["exchangeToLogUs"])
by_stage[row["stage"]].append(exchange_to_log_us)
grouped[build_pair_key(row)][row["stage"]].append(row)
print_stats("mdp3_received exchange_to_log_us", by_stage["mdp3_received"], "us")
print()
print_stats("shm_written exchange_to_log_us", by_stage["shm_written"], "us")
pipeline_us = []
for stages in grouped.values():
received = sorted(stages.get("mdp3_received", []), key=lambda r: to_int(r, "logMonoNs"))
written = sorted(stages.get("shm_written", []), key=lambda r: to_int(r, "logMonoNs"))
for recv_row, write_row in zip(received, written):
delta_us = (to_int(write_row, "logMonoNs") - to_int(recv_row, "logMonoNs")) / 1000.0
pipeline_us.append(delta_us)
print()
print_stats("pipeline mdp3_received_to_shm_written", pipeline_us, "us")
return 0
if __name__ == "__main__":
raise SystemExit(main())