Files
spark/tools/py/trade_latency_to_csv.py
2026-07-02 16:21:08 +00:00

100 lines
2.4 KiB
Python

#!/usr/bin/env python3
import argparse
import csv
import json
from pathlib import Path
CSV_FIELDS = [
"line_ts",
"line_mono_ns",
"stage",
"sec",
"key",
"tradeTsUnixNs",
"tradeTsIso",
"logTsUnixNs",
"logTsIso",
"logMonoNs",
"exchangeToLogUs",
"price",
"qty",
"agg",
"totalVolume",
"fromSpread",
]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Extract [trade_latency.json] log lines into CSV."
)
parser.add_argument("log_file", help="Path to the spark log file")
parser.add_argument(
"-o",
"--output",
help="Output CSV path. Defaults to <log_file>.trade_latency.csv",
)
return parser.parse_args()
def extract_json_payload(line: str) -> dict | None:
marker = "[trade_latency.json]"
idx = line.find(marker)
if idx < 0:
return None
payload = line[idx + len(marker) :].strip()
try:
return json.loads(payload)
except json.JSONDecodeError:
return None
def extract_line_columns(line: str) -> tuple[str, str]:
parts = line.rstrip("\n").split("|")
line_ts = parts[0].strip() if len(parts) > 0 else ""
line_mono_ns = parts[1].strip() if len(parts) > 1 else ""
return line_ts, line_mono_ns
def build_row(line: str, payload: dict) -> dict:
line_ts, line_mono_ns = extract_line_columns(line)
row = {
"line_ts": line_ts,
"line_mono_ns": line_mono_ns,
}
for field in CSV_FIELDS[2:]:
row[field] = payload.get(field, "")
return row
def main() -> int:
args = parse_args()
log_path = Path(args.log_file)
if args.output:
out_path = Path(args.output)
else:
suffix = log_path.suffix if log_path.suffix else ".log"
out_path = log_path.with_suffix(suffix + ".trade_latency.csv")
rows = []
with log_path.open("r", encoding="utf-8", errors="replace") as infile:
for line in infile:
payload = extract_json_payload(line)
if payload is None:
continue
rows.append(build_row(line, payload))
with out_path.open("w", newline="", encoding="utf-8") as outfile:
writer = csv.DictWriter(outfile, fieldnames=CSV_FIELDS)
writer.writeheader()
writer.writerows(rows)
print(f"wrote {len(rows)} rows to {out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())