100 lines
2.4 KiB
Python
100 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
CSV_FIELDS = [
|
|
"line_ts",
|
|
"line_mono_ns",
|
|
"stage",
|
|
"sec",
|
|
"key",
|
|
"tradeTsUnixNs",
|
|
"tradeTsIso",
|
|
"logTsUnixNs",
|
|
"logTsIso",
|
|
"logMonoNs",
|
|
"exchangeToLogUs",
|
|
"price",
|
|
"qty",
|
|
"agg",
|
|
"totalVolume",
|
|
"fromSpread",
|
|
]
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Extract [trade_latency.json] log lines into CSV."
|
|
)
|
|
parser.add_argument("log_file", help="Path to the spark log file")
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output",
|
|
help="Output CSV path. Defaults to <log_file>.trade_latency.csv",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def extract_json_payload(line: str) -> dict | None:
|
|
marker = "[trade_latency.json]"
|
|
idx = line.find(marker)
|
|
if idx < 0:
|
|
return None
|
|
payload = line[idx + len(marker) :].strip()
|
|
try:
|
|
return json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
|
|
|
|
def extract_line_columns(line: str) -> tuple[str, str]:
|
|
parts = line.rstrip("\n").split("|")
|
|
line_ts = parts[0].strip() if len(parts) > 0 else ""
|
|
line_mono_ns = parts[1].strip() if len(parts) > 1 else ""
|
|
return line_ts, line_mono_ns
|
|
|
|
|
|
def build_row(line: str, payload: dict) -> dict:
|
|
line_ts, line_mono_ns = extract_line_columns(line)
|
|
row = {
|
|
"line_ts": line_ts,
|
|
"line_mono_ns": line_mono_ns,
|
|
}
|
|
for field in CSV_FIELDS[2:]:
|
|
row[field] = payload.get(field, "")
|
|
return row
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
log_path = Path(args.log_file)
|
|
if args.output:
|
|
out_path = Path(args.output)
|
|
else:
|
|
suffix = log_path.suffix if log_path.suffix else ".log"
|
|
out_path = log_path.with_suffix(suffix + ".trade_latency.csv")
|
|
|
|
rows = []
|
|
with log_path.open("r", encoding="utf-8", errors="replace") as infile:
|
|
for line in infile:
|
|
payload = extract_json_payload(line)
|
|
if payload is None:
|
|
continue
|
|
rows.append(build_row(line, payload))
|
|
|
|
with out_path.open("w", newline="", encoding="utf-8") as outfile:
|
|
writer = csv.DictWriter(outfile, fieldnames=CSV_FIELDS)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
print(f"wrote {len(rows)} rows to {out_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|