add: trade latency logs

This commit is contained in:
arham committed 2026-07-02 16:21:08 +00:00
1 parent 2a8d407f08
commit 94bfdacf8b
6 files changed
+354 -11

No files matched your search

@@ -1,4 +1,5 @@
#include "MDP3Listener.h"
#include "MDP3Listener.h"
#include "spark/md/log_messages/trade_latency_log.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/utils/datetime.h"
@@ -179,19 +180,22 @@ void MDP3Listener::onBookUpdate(
// Trades
// Called from onTradeSummary and onTradeRecovery, ends up calling
// MDProducerHandle::add_trade
void on_trade(
SecurityData *data,
const onixs::Decimal &price,
void on_trade(
SecurityData *data,
const onixs::Decimal &price,
int size,
onixs::AggressorSide::Enum agside,
const onixs::Timestamp &onixs_date) {
auto ts = utils::TimeStamp::now();
PriceSize ps = pricesize_of_price_size(
price, data->handle.security().display_factor, size);
auto date = ll_of_timestamp(onixs_date);
data->handle.add_trade(
ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
}
auto ts = utils::TimeStamp::now();
PriceSize ps = pricesize_of_price_size(
price, data->handle.security().display_factor, size);
auto date = ll_of_timestamp(onixs_date);
log_trade_latency(
TradeLatencyStage::MDP3_RECEIVED, data->handle.security(), date, ps,
map_aggressor(agside), data->electronic_volume + size, false);
data->handle.add_trade(
ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
}
//
// RFQ
@@ -124,3 +124,8 @@ message VolSurfaceUsageUnit{
message VolSurfaceUsage{
repeated VolSurfaceUsageUnit items = 1;
}
message LogTrades{
string sec_name = 1;
bool log = 2;
}
@@ -1,4 +1,5 @@
#include "spark/logging/logging.h"
#include "spark/md/log_messages/trade_latency_log.h"
#ifndef NO_CPP0X
#include "MDProducerHandle.h"
#include "common.h"
@@ -133,6 +134,10 @@ void MDProducerHandle::add_trade(
trade.set_key(key());
_channel.add_trade(trade);
}
log_trade_latency(
TradeLatencyStage::SHM_WRITTEN, security(), tradetime, price, agside,
volume, from_spread);
}
void MDProducerHandle::set_last_trade(
TimeStamp time,
@@ -0,0 +1,124 @@
#pragma once
#include "spark/config/config.h"
#include "spark/config/protos/toggles.pb.h"
#include "spark/core/FMT_IMPL.h"
#include "spark/core/time_stamp.h"
#include "spark/logging/logging.h"
#include "spark/md/PriceSize.h"
#include "spark/md/Shm/FixedSecurity.h"
#include "spark/md/protos/aggressor_side.h"
#include "spark/utils/datetime.h"
#include <string>
#include <cstdint>
#include <utility>
namespace bts::md {
enum class TradeLatencyStage {
MDP3_RECEIVED,
SHM_WRITTEN,
};
inline const char *TradeLatencyStage_Name(TradeLatencyStage stage) {
switch (stage) {
case TradeLatencyStage::MDP3_RECEIVED:
return "mdp3_received";
case TradeLatencyStage::SHM_WRITTEN:
return "shm_written";
}
return "unknown";
}
inline std::string format_ll_datetime(datetime::LLDateTime ts) {
return datetime::iso_date_time_string(datetime::ll_to_ptime(ts));
}
inline bool should_log_trade_latency(const FixedSecurity &security) {
const auto &cfg = config::get<config::toggles::LogTrades>();
if (!cfg.log() || cfg.sec_name().empty()) {
return false;
}
return cfg.sec_name() == security.security_desc() ||
cfg.sec_name() == security.symbol();
}
struct TradeLatencyLog {
TradeLatencyStage stage;
std::string sec_name;
uint32_t security_key;
int64_t trade_ts_unix_ns;
std::string trade_ts_iso;
int64_t log_ts_unix_ns;
std::string log_ts_iso;
uint64_t monotonic_log_ts_ns;
int64_t exchange_to_log_us;
double price;
double quantity;
std::string aggressor_side;
uint32_t total_volume;
bool from_spread;
TradeLatencyLog(
TradeLatencyStage stage,
const FixedSecurity &security,
datetime::LLDateTime trade_ts,
core::TimeStamp log_ts,
utils::TimeStamp monotonic_log_ts,
const PriceSize &price_size,
co2::AggressorSide aggressor_side,
uint32_t total_volume,
bool from_spread)
: stage(stage), sec_name(security.security_desc()),
security_key(security.key), trade_ts_unix_ns(trade_ts),
trade_ts_iso(format_ll_datetime(trade_ts)),
log_ts_unix_ns(log_ts.nanos()), log_ts_iso(log_ts.to_iso_string()),
monotonic_log_ts_ns(monotonic_log_ts.to_uint64()),
exchange_to_log_us(
(log_ts - core::TimeStamp::from_ptime(datetime::ll_to_ptime(trade_ts)))
.total_micros()),
price(price_size.price()), quantity(price_size.size()),
aggressor_side(co2::AggressorSide_Name(aggressor_side)),
total_volume(total_volume), from_spread(from_spread) {}
};
inline void log_trade_latency(
TradeLatencyStage stage,
const FixedSecurity &security,
datetime::LLDateTime trade_ts,
const PriceSize &price_size,
co2::AggressorSide aggressor_side,
uint32_t total_volume,
bool from_spread) {
if (!should_log_trade_latency(security)) {
return;
}
const auto log_ts = core::TimeStamp::now();
const auto monotonic_log_ts = utils::TimeStamp::now();
LOG(md, Info, "[trade_latency.json]{}",
TradeLatencyLog(
stage, security, trade_ts, log_ts, monotonic_log_ts, price_size,
aggressor_side, total_volume, from_spread));
}
} // namespace bts::md
FMT_IMPL(
bts::md::TradeLatencyLog,
l,
R"({{ "stage": "{}", "sec": "{}", "key": {}, "tradeTsUnixNs": {}, "tradeTsIso": "{}", "logTsUnixNs": {}, "logTsIso": "{}", "logMonoNs": {}, "exchangeToLogUs": {}, "price": {}, "qty": {}, "agg": "{}", "totalVolume": {}, "fromSpread": {} }})",
bts::md::TradeLatencyStage_Name(l.stage),
l.sec_name,
l.security_key,
l.trade_ts_unix_ns,
l.trade_ts_iso,
l.log_ts_unix_ns,
l.log_ts_iso,
l.monotonic_log_ts_ns,
l.exchange_to_log_us,
l.price,
l.quantity,
l.aggressor_side,
l.total_volume,
l.from_spread);
+106
View File
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
import argparse
import csv
import math
from collections import defaultdict
from pathlib import Path
PAIR_KEY_FIELDS = (
"sec",
"key",
"tradeTsUnixNs",
"price",
"qty",
"agg",
"totalVolume",
"fromSpread",
)
PERCENTILES = (50, 90, 95, 99)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Print simple stats for trade latency CSV output."
)
parser.add_argument("csv_file", help="CSV file created by trade_latency_to_csv.py")
return parser.parse_args()
def to_int(row: dict, key: str) -> int:
value = row.get(key, "")
return int(value) if value not in ("", None) else 0
def percentile(values: list[float], pct: int) -> float:
if not values:
return float("nan")
if len(values) == 1:
return values[0]
rank = (len(values) - 1) * (pct / 100.0)
lower = math.floor(rank)
upper = math.ceil(rank)
if lower == upper:
return values[lower]
weight = rank - lower
return values[lower] * (1 - weight) + values[upper] * weight
def print_stats(title: str, values: list[float], unit: str) -> None:
values = sorted(values)
if not values:
print(f"{title}: no data")
return
print(f"{title}:")
print(f" count={len(values)}")
print(f" avg={sum(values) / len(values):.3f} {unit}")
print(f" min={values[0]:.3f} {unit}")
print(f" max={values[-1]:.3f} {unit}")
for pct in PERCENTILES:
print(f" p{pct}={percentile(values, pct):.3f} {unit}")
def build_pair_key(row: dict) -> tuple[str, ...]:
return tuple(str(row.get(field, "")) for field in PAIR_KEY_FIELDS)
def main() -> int:
args = parse_args()
csv_path = Path(args.csv_file)
rows = []
with csv_path.open("r", encoding="utf-8", newline="") as infile:
reader = csv.DictReader(infile)
for row in reader:
rows.append(row)
by_stage = defaultdict(list)
grouped = defaultdict(lambda: defaultdict(list))
for row in rows:
exchange_to_log_us = float(row["exchangeToLogUs"])
by_stage[row["stage"]].append(exchange_to_log_us)
grouped[build_pair_key(row)][row["stage"]].append(row)
print_stats("mdp3_received exchange_to_log_us", by_stage["mdp3_received"], "us")
print()
print_stats("shm_written exchange_to_log_us", by_stage["shm_written"], "us")
pipeline_us = []
for stages in grouped.values():
received = sorted(stages.get("mdp3_received", []), key=lambda r: to_int(r, "logMonoNs"))
written = sorted(stages.get("shm_written", []), key=lambda r: to_int(r, "logMonoNs"))
for recv_row, write_row in zip(received, written):
delta_us = (to_int(write_row, "logMonoNs") - to_int(recv_row, "logMonoNs")) / 1000.0
pipeline_us.append(delta_us)
print()
print_stats("pipeline mdp3_received_to_shm_written", pipeline_us, "us")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
import argparse
import csv
import json
from pathlib import Path
CSV_FIELDS = [
"line_ts",
"line_mono_ns",
"stage",
"sec",
"key",
"tradeTsUnixNs",
"tradeTsIso",
"logTsUnixNs",
"logTsIso",
"logMonoNs",
"exchangeToLogUs",
"price",
"qty",
"agg",
"totalVolume",
"fromSpread",
]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Extract [trade_latency.json] log lines into CSV."
)
parser.add_argument("log_file", help="Path to the spark log file")
parser.add_argument(
"-o",
"--output",
help="Output CSV path. Defaults to <log_file>.trade_latency.csv",
)
return parser.parse_args()
def extract_json_payload(line: str) -> dict | None:
marker = "[trade_latency.json]"
idx = line.find(marker)
if idx < 0:
return None
payload = line[idx + len(marker) :].strip()
try:
return json.loads(payload)
except json.JSONDecodeError:
return None
def extract_line_columns(line: str) -> tuple[str, str]:
parts = line.rstrip("\n").split("|")
line_ts = parts[0].strip() if len(parts) > 0 else ""
line_mono_ns = parts[1].strip() if len(parts) > 1 else ""
return line_ts, line_mono_ns
def build_row(line: str, payload: dict) -> dict:
line_ts, line_mono_ns = extract_line_columns(line)
row = {
"line_ts": line_ts,
"line_mono_ns": line_mono_ns,
}
for field in CSV_FIELDS[2:]:
row[field] = payload.get(field, "")
return row
def main() -> int:
args = parse_args()
log_path = Path(args.log_file)
if args.output:
out_path = Path(args.output)
else:
suffix = log_path.suffix if log_path.suffix else ".log"
out_path = log_path.with_suffix(suffix + ".trade_latency.csv")
rows = []
with log_path.open("r", encoding="utf-8", errors="replace") as infile:
for line in infile:
payload = extract_json_payload(line)
if payload is None:
continue
rows.append(build_row(line, payload))
with out_path.open("w", newline="", encoding="utf-8") as outfile:
writer = csv.DictWriter(outfile, fieldnames=CSV_FIELDS)
writer.writeheader()
writer.writerows(rows)
print(f"wrote {len(rows)} rows to {out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())