add: trade latency logs
This commit is contained in:
1 parent
2a8d407f08
commit
94bfdacf8b
6 files changed
+354
-11
No files matched your search
@@ -1,4 +1,5 @@
|
||||
#include "MDP3Listener.h"
|
||||
#include "MDP3Listener.h"
|
||||
#include "spark/md/log_messages/trade_latency_log.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "spark/md/protos/co2.pb.h"
|
||||
#include "spark/utils/datetime.h"
|
||||
@@ -179,19 +180,22 @@ void MDP3Listener::onBookUpdate(
|
||||
// Trades
|
||||
// Called from onTradeSummary and onTradeRecovery, ends up calling
|
||||
// MDProducerHandle::add_trade
|
||||
void on_trade(
|
||||
SecurityData *data,
|
||||
const onixs::Decimal &price,
|
||||
void on_trade(
|
||||
SecurityData *data,
|
||||
const onixs::Decimal &price,
|
||||
int size,
|
||||
onixs::AggressorSide::Enum agside,
|
||||
const onixs::Timestamp &onixs_date) {
|
||||
auto ts = utils::TimeStamp::now();
|
||||
PriceSize ps = pricesize_of_price_size(
|
||||
price, data->handle.security().display_factor, size);
|
||||
auto date = ll_of_timestamp(onixs_date);
|
||||
data->handle.add_trade(
|
||||
ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
|
||||
}
|
||||
auto ts = utils::TimeStamp::now();
|
||||
PriceSize ps = pricesize_of_price_size(
|
||||
price, data->handle.security().display_factor, size);
|
||||
auto date = ll_of_timestamp(onixs_date);
|
||||
log_trade_latency(
|
||||
TradeLatencyStage::MDP3_RECEIVED, data->handle.security(), date, ps,
|
||||
map_aggressor(agside), data->electronic_volume + size, false);
|
||||
data->handle.add_trade(
|
||||
ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
|
||||
}
|
||||
|
||||
//
|
||||
// RFQ
|
||||
|
||||
@@ -124,3 +124,8 @@ message VolSurfaceUsageUnit{
|
||||
message VolSurfaceUsage{
|
||||
repeated VolSurfaceUsageUnit items = 1;
|
||||
}
|
||||
|
||||
message LogTrades{
|
||||
string sec_name = 1;
|
||||
bool log = 2;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/md/log_messages/trade_latency_log.h"
|
||||
#ifndef NO_CPP0X
|
||||
#include "MDProducerHandle.h"
|
||||
#include "common.h"
|
||||
@@ -133,6 +134,10 @@ void MDProducerHandle::add_trade(
|
||||
trade.set_key(key());
|
||||
_channel.add_trade(trade);
|
||||
}
|
||||
|
||||
log_trade_latency(
|
||||
TradeLatencyStage::SHM_WRITTEN, security(), tradetime, price, agside,
|
||||
volume, from_spread);
|
||||
}
|
||||
void MDProducerHandle::set_last_trade(
|
||||
TimeStamp time,
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/toggles.pb.h"
|
||||
#include "spark/core/FMT_IMPL.h"
|
||||
#include "spark/core/time_stamp.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/md/PriceSize.h"
|
||||
#include "spark/md/Shm/FixedSecurity.h"
|
||||
#include "spark/md/protos/aggressor_side.h"
|
||||
#include "spark/utils/datetime.h"
|
||||
|
||||
#include <string>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
enum class TradeLatencyStage {
|
||||
MDP3_RECEIVED,
|
||||
SHM_WRITTEN,
|
||||
};
|
||||
|
||||
inline const char *TradeLatencyStage_Name(TradeLatencyStage stage) {
|
||||
switch (stage) {
|
||||
case TradeLatencyStage::MDP3_RECEIVED:
|
||||
return "mdp3_received";
|
||||
case TradeLatencyStage::SHM_WRITTEN:
|
||||
return "shm_written";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
inline std::string format_ll_datetime(datetime::LLDateTime ts) {
|
||||
return datetime::iso_date_time_string(datetime::ll_to_ptime(ts));
|
||||
}
|
||||
|
||||
inline bool should_log_trade_latency(const FixedSecurity &security) {
|
||||
const auto &cfg = config::get<config::toggles::LogTrades>();
|
||||
if (!cfg.log() || cfg.sec_name().empty()) {
|
||||
return false;
|
||||
}
|
||||
return cfg.sec_name() == security.security_desc() ||
|
||||
cfg.sec_name() == security.symbol();
|
||||
}
|
||||
|
||||
struct TradeLatencyLog {
|
||||
TradeLatencyStage stage;
|
||||
std::string sec_name;
|
||||
uint32_t security_key;
|
||||
int64_t trade_ts_unix_ns;
|
||||
std::string trade_ts_iso;
|
||||
int64_t log_ts_unix_ns;
|
||||
std::string log_ts_iso;
|
||||
uint64_t monotonic_log_ts_ns;
|
||||
int64_t exchange_to_log_us;
|
||||
double price;
|
||||
double quantity;
|
||||
std::string aggressor_side;
|
||||
uint32_t total_volume;
|
||||
bool from_spread;
|
||||
|
||||
TradeLatencyLog(
|
||||
TradeLatencyStage stage,
|
||||
const FixedSecurity &security,
|
||||
datetime::LLDateTime trade_ts,
|
||||
core::TimeStamp log_ts,
|
||||
utils::TimeStamp monotonic_log_ts,
|
||||
const PriceSize &price_size,
|
||||
co2::AggressorSide aggressor_side,
|
||||
uint32_t total_volume,
|
||||
bool from_spread)
|
||||
: stage(stage), sec_name(security.security_desc()),
|
||||
security_key(security.key), trade_ts_unix_ns(trade_ts),
|
||||
trade_ts_iso(format_ll_datetime(trade_ts)),
|
||||
log_ts_unix_ns(log_ts.nanos()), log_ts_iso(log_ts.to_iso_string()),
|
||||
monotonic_log_ts_ns(monotonic_log_ts.to_uint64()),
|
||||
exchange_to_log_us(
|
||||
(log_ts - core::TimeStamp::from_ptime(datetime::ll_to_ptime(trade_ts)))
|
||||
.total_micros()),
|
||||
price(price_size.price()), quantity(price_size.size()),
|
||||
aggressor_side(co2::AggressorSide_Name(aggressor_side)),
|
||||
total_volume(total_volume), from_spread(from_spread) {}
|
||||
};
|
||||
|
||||
inline void log_trade_latency(
|
||||
TradeLatencyStage stage,
|
||||
const FixedSecurity &security,
|
||||
datetime::LLDateTime trade_ts,
|
||||
const PriceSize &price_size,
|
||||
co2::AggressorSide aggressor_side,
|
||||
uint32_t total_volume,
|
||||
bool from_spread) {
|
||||
if (!should_log_trade_latency(security)) {
|
||||
return;
|
||||
}
|
||||
const auto log_ts = core::TimeStamp::now();
|
||||
const auto monotonic_log_ts = utils::TimeStamp::now();
|
||||
LOG(md, Info, "[trade_latency.json]{}",
|
||||
TradeLatencyLog(
|
||||
stage, security, trade_ts, log_ts, monotonic_log_ts, price_size,
|
||||
aggressor_side, total_volume, from_spread));
|
||||
}
|
||||
|
||||
} // namespace bts::md
|
||||
|
||||
FMT_IMPL(
|
||||
bts::md::TradeLatencyLog,
|
||||
l,
|
||||
R"({{ "stage": "{}", "sec": "{}", "key": {}, "tradeTsUnixNs": {}, "tradeTsIso": "{}", "logTsUnixNs": {}, "logTsIso": "{}", "logMonoNs": {}, "exchangeToLogUs": {}, "price": {}, "qty": {}, "agg": "{}", "totalVolume": {}, "fromSpread": {} }})",
|
||||
bts::md::TradeLatencyStage_Name(l.stage),
|
||||
l.sec_name,
|
||||
l.security_key,
|
||||
l.trade_ts_unix_ns,
|
||||
l.trade_ts_iso,
|
||||
l.log_ts_unix_ns,
|
||||
l.log_ts_iso,
|
||||
l.monotonic_log_ts_ns,
|
||||
l.exchange_to_log_us,
|
||||
l.price,
|
||||
l.quantity,
|
||||
l.aggressor_side,
|
||||
l.total_volume,
|
||||
l.from_spread);
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PAIR_KEY_FIELDS = (
|
||||
"sec",
|
||||
"key",
|
||||
"tradeTsUnixNs",
|
||||
"price",
|
||||
"qty",
|
||||
"agg",
|
||||
"totalVolume",
|
||||
"fromSpread",
|
||||
)
|
||||
|
||||
PERCENTILES = (50, 90, 95, 99)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Print simple stats for trade latency CSV output."
|
||||
)
|
||||
parser.add_argument("csv_file", help="CSV file created by trade_latency_to_csv.py")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def to_int(row: dict, key: str) -> int:
|
||||
value = row.get(key, "")
|
||||
return int(value) if value not in ("", None) else 0
|
||||
|
||||
|
||||
def percentile(values: list[float], pct: int) -> float:
|
||||
if not values:
|
||||
return float("nan")
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
rank = (len(values) - 1) * (pct / 100.0)
|
||||
lower = math.floor(rank)
|
||||
upper = math.ceil(rank)
|
||||
if lower == upper:
|
||||
return values[lower]
|
||||
weight = rank - lower
|
||||
return values[lower] * (1 - weight) + values[upper] * weight
|
||||
|
||||
|
||||
def print_stats(title: str, values: list[float], unit: str) -> None:
|
||||
values = sorted(values)
|
||||
if not values:
|
||||
print(f"{title}: no data")
|
||||
return
|
||||
|
||||
print(f"{title}:")
|
||||
print(f" count={len(values)}")
|
||||
print(f" avg={sum(values) / len(values):.3f} {unit}")
|
||||
print(f" min={values[0]:.3f} {unit}")
|
||||
print(f" max={values[-1]:.3f} {unit}")
|
||||
for pct in PERCENTILES:
|
||||
print(f" p{pct}={percentile(values, pct):.3f} {unit}")
|
||||
|
||||
|
||||
def build_pair_key(row: dict) -> tuple[str, ...]:
|
||||
return tuple(str(row.get(field, "")) for field in PAIR_KEY_FIELDS)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
csv_path = Path(args.csv_file)
|
||||
|
||||
rows = []
|
||||
with csv_path.open("r", encoding="utf-8", newline="") as infile:
|
||||
reader = csv.DictReader(infile)
|
||||
for row in reader:
|
||||
rows.append(row)
|
||||
|
||||
by_stage = defaultdict(list)
|
||||
grouped = defaultdict(lambda: defaultdict(list))
|
||||
|
||||
for row in rows:
|
||||
exchange_to_log_us = float(row["exchangeToLogUs"])
|
||||
by_stage[row["stage"]].append(exchange_to_log_us)
|
||||
grouped[build_pair_key(row)][row["stage"]].append(row)
|
||||
|
||||
print_stats("mdp3_received exchange_to_log_us", by_stage["mdp3_received"], "us")
|
||||
print()
|
||||
print_stats("shm_written exchange_to_log_us", by_stage["shm_written"], "us")
|
||||
|
||||
pipeline_us = []
|
||||
for stages in grouped.values():
|
||||
received = sorted(stages.get("mdp3_received", []), key=lambda r: to_int(r, "logMonoNs"))
|
||||
written = sorted(stages.get("shm_written", []), key=lambda r: to_int(r, "logMonoNs"))
|
||||
for recv_row, write_row in zip(received, written):
|
||||
delta_us = (to_int(write_row, "logMonoNs") - to_int(recv_row, "logMonoNs")) / 1000.0
|
||||
pipeline_us.append(delta_us)
|
||||
|
||||
print()
|
||||
print_stats("pipeline mdp3_received_to_shm_written", pipeline_us, "us")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CSV_FIELDS = [
|
||||
"line_ts",
|
||||
"line_mono_ns",
|
||||
"stage",
|
||||
"sec",
|
||||
"key",
|
||||
"tradeTsUnixNs",
|
||||
"tradeTsIso",
|
||||
"logTsUnixNs",
|
||||
"logTsIso",
|
||||
"logMonoNs",
|
||||
"exchangeToLogUs",
|
||||
"price",
|
||||
"qty",
|
||||
"agg",
|
||||
"totalVolume",
|
||||
"fromSpread",
|
||||
]
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract [trade_latency.json] log lines into CSV."
|
||||
)
|
||||
parser.add_argument("log_file", help="Path to the spark log file")
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="Output CSV path. Defaults to <log_file>.trade_latency.csv",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def extract_json_payload(line: str) -> dict | None:
|
||||
marker = "[trade_latency.json]"
|
||||
idx = line.find(marker)
|
||||
if idx < 0:
|
||||
return None
|
||||
payload = line[idx + len(marker) :].strip()
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def extract_line_columns(line: str) -> tuple[str, str]:
|
||||
parts = line.rstrip("\n").split("|")
|
||||
line_ts = parts[0].strip() if len(parts) > 0 else ""
|
||||
line_mono_ns = parts[1].strip() if len(parts) > 1 else ""
|
||||
return line_ts, line_mono_ns
|
||||
|
||||
|
||||
def build_row(line: str, payload: dict) -> dict:
|
||||
line_ts, line_mono_ns = extract_line_columns(line)
|
||||
row = {
|
||||
"line_ts": line_ts,
|
||||
"line_mono_ns": line_mono_ns,
|
||||
}
|
||||
for field in CSV_FIELDS[2:]:
|
||||
row[field] = payload.get(field, "")
|
||||
return row
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
log_path = Path(args.log_file)
|
||||
if args.output:
|
||||
out_path = Path(args.output)
|
||||
else:
|
||||
suffix = log_path.suffix if log_path.suffix else ".log"
|
||||
out_path = log_path.with_suffix(suffix + ".trade_latency.csv")
|
||||
|
||||
rows = []
|
||||
with log_path.open("r", encoding="utf-8", errors="replace") as infile:
|
||||
for line in infile:
|
||||
payload = extract_json_payload(line)
|
||||
if payload is None:
|
||||
continue
|
||||
rows.append(build_row(line, payload))
|
||||
|
||||
with out_path.open("w", newline="", encoding="utf-8") as outfile:
|
||||
writer = csv.DictWriter(outfile, fieldnames=CSV_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
print(f"wrote {len(rows)} rows to {out_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user