Files
spark/src/server/sparkd/cmds/ParseHistoricalFIX.cpp
T

417 lines
15 KiB
C++

#include "ParseHistoricalFIX.h"
#include "lib/jetfix/codec/FIXMsg.h"
#include "lib/jetfix/codec/FIXParser.h"
#include "lib/jetfix/codec/Fields.h"
#include "lib/jetfix/codec/Group.h"
#include "lib/jetfix/codec/MessageTypes.h"
#include "lib/utils/bts_chrono.h"
#include <fstream>
#include <unordered_map>
namespace bts {
namespace cmd {
using namespace jetfix;
typedef StringField<75> TradeDate;
typedef IntField<34> SeqNum;
typedef StringField<52> SendingTime;
typedef StringField<60> TransactTime;
typedef StringField<5799> MatchEventIndicator;
typedef IntField<268> NoMDEntries;
typedef StringField<279> MDUpateAction;
typedef StringField<269> MDEntryType;
typedef StringField<48> SecurityID;
typedef StringField<55> Symbol;
typedef StringField<83> ReptSeq;
typedef FloatField<270> MDEntryPx;
typedef IntField<271> MDEntrySize;
typedef IntField<346> NumberOfOrders;
typedef IntField<1023> MDPriceLevel;
typedef StringField<286> OpenCloseSettlFlag;
typedef StringField<731> SettlPriceType;
typedef StringField<5797> AggressorSide;
typedef StringField<37711> MDTradeEntryID;
typedef StringField<5796> TradingReferenceDate;
typedef StringField<1149> HighLimitPrice;
typedef StringField<1148> LowLimitPrice;
typedef StringField<1143> MaxPriceVariation;
typedef StringField<1180> ApplID;
typedef IntField<9633> ReferenceID;
typedef IntField<32> LastQty;
typedef IntField<37705> NoOrderIDEntries;
typedef StringField<37> OrderID;
typedef IntField<37707> MDOrderPriority;
typedef IntField<37706> MDDisplayQty;
typedef StringField<37708> OrderUpdateAction;
typedef Group<NoOrderIDEntries, OrderID, MDDisplayQty, MDOrderPriority,
OrderUpdateAction, ReferenceID, LastQty>
OrderIDEntries;
typedef Group<NoMDEntries, MDUpateAction, MDEntryType, SecurityID, Symbol,
ReptSeq, MDEntryPx, MDEntrySize, NumberOfOrders, MDPriceLevel,
OpenCloseSettlFlag, SettlPriceType, AggressorSide, MDTradeEntryID,
TradingReferenceDate, HighLimitPrice, LowLimitPrice,
MaxPriceVariation, ApplID, ReferenceID, LastQty, OrderID,
MDOrderPriority, MDDisplayQty>
MDEntries;
typedef FIXMsg<MessageTypes::MDIncrementalUpdate, TradeDate, SeqNum,
SendingTime, TransactTime, MatchEventIndicator, MDEntries,
OrderIDEntries>
MDIncrementalUpdate;
std::string to_str(std::string const &s) { return s; }
template <class V> std::string to_str(V const &v) { return std::to_string(v); }
template <class F, class G> std::string get_if_valid(G const &update) {
if (update.template has<F>())
return to_str(update.template get<F>());
else
return "";
}
void print_msg_headers(std::ostream &out) {
out << "MsgID,"
<< "TradeDate,"
<< "SeqNum,"
<< "SendingTime,"
<< "TransactTime,"
<< "MatchEventIndicator,"
<< "NoMDEntries,"
<< "NoOrderIDEntries\n";
}
void print_msgs(size_t msg_idx, MDIncrementalUpdate const &line,
std::ostream &out) {
line.get<SeqNum>();
out << msg_idx << "," << get_if_valid<TradeDate>(line) << ","
<< get_if_valid<SeqNum>(line) << "," << get_if_valid<SendingTime>(line)
<< "," << get_if_valid<TransactTime>(line) << ","
<< get_if_valid<MatchEventIndicator>(line) << ","
<< (line.has<MDEntries>() ? line.get<MDEntries>().size() : 0) << ","
<< (line.has<OrderIDEntries>() ? line.get<OrderIDEntries>().size() : 0)
<< "\n";
}
struct PriceSize {
double price = 0;
int32_t size = 0;
};
struct Book {
Book() {
book[0].resize(10);
book[1].resize(10);
}
std::string symbol;
std::string transact_time;
std::string send_time;
uint32_t seqnum;
std::array<std::vector<PriceSize>, 2> book;
};
size_t n_printed_books = 2;
void print_book_side(std::vector<PriceSize> const &side,
std::ostream &book_stream) {
for (size_t i = 0; i < n_printed_books; i++)
if (i < side.size())
book_stream << "," << side[i].price << "," << side[i].size;
else
book_stream << "," << 0 << "," << 0;
}
void print_book_header(std::ostream &book_stream) {
book_stream << "Symbol,SendingTime,TransactTime, SeqnNum";
for (size_t i = 0; i < n_printed_books; i++)
book_stream << ",bid_p_" << i << ",bid_sz_" << i;
for (size_t i = 0; i < n_printed_books; i++)
book_stream << ",ask_p_" << i << ",ask_sz_" << i;
book_stream << ",\n";
}
void print_book(Book &b, std::ostream &book_stream) {
book_stream << b.symbol << "," << b.send_time << "," << b.transact_time
<< "," << b.seqnum;
print_book_side(b.book[0], book_stream);
print_book_side(b.book[1], book_stream);
book_stream << "\n";
}
void update_and_maybe_print_book(MDIncrementalUpdate const &update,
std::unordered_map<std::string, Book> &books,
std::ostream &book_stream) {
std::set<Book *> to_update;
if (update.has<MDEntries>()) {
auto &md_entries = update.get<MDEntries>();
for (size_t i = 0; i < md_entries.size(); i++) {
auto &entry = md_entries[i];
if (entry.has<MDPriceLevel>()) {
if (entry.get<MDEntryType>() == "0" ||
entry.get<MDEntryType>() == "1") {
auto side = entry.get<MDEntryType>()[0] - '0';
auto &sym = entry.get<Symbol>();
auto &book = books[sym];
to_update.insert(&book);
auto &book_side = book.book[side];
book.symbol = sym;
book.transact_time = update.get<TransactTime>();
book.send_time = update.get<SendingTime>();
book.transact_time = update.get<TransactTime>();
book.seqnum = update.get<SeqNum>();
auto level = entry.get<MDPriceLevel>() - 1;
if (entry.get<MDUpateAction>() == "0") // new
book_side.insert(
book_side.begin() + level,
{entry.get<MDEntryPx>(), entry.get<MDEntrySize>()});
else if (entry.get<MDUpateAction>() == "1")
book_side[level] = {entry.get<MDEntryPx>(),
entry.get<MDEntrySize>()};
else
book_side.erase(book_side.begin() + level);
}
}
}
auto indicator = update.get<MatchEventIndicator>();
if (indicator[5] == '1')
for (auto b : to_update)
print_book(*b, book_stream);
}
}
void print_trades_header(std::ostream &trade_stream) {
trade_stream << "Symbol,SendingTime,TransactTime,MDEntryPx,MDEntrySize,"
"NumberOfOrders,AggressorSide,MDTradeEntryID,AgressorOID,"
"RestingOID,PartialQty";
trade_stream << "\n";
}
struct TradeSummaryItem {
std::string symbol;
std::string sending_time;
std::string transact_time;
double md_entry_px;
uint32_t md_entry_size;
uint32_t n_orders;
std::string agressor_side;
std::string md_trade_entry_id;
};
struct TradeOrder {
std::string order_id;
uint32_t last_qty;
};
struct TradeSummaryEvent {
std::vector<TradeSummaryItem> md_entries;
std::vector<TradeOrder> orderid_entries;
};
void print_trades(TradeSummaryEvent &event, std::ostream &trade_stream) {
size_t current_order = 0;
for (auto &trade : event.md_entries) {
for (auto i = current_order; i < current_order + trade.n_orders; i++) {
trade_stream << trade.symbol << "," << trade.sending_time << ","
<< trade.transact_time << "," << trade.md_entry_px << ","
<< trade.md_entry_size << "," << trade.n_orders << ","
<< trade.agressor_side << "," << trade.md_trade_entry_id
<< "," << event.orderid_entries[current_order].order_id
<< "," << event.orderid_entries[i].order_id << ","
<< event.orderid_entries[i].last_qty << "\n";
}
current_order += trade.n_orders;
}
}
void update_and_maybe_print_trades(MDIncrementalUpdate const &update,
TradeSummaryEvent &event,
std::ostream &trade_stream) {
if (update.has<MDEntries>()) {
auto &md_entries = update.get<MDEntries>();
bool is_trade_summary = false;
for (size_t i = 0; i < md_entries.size(); i++) {
auto &entry = md_entries[i];
if (entry.get<MDEntryType>() == "2") {
is_trade_summary = true;
TradeSummaryItem item;
item.symbol = entry.get<Symbol>();
item.sending_time = update.get<SendingTime>();
item.transact_time = update.get<TransactTime>();
item.md_entry_px = entry.get<MDEntryPx>();
item.md_entry_size = entry.get<MDEntrySize>();
item.n_orders = entry.get<NumberOfOrders>();
item.agressor_side = entry.get<AggressorSide>();
item.md_trade_entry_id = entry.get<MDTradeEntryID>();
event.md_entries.emplace_back(std::move(item));
}
if (is_trade_summary) {
if (update.has<OrderIDEntries>()) {
auto &oid_entries = update.get<OrderIDEntries>();
for (size_t i = 0; i < oid_entries.size(); i++) {
auto &oid = oid_entries[i];
TradeOrder order_id_entry;
order_id_entry.order_id = oid.get<OrderID>();
order_id_entry.last_qty = oid.get<LastQty>();
event.orderid_entries.emplace_back(std::move(order_id_entry));
}
}
auto indicator = update.get<MatchEventIndicator>();
if (indicator[7] == '1') {
print_trades(event, trade_stream);
event.md_entries.clear();
event.orderid_entries.clear();
}
}
}
}
}
void print_orders_header(std::ostream &orders_stream) {
orders_stream << "Symbol,"
<< "SendingTime,"
<< "TransactTime,"
<< "MDEntryPx,"
<< "OrderID,"
<< "MDDisplayQty,"
<< "MDOrderPriority,"
<< "OrderUpdateAction,"
<< "Side"
<< "\n";
}
struct Order {
std::string sending_time;
std::string transact_time;
std::string side;
std::string order_id;
std::string order_update_action;
uint32_t md_order_priority;
uint32_t display_qty;
std::string symbol;
double md_entry_px;
};
void print_order(Order const& order, std::ostream &orders_stream) {
orders_stream << order.symbol << ","
<< order.sending_time << ","
<< order.transact_time << ","
<< order.md_entry_px << ","
<< order.order_id << ","
<< order.display_qty << ","
<< order.md_order_priority << ","
<< order.order_update_action << ","
<< order.side << "\n";
}
void print_orders(MDIncrementalUpdate const &update,std::ostream &orders_stream) {
if (update.has<MDEntries>()) {
auto &md_entries = update.get<MDEntries>();
auto & first_entry = md_entries[0];
if (first_entry.get<MDEntryType> () == "0" ||
first_entry.get<MDEntryType> () == "1") {
Order order;
order.sending_time = update.get<SendingTime>();
order.transact_time = update.get<TransactTime>();
order.side = first_entry.get<MDEntryType> ();
if (first_entry.has<MDPriceLevel>()) {
// combined template https://www.cmegroup.com/confluence/display/EPICSANDBOX/Incremental+Refresh+SBE+Template+Book+Processing
if (update.has<OrderIDEntries>()) {
auto &oid_entries = update.get<OrderIDEntries>();
for (size_t i = 0; i < oid_entries.size(); i++) {
auto &oid = oid_entries[i];
order.order_id = oid.get<OrderID>();
order.order_update_action = oid.get<OrderUpdateAction>();
order.md_order_priority = oid.get<MDOrderPriority>();
order.display_qty = oid.get<MDDisplayQty>();
auto &md_entry = update.get<MDEntries>()[oid.get<ReferenceID>()-1];
order.symbol = md_entry.get<Symbol>();
order.md_entry_px = md_entry.get<MDEntryPx>();
print_order(order, orders_stream);
}
}
}
else {
// MBO only template
for (size_t i = 0; i < md_entries.size(); i++) {
auto &entry = md_entries[i];
order.order_id = entry.get<OrderID>();
order.order_update_action = entry.get<MDUpateAction>();
order.md_order_priority = entry.get<MDOrderPriority>();
order.display_qty = entry.get<MDDisplayQty>();
order.symbol = entry.get<Symbol>();
order.md_entry_px = entry.get<MDEntryPx>();
print_order(order,orders_stream);
}
}
}
}
}
int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
std::ifstream input(args[0]);
std::ofstream books_stream(args[0] + "_books.csv");
std::ofstream msgs_stream(args[0] + "_msgs.csv");
std::ofstream trades_stream(args[0] + "_trades.csv");
std::ofstream orders_stream(args[0] + "_orders.csv");
std::unordered_map<std::string, Book> books;
TradeSummaryEvent current_trade_summary;
MDIncrementalUpdate update;
FIXParser<MDIncrementalUpdate> parser;
std::string line;
size_t c = 0;
auto ok = 0;
auto start = Timer::now();
print_book_header(books_stream);
print_msg_headers(msgs_stream);
print_trades_header(trades_stream);
print_orders_header(orders_stream);
while (std::getline(input, line)) {
c++;
auto parsed_ptr = parser.parse(InMsgBuffer(
line.data(), line.size(), std::shared_ptr<std::string>{}));
if (parsed_ptr) {
auto msg =
std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
update_and_maybe_print_book(*msg, books, books_stream);
print_msgs(c, *msg, msgs_stream);
update_and_maybe_print_trades(*msg, current_trade_summary,
trades_stream);
print_orders(*msg, orders_stream);
}
ok += (bool)(parsed_ptr);
}
auto sec_duration = sec_diff(Timer::now(), start);
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
<< " out of " << c << " time: " << sec_duration << " secs\n";
return 0;
}
std::string ParseHistoricalFIX::help() {
return "ParseHistoricalFIX <input> will produce 2 files, "
"<input>_mdentries.csv and <input>order_entries.csv";
}
} // namespace cmd
} // namespace bts