417 lines
15 KiB
C++
417 lines
15 KiB
C++
#include "ParseHistoricalFIX.h"
|
|
#include "lib/jetfix/codec/FIXMsg.h"
|
|
#include "lib/jetfix/codec/FIXParser.h"
|
|
#include "lib/jetfix/codec/Fields.h"
|
|
#include "lib/jetfix/codec/Group.h"
|
|
#include "lib/jetfix/codec/MessageTypes.h"
|
|
#include "lib/utils/bts_chrono.h"
|
|
|
|
#include <fstream>
|
|
#include <unordered_map>
|
|
|
|
namespace bts {
|
|
namespace cmd {
|
|
|
|
using namespace jetfix;
|
|
|
|
typedef StringField<75> TradeDate;
|
|
typedef IntField<34> SeqNum;
|
|
typedef StringField<52> SendingTime;
|
|
typedef StringField<60> TransactTime;
|
|
typedef StringField<5799> MatchEventIndicator;
|
|
|
|
typedef IntField<268> NoMDEntries;
|
|
typedef StringField<279> MDUpateAction;
|
|
typedef StringField<269> MDEntryType;
|
|
typedef StringField<48> SecurityID;
|
|
typedef StringField<55> Symbol;
|
|
typedef StringField<83> ReptSeq;
|
|
typedef FloatField<270> MDEntryPx;
|
|
typedef IntField<271> MDEntrySize;
|
|
typedef IntField<346> NumberOfOrders;
|
|
typedef IntField<1023> MDPriceLevel;
|
|
typedef StringField<286> OpenCloseSettlFlag;
|
|
typedef StringField<731> SettlPriceType;
|
|
typedef StringField<5797> AggressorSide;
|
|
typedef StringField<37711> MDTradeEntryID;
|
|
typedef StringField<5796> TradingReferenceDate;
|
|
typedef StringField<1149> HighLimitPrice;
|
|
typedef StringField<1148> LowLimitPrice;
|
|
typedef StringField<1143> MaxPriceVariation;
|
|
typedef StringField<1180> ApplID;
|
|
typedef IntField<9633> ReferenceID;
|
|
typedef IntField<32> LastQty;
|
|
|
|
typedef IntField<37705> NoOrderIDEntries;
|
|
typedef StringField<37> OrderID;
|
|
typedef IntField<37707> MDOrderPriority;
|
|
typedef IntField<37706> MDDisplayQty;
|
|
typedef StringField<37708> OrderUpdateAction;
|
|
|
|
typedef Group<NoOrderIDEntries, OrderID, MDDisplayQty, MDOrderPriority,
|
|
OrderUpdateAction, ReferenceID, LastQty>
|
|
OrderIDEntries;
|
|
|
|
typedef Group<NoMDEntries, MDUpateAction, MDEntryType, SecurityID, Symbol,
|
|
ReptSeq, MDEntryPx, MDEntrySize, NumberOfOrders, MDPriceLevel,
|
|
OpenCloseSettlFlag, SettlPriceType, AggressorSide, MDTradeEntryID,
|
|
TradingReferenceDate, HighLimitPrice, LowLimitPrice,
|
|
MaxPriceVariation, ApplID, ReferenceID, LastQty, OrderID,
|
|
MDOrderPriority, MDDisplayQty>
|
|
MDEntries;
|
|
|
|
typedef FIXMsg<MessageTypes::MDIncrementalUpdate, TradeDate, SeqNum,
|
|
SendingTime, TransactTime, MatchEventIndicator, MDEntries,
|
|
OrderIDEntries>
|
|
MDIncrementalUpdate;
|
|
|
|
std::string to_str(std::string const &s) { return s; }
|
|
|
|
template <class V> std::string to_str(V const &v) { return std::to_string(v); }
|
|
|
|
template <class F, class G> std::string get_if_valid(G const &update) {
|
|
if (update.template has<F>())
|
|
return to_str(update.template get<F>());
|
|
else
|
|
return "";
|
|
}
|
|
|
|
void print_msg_headers(std::ostream &out) {
|
|
out << "MsgID,"
|
|
<< "TradeDate,"
|
|
<< "SeqNum,"
|
|
<< "SendingTime,"
|
|
<< "TransactTime,"
|
|
<< "MatchEventIndicator,"
|
|
<< "NoMDEntries,"
|
|
<< "NoOrderIDEntries\n";
|
|
}
|
|
|
|
void print_msgs(size_t msg_idx, MDIncrementalUpdate const &line,
|
|
std::ostream &out) {
|
|
line.get<SeqNum>();
|
|
out << msg_idx << "," << get_if_valid<TradeDate>(line) << ","
|
|
<< get_if_valid<SeqNum>(line) << "," << get_if_valid<SendingTime>(line)
|
|
<< "," << get_if_valid<TransactTime>(line) << ","
|
|
<< get_if_valid<MatchEventIndicator>(line) << ","
|
|
<< (line.has<MDEntries>() ? line.get<MDEntries>().size() : 0) << ","
|
|
<< (line.has<OrderIDEntries>() ? line.get<OrderIDEntries>().size() : 0)
|
|
<< "\n";
|
|
}
|
|
|
|
struct PriceSize {
|
|
double price = 0;
|
|
int32_t size = 0;
|
|
};
|
|
|
|
struct Book {
|
|
Book() {
|
|
book[0].resize(10);
|
|
book[1].resize(10);
|
|
}
|
|
std::string symbol;
|
|
std::string transact_time;
|
|
std::string send_time;
|
|
uint32_t seqnum;
|
|
std::array<std::vector<PriceSize>, 2> book;
|
|
};
|
|
|
|
size_t n_printed_books = 2;
|
|
|
|
void print_book_side(std::vector<PriceSize> const &side,
|
|
std::ostream &book_stream) {
|
|
for (size_t i = 0; i < n_printed_books; i++)
|
|
if (i < side.size())
|
|
book_stream << "," << side[i].price << "," << side[i].size;
|
|
else
|
|
book_stream << "," << 0 << "," << 0;
|
|
}
|
|
|
|
void print_book_header(std::ostream &book_stream) {
|
|
book_stream << "Symbol,SendingTime,TransactTime, SeqnNum";
|
|
for (size_t i = 0; i < n_printed_books; i++)
|
|
book_stream << ",bid_p_" << i << ",bid_sz_" << i;
|
|
for (size_t i = 0; i < n_printed_books; i++)
|
|
book_stream << ",ask_p_" << i << ",ask_sz_" << i;
|
|
book_stream << ",\n";
|
|
}
|
|
|
|
void print_book(Book &b, std::ostream &book_stream) {
|
|
book_stream << b.symbol << "," << b.send_time << "," << b.transact_time
|
|
<< "," << b.seqnum;
|
|
print_book_side(b.book[0], book_stream);
|
|
print_book_side(b.book[1], book_stream);
|
|
book_stream << "\n";
|
|
}
|
|
|
|
void update_and_maybe_print_book(MDIncrementalUpdate const &update,
|
|
std::unordered_map<std::string, Book> &books,
|
|
std::ostream &book_stream) {
|
|
std::set<Book *> to_update;
|
|
if (update.has<MDEntries>()) {
|
|
auto &md_entries = update.get<MDEntries>();
|
|
for (size_t i = 0; i < md_entries.size(); i++) {
|
|
auto &entry = md_entries[i];
|
|
if (entry.has<MDPriceLevel>()) {
|
|
if (entry.get<MDEntryType>() == "0" ||
|
|
entry.get<MDEntryType>() == "1") {
|
|
auto side = entry.get<MDEntryType>()[0] - '0';
|
|
auto &sym = entry.get<Symbol>();
|
|
auto &book = books[sym];
|
|
to_update.insert(&book);
|
|
auto &book_side = book.book[side];
|
|
book.symbol = sym;
|
|
book.transact_time = update.get<TransactTime>();
|
|
book.send_time = update.get<SendingTime>();
|
|
book.transact_time = update.get<TransactTime>();
|
|
book.seqnum = update.get<SeqNum>();
|
|
auto level = entry.get<MDPriceLevel>() - 1;
|
|
|
|
if (entry.get<MDUpateAction>() == "0") // new
|
|
book_side.insert(
|
|
book_side.begin() + level,
|
|
{entry.get<MDEntryPx>(), entry.get<MDEntrySize>()});
|
|
else if (entry.get<MDUpateAction>() == "1")
|
|
book_side[level] = {entry.get<MDEntryPx>(),
|
|
entry.get<MDEntrySize>()};
|
|
else
|
|
book_side.erase(book_side.begin() + level);
|
|
}
|
|
}
|
|
}
|
|
auto indicator = update.get<MatchEventIndicator>();
|
|
if (indicator[5] == '1')
|
|
for (auto b : to_update)
|
|
print_book(*b, book_stream);
|
|
}
|
|
}
|
|
void print_trades_header(std::ostream &trade_stream) {
|
|
trade_stream << "Symbol,SendingTime,TransactTime,MDEntryPx,MDEntrySize,"
|
|
"NumberOfOrders,AggressorSide,MDTradeEntryID,AgressorOID,"
|
|
"RestingOID,PartialQty";
|
|
trade_stream << "\n";
|
|
}
|
|
|
|
struct TradeSummaryItem {
|
|
std::string symbol;
|
|
std::string sending_time;
|
|
std::string transact_time;
|
|
double md_entry_px;
|
|
uint32_t md_entry_size;
|
|
uint32_t n_orders;
|
|
std::string agressor_side;
|
|
std::string md_trade_entry_id;
|
|
};
|
|
|
|
struct TradeOrder {
|
|
std::string order_id;
|
|
uint32_t last_qty;
|
|
};
|
|
|
|
struct TradeSummaryEvent {
|
|
std::vector<TradeSummaryItem> md_entries;
|
|
std::vector<TradeOrder> orderid_entries;
|
|
};
|
|
|
|
void print_trades(TradeSummaryEvent &event, std::ostream &trade_stream) {
|
|
size_t current_order = 0;
|
|
for (auto &trade : event.md_entries) {
|
|
for (auto i = current_order; i < current_order + trade.n_orders; i++) {
|
|
trade_stream << trade.symbol << "," << trade.sending_time << ","
|
|
<< trade.transact_time << "," << trade.md_entry_px << ","
|
|
<< trade.md_entry_size << "," << trade.n_orders << ","
|
|
<< trade.agressor_side << "," << trade.md_trade_entry_id
|
|
<< "," << event.orderid_entries[current_order].order_id
|
|
<< "," << event.orderid_entries[i].order_id << ","
|
|
<< event.orderid_entries[i].last_qty << "\n";
|
|
}
|
|
current_order += trade.n_orders;
|
|
}
|
|
}
|
|
|
|
void update_and_maybe_print_trades(MDIncrementalUpdate const &update,
|
|
TradeSummaryEvent &event,
|
|
std::ostream &trade_stream) {
|
|
if (update.has<MDEntries>()) {
|
|
auto &md_entries = update.get<MDEntries>();
|
|
bool is_trade_summary = false;
|
|
for (size_t i = 0; i < md_entries.size(); i++) {
|
|
|
|
auto &entry = md_entries[i];
|
|
if (entry.get<MDEntryType>() == "2") {
|
|
is_trade_summary = true;
|
|
TradeSummaryItem item;
|
|
item.symbol = entry.get<Symbol>();
|
|
item.sending_time = update.get<SendingTime>();
|
|
item.transact_time = update.get<TransactTime>();
|
|
item.md_entry_px = entry.get<MDEntryPx>();
|
|
item.md_entry_size = entry.get<MDEntrySize>();
|
|
item.n_orders = entry.get<NumberOfOrders>();
|
|
item.agressor_side = entry.get<AggressorSide>();
|
|
item.md_trade_entry_id = entry.get<MDTradeEntryID>();
|
|
event.md_entries.emplace_back(std::move(item));
|
|
}
|
|
|
|
if (is_trade_summary) {
|
|
if (update.has<OrderIDEntries>()) {
|
|
auto &oid_entries = update.get<OrderIDEntries>();
|
|
for (size_t i = 0; i < oid_entries.size(); i++) {
|
|
auto &oid = oid_entries[i];
|
|
TradeOrder order_id_entry;
|
|
order_id_entry.order_id = oid.get<OrderID>();
|
|
order_id_entry.last_qty = oid.get<LastQty>();
|
|
event.orderid_entries.emplace_back(std::move(order_id_entry));
|
|
}
|
|
}
|
|
auto indicator = update.get<MatchEventIndicator>();
|
|
if (indicator[7] == '1') {
|
|
print_trades(event, trade_stream);
|
|
event.md_entries.clear();
|
|
event.orderid_entries.clear();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void print_orders_header(std::ostream &orders_stream) {
|
|
orders_stream << "Symbol,"
|
|
<< "SendingTime,"
|
|
<< "TransactTime,"
|
|
<< "MDEntryPx,"
|
|
<< "OrderID,"
|
|
<< "MDDisplayQty,"
|
|
<< "MDOrderPriority,"
|
|
<< "OrderUpdateAction,"
|
|
<< "Side"
|
|
<< "\n";
|
|
}
|
|
|
|
struct Order {
|
|
std::string sending_time;
|
|
std::string transact_time;
|
|
std::string side;
|
|
|
|
std::string order_id;
|
|
std::string order_update_action;
|
|
uint32_t md_order_priority;
|
|
uint32_t display_qty;
|
|
|
|
std::string symbol;
|
|
double md_entry_px;
|
|
};
|
|
|
|
void print_order(Order const& order, std::ostream &orders_stream) {
|
|
orders_stream << order.symbol << ","
|
|
<< order.sending_time << ","
|
|
<< order.transact_time << ","
|
|
<< order.md_entry_px << ","
|
|
<< order.order_id << ","
|
|
<< order.display_qty << ","
|
|
<< order.md_order_priority << ","
|
|
<< order.order_update_action << ","
|
|
<< order.side << "\n";
|
|
}
|
|
|
|
|
|
void print_orders(MDIncrementalUpdate const &update,std::ostream &orders_stream) {
|
|
if (update.has<MDEntries>()) {
|
|
auto &md_entries = update.get<MDEntries>();
|
|
auto & first_entry = md_entries[0];
|
|
|
|
if (first_entry.get<MDEntryType> () == "0" ||
|
|
first_entry.get<MDEntryType> () == "1") {
|
|
|
|
Order order;
|
|
order.sending_time = update.get<SendingTime>();
|
|
order.transact_time = update.get<TransactTime>();
|
|
order.side = first_entry.get<MDEntryType> ();
|
|
|
|
if (first_entry.has<MDPriceLevel>()) {
|
|
// combined template https://www.cmegroup.com/confluence/display/EPICSANDBOX/Incremental+Refresh+SBE+Template+Book+Processing
|
|
if (update.has<OrderIDEntries>()) {
|
|
auto &oid_entries = update.get<OrderIDEntries>();
|
|
for (size_t i = 0; i < oid_entries.size(); i++) {
|
|
|
|
auto &oid = oid_entries[i];
|
|
order.order_id = oid.get<OrderID>();
|
|
order.order_update_action = oid.get<OrderUpdateAction>();
|
|
order.md_order_priority = oid.get<MDOrderPriority>();
|
|
order.display_qty = oid.get<MDDisplayQty>();
|
|
|
|
auto &md_entry = update.get<MDEntries>()[oid.get<ReferenceID>()-1];
|
|
order.symbol = md_entry.get<Symbol>();
|
|
order.md_entry_px = md_entry.get<MDEntryPx>();
|
|
print_order(order, orders_stream);
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
// MBO only template
|
|
for (size_t i = 0; i < md_entries.size(); i++) {
|
|
auto &entry = md_entries[i];
|
|
order.order_id = entry.get<OrderID>();
|
|
order.order_update_action = entry.get<MDUpateAction>();
|
|
order.md_order_priority = entry.get<MDOrderPriority>();
|
|
order.display_qty = entry.get<MDDisplayQty>();
|
|
order.symbol = entry.get<Symbol>();
|
|
order.md_entry_px = entry.get<MDEntryPx>();
|
|
print_order(order,orders_stream);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
|
|
std::ifstream input(args[0]);
|
|
std::ofstream books_stream(args[0] + "_books.csv");
|
|
std::ofstream msgs_stream(args[0] + "_msgs.csv");
|
|
std::ofstream trades_stream(args[0] + "_trades.csv");
|
|
std::ofstream orders_stream(args[0] + "_orders.csv");
|
|
|
|
std::unordered_map<std::string, Book> books;
|
|
TradeSummaryEvent current_trade_summary;
|
|
|
|
MDIncrementalUpdate update;
|
|
FIXParser<MDIncrementalUpdate> parser;
|
|
|
|
std::string line;
|
|
size_t c = 0;
|
|
auto ok = 0;
|
|
auto start = Timer::now();
|
|
|
|
print_book_header(books_stream);
|
|
print_msg_headers(msgs_stream);
|
|
print_trades_header(trades_stream);
|
|
print_orders_header(orders_stream);
|
|
|
|
while (std::getline(input, line)) {
|
|
c++;
|
|
auto parsed_ptr = parser.parse(InMsgBuffer(
|
|
line.data(), line.size(), std::shared_ptr<std::string>{}));
|
|
if (parsed_ptr) {
|
|
auto msg =
|
|
std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
|
|
update_and_maybe_print_book(*msg, books, books_stream);
|
|
print_msgs(c, *msg, msgs_stream);
|
|
update_and_maybe_print_trades(*msg, current_trade_summary,
|
|
trades_stream);
|
|
print_orders(*msg, orders_stream);
|
|
}
|
|
ok += (bool)(parsed_ptr);
|
|
}
|
|
auto sec_duration = sec_diff(Timer::now(), start);
|
|
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
|
|
<< " out of " << c << " time: " << sec_duration << " secs\n";
|
|
return 0;
|
|
}
|
|
|
|
std::string ParseHistoricalFIX::help() {
|
|
return "ParseHistoricalFIX <input> will produce 2 files, "
|
|
"<input>_mdentries.csv and <input>order_entries.csv";
|
|
}
|
|
|
|
} // namespace cmd
|
|
} // namespace bts
|