Files
spark/src/server/sparkd/md/taifex/TaifexDataHandler.cpp
T

1227 lines
43 KiB
C++

#include "TaifexDataHandler.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/config/config.h"
#include "lib/domain/Price.h"
#include "lib/domain/Security.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/persistence/IDGenerator.h"
#include <emumba-deribit-connector/deribit_utils.h>
#include <algorithm>
#include <bit>
#include <bitset>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <map>
#include <sstream>
#include <string>
#include <unordered_set>
#include <vector>
#include <set>
#include <lib/config/protos/system_config.pb.h>
namespace bts::md::taifex {
const size_t BUFFER_SIZE = 1024 * 1024;
const size_t FOOTER_SIZE = 3;
const size_t HEADER_SIZE = 19;
void printBuffer(const std::vector<unsigned char> &buffer, size_t offset = 0,
size_t length = -1) {
if ((int)length == -1) {
length = buffer.size();
}
for (size_t i = 0; i < length; ++i) {
// Print each byte in hexadecimal format
// std::cout << std::hex << std::setw(2) << std::setfill('0')
// << (int)buffer[i];
printf(" offset: %zu, i: %zu, value: %.2X|", offset, i,
(unsigned int)buffer[offset + i]);
}
// std::cout << std::dec << std::endl;
}
struct MessageIdentifier {
enum MessageID {
I010 = 1,
I030 = 2,
I011 = 3,
I050 = 4,
I060 = 5,
I120 = 6,
I130 = 7,
I064 = 8,
I012 = 9,
I083 = 10,
I140 = 11,
I110 = 12,
I014 = 13,
I024 = 14,
I025 = 15,
I100 = 16,
I081 = 17, // Add I081
I084 = 18, // Add I084
I072 = 19, // Add I072
I071 = 20, // Add I071
I073 = 21, // Add I073
I070 = 22,
I001 = 23,
I002 = 24,
I999 = 0 // unknown
};
static std::string messageIDToString(MessageID id) {
switch (id) {
case I010:
return "I010";
case I030:
return "I030";
case I011:
return "I011";
case I050:
return "I050";
case I060:
return "I060";
case I120:
return "I120";
case I130:
return "I130";
case I064:
return "I064";
case I012:
return "I012";
case I083:
return "I083";
case I140:
return "I140";
case I110:
return "I110";
case I014:
return "I014";
case I024:
return "I024";
case I025:
return "I025";
case I100:
return "I100";
case I081:
return "I081";
case I084:
return "I084";
case I072:
return "I072";
case I071:
return "I071";
case I073:
return "I073";
case I070:
return "I070";
case I001:
return "I001";
case I002:
return "I002";
default:
return "Unknown";
}
}
void set_identifier_description(const unsigned char transmissionCode,
const unsigned char messageKind) {
switch (transmissionCode) {
case '0':
switch (messageKind) {
case '1':
identifier = I001;
description = "Heartbeat";
break;
case '2':
identifier = I002;
description = "Reset message";
break;
default:
identifier = I999;
description = "unknown";
break;
}
break;
case '1':
switch (messageKind) {
case '1':
identifier = I010;
description = "Product basic data";
break;
case '2':
identifier = I030;
description = "Sum of order data";
break;
case '3':
identifier = I011;
description = "Contract basic data";
break;
case '4':
identifier = I050;
description = "System message";
break;
case '5':
identifier = I060;
description = "Underlying index or stock reference data";
break;
case '6':
identifier = I120;
description = "Reference of stock or underlying index";
break;
case '7':
identifier = I130;
description = "Contract Adjustment";
break;
case '8':
identifier = I064;
description =
"Calculated opening and closing price for underlying stocks";
break;
case 'A':
identifier = I012;
description = "Price limits";
break;
default:
identifier = I999;
description = "unknown";
break;
}
break;
case '2':
switch (messageKind) {
case 'B':
identifier = I083;
description = "Snapshot order book";
break;
case '3':
identifier = I140;
description = "System message";
break;
case '4':
identifier = I100;
description = "Closing market data (settlement price included)";
break;
case 'A':
identifier = I081;
description = "Aggregated order book";
break;
case 'C':
identifier = I084;
description = "Market data snapshot refresh";
break;
case 'D':
identifier = I024;
description = "Matched prices and quantities";
break;
case 'E':
identifier = I025;
description = "Closing market data";
break;
default:
identifier = I999;
description = "unknown";
break;
}
break;
case '3':
switch (messageKind) {
case '3':
identifier = I072;
description = "Closing market data";
break;
case '1':
identifier = I070;
description = "Closing market data";
break;
case '2':
identifier = I071;
description = "Closing market data with settlement price";
break;
case '4':
identifier = I073;
description = "Combination product closing market data";
break;
default:
identifier = I999;
description = "unknown";
break;
}
break;
default:
identifier = I999;
description = "unknown";
break;
}
}
MessageIdentifier(const unsigned char transmissionCode,
const unsigned char messageKind)
: transmissionCode(transmissionCode), messageKind(messageKind) {
set_identifier_description(transmissionCode, messageKind);
}
unsigned char transmissionCode;
unsigned char messageKind;
MessageID identifier;
std::string description;
friend std::ostream &operator<<(std::ostream &os,
const MessageIdentifier &MI) {
os << "TransmissionCode: " << MI.transmissionCode
<< "\nMessageKind: " << MI.messageKind
<< "\nDescription: " << MI.description << "\nidentifier: "
<< MessageIdentifier::messageIDToString(MI.identifier);
return os;
}
};
// need to change parameters
std::string decodePackBCD(const unsigned char *data, size_t length) {
std::stringstream ss;
for (size_t i = 0; i < length; ++i) {
// Extract the upper and lower nibble (4 bits each)
int upper = (data[i] >> 4) & 0x0F;
int lower = data[i] & 0x0F;
// Append both nibbles to the stringstream
ss << upper << lower;
}
std::string result = ss.str();
// Remove leading zeros
result.erase(0, result.find_first_not_of('0'));
// If the result is empty (all zeros), return "0"
return result.empty() ? "0" : result;
}
struct Header_Fields {
uint8_t Esc_Code; // 1 byte
MessageIdentifier message_type;
std::string Information_Time;
int Channel_Id;
std::string Channel_Seq;
uint8_t Version_No;
uint Body_Length;
bool valid = false;
Header_Fields(const std::vector<unsigned char> &buffer, const size_t &offset)
: Esc_Code(buffer[offset]),
message_type(buffer[offset + 1], buffer[offset + 2]) {
// creates header feild struct
// and populates all fields
// Header_Fields header;
// Read single-byte fields
assert(message_type.identifier != MessageIdentifier::MessageID::I999);
// Read 2 bytes for Body_Length
unsigned char bl[2] = {buffer[offset + 17], buffer[offset + 18]};
Body_Length = std::stoul(decodePackBCD(bl, 2));
valid = true;
}
friend std::ostream &operator<<(std::ostream &os, const Header_Fields &hf) {
os << "Esc_Code: " << static_cast<int>(hf.Esc_Code) << "\n"
<< hf.message_type
<< "\n"
// << "Information_Time: " << hf.Information_Time << "\n"
// << "Channel_Id: " << hf.Channel_Id << "\n"
// << "Channel_Seq: " << hf.Channel_Seq << "\n"
// << "Version_No: " << static_cast<int>(hf.Version_No) << "\n"
<< "Body_Length: " << hf.Body_Length << "\n"
<< "Valid: " << (hf.valid ? "true" : "false");
return os;
}
};
std::string extractString(const std::vector<unsigned char> &buffer,
size_t &position, size_t length) {
// Extract the substring from the buffer
std::string result(buffer.begin() + position,
buffer.begin() + position + length);
position += length;
// Log the raw result
return result;
}
// Helper function to extract a BCD value from the buffer and convert to
// unsigned long
unsigned long extractBCD_STOUL(const std::vector<unsigned char> &buffer,
size_t &position, size_t length) {
std::string bcd_str = decodePackBCD(&buffer[position], length);
position += length;
return std::stoul(bcd_str);
}
std::string extractBCD_STRING(const std::vector<unsigned char> &buffer,
size_t &position, size_t length) {
std::string bcd_str = decodePackBCD(&buffer[position], length);
position += length;
return bcd_str;
}
// Helper function to extract a single character and convert to bool
bool extractBool(const std::vector<unsigned char> &buffer, size_t &position) {
bool result = (buffer[position] == 'Y');
position += 1;
return result;
}
std::string extractBCDDate(const std::vector<unsigned char> &buffer,
size_t &current_position, size_t size) {
if (size != 4) {
throw std::invalid_argument("BCD date must be 4 bytes long");
}
std::string date;
for (size_t i = 0; i < size; ++i) {
unsigned char byte = buffer[current_position++];
char high = (byte >> 4) + '0';
char low = (byte & 0x0F) + '0';
date.push_back(high);
date.push_back(low);
}
// Ensure the date is in YYYYMMDD format
if (date.length() != 8) {
throw std::runtime_error("Extracted date is not in YYYYMMDD format");
}
return date;
}
struct TickerIdentifier {
// Enum for Ticker Symbols
enum TickerSymbol {
TXF,
MXF,
MX1,
MX2,
MX4,
MX5,
EXF,
FXF,
GDF,
XIF,
GTF,
TGF,
RHF,
RTF,
TJF,
XEF,
XJF,
UDF,
SPF,
XAF,
XBF,
BRF,
UNF,
G2F,
E4F,
BTF,
F1F,
ZEF,
ZFF,
SOF,
SHF,
SXF,
TMF,
UNKNOWN
};
// Method to convert TickerSymbol to string
static std::string tickerSymbolToString(TickerSymbol symbol) {
switch (symbol) {
case TXF: return "TXF TAIEX Index Futures";
case MXF: return "MXF Mini-TAIEX Index Futures";
case MX1: return "MX1 Mini-TAIEX Index Futures expire at 1st week";
case MX2: return "MX2 Mini-TAIEX Index Futures expire at 2nd week";
case MX4: return "MX4 Mini-TAIEX Index Futures expire at 4th week";
case MX5: return "MX5 Mini-TAIEX Index Futures expire at 5th week";
case EXF: return "EXF TWSE Electronic Sector Index Futures";
case FXF: return "FXF TWSE Finance Sector Index Futures";
case GDF: return "GDF Gold Futures";
case XIF: return "XIF NonFinance NonElectronics Sub-Index Futures";
case GTF: return "GTF Taipei Exchange Stock Index Futures";
case TGF: return "TGF NTD Gold Futures";
case RHF: return "RHF USD/CNH FX Futures";
case RTF: return "RTF USD/CNT FX Futures";
case TJF: return "TJF TOPIX Futures";
case XEF: return "XEF EUR/USD Futures";
case XJF: return "XJF USD/JPY Futures";
case UDF: return "UDF DJIA Futures";
case SPF: return "SPF S&P 500 Futures";
case XAF: return "XAF AUD/USD Futures";
case XBF: return "XBF GBP/USD Futures";
case BRF: return "BRF Brent Crude Oil Futures";
case UNF: return "UNF NASDAQ 100 Index Futures";
case G2F: return "G2F TPEx 200 Index Futures";
case E4F: return "E4F FTSE4Good TIP Taiwan ESG Index Futures";
case BTF: return "BTF TIP Taiwan Market Biotechnology and Medical Care Index Futures";
case F1F: return "F1F FTSE 100 Index Futures";
case ZEF: return "ZEF Mini Electronics Sector Futures";
case ZFF: return "ZFF Mini Finance Sector Futures";
case SOF: return "SOF Semiconductor 30 Futures";
case SHF: return "SHF Shipping and Transportation Sector Futures";
case SXF: return "SXF PHLX Semiconductor Sector Futures";
case TMF: return "TMF Micro TAIEX Futures";
default: return "Unknown";
}
}
// Public members
TickerSymbol symbol = UNKNOWN;
std::string description = "Unknown";
};
struct I024Data {
struct trade {
enum Sign { POSITIVE = 0, NEGATIVE = 1 };
Sign sign;
std::string match_price; // need to determine type(appendix 2)
std::string match_qty; // keeping str for simplicity
trade(const std::vector<unsigned char> &buffer, const size_t &offset = 0,
const size_t match_qty_size = 2) {
const size_t SIGN_SIZE = 1;
const size_t MATCH_PRICE_SIZE = 5;
const size_t MATCH_QTY_SIZE = match_qty_size;
auto current_position = offset;
// Extract Sign
sign = (buffer[current_position] == '0') ? POSITIVE : NEGATIVE;
current_position += SIGN_SIZE;
match_price =
decodePackBCD(buffer.data() + current_position, MATCH_PRICE_SIZE);
current_position += MATCH_PRICE_SIZE;
match_qty = extractBCD_STRING(buffer, current_position, MATCH_QTY_SIZE);
}
friend std::ostream &operator<<(std::ostream &os, const trade &data) {
os << "Sign: " << (data.sign == Sign::POSITIVE ? "Positive" : "Negative")
<< '\n'
<< "Match Price: " << data.match_price << '\n'
<< "Match Quantity: " << data.match_qty << '\n';
return os;
}
};
std::string prod_id;
unsigned long sequence_number;
bool calculated_flag;
std::string match_time; // make another struct for time?
char match_display_item; // need to discuss
std::vector<trade> trades;
// next fields to add
std::string match_total_qty;
std::string match_buy_count;
std::string match_sell_count;
I024Data(const std::vector<unsigned char> &buffer, const size_t &offset = 0,
const size_t &length = -1) {
const size_t PROD_ID_SIZE = 20;
const size_t SEQ_NUMBER_SIZE = 5;
const size_t CALCULATED_FLAG = 1;
const size_t MATCH_TIME = 6;
const size_t MATCH_DISPLAY_ITEM_SIZE = 1;
const size_t TRADE_SIZE = 8;
const size_t MATCH_TOTAL_QTY_SIZE = 4;
const size_t MATCH_BUY_COUNT_SIZE = 4;
const size_t MATCH_SELL_COUNT_SIZE = 4;
const size_t FIRST_MATCH_QTY_SIZE = 4;
if (buffer.size() < offset + length) {
throw std::out_of_range("Buffer is smaller than the required length");
}
// do we need this?
size_t current_position = offset;
prod_id = extractString(buffer, current_position, PROD_ID_SIZE);
sequence_number =
extractBCD_STOUL(buffer, current_position, SEQ_NUMBER_SIZE);
// Extract Calculated Flag
std::string calculated_flag_str =
extractString(buffer, current_position, CALCULATED_FLAG);
calculated_flag = (calculated_flag_str[0] == '1');
// Extract Match Time
match_time = decodePackBCD(buffer.data() + current_position, MATCH_TIME);
; // extractBCD(buffer, current_position, MATCH_TIME);
current_position += MATCH_TIME;
trades.push_back(trade(buffer, current_position, FIRST_MATCH_QTY_SIZE));
current_position += TRADE_SIZE + 2;
// Extract Match Display Item
auto getFirstBitAndRemainingValue =
[](char input) -> std::tuple<bool, int> {
bool firstBit = (input & 0b10000000) != 0;
int remainingValue = input & 0b01111111;
return std::make_tuple(firstBit, remainingValue);
};
match_display_item = buffer[current_position];
current_position += MATCH_DISPLAY_ITEM_SIZE;
auto [new_message, trades_count] =
getFirstBitAndRemainingValue(match_display_item);
// assert(new_message);
for (int i = 0; i < trades_count; i++) {
trades.push_back(trade(buffer, current_position));
current_position += TRADE_SIZE;
}
// match quantity
match_total_qty =
extractBCD_STRING(buffer, current_position, MATCH_TOTAL_QTY_SIZE);
// match buy count
match_buy_count =
extractBCD_STRING(buffer, current_position, MATCH_BUY_COUNT_SIZE);
// match sell count
match_sell_count =
extractBCD_STRING(buffer, current_position, MATCH_SELL_COUNT_SIZE);
std::cout << "trades count: " << trades_count << std::endl;
assert(buffer[current_position + 1] == 0x0D &&
buffer[current_position + 2] == 0x0A);
}
// 833 251 1142
friend std::ostream &operator<<(std::ostream &os, const I024Data &data) {
os << "Product ID: " << data.prod_id << '\n'
<< "Sequence Number: " << data.sequence_number << '\n'
<< "Calculated Flag: " << (data.calculated_flag ? '1' : '0') << '\n'
<< "Match Time: " << data.match_time << '\n'
<< "Match Display Item: "
<< std::bitset<8>{static_cast<unsigned long long>(
data.match_display_item)}
<< "\n"; // Output as integer to show value
for (auto t : data.trades) {
os << "Trade: " << t << "\n";
}
os << "Match Total Quantity: " << data.match_total_qty << '\n'
<< "Match Buy Count: " << data.match_buy_count << '\n'
<< "Match Sell Count: " << data.match_sell_count << '\n';
return os;
}
};
struct I010Data {
std::string prod_id; // PROD-ID-S X(10) 10 Product ID
unsigned long
reference_price; // REFERENCE-PRICE 9(9) 5 Reference price PACK BCD
std::string prod_kind; // PROD-KIND X(1) 1 Type of contract
unsigned int decimal_locator; // DECIMAL-LOCATOR 9(1) 1 Number of decimal
// digits in price fields PACK BCD
unsigned int
strike_price_decimal_locator; // STRIKE-PRICE-DECIMAL-LOCATOR 9(1) 1
// Number of decimal digits in strike price
// of options product PACK BCD
std::string begin_date; // BEGIN-DATE 9(8) 4 Listed date of product (YYYYMMDD)
// PACK BCD
std::string
end_date; // END-DATE 9(8) 4 Expiry date of product (YYYYMMDD) PACK BCD
unsigned int flow_group; // FLOW-GROUP 9(2) 1 Flow group PACK BCD
std::string
delivery_date; // DELIVERY-DATE 9(8) 4 Delivery date of product PACK BCD
bool dynamic_banding; // DYNAMIC-BANDING X(1) 1 Is dynamic banding applicable?
// Y: Yes, N: No
// Embedded ProductInfo struct
struct ProductInfo {
enum class ProductType {
FUTURES,
OPTIONS
} productType;
std::string productID;
int expirationYear;
int expirationMonth;
bool isCall;
double strikePrice;
bool isWeekly;
bool isSpread;
std::string spreadSecondLeg;
// Convert month code to month number
static int getMonthFromCode(char code) {
if (code >= 'A' && code <= 'L')
return code - 'A' + 1; // 'A' corresponds to 1 (January), 'B' to 2, ..., 'L' to 12 (December)
throw std::runtime_error("Invalid month code");
}
// Method to parse Product ID and initialize ProductInfo
static ProductInfo parseProductID(const std::string &prodID) {
ProductInfo info;
if (prodID.length() == 10 || prodID.length() == 20) {
// Futures product parsing
info.productType = ProductType::FUTURES;
info.productID = prodID.substr(0, 3);
info.expirationMonth = getMonthFromCode(prodID[3]);
info.expirationYear = 2020 + (prodID[4] - '0'); // Assuming the year is between 2020 and 2029
info.isWeekly = prodID[2] >= '1' && prodID[2] <= '5'; // Weekly expired contract
info.isSpread = prodID.length() > 10; // Check if it's a spread
if (info.isSpread) {
info.spreadSecondLeg = prodID.substr(11, 6); // Parsing the second leg of the spread
}
} else if (prodID.length() == 11) {
// Weekly time spread product parsing
info.productType = ProductType::FUTURES;
info.isWeekly = true;
info.isSpread = true;
info.productID = prodID.substr(0, 3);
info.expirationMonth = getMonthFromCode(prodID[3]);
info.expirationYear = 2020 + (prodID[4] - '0');
info.spreadSecondLeg = prodID.substr(6, 5);
} else if (prodID.length() == 12) {
// Options product parsing
info.productType = ProductType::OPTIONS;
info.productID = prodID.substr(0, 3);
info.strikePrice = std::stod(prodID.substr(3, 5)); // Assuming no decimal point for now
info.isCall = prodID[8] >= 'A' && prodID[8] <= 'L'; // Call options
info.expirationMonth = getMonthFromCode(prodID[8]);
info.expirationYear = 2020 + (prodID[9] - '0');
} else {
throw std::invalid_argument("Invalid Product ID length");
}
return info;
}
// Overload the << operator to print ProductInfo
friend std::ostream &operator<<(std::ostream &os, const ProductInfo &info) {
os << "Product Type: " << (info.productType == ProductType::FUTURES ? "Futures" : "Options") << '\n'
<< "Product ID: " << info.productID << '\n'
<< "Expiration Month: " << info.expirationMonth << " ("
<< (info.expirationMonth == 1 ? "January" :
info.expirationMonth == 2 ? "February" :
info.expirationMonth == 3 ? "March" :
info.expirationMonth == 4 ? "April" :
info.expirationMonth == 5 ? "May" :
info.expirationMonth == 6 ? "June" :
info.expirationMonth == 7 ? "July" :
info.expirationMonth == 8 ? "August" :
info.expirationMonth == 9 ? "September" :
info.expirationMonth == 10 ? "October" :
info.expirationMonth == 11 ? "November" :
info.expirationMonth == 12 ? "December" : "Unknown") << ")\n"
<< "Expiration Year: " << info.expirationYear << '\n'
<< "Strike Price: " << info.strikePrice << '\n'
<< "Is Weekly: " << (info.isWeekly ? "Yes" : "No") << '\n'
<< "Is Call: " << (info.isCall ? "Yes" : "No") << '\n'
<< "Is Spread: " << (info.isSpread ? "Yes" : "No") << '\n';
if (info.isSpread) {
os << "Spread Second Leg: " << info.spreadSecondLeg << '\n';
}
return os;
}
}parsedProductInfo;
// Constructor to initialize from a buffer
I010Data(const std::vector<unsigned char> &buffer, size_t offset = 0,
size_t length = -1) {
const size_t PROD_ID_SIZE = 10;
const size_t REFERENCE_PRICE_SIZE = 5;
const size_t PROD_KIND_SIZE = 1;
const size_t DECIMAL_LOCATOR_SIZE = 1;
const size_t STRIKE_PRICE_DECIMAL_LOCATOR_SIZE = 1;
const size_t BEGIN_DATE_SIZE = 4;
const size_t END_DATE_SIZE = 4;
const size_t FLOW_GROUP_SIZE = 1;
const size_t DELIVERY_DATE_SIZE = 4;
// const size_t DYNAMIC_BANDING_SIZE = 1;
// Parse ProductInfo using the extracted product ID
// parsedProductInfo = ProductInfo::parseProductID(prod_id, strike_price_decimal_locator);
if (buffer.size() < offset + length) {
throw std::out_of_range("Buffer is smaller than the required length");
}
size_t current_position = offset;
prod_id = extractString(buffer, current_position, PROD_ID_SIZE);
reference_price =
extractBCD_STOUL(buffer, current_position, REFERENCE_PRICE_SIZE);
prod_kind = extractString(buffer, current_position, PROD_KIND_SIZE);
decimal_locator =
extractBCD_STOUL(buffer, current_position, DECIMAL_LOCATOR_SIZE);
strike_price_decimal_locator = extractBCD_STOUL(
buffer, current_position, STRIKE_PRICE_DECIMAL_LOCATOR_SIZE);
begin_date = extractBCDDate(buffer, current_position, BEGIN_DATE_SIZE);
end_date = extractBCDDate(buffer, current_position, END_DATE_SIZE);
flow_group = extractBCD_STOUL(buffer, current_position, FLOW_GROUP_SIZE);
delivery_date =
extractBCDDate(buffer, current_position, DELIVERY_DATE_SIZE);
dynamic_banding = extractBool(buffer, current_position);
parsedProductInfo = ProductInfo::parseProductID(prod_id);
}
// Overload the << operator for ProductInfo to print its contents
// Friend function to overload the << operator for outputting the struct data
friend std::ostream &operator<<(std::ostream &os, const I010Data &data) {
os << "Product ID: " << data.prod_id << '\n'
<< "Reference Price: " << data.reference_price << '\n'
<< "Product Kind: " << data.prod_kind << '\n'
<< "Decimal Locator: " << data.decimal_locator << '\n'
<< "Strike Price Decimal Locator: " << data.strike_price_decimal_locator
<< '\n'
<< "Begin Date: " << data.begin_date << '\n'
<< "End Date: " << data.end_date << '\n'
<< "Flow Group: " << data.flow_group << '\n'
<< "Delivery Date: " << data.delivery_date << '\n'
<< "Dynamic Banding: " << (data.dynamic_banding ? 'Y' : 'N')
<< "\nParsed Product Info:\n" << data.parsedProductInfo;
return os;
}
// Overload the << operator for ProductInfo to print its contents
// friend std::ostream &operator<<(std::ostream &os, const ProductInfo &info) {
// os << "Product Type: " << info.productType << '\n'
// << "Product ID: " << info.productID << '\n'
// << "Contract Type: " << info.contractType << '\n'
// << "Expiration Year: " << info.expirationYear << '\n'
// << "Expiration Month: " << info.expirationMonth << '\n'
// << "Is Call: " << (info.isCall ? "Yes" : "No") << '\n'
// << "Strike Price: " << info.strikePrice << '\n'
// << "Is Weekly: " << (info.isWeekly ? "Yes" : "No") << '\n'
// << "Is Spread: " << (info.isSpread ? "Yes" : "No") << '\n'
// << "Spread Second Leg: " << info.spreadSecondLeg;
// return os;
// }
};
struct I011Data {
std::string kind_id; // KIND-ID X(4) 4 Contract ID
std::string
chinese_name; // NAME X(30) 30 Contract name contract's Chinese name
std::string
stock_id; // STOCK-ID X(6) 6 Underlying stock ID Only for stock type
unsigned long
contract_size; // CONTRACT-SIZE 9(7)V9(4) 6 Contract size PACK BCD
unsigned int decimal_locator; // DECIMAL-LOCATOR 9(1) 1 Decimals of strike
// price PACK BCD
unsigned int
strike_price_decimal_locator; // STRIKE-PRICE-DECIMAL-LOCATOR 9(1) 1
// Decimals of strike price PACK BCD
std::string begin_date; // BEGIN-DATE X(8) 8 List date Format: YYYYMMDD Only
// for stock type
enum CONTRACT_TYPE { // Contract types
INDEX = 1,
INTEREST_RATE, // 2
BOND, // 3
COMMODITY, // 4
STOCK, // 5
CURRENCY // 6
} contract_type;
enum CURRENCY_TYPE { // Currency types
TWD = 1, // Taiwan Dollar
USD, // US Dollar
EUR, // Euro
JPY, // Japanese Yen
GBP, // British Pound
AUD, // Australian Dollar
HKD, // Hong Kong Dollar
CNY // Chinese Yuan
} currency_enum_type;
enum STATUS_CODE { // Status codes
NORMAL = 'N',
PAUSE = 'P',
UNLISTED = 'U'
} status_code;
enum EXPIRY_TYPE { // Expiry types
STANDARD = 'S',
WEEKLY = 'W'
} expiry_type;
enum UNDERLYING_TYPE { // Underlying types
ETF = 'E',
SINGLE_STOCK = 'S'
} underlying_type;
bool accept_quote_flag; // ACCEPT-QUOTE-FLAG X(1) 1 Quote acceptable? true:
// Yes, false: No
bool block_trade_flag; // BLOCK-TRADE-FLAG X(1) 1 Can block trade? true:
// Yes, false: No
// Constructor to initialize from a buffer
I011Data(const std::vector<unsigned char> &buffer, size_t offset = 0,
size_t length = -1) {
const size_t KIND_ID_SIZE = 4;
const size_t CHINESE_NAME_SIZE = 30;
const size_t STOCK_ID_SIZE = 6;
const size_t CONTRACT_SIZE_SIZE = 6;
const size_t DECIMAL_LOCATOR_SIZE = 1;
const size_t STRIKE_PRICE_DECIMAL_LOCATOR_SIZE = 1;
const size_t BEGIN_DATE_SIZE = 8;
if (buffer.size() < offset + length) {
throw std::out_of_range("Buffer is smaller than the required length");
}
size_t current_position = offset;
kind_id = extractString(buffer, current_position, KIND_ID_SIZE);
chinese_name = extractString(buffer, current_position, CHINESE_NAME_SIZE);
stock_id = extractString(buffer, current_position, STOCK_ID_SIZE);
contract_size =
extractBCD_STOUL(buffer, current_position, CONTRACT_SIZE_SIZE);
status_code = static_cast<STATUS_CODE>(buffer[current_position++]);
currency_enum_type = static_cast<CURRENCY_TYPE>(buffer[current_position++]);
decimal_locator =
extractBCD_STOUL(buffer, current_position, DECIMAL_LOCATOR_SIZE);
strike_price_decimal_locator = extractBCD_STOUL(
buffer, current_position, STRIKE_PRICE_DECIMAL_LOCATOR_SIZE);
accept_quote_flag = extractBool(buffer, current_position);
begin_date = extractString(buffer, current_position, BEGIN_DATE_SIZE);
block_trade_flag = extractBool(buffer, current_position);
expiry_type = static_cast<EXPIRY_TYPE>(buffer[current_position++]);
underlying_type = static_cast<UNDERLYING_TYPE>(buffer[current_position++]);
}
std::string getCurrencyString() const {
switch (currency_enum_type) {
case TWD:
return "Taiwan Dollar";
case USD:
return "US Dollar";
case EUR:
return "Euro";
case JPY:
return "Japanese Yen";
case GBP:
return "British Pound";
case AUD:
return "Australian Dollar";
case HKD:
return "Hong Kong Dollar";
case CNY:
return "Chinese Yuan";
default:
return "Unknown Currency";
}
}
// Friend function to overload the << operator for outputting the struct
// data
friend std::ostream &operator<<(std::ostream &os, const I011Data &data) {
os << "Kind ID: " << data.kind_id << '\n'
<< "Chinese Name: " << data.chinese_name << '\n'
<< "Stock ID: " << data.stock_id << '\n'
<< "Contract Size: " << data.contract_size << '\n'
<< "Status Code: " << static_cast<char>(data.status_code) << '\n'
<< "Currency Type: " << static_cast<int>(data.currency_enum_type) << '\n'
<< "Decimal Locator: " << data.decimal_locator << '\n'
<< "Strike Price Decimal Locator: " << data.strike_price_decimal_locator
<< '\n'
<< "Accept Quote Flag: " << (data.accept_quote_flag ? 'Y' : 'N') << '\n'
<< "Begin Date: " << data.begin_date << '\n'
<< "Block Trade Flag: " << (data.block_trade_flag ? 'Y' : 'N') << '\n'
<< "Expiry Type: " << static_cast<char>(data.expiry_type) << '\n'
<< "Underlying Type: " << static_cast<char>(data.underlying_type);
return os;
}
};
struct I081Data {
struct MD_ENTRY {
enum Sign { POSITIVE = 0, NEGATIVE = 1 };
int md_update_action;
char md_entry_type;
Sign sign;
double price;
size_t level;
size_t quantity;
// operator overload for MDentry
friend std::ostream &operator<<(std::ostream &os,
const I081Data::MD_ENTRY &entry) {
os << "MD Update Action: " << entry.md_update_action << "\n"
<< "MD Entry Type: " << entry.md_entry_type << "\n"
<< "Sign: "
<< (entry.sign == I081Data::MD_ENTRY::POSITIVE ? "Positive"
: "Negative")
<< "\n"
<< "Price: " << entry.price << "\n"
<< "Level: " << entry.level << "\n"
<< "Quantity: " << entry.quantity << "\n";
return os;
}
MD_ENTRY(const std::vector<unsigned char> &buffer, size_t offset = 0) {
md_update_action = buffer[offset];
md_entry_type = buffer[offset + 1];
sign = static_cast<Sign>(buffer[offset + 2]);
// price
std::string price_str = decodePackBCD(&buffer[offset + 3], 5);
price = std::stod(price_str);
// quantity
std::string size_str = decodePackBCD(&buffer[offset + 8], 4);
quantity = std::stoul(size_str);
// level
std::string level_str = decodePackBCD(&buffer[offset + 12], 1);
level = std::stoul(level_str);
}
};
std::vector<MD_ENTRY> entries;
std::string prod_id;
size_t entries_count;
unsigned long seq_number;
// overload that prints dummy values
friend std::ostream &operator<<(std::ostream &os, const I081Data &data) {
os << "Product ID: " << data.prod_id << "\n"
<< "Sequence Number: " << data.seq_number << "\n"
<< "Entries Count: " << data.entries_count << "\n"
<< "MD entries: ";
for (auto entry : data.entries) {
std::cout << " Entry: " << entry;
}
return os;
}
I081Data(const std::vector<unsigned char> &buffer, const size_t &offset = 0) {
const size_t ENTRIES_COUNT_SIZE = 1;
const size_t PROD_ID_SIZE = 20;
const size_t SEQ_NUMBER_SIZE = 5;
auto current_position = buffer.begin() + offset;
prod_id.assign(current_position, current_position + PROD_ID_SIZE);
current_position += PROD_ID_SIZE;
std::string seq_number_str =
decodePackBCD(&(*current_position), SEQ_NUMBER_SIZE);
seq_number = std::stoul(seq_number_str);
current_position += SEQ_NUMBER_SIZE;
std::string entries_count_string =
decodePackBCD(&(*current_position), ENTRIES_COUNT_SIZE);
entries_count = std::stoul(entries_count_string);
current_position += ENTRIES_COUNT_SIZE;
// Following fields group repeat NO-MD-ENTRIES times
auto position = PROD_ID_SIZE + SEQ_NUMBER_SIZE + ENTRIES_COUNT_SIZE;
for (size_t i = 0; i < entries_count; ++i) {
entries.push_back(MD_ENTRY(buffer, offset + position));
position += 13;
// current_position += ENTRIES_COUNT_SIZE;//need to fix
}
}
};
bool readBinaryFile(const std::string &file_name,
std::shared_ptr<bts::md::IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store ) {
std::ifstream file(file_name, std::ios::binary);
UNUSED(listener);
if (!file) {
std::cerr << "Failed to open file: " << file_name << std::endl;
assert (false);
}
std::vector<unsigned char> buffer(BUFFER_SIZE);
while (file.read((reinterpret_cast<char *>(buffer.data())), BUFFER_SIZE)) {
// printBuffer(buffer, 0, 20);
std::cout << std::endl << std::endl;
size_t position = 0;
// there needs to be atleast 19 bytes left in buffer to read a complete
// header
size_t counter = 0;
size_t hits = 0;
const size_t hits_limit = 5;
const size_t limit = 10000;
std::unordered_set<std::string> names;
while (position < BUFFER_SIZE - 19 && counter < limit &&
hits < hits_limit) {
auto Header = Header_Fields(buffer, position);
// printBuffer(buffer, position,
// Header.Body_Length + HEADER_SIZE + FOOTER_SIZE); //
// UNCOMMENT TO PRINT HEX VALUES OF MESSAGE
// HEADER VALUE
switch (Header.message_type.identifier) {
case MessageIdentifier::I024: {
// printBuffer(buffer, position,
// Header.Body_Length + HEADER_SIZE + FOOTER_SIZE);
//std::cout << std::endl << std::endl;
//std::cout << "Header: " << Header << std::endl; // UNCOMMENT TO PRINT
auto i024_message = I024Data(buffer, position + HEADER_SIZE,
Header.Body_Length + FOOTER_SIZE);
//std::cout << i024_message << std::endl;
// char c = 0b10000000;
// assert((int)i024_message.match_display_item == c);
// hits++;
}
case MessageIdentifier::I010: {
// std::cout << "we are looking for this!" << std::endl;
auto i010_message = I010Data(buffer, position + HEADER_SIZE,
Header.Body_Length + FOOTER_SIZE);
const auto remove_spaces= [](const std::string &name){
std::string removed = "";
const char space = ' ';
for (auto c:name){
if(c!=space){
removed+=c;
}
}
return removed;
};
const auto id = remove_spaces(i010_message.prod_id);
const auto symbol = id.substr(0,2);
bts::domain::SecurityID sec_id{
sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, id)};
std::vector<bts::domain::Security> complete_security_list;
bts::domain::Security &sec = complete_security_list.emplace_back();
sec.security_common.venue = bts::api::domain::VENUE_TAIFEX;
sec.security_common.id = sec_id;
sec.security_common.exchange_sec_id = id; // Need to fix this
sec.security_common.desc = id;
sec.security_common.symbol = symbol;
sec.security_common.asset = id;
sec.security_common.group = id;
sec.security_common.min_price_increment = 0.1;
sec.security_common.min_quantity = 1;
sec.security_common.min_quantity_increment = 0.1;
sec.security_common.currency = "USD";
switch(i010_message.parsedProductInfo.productType){
case I010Data::ProductInfo::ProductType::FUTURES: {
auto &t = sec.security.emplace<domain::FutureSecurity>();
//assuming time stamp
// std::tm tm = {};
// std::istringstream iss(i010_message.end_date);
// iss >> std::get_time(&tm, "%Y%m%d");
// std::time_t time = std::mktime(&tm);
LOG(md, Error, "DATE {}",i010_message.end_date);
std::cout<<"hello world "<< i010_message.end_date<<std::endl;
auto toISODate = [](const std::string& date) {
return date.substr(0, 4) + "-" + date.substr(4, 2) + "-" + date.substr(6, 2);
};
t.expiration = bts::core::TimeStamp::from_iso_string(toISODate(i010_message.end_date));
// t.future().set_expiration_year(std::stoi(i010_message.end_date.substr(0, 4)));
// t.future().set_expiration_month(std::stoi(i010_message.end_date.substr(4, 2)));
// t.future().set_expiration_day(std::stoi(i010_message.end_date.substr(6, 2)));
break;
}
case I010Data::ProductInfo::ProductType::OPTIONS: {
auto &t = sec.security.emplace<domain::OptionSecurity>();
std::tm tm = {};
std::istringstream iss(i010_message.end_date);
iss >> std::get_time(&tm, "%Y%m%d");
std::time_t time = std::mktime(&tm);
t.expiration = bts::core::TimeStamp(static_cast<uint64_t>(time));
t.strike = domain::Price::from_double(i010_message.parsedProductInfo.strikePrice);
break;
}
}
std::vector<const domain::Security *> const_security_list;
for (const auto &security : complete_security_list) {
const_security_list.push_back(&security);
}
listener->on_securities(const_security_list);
// hits++;
break;
//std::cout << i010_message << std::endl;
}
case MessageIdentifier::I011: {
// std::cout << "we are looking for this!" << std::endl;
auto i011_message = I011Data(buffer, position + HEADER_SIZE,
Header.Body_Length + FOOTER_SIZE);
// bts::domain::SecurityID sec_id{
// sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, i011_message.kind_id)};
// std::vector<bts::domain::Security> complete_security_list;
// bts::domain::Security &sec = complete_security_list.emplace_back();
// sec.security_common.venue = bts::api::domain::VENUE_TAIFEX;
// // sec.security_common.id = sec_id;
// sec.security_common.exchange_sec_id = i011_message.kind_id; // Need to fix this
// sec.security_common.desc = i011_message.kind_id;
// sec.security_common.symbol = i011_message.kind_id;
// sec.security_common.asset = i011_message.kind_id;
// sec.security_common.group = i011_message.kind_id;
// sec.security_common.min_price_increment = 0.1;
// sec.security_common.min_quantity = 1;
// sec.security_common.min_quantity_increment = 0.1;
// sec.security_common.currency = i011_message.getCurrencyString();
// switch(i011_message.contract_type){
// case I011Data::STOCK:
// case I011Data::INDEX:
// case I011Data::INTEREST_RATE:
// case I011Data::BOND:
// case I011Data::COMMODITY:
// case I011Data::CURRENCY:
// break;
// }
// auto &t = sec.security.emplace<domain::FutureSecurity>();
// t.expiration = exp_time;
// std::vector<const domain::Security *> const_security_list;
// for (const auto &security : complete_security_list) {
// const_security_list.push_back(&security);
// }
// listner->on_securities(const_security_list);
// std::cout << i011_message << std::endl;
break;
}
case MessageIdentifier::I081: {
// TODO:
// write overload that just print dummy values
// fix i081 constructor so it can parse id, seq and no of entries
// make overload print those 3
// uncomment constructor 2 lines below
// uncomment cout 3 lines below and make sure output makes sense
// tell arham
// start working on MD_ENTRY
// std::cout << "Handling " << Header.message_type.description << "...
// "
// << std::endl;
// auto i081_message = I081Data(buffer, position + HEADER_SIZE,
// Header.Body_Length + FOOTER_SIZE);
// std::cout << "I081 " << i081_message
// << std::endl; // Uncomment once overload
// // written
break;
}
default: {
// std::cout << "Case not Handled!" << std::endl;
}
}
position += Header.Body_Length + HEADER_SIZE + FOOTER_SIZE;
// assert(Header.message_type.identifier != MessageIdentifier::I024);
counter++;
}
return false;
}
return true;
}
//const std::string &filename,
void bts::md::taifex::TaifexDatahandler::run(
bool use_file,
std::shared_ptr<IMarketDataListener> listner,
std::shared_ptr<persistence::IDGenerator> sec_id_store) {
std::string file_name = "/reference_data_20240806_1";
if (use_file) {
std::string base_path =
config::get<bts::config::SystemConfig>().data_path();
readBinaryFile(base_path + file_name, listner, sec_id_store);
}
// bts_assert_msg(false, "inside run taifex");
}
}//namespace bts::md::taifex