ADD: TWSE Code Complete

This commit is contained in:
Adeel Khan committed 2025-03-26 18:20:39 +00:00
1 parent 022546a6c1
commit 06bfe9c29f
8 files changed
+1328 -1

No files matched your search

+33 -1
View File
@@ -36,6 +36,7 @@
#include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
#include "server/sparkd/md/recording/MDReplayer.h"
#include "server/sparkd/md/taifex/TaifexDataGateway.h"
#include "server/sparkd/md/twse/TwseDataGateway.h"
#include <bits/stdint-uintn.h>
#include <boost/filesystem.hpp>
@@ -139,8 +140,9 @@ void MarketDataGateway::run() {
bts_assert(me_->md_config.emumba_deribit().size() == 1);
_start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
}
start_glbx(me_->md_config);
start_glbx(me_->md_config);
start_taifex(me_->md_config);
start_twse(me_->md_config);
start_cboe(me_->md_config);
_start_replays(me_->md_config);
while (true) {
@@ -377,6 +379,36 @@ void MarketDataGateway::_start_replays(
os::get_thread_id(), os::get_thread_name());
}
void MarketDataGateway::start_twse(const bts::api::md::MarketDataConfig &cfg) {
// twse
// int counter = 0;
// LOG(md, Info, "In start, mdgtway.cpp");
// for (const auto &twse : cfg.twse()) {
// counter++;
// auto p = me_->venue_listeners.emplace_back(
// std::make_shared<VenueListenerData>(bts::api::VENUE_TWSE));
// std::string tid_name = "twse" + std::to_string(counter);
// bts::threading::make_thread(tid_name, [&]() {
// bts::md::twse::TwseDataGateway::run(
// twse, p->listener, me_->sec_id_store);
// }).detach();
// }
for (const auto &twse : cfg.twse()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_TWSE));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "twse";
bts::threading::make_thread(tid_name, [listener, sec_id_store, twse]() {
bts::md::twse::TwseDataGateway::run(
twse, listener, sec_id_store);
}).detach();
}
}
void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
// taifex
for (const auto &taifex : cfg.taifex()) {
+1
View File
@@ -39,6 +39,7 @@ class MarketDataGateway {
void start_glbx(const bts::api::md::MarketDataConfig &cfg);
void start_cboe(const bts::api::md::MarketDataConfig &cfg);
void start_taifex(const bts::api::md::MarketDataConfig &cfg);
void start_twse(const bts::api::md::MarketDataConfig &cfg);
};
} // namespace bts::md
@@ -0,0 +1,663 @@
#include "TwseDataGateway.h"
#include "TwseMessages.h"
#include "lib/api/domain/SecurityType.pb.h"
#include "lib/domain/Security.h"
#include "lib/domain/TradeRecord.h"
#include "lib/logging/logging.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/helper_macros.h"
#include <c++/11/bits/fs_fwd.h>
#include <lib/utils/bts_assert.h>
namespace bts::md::twse {
using UNPROCESSED_DATA = std::vector<unsigned char>;
bool runOnBinaryFile(
const std::string &file_name,
std::set<std::string> &securities_set,
const std::unordered_set<std::string> required_securities,
const std::string &twse_securities_path,
std::shared_ptr<bts::md::IMarketDataListener> &listener,
std::shared_ptr<persistence::IDGenerator> &sec_id_store) {
LOG(md, Diag, "In runOnBinaryFile");
LOG(md, Diag, "The path: {}", file_name);
LOG(md, Diag, "Absolute path: {}",
boost::filesystem::absolute(file_name).string());
std::ifstream file(file_name, std::ios::binary);
if (!file) {
LOG(md, Error, "File not found: {}", file_name);
bts_assert(false);
}
size_t BUFFER_SIZE = 20000;
[[maybe_unused]] size_t counter = 0;
UNPROCESSED_DATA buffer(BUFFER_SIZE);
while (file.read((reinterpret_cast<char *>(buffer.data())), BUFFER_SIZE)) {
size_t position = 0;
[[maybe_unused]] size_t format_counter = 0;
std::cout << std::dec << std::endl;
while (position < BUFFER_SIZE && buffer[position] != 0x1B) {
++position;
}
if (position >= BUFFER_SIZE) {
LOG(md, Error, "Escape code (0x0B) not found in buffer");
break;
}
while (position < BUFFER_SIZE - 10) {
LOG(md, Diag, "Esc Code in Hexadecimal: 0x{:02X}, format counter: {}",
static_cast<int>(buffer[position]), format_counter);
std::cout << std::dec;
LOG(md, Diag, "Position: {}", position);
format_counter++;
position++;
auto Header = TwseMessages::Header_Fields(buffer, position);
std::ostringstream oss;
oss << Header;
std::string headerStr = oss.str();
// LOG(md, Diag, "Header: {}", headerStr);
//std::cout << "Header: \n" << Header;
if (Header.transmission_format_code == 06) {
LOG(md, Diag, "We're in Case 06");
auto format_6 = TwseMessages::Format_6(buffer, position);
std::ostringstream oss;
oss << format_6;
std::string format6Str = oss.str();
LOG(md, Diag, "Format 6: {}", format6Str);
if(format_6.stock_code == "0050"){
LOG(md, Diag, "0050 book!: {}", format6Str);
}
domain::Book book;
auto &bids = book.bids();
auto &asks = book.asks();
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
format_6.stock_code)};
// std::vector<bts::domain::TradeRecord> trades;
book.set_security_id(sec_id);
for (unsigned long i = 0; i < format_6.bids.size(); ++i) {
const auto &entry = format_6.bids[i];
auto price = bts::domain::Price::from_double(entry.first / 100.0);
bts::domain::Quantity quantity(entry.second);
bids[i].price =
price; //checck if the indexing starts from 1 or 0
bids[i].total_quantity = quantity;
}
// Similarly for asks
for (unsigned int i = 0; i < format_6.asks.size(); ++i) {
const auto &entry = format_6.asks[i];
auto price = bts::domain::Price::from_double(entry.first / 100.0);
bts::domain::Quantity quantity(entry.second);
asks[i].price = price;
asks[i].total_quantity = quantity;
}
std::vector<bts::domain::TradeRecord> trades;
// First, add the real-time quote if present
for (const auto &quote : format_6.quotes) {
auto &record = trades.emplace_back();
record.sec_id = sec_id;
// record.time_stamp = bts::core::TimeStamp::from_iso_string(
// format_6.matching_time);
record.time_stamp = bts::core::TimeStamp::from_iso_string(
TwseUtils::convertToISO(format_6.matching_time));
record.trade_quantity =
bts::domain::Quantity(quote.volume_field);
record.trade_price = bts::domain::Price(
stod(quote.price_field) * std::pow(10, 9 - 4));
}
std::vector<const domain::TradeRecord *> trade_ptrs;
trade_ptrs.reserve(trades.size());
for (const auto &trade : trades) {
trade_ptrs.push_back(&trade);
}
std::ostringstream ptr_oss;
ptr_oss << "Trade Pointers: [";
for (const auto *trade_ptr : trade_ptrs) {
if (trade_ptr) { // Ensure the pointer is not null
ptr_oss << "{Sec ID: " << trade_ptr->sec_id
<< ", Timestamp: " << trade_ptr->time_stamp
<< ", Quantity: " << trade_ptr->trade_quantity.value()
<< ", Price: " << trade_ptr->trade_price
<< ", Aggressor Side: "
<< ((trade_ptr->agg_side ==
domain::AggressorSide::Buy)
? "Buy"
: "Sell")
<< "}, ";
}
}
// std::string ptr_log = ptr_oss.str();
if (!trade_ptrs.empty()) {
// ptr_log.pop_back(); // Remove trailing comma
// ptr_log.pop_back(); // Remove trailing space
}
// ptr_log += "]";
// Log the string
// LOG(md, Info, "TWSE Sending to Listner {}", ptr_log);
listener->on_book(book);
listener->on_trades(trade_ptrs);
position = position + Header.information_length - 10;
} else if (Header.transmission_format_code == 01) {
LOG(md, Diag, "We're in Case 01");
// Parse Format 1 message
auto format_1 = TwseMessages::Format_1(buffer, position);
// Convert Format_1 to string for logging
std::ostringstream oss;
oss << format_1;
std::string format1Str = oss.str();
LOG(md, Diag, "Format 1: {}", format1Str);
// Extract security ID
const auto id = format_1.sec_code;
const auto symbol = id;
// LOG(md, Info, "Extracted security ID: {}, Symbol: {}", id, symbol);
const std::string trimmed_id = TwseUtils::trim(id);
if (required_securities.find(trimmed_id) ==
required_securities.end()) {
LOG(md, Diag, "Security {} not in required list. Skipping.",
trimmed_id);
continue;
}
if (securities_set.find(trimmed_id) != securities_set.end()) {
LOG(md, Diag, "Security {} already processed. Skipping.", id);
} else {
securities_set.insert(trimmed_id);
// Generate or retrieve SecurityID
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
id)};
// LOG(md, Info, "SecurityID in store: {}", sec_id);
// Populate security details
bts::domain::Security sec;
sec.security_common.venue = bts::api::domain::VENUE_TWSE;
sec.security_common.id = sec_id;
sec.security_common.exchange_sec_id = id;
sec.security_common.desc = id;
sec.security_common.symbol = symbol;
sec.security_common.asset = id;
sec.security_common.group = id;
auto get_min_inc = [](const std::string &id) {
// LOG(md, Info, "Getting minimum increment for ID: {}", id);
UNUSED(id);
return 0.1;
};
// Set minimum increments
sec.security_common.min_price_increment = get_min_inc(id);
sec.security_common.min_quantity = get_min_inc(id);
sec.security_common.min_quantity_increment = get_min_inc(id);
LOG(md, Info, "Set minimum increments for security.");
sec.security_common.cash_per_point = stod(format_1.trading_unit);
sec.security_common.currency = format_1.trading_currency_code;
// LOG(md, Info, "Set cash per point: {}, Currency: {}",
// sec.security_common.cash_per_point,
// sec.security_common.currency);
// Serialize Security to JSON
nlohmann::json j;
to_json(j, sec);
std::string json_str = j.dump() + "\n";
// Write JSON to file
std::ofstream outfile(twse_securities_path, std::ios::app);
if (outfile.is_open()) {
outfile << json_str;
outfile.close();
// LOG(md, Info, "Security data written to file.");
} else {
// LOG(md, Info, "Failed to open securities file for writing.");
}
LOG(md, Diag, "securities_set size: {}", securities_set.size());
// Notify listener
std::vector<const domain::Security *> const_security_list;
const_security_list.push_back(&sec);
listener->on_securities(const_security_list);
}
// Update position
position = position + Header.information_length - 10;
} else {
LOG(md, Diag, "Unknown Format");
auto payload_length = Header.information_length;
position += payload_length - 10;
}
}
return true;
}
return false;
}
int runLive(
[[maybe_unused]] std::set<std::string> &securities_set,
[[maybe_unused]] const std::unordered_set<std::string> required_securities,
[[maybe_unused]] const std::string &twse_securities_path,
std::shared_ptr<bts::md::IMarketDataListener> &listener,
std::shared_ptr<persistence::IDGenerator> &sec_id_store) {
LOG(md, Diag, "In runLive");
//live connection
const std::string multicastIp = "224.0.100.100";
const int port = 10000;
const std::string interfaceIp = "10.102.167.131";
// Create a UDP socket
int sock = socket(AF_INET, SOCK_DGRAM, 0);
if (sock < 0) {
LOG(md, Error, "Socket creation failed");
//perror("socket creation failed");
return EXIT_FAILURE;
}
LOG(md, Diag, "Socket created successfully");
int reuse = 1;
if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) < 0) {
LOG(md, Error, "Failed to set SO_REUSEADDR on socket");
// perror("setsockopt(SO_REUSEADDR) failed");
close(sock);
return EXIT_FAILURE;
}
LOG(md, Diag, "Socket option SO_REUSEADDR set successfully");
struct sockaddr_in localAddr = {};
localAddr.sin_family = AF_INET;
localAddr.sin_port = htons(port);
localAddr.sin_addr.s_addr = INADDR_ANY;
if (bind(sock, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
LOG(md, Error, "Socket bind failed");
// perror("bind failed");
close(sock);
return EXIT_FAILURE;
}
LOG(md, Diag, "Socket bound successfully to port");
// Join multicast group
struct ip_mreq mreq;
mreq.imr_multiaddr.s_addr = inet_addr(multicastIp.c_str());
mreq.imr_interface.s_addr = inet_addr(interfaceIp.c_str());
if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) <
0) {
LOG(md, Error, "Failed to join multicast group");
// perror("setsockopt(IP_ADD_MEMBERSHIP) failed");
close(sock);
return EXIT_FAILURE;
}
LOG(md, Diag, "Joined multicast group successfully");
const size_t BUFFER_SIZE = 1024 * 1024;
std::vector<unsigned char> buffer(BUFFER_SIZE);
LOG(md, Diag, "Entering data reception loop");
while (true) {
struct sockaddr_in srcAddr;
socklen_t addrLen = sizeof(srcAddr);
size_t bytesRead = recvfrom(
sock, buffer.data(), BUFFER_SIZE, 0, (struct sockaddr *)&srcAddr,
&addrLen);
if (bytesRead < 0) {
std::string err = strerror(errno);
LOG(md, Error, "recvfrom failed: {}", err);
break;
}
// LOG(md, Info, "Received {} bytes from {}:{}", bytesRead,
// inet_ntoa(srcAddr.sin_addr), ntohs(srcAddr.sin_port));
// Parse received data
size_t position = 0;
size_t format_counter = 0;
while (position < bytesRead && buffer[position] != 0x1B) {
++position; // Search for escape code (0x1B)
}
if (position >= bytesRead) {
// std::cerr << "Escape code (0x1B) not found in buffer" << std::endl;
continue;
}
while (position < bytesRead - 10) {
std::ostringstream sink;
sink << "Escape Code in Hexadecimal: 0x" << std::hex
<< std::setw(2) << std::setfill('0')
<< static_cast<int>(buffer[position]) << " for format #"
<< format_counter << std::endl;
++format_counter;
++position;
auto Header = TwseMessages::Header_Fields(buffer, position);
std::ostringstream oss;
oss << Header;
std::string headerStr = oss.str();
LOG(md, Diag, "Header: {}", headerStr);
if (Header.transmission_format_code == 6) {
LOG(md, Info, "Book Message --> Case 06");
LOG(md, Diag, "Book-->Case 06");
auto format_6 = TwseMessages::Format_6(buffer, position);
std::ostringstream oss;
oss << format_6;
std::string format6Str = oss.str();
LOG(md, Diag, "Format 6: {}", format6Str);
if(format_6.stock_code == "0050"){
// LOG(md, Diag, "0050 book!: {}", format6Str);
}
domain::Book book;
auto &bids = book.bids();
auto &asks = book.asks();
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
format_6.stock_code)};
// std::vector<bts::domain::TradeRecord> trades;
book.set_security_id(sec_id);
for (unsigned long i = 0; i < format_6.bids.size(); ++i) {
const auto &entry = format_6.bids[i];
auto price = bts::domain::Price::from_double(entry.first / 100.0);
bts::domain::Quantity quantity(entry.second);
bids[i].price =
price; //checck if the indexing starts from 1 or 0
bids[i].total_quantity = quantity;
}
// Similarly for asks
for (unsigned int i = 0; i < format_6.asks.size(); ++i) {
const auto &entry = format_6.asks[i];
auto price = bts::domain::Price::from_double(entry.first / 100.0);
bts::domain::Quantity quantity(entry.second);
asks[i].price = price;
asks[i].total_quantity = quantity;
}
std::vector<bts::domain::TradeRecord> trades;
// First, add the real-time quote if present
for (const auto &quote : format_6.quotes) {
auto &record = trades.emplace_back();
record.sec_id = sec_id;
// record.time_stamp = bts::core::TimeStamp::from_iso_string(
// format_6.matching_time);
record.time_stamp = bts::core::TimeStamp::from_iso_string(
TwseUtils::convertToISO(format_6.matching_time));
record.trade_quantity =
bts::domain::Quantity(quote.volume_field);
record.trade_price = bts::domain::Price(
stod(quote.price_field) * std::pow(10, 9 - 4));
}
std::vector<const domain::TradeRecord *> trade_ptrs;
trade_ptrs.reserve(trades.size());
for (const auto &trade : trades) {
trade_ptrs.push_back(&trade);
}
std::ostringstream ptr_oss;
ptr_oss << "Trade Pointers: [";
for (const auto *trade_ptr : trade_ptrs) {
if (trade_ptr) { // Ensure the pointer is not null
ptr_oss << "{Sec ID: " << trade_ptr->sec_id
<< ", Timestamp: " << trade_ptr->time_stamp
<< ", Quantity: " << trade_ptr->trade_quantity.value()
<< ", Price: " << trade_ptr->trade_price
<< ", Aggressor Side: "
<< ((trade_ptr->agg_side == domain::AggressorSide::Buy) ? "Buy" : "Sell")
<< "}, ";
}
}
std::string ptr_log = ptr_oss.str();
if (!trade_ptrs.empty()) {
ptr_log.pop_back(); // Remove trailing comma
ptr_log.pop_back(); // Remove trailing space
}
ptr_log += "]";
// Log the string
LOG(md, Diag, "TWSE Sending to Listner {}", ptr_log);
listener->on_book(book);
listener->on_trades(trade_ptrs);
position += Header.information_length - 10;
} else if (Header.transmission_format_code == 1) {
LOG(md, Info, "Case 01 --> Securities Message");
LOG(md, Diag, "We're in Case 01");
// Parse Format 1 message
auto format_1 = TwseMessages::Format_1(buffer, position);
// Convert Format_1 to string for logging
std::ostringstream oss;
oss << format_1;
std::string format1Str = oss.str();
LOG(md, Diag, "Format 1: {}", format1Str);
// Extract security ID
const auto id = format_1.sec_code;
const auto symbol = id;
LOG(md, Diag, "Extracted security ID: {}, Symbol: {}", id, symbol);
const std::string trimmed_id = TwseUtils::trim(id);
if (required_securities.find(trimmed_id) ==
required_securities.end()) {
LOG(md, Diag, "Security {} not in required list. Skipping.",
trimmed_id);
continue;
}
if (securities_set.find(trimmed_id) != securities_set.end()) {
LOG(md, Diag, "Security {} already processed. Skipping.", id);
} else {
securities_set.insert(trimmed_id);
// Generate or retrieve SecurityID
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT,
id)};
// LOG(md, Info, "SecurityID in store: {}", sec_id);
// Populate security details
bts::domain::Security sec;
sec.security_common.venue = bts::api::domain::VENUE_TWSE;
sec.security_common.id = sec_id;
sec.security_common.exchange_sec_id = id;
sec.security_common.desc = id;
sec.security_common.symbol = symbol;
sec.security_common.asset = id;
sec.security_common.group = id;
// Helper function for minimum increment
auto get_min_inc = [](const std::string &id) {
// LOG(md, Info, "Getting minimum increment for ID: {}", id);
UNUSED(id);
return 0.1;
};
// Populate other fields
sec.security_common.min_price_increment = get_min_inc(id);
sec.security_common.min_quantity = get_min_inc(id);
sec.security_common.min_quantity_increment = get_min_inc(id);
// LOG(md, Info, "Set minimum increments for security.");
sec.security_common.cash_per_point = stod(format_1.trading_unit);
sec.security_common.currency = format_1.trading_currency_code;
// LOG(md, Info, "Set cash per point: {}, Currency: {}",
// sec.security_common.cash_per_point,
// sec.security_common.currency);
// Serialize Security to JSON
nlohmann::json j;
to_json(j, sec);
std::string json_str = j.dump() + "\n";
// Write JSON to file
std::ofstream outfile(twse_securities_path, std::ios::app);
if (outfile.is_open()) {
outfile << json_str;
outfile.close();
// LOG(md, Info, "Security data written to file.");
} else {
// LOG(md, Info, "Failed to open securities file for writing.");
}
std::vector<const domain::Security *> const_security_list;
const_security_list.push_back(&sec);
listener->on_securities(const_security_list);
}
position += Header.information_length - 10;
} else {
LOG(md, Diag, "Unknown Format");
auto payload_length = Header.information_length;
position += payload_length - 10;
}
}
}
close(sock);
return 0;
}
void loadSecurities(
const std::string &filePath,
std::set<std::string> &securitiesSet,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
std::vector<std::shared_ptr<bts::domain::Security>> &securitiesVector,
std::shared_ptr<IMarketDataListener> listener) {
std::ifstream infile(filePath);
if (!infile.is_open()) {
LOG(md, Error, "Failed to open securities file for reading: {}",
filePath);
return;
}
std::string line;
while (std::getline(infile, line)) {
try {
nlohmann::json j = nlohmann::json::parse(line);
std::shared_ptr<bts::domain::Security> sec =
std::make_shared<bts::domain::Security>();
from_json(j, *sec);
const std::string id = sec->security_common.exchange_sec_id;
if (securitiesSet.find(id) == securitiesSet.end()) {
// Generate or retrieve SecurityID
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, id)};
sec->security_common.id = sec_id;
securitiesSet.insert(id);
securitiesVector.push_back(sec);
LOG(md, Diag, "Loaded security: {}", id);
} else {
LOG(md, Diag, "Duplicate security ID: {}", id);
}
} catch (const nlohmann::json::parse_error &e) {
std::string err = e.what();
LOG(md, Error, "JSON parse error: {}", err);
} catch (const std::exception &e) {
std::string err = e.what();
LOG(md, Error, "Error loading security: {}", err);
}
}
infile.close();
// Notify listener about the loaded securities
if (!securitiesVector.empty()) {
std::vector<const bts::domain::Security *> securitiesToNotify;
for (const auto &sec : securitiesVector) {
securitiesToNotify.push_back(sec.get());
}
listener->on_securities(securitiesToNotify);
}
}
void bts::md::twse::TwseDataGateway::run(
[[maybe_unused]] const api::md::TwseMarketDataConfig &config,
[[maybe_unused]] std::shared_ptr<IMarketDataListener> listner,
[[maybe_unused]] std::shared_ptr<persistence::IDGenerator> sec_id_store) {
LOG(md, Diag, "In run, twsedatagateway.cpp");
std::string twse_securities_path =
"/home/bts/sparkdata/data/twse_load_securities.jsonl";
std::string sec_path = "/home/bts/sparkdata/data/1st_ip_securities_20250116_4";
std::string book_path =
"/home/bts/sparkdata/data/1st_ip_securities_20250116_27";
std::set<std::string> securities_set;
LOG(md, Diag, "securities_set size: {}", securities_set.size());
// const std::unordered_set<std::string> required_securities = {
// "0050", "2317", "2454", "2603", "2615", "00631L", "00633L",
// "00634R", "00637L", "00638R", "00653L", "00654R", "00675L", "00676R",
// "00637L", "006204", "00929", "00919", "006208"};
const std::unordered_set<std::string> required_securities = {
"0050", "2317", "2454", "2603", "2615", "00631L", "00633L",
"00634R", "00637L", "00638R", "00653L", "00654R", "00675L", "00676R",
"00637L", "006204", "00929", "00919", "006208",/*
"6239", "0050", "00703", "00706L", "00707R", "00961", "01001T", "2613",
"0056", "0055", "6239", "00703", "00706L", "00707R", "00961", "01001T",
"2613", "0056", "0055", "00657K", "00690", "00708L", "00712", "00715L",
"00919", "00934", "00938", "00943", "00952", "00685L",*/"054276", "054277", "054278"
};
std::vector<std::shared_ptr<bts::domain::Security>> securities_vector;
loadSecurities(
twse_securities_path, securities_set, sec_id_store, securities_vector,
listner);
bool run_on_live = true;
if (run_on_live) {
runLive(
securities_set, required_securities, twse_securities_path, listner,
sec_id_store);
} else {
runOnBinaryFile(
sec_path, securities_set, required_securities, twse_securities_path,
listner, sec_id_store);
runOnBinaryFile(
book_path, securities_set, required_securities, twse_securities_path,
listner, sec_id_store);
}
}
} // namespace bts::md::twse
@@ -0,0 +1,27 @@
#include "lib/api/md/TwseMarketDataConfig.pb.h"
#include <memory> // Add this line for std::shared_ptr
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
class IMarketDataListener;
}
namespace bts::md::twse {
class TwseDataGateway {
public:
static void run(const api::md::TwseMarketDataConfig &config,
std::shared_ptr<IMarketDataListener> listner,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
// (void)config;
// (void)listener;
// (void)sec_id_store;
// // Dummy implementation
};
} // namespace bts::md::twse
@@ -0,0 +1,37 @@
// TwseDataProcessors.h
#ifndef TWSE_DATA_PROCESSORS_H
#define TWSE_DATA_PROCESSORS_H
#include "TwseMessages.h"
#include "lib/domain/Security.h"
#include "lib/md/IMarketDataListener.h"
namespace bts::md::twse {
class TwseDataProcessors {
public:
// TwseMessages::Format_1 format_1;
// static bool processBookMessage(
// const TwseMessages::Format_6 &format_6,
// const TwseMessages::Format_1 &format_1,
// std::shared_ptr<bts::md::IMarketDataListener> listener) {
// auto sec_id = format_1.sec_code;
// if (!sec_id)
// return false;
// domain::Book book;
// auto &bids = book.bids();
// auto &asks = book.asks();
// book.set_security_id(sec_id.value());
// for(const auto &quote : format_6.quotes){
// bts::domain::Price price(quote.price_field);
// bts::domain::Quantity quantity(quote.volume_field);
// }
// listener -> on_book(book);
// return true;
// }
};
} // namespace bts::md::twse
#endif // TWSE_DATA_PROCESSORS_H
+430
View File
@@ -0,0 +1,430 @@
// TwseMessages.h
#include "TwseUtils.h"
#include "lib/logging/logging.h"
#include <bitset>
#include <cstdint>
#include <ostream>
#include <vector>
#ifndef TWSE_MESSAGES_H
#define TWSE_MESSAGES_H
namespace bts::md::twse {
class TwseMessages {
public:
struct Header_Fields {
size_t information_length;
size_t business_type;
size_t transmission_format_code;
size_t transmission_format_version;
size_t transmission_sn;
static const size_t HEADER_SIZE =
9; // Total size of header including all fields
Header_Fields(const std::vector<unsigned char> &buffer, size_t &offset) {
information_length =
stoull(TwseUtils::decodePackBCD(&buffer[offset], 2));
business_type =
stoull(TwseUtils::decodePackBCD(&buffer[offset + 2], 1));
transmission_format_code =
stoull(TwseUtils::decodePackBCD(&buffer[offset + 3], 1));
transmission_format_version =
stoull(TwseUtils::decodePackBCD(&buffer[offset + 4], 1));
transmission_sn =
stoull(TwseUtils::decodePackBCD(&buffer[offset + 5], 4));
// move offset past header
offset += HEADER_SIZE;
}
friend std::ostream &operator<<(
std::ostream &os, const Header_Fields &hf) {
// std::cout<<std::dec;
os << "Information Length: " << hf.information_length << "\n"
<< "Business Type: " << hf.business_type << "\n"
<< "Transmission Format Code: " << hf.transmission_format_code
<< "\n"
<< "Transmission Format Version: " << hf.transmission_format_version
<< "\n"
<< "Transmission S/N: " << hf.transmission_sn << "\n";
return os;
}
};
struct Format_6 {
std::string stock_code;
std::string matching_time;
uint8_t disclosed_item_remarks;
uint8_t rise_fall_remarks;
uint8_t status_remarks;
unsigned long accumalative_trading_volume;
std::vector<std::pair<double, uint32_t>> bids;
std::vector<std::pair<double, uint32_t>> asks;
struct RealTimeQuotes {
std::string price_field;
uint32_t volume_field;
RealTimeQuotes(
const std::vector<unsigned char> &buffer,
const size_t &offset = 0) {
price_field = TwseUtils::decodePackBCD(&buffer[offset], 5);
volume_field =
stoul(TwseUtils::decodePackBCD(&buffer[offset + 5], 4));
//offset = offset + 9;
}
friend std::ostream &operator<<(
std::ostream &os,
const Format_6::RealTimeQuotes &real_time_quotes) {
os << "\n Price Field: " << real_time_quotes.price_field << "\n"
<< "Volume Field: " << real_time_quotes.volume_field << "\n";
return os;
}
};
std::vector<RealTimeQuotes> quotes;
// std::vector<RealTimeQuotes> quotes;
friend std::ostream &operator<<(
std::ostream &os, const Format_6 &format) {
os << "\nStock Code: ." << format.stock_code << ".\n"
<< "Matching Time: " << format.matching_time << "\n"
<< "Disclosed Item Remarks (Binary): "
<< std::bitset<8>(format.disclosed_item_remarks)
<< " (Decimal: " << static_cast<int>(format.disclosed_item_remarks)
<< ")\n"
<< "Rise/Fall Remarks (Binary): "
<< std::bitset<8>(format.rise_fall_remarks)
<< " (Decimal: " << static_cast<int>(format.rise_fall_remarks)
<< ")\n"
<< "Status Remarks (Binary): "
<< std::bitset<8>(format.status_remarks)
<< " (Decimal: " << static_cast<int>(format.status_remarks) << ")\n"
<< "Accumulative Trading Volume: "
<< format.accumalative_trading_volume << "\n";
os << "Real Time Quotes:\n";
for (const auto &quote : format.quotes) {
os << quote;
}
os << "Bids: \n";
for (const auto &bid : format.bids) {
os << "Price = " << bid.first << " Quantity = " << bid.second
<< "\n";
}
os << "\nAsks: \n";
for (const auto &ask : format.asks) {
os << "Price = " << ask.first << " Quantity = " << ask.second
<< "\n";
}
// os << "Quotes:\n";
// for (const auto &real_time_quotes : format.quotes) {
// os << real_time_quotes;
// }
return os;
}
Format_6(
const std::vector<unsigned char> &buffer, const size_t &offset = 0) {
size_t position = offset;
stock_code = std::string(
buffer.begin() + position, buffer.begin() + position + 6);
stock_code = TwseUtils::trim(stock_code);
position += 6;
matching_time = TwseUtils::decodePackBCD(&buffer[position], 6);
position += 6;
disclosed_item_remarks = buffer[position];
position += 1;
rise_fall_remarks = buffer[position];
position += 1;
status_remarks = buffer[position];
position += 1;
accumalative_trading_volume =
stoul(TwseUtils::decodePackBCD(&buffer[position], 4));
position += 4;
if (disclosed_item_remarks & 0x80) { // If Bit 7 is 1
LOG(md, Diag, "Price Field and Volume Field present");
// std::cin.ignore();
auto rtq = RealTimeQuotes(buffer, position);
quotes.push_back(rtq);
} else {
LOG(md, Diag, "No Price Field and Volume Field");
}
LOG(md, Diag, "Purchasing Price and Volumes");
std::cout << std::dec;
// Read Purchase Price and Volume
uint8_t purchasing_count =
(disclosed_item_remarks >> 4) & 0x07; // Get bits 6-4
for (uint8_t i = 0; i < purchasing_count; ++i) {
std::string purchase_price =
TwseUtils::decodePackBCD(&buffer[position], 5); // Price Field
double bid_price =
std::stod(TwseUtils::decodePackBCD(&buffer[position], 5));
position += 5; // Move to the volume field
std::string volume_field =
TwseUtils::decodePackBCD(&buffer[position], 4); // Volume Field
uint32_t bid_volume =
stoul(TwseUtils::decodePackBCD(&buffer[position], 4));
position += 4; // Move to the next price field
LOG(md, Info, "Purchase Price {}: {} Volume {}: {}", i + 1,
purchase_price, i + 1, volume_field);
bids.emplace_back(bid_price, bid_volume);
}
// Read Sell Price and Volume if applicable
uint8_t selling_count =
(disclosed_item_remarks >> 1) & 0x07; // Get bits 3-1
for (uint8_t i = 0; i < selling_count; ++i) {
std::string sell_price = TwseUtils::decodePackBCD(
&buffer[position], 5); // Price Field: 5 bytes
double ask_price =
std::stod(sell_price); // Use the 5-byte decoded value
position += 5; // Move to the volume field
std::string volume_field = TwseUtils::decodePackBCD(
&buffer[position], 4); // Volume Field: 4 bytes
uint32_t ask_volume =
stoul(volume_field); // Use the 4-byte decoded value
position += 4; // Move to the next price field
LOG(md, Info, "Sell Price {}: {} Volume {}: {}", i + 1, sell_price,
i + 1, volume_field);
asks.emplace_back(ask_price, ask_volume);
}
}
};
struct Format_1 {
std::string sec_code;
std::string stock_abbreviation;
std::string industry_category;
std::string stock_category; //can be blank
std::string stock_entries; //can be blank
std::string stock_anomaly_code;
std::string board_code;
std::string today_currency_price;
std::string trading_currency_code;
std::string rise_stop_price;
std::string fall_stop_price;
std::string non_10_face_value_indicator;
std::string abnormal_recommendation_indicator;
std::string abnormal_securities_indicator;
std::string day_trading_indicator;
std::string exemption_unchanged_market_margin_sale_indicator;
std::string exemption_unchanged_market_securities_lending_sale_indicator;
std::string matching_cycle_seconds;
std::string warrant_id;
std::string
exercise_strike_price; // This will need to be decoded from PACK BCD
std::string
prev_business_day_exercise_volume; // This will also need PACK BCD decoding
std::string prev_business_day_cancellation_volume; // PACK BCD format
std::string issuing_balance_volume; // PACK BCD format
std::string strike_ratio; // PACK BCD format
std::string upper_limit_price; // PACK BCD format
std::string lower_limit_price; // PACK BCD format
std::string maturity_date;
std::string foreign_stock_id;
std::string trading_unit; // PACK BCD format
std::string trading_currency_code_foreign_stock;
std::string market_information_note;
Format_1(
const std::vector<unsigned char> &buffer, const size_t &offset = 0) {
size_t position = offset;
sec_code = std::string(
buffer.begin() + position, buffer.begin() + position + 6);
sec_code = TwseUtils::trim(sec_code);
position += 6;
stock_abbreviation = std::string(
buffer.begin() + position, buffer.begin() + position + 16);
position += 16;
industry_category = std::string(
buffer.begin() + position, buffer.begin() + position + 2);
// std::cout << "Industry Category (Hex): ";
// for (char c : industry_category_str) {
// std::cout << std::setw(2) << std::setfill('0') << std::hex <<
// (int)(unsigned char)c << " ";
// }
// std::cout << std::endl;
// Convert the ASCII string to an integer
position += 2;
//hex values show pointing at the right place
stock_category = std::string(
buffer.begin() + position, buffer.begin() + position + 2);
// std::cout << "Stock Category (String): " << stock_category << std::endl;
// std::cout << "Stock Category (ASCII Values): ";
// for (char c : stock_category) {
// std::cout << (int)(unsigned char)c << " "; // Prints the ASCII value of each character
// }
// std::cout << std::endl;
// std::cout << "Stock Category (Hex): ";
// for (char c : stock_category) {
// std::cout << std::setw(2) << std::setfill('0') << std::hex <<
// (int)(unsigned char)c << " ";
// }
// std::cout << std::endl;
position += 2;
stock_entries = std::string(
buffer.begin() + position, buffer.begin() + position + 2);
position += 2;
stock_anomaly_code = TwseUtils::decodePackBCD(&buffer[position], 1);
position += 1;
board_code = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
today_currency_price = TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
rise_stop_price = TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
fall_stop_price = TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
non_10_face_value_indicator = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
abnormal_recommendation_indicator = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
abnormal_securities_indicator = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
day_trading_indicator = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
exemption_unchanged_market_margin_sale_indicator = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
exemption_unchanged_market_securities_lending_sale_indicator =
std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
matching_cycle_seconds =
TwseUtils::decodePackBCD(&buffer[position], 3);
position += 3;
warrant_id = std::string(
buffer.begin() + position, buffer.begin() + position + 1);
position += 1;
exercise_strike_price = TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
prev_business_day_exercise_volume =
TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
prev_business_day_cancellation_volume =
TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
issuing_balance_volume =
TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
strike_ratio = TwseUtils::decodePackBCD(&buffer[position], 4);
position += 4;
upper_limit_price = TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
lower_limit_price = TwseUtils::decodePackBCD(&buffer[position], 5);
position += 5;
maturity_date = TwseUtils::decodePackBCD(&buffer[position], 4);
position += 4;
foreign_stock_id = TwseUtils::decodePackBCD(&buffer[position], 1);
position += 1;
trading_unit = TwseUtils::decodePackBCD(&buffer[position], 3);
position += 3;
trading_currency_code = TwseUtils::decodePackBCD(&buffer[position], 3);
position += 3;
market_information_note =
TwseUtils::decodePackBCD(&buffer[position], 1);
position += 1;
}
friend std::ostream &operator<<(
std::ostream &os, const Format_1 &format) {
os << "\nSEC Code: " << format.sec_code << "\n"
<< "Stock Abbreviation: " << format.stock_abbreviation << "\n"
<< "Industry Category: " << format.industry_category << "\n"
<< "Stock Category(can be blank): " << format.stock_category << "\n"
<< "Stock Entries(can be blank): " << format.stock_entries << "\n"
<< "Stock Anomaly code: " << format.stock_anomaly_code << "\n"
<< "Board Code: " << format.board_code << "\n"
<< "Today Currency Price: " << format.today_currency_price << "\n"
<< "Rise Stop Price: " << format.rise_stop_price << "\n"
<< "Fall Stop Price: " << format.fall_stop_price << "\n"
<< "Non-$10 Face Value Indicator: "
<< format.non_10_face_value_indicator << "\n"
<< "Abnormal Recommendation Indicator: "
<< format.abnormal_recommendation_indicator << "\n"
<< "Abnormal Securities Indicator: "
<< format.abnormal_securities_indicator << "\n"
<< "Day Trading Indicator: " << format.day_trading_indicator << "\n"
<< "Exemption of Unchanged Market Margin Sale Indicator: "
<< format.exemption_unchanged_market_margin_sale_indicator << "\n"
<< "Exemption of Unchanged Market Securities Lending Sale "
"Indicator: "
<< format
.exemption_unchanged_market_securities_lending_sale_indicator
<< "\n"
<< "Matching Cycle Seconds: " << format.matching_cycle_seconds
<< "\n"
<< "Warrant ID: " << format.warrant_id << "\n"
<< "Exercise (Strike) Price: " << format.exercise_strike_price
<< "\n"
<< "Previous Business Day Exercise Volume: "
<< format.prev_business_day_exercise_volume << "\n"
<< "Previous Business Day Cancellation Volume: "
<< format.prev_business_day_cancellation_volume << "\n"
<< "Issuing Balance Volume: " << format.issuing_balance_volume
<< "\n"
<< "Strike Ratio: " << format.strike_ratio << "\n"
<< "Upper Limit Price: " << format.upper_limit_price << "\n"
<< "Lower Limit Price: " << format.lower_limit_price << "\n"
<< "Maturity Date: " << format.maturity_date << "\n"
<< "Foreign Stock ID: " << format.foreign_stock_id << "\n"
<< "Trading Unit: " << format.trading_unit << "\n"
<< "Trading Currency Code: " << format.trading_currency_code << "\n"
<< "Market Information Note: " << format.market_information_note
<< "\n";
return os;
}
};
};
} //namespace bts::md::twse
#endif // TWSE_MESSAGES_H
@@ -0,0 +1,13 @@
// TwseSecurityDetailsMapper.h
#ifndef TWSE_SECURITY_DETAILS_MAPPER_H
#define TWSE_SECURITY_DETAILS_MAPPER_H
namespace TWSE {
class TwseSecurityDetailsMapper {
public:
TwseSecurityDetailsMapper() = default;
~TwseSecurityDetailsMapper() = default;
};
}
#endif // TWSE_SECURITY_DETAILS_MAPPER_H
+124
View File
@@ -0,0 +1,124 @@
// TwseUtils.h
#include <boost/date_time/posix_time/ptime.hpp>
#include <boost/date_time/posix_time/time_parsers.hpp>
#include <chrono>
#include <sstream>
#ifndef TWSE_UTILS_H
#define TWSE_UTILS_H
namespace bts::md::twse {
class TwseUtils {
public:
// Function to trim whitespace from a string
static std::string trim(const std::string &str) {
size_t first = str.find_first_not_of(' ');
if (std::string::npos == first) {
return str;
}
size_t last = str.find_last_not_of(' ');
return str.substr(first, (last - first + 1));
}
static std::string getCurrentDate() {
auto now = std::chrono::system_clock::now();
std::time_t now_c = std::chrono::system_clock::to_time_t(now);
std::tm local_tm = *std::localtime(&now_c);
char buffer[11]; // Buffer to hold formatted date
std::strftime(buffer, sizeof(buffer), "%Y-%m-%d", &local_tm);
return std::string(buffer);
}
static std::string convertToISO(
const std::string &match_time,
const std::string &format = "datetime_no_timezone") {
// Ensure match_time is not longer than 15 characters
if (match_time.length() > 15) {
throw std::invalid_argument(
"in function convertToISO: match_time must not be longer than 15 "
"characters: -> " +
match_time);
}
// If match_time is shorter than 12 characters, append '0's to the end (for microseconds)
std::string adjusted_match_time = match_time;
while (adjusted_match_time.length() < 12) {
adjusted_match_time += '0'; // Append '0' for microseconds
}
// Remove non-numeric characters (e.g., ":" and ".")
adjusted_match_time.erase(
std::remove_if(
adjusted_match_time.begin(), adjusted_match_time.end(),
[](char c) { return !std::isdigit(c); }),
adjusted_match_time.end());
// Validate time components
try {
if (adjusted_match_time.length() != 12) {
throw std::invalid_argument(
"Invalid length for time format: " + match_time);
}
std::string hours = adjusted_match_time.substr(0, 2);
std::string minutes = adjusted_match_time.substr(2, 2);
std::string seconds = adjusted_match_time.substr(4, 2);
std::string microseconds =
adjusted_match_time.substr(6, 6); // Ensure this is 6 digits
// Now safely convert to integers
int h = std::abs(std::stoi(hours)) % 24;
int m = std::abs(std::stoi(minutes)) % 60;
int s = std::abs(std::stoi(seconds)) % 60;
// Get the current date dynamically
std::string current_date = getCurrentDate();
// Construct the desired ISO format
std::ostringstream iso_stream;
if (format == "datetime_with_timezone") {
iso_stream << current_date << "T" << std::setfill('0')
<< std::setw(2) << h << ":" << std::setfill('0')
<< std::setw(2) << m << ":" << std::setfill('0')
<< std::setw(2) << s << "." << microseconds
<< "Z"; // Assuming UTC for now
} else if (format == "datetime_no_timezone") {
iso_stream << current_date << "T" << std::setfill('0')
<< std::setw(2) << h << ":" << std::setfill('0')
<< std::setw(2) << m << ":" << std::setfill('0')
<< std::setw(2) << s << "." << microseconds;
} else if (format == "date_only") {
iso_stream << current_date;
} else {
throw std::invalid_argument(
"Unknown format specified for convertToISO: " + format);
}
return iso_stream.str();
} catch (const std::exception &e) {
std::cerr << "Error in convertToISO: " << e.what() << std::endl;
throw;
}
}
static std::string decodePackBCD(const unsigned char *data, size_t length) {
std::stringstream ss;
for (size_t i = 0; i < length; ++i) {
// Extract the upper and lower nibble (4 bits each)
int upper = (data[i] >> 4) & 0x0F;
int lower = data[i] & 0x0F;
// Append both nibbles to the stringstream
ss << upper << lower;
}
std::string result = ss.str();
// Remove leading zeros
// result.erase(0, result.find_first_not_of('0'));
// If the result is empty (all zeros), return "0"
return result.empty() ? "0" : result;
}
};
} //namespace bts::md::twse
#endif // TWSE_UTILS_H