diff --git a/src/server/sparkd/md/MarketDataGateway.cpp b/src/server/sparkd/md/MarketDataGateway.cpp index 2303c7ba8..757083404 100644 --- a/src/server/sparkd/md/MarketDataGateway.cpp +++ b/src/server/sparkd/md/MarketDataGateway.cpp @@ -36,6 +36,7 @@ #include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h" #include "server/sparkd/md/recording/MDReplayer.h" #include "server/sparkd/md/taifex/TaifexDataGateway.h" +#include "server/sparkd/md/twse/TwseDataGateway.h" #include #include @@ -139,8 +140,9 @@ void MarketDataGateway::run() { bts_assert(me_->md_config.emumba_deribit().size() == 1); _start_emumba_deribit(me_->md_config.emumba_deribit().at(0)); } - start_glbx(me_->md_config); + start_glbx(me_->md_config); start_taifex(me_->md_config); + start_twse(me_->md_config); start_cboe(me_->md_config); _start_replays(me_->md_config); while (true) { @@ -377,6 +379,36 @@ void MarketDataGateway::_start_replays( os::get_thread_id(), os::get_thread_name()); } +void MarketDataGateway::start_twse(const bts::api::md::MarketDataConfig &cfg) { + // twse +// int counter = 0; +// LOG(md, Info, "In start, mdgtway.cpp"); + +// for (const auto &twse : cfg.twse()) { +// counter++; +// auto p = me_->venue_listeners.emplace_back( +// std::make_shared(bts::api::VENUE_TWSE)); +// std::string tid_name = "twse" + std::to_string(counter); +// bts::threading::make_thread(tid_name, [&]() { +// bts::md::twse::TwseDataGateway::run( +// twse, p->listener, me_->sec_id_store); +// }).detach(); +// } + for (const auto &twse : cfg.twse()) { + auto p = me_->venue_listeners.emplace_back( + std::make_shared(bts::api::VENUE_TWSE)); + // Capture `p->listener` and `me_->sec_id_store` by value + auto listener = p->listener; + auto sec_id_store = me_->sec_id_store; + std::string tid_name = "twse"; + bts::threading::make_thread(tid_name, [listener, sec_id_store, twse]() { + bts::md::twse::TwseDataGateway::run( + twse, listener, sec_id_store); + }).detach(); + + } +} + void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) { // taifex for (const auto &taifex : cfg.taifex()) { diff --git a/src/server/sparkd/md/MarketDataGateway.h b/src/server/sparkd/md/MarketDataGateway.h index 22a88d1c6..473b063c5 100644 --- a/src/server/sparkd/md/MarketDataGateway.h +++ b/src/server/sparkd/md/MarketDataGateway.h @@ -39,6 +39,7 @@ class MarketDataGateway { void start_glbx(const bts::api::md::MarketDataConfig &cfg); void start_cboe(const bts::api::md::MarketDataConfig &cfg); void start_taifex(const bts::api::md::MarketDataConfig &cfg); + void start_twse(const bts::api::md::MarketDataConfig &cfg); }; } // namespace bts::md diff --git a/src/server/sparkd/md/twse/TwseDataGateway.cpp b/src/server/sparkd/md/twse/TwseDataGateway.cpp new file mode 100644 index 000000000..366b67547 --- /dev/null +++ b/src/server/sparkd/md/twse/TwseDataGateway.cpp @@ -0,0 +1,663 @@ +#include "TwseDataGateway.h" +#include "TwseMessages.h" +#include "lib/api/domain/SecurityType.pb.h" +#include "lib/domain/Security.h" +#include "lib/domain/TradeRecord.h" +#include "lib/logging/logging.h" +#include "lib/md/IMarketDataListener.h" +#include "lib/persistence/IDGenerator.h" +#include "lib/utils/helper_macros.h" +#include +#include +namespace bts::md::twse { + +using UNPROCESSED_DATA = std::vector; + +bool runOnBinaryFile( + const std::string &file_name, + std::set &securities_set, + const std::unordered_set required_securities, + const std::string &twse_securities_path, + std::shared_ptr &listener, + std::shared_ptr &sec_id_store) { + LOG(md, Diag, "In runOnBinaryFile"); + LOG(md, Diag, "The path: {}", file_name); + LOG(md, Diag, "Absolute path: {}", + boost::filesystem::absolute(file_name).string()); + + std::ifstream file(file_name, std::ios::binary); + if (!file) { + LOG(md, Error, "File not found: {}", file_name); + bts_assert(false); + } + + size_t BUFFER_SIZE = 20000; + [[maybe_unused]] size_t counter = 0; + UNPROCESSED_DATA buffer(BUFFER_SIZE); + + while (file.read((reinterpret_cast(buffer.data())), BUFFER_SIZE)) { + size_t position = 0; + [[maybe_unused]] size_t format_counter = 0; + std::cout << std::dec << std::endl; + while (position < BUFFER_SIZE && buffer[position] != 0x1B) { + ++position; + } + + if (position >= BUFFER_SIZE) { + LOG(md, Error, "Escape code (0x0B) not found in buffer"); + break; + } + + while (position < BUFFER_SIZE - 10) { + LOG(md, Diag, "Esc Code in Hexadecimal: 0x{:02X}, format counter: {}", + static_cast(buffer[position]), format_counter); + std::cout << std::dec; + LOG(md, Diag, "Position: {}", position); + format_counter++; + position++; + + auto Header = TwseMessages::Header_Fields(buffer, position); + std::ostringstream oss; + oss << Header; + std::string headerStr = oss.str(); + // LOG(md, Diag, "Header: {}", headerStr); + //std::cout << "Header: \n" << Header; + if (Header.transmission_format_code == 06) { + + LOG(md, Diag, "We're in Case 06"); + auto format_6 = TwseMessages::Format_6(buffer, position); + std::ostringstream oss; + oss << format_6; + std::string format6Str = oss.str(); + LOG(md, Diag, "Format 6: {}", format6Str); + + + if(format_6.stock_code == "0050"){ + LOG(md, Diag, "0050 book!: {}", format6Str); + } + + domain::Book book; + auto &bids = book.bids(); + auto &asks = book.asks(); + + bts::domain::SecurityID sec_id{sec_id_store->get_or_add( + api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, + format_6.stock_code)}; + + // std::vector trades; + + book.set_security_id(sec_id); + + for (unsigned long i = 0; i < format_6.bids.size(); ++i) { + const auto &entry = format_6.bids[i]; + + auto price = bts::domain::Price::from_double(entry.first / 100.0); + bts::domain::Quantity quantity(entry.second); + + bids[i].price = + price; //checck if the indexing starts from 1 or 0 + bids[i].total_quantity = quantity; + } + + // Similarly for asks + for (unsigned int i = 0; i < format_6.asks.size(); ++i) { + const auto &entry = format_6.asks[i]; + + auto price = bts::domain::Price::from_double(entry.first / 100.0); + bts::domain::Quantity quantity(entry.second); + + asks[i].price = price; + asks[i].total_quantity = quantity; + } + + std::vector trades; + // First, add the real-time quote if present + for (const auto "e : format_6.quotes) { + auto &record = trades.emplace_back(); + record.sec_id = sec_id; + // record.time_stamp = bts::core::TimeStamp::from_iso_string( + // format_6.matching_time); + record.time_stamp = bts::core::TimeStamp::from_iso_string( + TwseUtils::convertToISO(format_6.matching_time)); + + record.trade_quantity = + bts::domain::Quantity(quote.volume_field); + record.trade_price = bts::domain::Price( + stod(quote.price_field) * std::pow(10, 9 - 4)); + } + + std::vector trade_ptrs; + trade_ptrs.reserve(trades.size()); + for (const auto &trade : trades) { + trade_ptrs.push_back(&trade); + } + + std::ostringstream ptr_oss; + ptr_oss << "Trade Pointers: ["; + for (const auto *trade_ptr : trade_ptrs) { + if (trade_ptr) { // Ensure the pointer is not null + ptr_oss << "{Sec ID: " << trade_ptr->sec_id + << ", Timestamp: " << trade_ptr->time_stamp + << ", Quantity: " << trade_ptr->trade_quantity.value() + << ", Price: " << trade_ptr->trade_price + << ", Aggressor Side: " + << ((trade_ptr->agg_side == + domain::AggressorSide::Buy) + ? "Buy" + : "Sell") + << "}, "; + } + } + // std::string ptr_log = ptr_oss.str(); + if (!trade_ptrs.empty()) { + // ptr_log.pop_back(); // Remove trailing comma + // ptr_log.pop_back(); // Remove trailing space + } + // ptr_log += "]"; + + // Log the string + // LOG(md, Info, "TWSE Sending to Listner {}", ptr_log); + + listener->on_book(book); + listener->on_trades(trade_ptrs); + position = position + Header.information_length - 10; + } else if (Header.transmission_format_code == 01) { + LOG(md, Diag, "We're in Case 01"); + // Parse Format 1 message + auto format_1 = TwseMessages::Format_1(buffer, position); + + // Convert Format_1 to string for logging + std::ostringstream oss; + oss << format_1; + std::string format1Str = oss.str(); + LOG(md, Diag, "Format 1: {}", format1Str); + + // Extract security ID + const auto id = format_1.sec_code; + const auto symbol = id; + // LOG(md, Info, "Extracted security ID: {}, Symbol: {}", id, symbol); + + const std::string trimmed_id = TwseUtils::trim(id); + if (required_securities.find(trimmed_id) == + required_securities.end()) { + LOG(md, Diag, "Security {} not in required list. Skipping.", + trimmed_id); + continue; + } + + if (securities_set.find(trimmed_id) != securities_set.end()) { + LOG(md, Diag, "Security {} already processed. Skipping.", id); + } else { + securities_set.insert(trimmed_id); + + // Generate or retrieve SecurityID + bts::domain::SecurityID sec_id{sec_id_store->get_or_add( + api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, + id)}; + // LOG(md, Info, "SecurityID in store: {}", sec_id); + + // Populate security details + bts::domain::Security sec; + sec.security_common.venue = bts::api::domain::VENUE_TWSE; + sec.security_common.id = sec_id; + sec.security_common.exchange_sec_id = id; + sec.security_common.desc = id; + sec.security_common.symbol = symbol; + sec.security_common.asset = id; + sec.security_common.group = id; + + auto get_min_inc = [](const std::string &id) { + // LOG(md, Info, "Getting minimum increment for ID: {}", id); + UNUSED(id); + return 0.1; + }; + + // Set minimum increments + sec.security_common.min_price_increment = get_min_inc(id); + sec.security_common.min_quantity = get_min_inc(id); + sec.security_common.min_quantity_increment = get_min_inc(id); + LOG(md, Info, "Set minimum increments for security."); + + sec.security_common.cash_per_point = stod(format_1.trading_unit); + sec.security_common.currency = format_1.trading_currency_code; + // LOG(md, Info, "Set cash per point: {}, Currency: {}", + // sec.security_common.cash_per_point, + // sec.security_common.currency); + + // Serialize Security to JSON + nlohmann::json j; + to_json(j, sec); + std::string json_str = j.dump() + "\n"; + + // Write JSON to file + std::ofstream outfile(twse_securities_path, std::ios::app); + if (outfile.is_open()) { + outfile << json_str; + outfile.close(); + // LOG(md, Info, "Security data written to file."); + } else { + // LOG(md, Info, "Failed to open securities file for writing."); + } + LOG(md, Diag, "securities_set size: {}", securities_set.size()); + + // Notify listener + std::vector const_security_list; + const_security_list.push_back(&sec); + listener->on_securities(const_security_list); + } + // Update position + position = position + Header.information_length - 10; + } else { + LOG(md, Diag, "Unknown Format"); + auto payload_length = Header.information_length; + position += payload_length - 10; + } + } + return true; + } + return false; +} + +int runLive( + [[maybe_unused]] std::set &securities_set, + [[maybe_unused]] const std::unordered_set required_securities, + [[maybe_unused]] const std::string &twse_securities_path, + std::shared_ptr &listener, + std::shared_ptr &sec_id_store) { + LOG(md, Diag, "In runLive"); + + //live connection + const std::string multicastIp = "224.0.100.100"; + const int port = 10000; + const std::string interfaceIp = "10.102.167.131"; + + // Create a UDP socket + int sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock < 0) { + LOG(md, Error, "Socket creation failed"); + + //perror("socket creation failed"); + return EXIT_FAILURE; + } + LOG(md, Diag, "Socket created successfully"); + + int reuse = 1; + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)) < 0) { + LOG(md, Error, "Failed to set SO_REUSEADDR on socket"); + + // perror("setsockopt(SO_REUSEADDR) failed"); + close(sock); + return EXIT_FAILURE; + } + + LOG(md, Diag, "Socket option SO_REUSEADDR set successfully"); + + struct sockaddr_in localAddr = {}; + localAddr.sin_family = AF_INET; + localAddr.sin_port = htons(port); + localAddr.sin_addr.s_addr = INADDR_ANY; + + if (bind(sock, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { + LOG(md, Error, "Socket bind failed"); + + // perror("bind failed"); + close(sock); + return EXIT_FAILURE; + } + + LOG(md, Diag, "Socket bound successfully to port"); + + // Join multicast group + struct ip_mreq mreq; + mreq.imr_multiaddr.s_addr = inet_addr(multicastIp.c_str()); + mreq.imr_interface.s_addr = inet_addr(interfaceIp.c_str()); + + if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < + 0) { + LOG(md, Error, "Failed to join multicast group"); + + // perror("setsockopt(IP_ADD_MEMBERSHIP) failed"); + close(sock); + return EXIT_FAILURE; + } + + LOG(md, Diag, "Joined multicast group successfully"); + + const size_t BUFFER_SIZE = 1024 * 1024; + std::vector buffer(BUFFER_SIZE); + + LOG(md, Diag, "Entering data reception loop"); + + while (true) { + struct sockaddr_in srcAddr; + socklen_t addrLen = sizeof(srcAddr); + size_t bytesRead = recvfrom( + sock, buffer.data(), BUFFER_SIZE, 0, (struct sockaddr *)&srcAddr, + &addrLen); + if (bytesRead < 0) { + std::string err = strerror(errno); + LOG(md, Error, "recvfrom failed: {}", err); + break; + } + // LOG(md, Info, "Received {} bytes from {}:{}", bytesRead, + // inet_ntoa(srcAddr.sin_addr), ntohs(srcAddr.sin_port)); + // Parse received data + size_t position = 0; + size_t format_counter = 0; + + while (position < bytesRead && buffer[position] != 0x1B) { + ++position; // Search for escape code (0x1B) + } + + if (position >= bytesRead) { + // std::cerr << "Escape code (0x1B) not found in buffer" << std::endl; + continue; + } + + while (position < bytesRead - 10) { + std::ostringstream sink; + sink << "Escape Code in Hexadecimal: 0x" << std::hex + << std::setw(2) << std::setfill('0') + << static_cast(buffer[position]) << " for format #" + << format_counter << std::endl; + + ++format_counter; + ++position; + + auto Header = TwseMessages::Header_Fields(buffer, position); + std::ostringstream oss; + oss << Header; + std::string headerStr = oss.str(); + LOG(md, Diag, "Header: {}", headerStr); + + if (Header.transmission_format_code == 6) { + LOG(md, Info, "Book Message --> Case 06"); + LOG(md, Diag, "Book-->Case 06"); + auto format_6 = TwseMessages::Format_6(buffer, position); + std::ostringstream oss; + oss << format_6; + std::string format6Str = oss.str(); + LOG(md, Diag, "Format 6: {}", format6Str); + if(format_6.stock_code == "0050"){ + // LOG(md, Diag, "0050 book!: {}", format6Str); + } + domain::Book book; + auto &bids = book.bids(); + auto &asks = book.asks(); + + bts::domain::SecurityID sec_id{sec_id_store->get_or_add( + api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, + format_6.stock_code)}; + + // std::vector trades; + + book.set_security_id(sec_id); + + for (unsigned long i = 0; i < format_6.bids.size(); ++i) { + const auto &entry = format_6.bids[i]; + + auto price = bts::domain::Price::from_double(entry.first / 100.0); + bts::domain::Quantity quantity(entry.second); + + bids[i].price = + price; //checck if the indexing starts from 1 or 0 + bids[i].total_quantity = quantity; + } + + // Similarly for asks + for (unsigned int i = 0; i < format_6.asks.size(); ++i) { + const auto &entry = format_6.asks[i]; + + auto price = bts::domain::Price::from_double(entry.first / 100.0); + bts::domain::Quantity quantity(entry.second); + + asks[i].price = price; + asks[i].total_quantity = quantity; + } + + std::vector trades; + // First, add the real-time quote if present + for (const auto "e : format_6.quotes) { + auto &record = trades.emplace_back(); + record.sec_id = sec_id; + // record.time_stamp = bts::core::TimeStamp::from_iso_string( + // format_6.matching_time); + record.time_stamp = bts::core::TimeStamp::from_iso_string( + TwseUtils::convertToISO(format_6.matching_time)); + + record.trade_quantity = + bts::domain::Quantity(quote.volume_field); + record.trade_price = bts::domain::Price( + stod(quote.price_field) * std::pow(10, 9 - 4)); + } + + std::vector trade_ptrs; + trade_ptrs.reserve(trades.size()); + for (const auto &trade : trades) { + trade_ptrs.push_back(&trade); + } + + std::ostringstream ptr_oss; + ptr_oss << "Trade Pointers: ["; + for (const auto *trade_ptr : trade_ptrs) { + if (trade_ptr) { // Ensure the pointer is not null + ptr_oss << "{Sec ID: " << trade_ptr->sec_id + << ", Timestamp: " << trade_ptr->time_stamp + << ", Quantity: " << trade_ptr->trade_quantity.value() + << ", Price: " << trade_ptr->trade_price + << ", Aggressor Side: " + << ((trade_ptr->agg_side == domain::AggressorSide::Buy) ? "Buy" : "Sell") + << "}, "; + } + } + std::string ptr_log = ptr_oss.str(); + if (!trade_ptrs.empty()) { + ptr_log.pop_back(); // Remove trailing comma + ptr_log.pop_back(); // Remove trailing space + } + ptr_log += "]"; + + // Log the string + LOG(md, Diag, "TWSE Sending to Listner {}", ptr_log); + + listener->on_book(book); + listener->on_trades(trade_ptrs); + position += Header.information_length - 10; + } else if (Header.transmission_format_code == 1) { + LOG(md, Info, "Case 01 --> Securities Message"); + LOG(md, Diag, "We're in Case 01"); + // Parse Format 1 message + auto format_1 = TwseMessages::Format_1(buffer, position); + + // Convert Format_1 to string for logging + std::ostringstream oss; + oss << format_1; + std::string format1Str = oss.str(); + LOG(md, Diag, "Format 1: {}", format1Str); + + // Extract security ID + const auto id = format_1.sec_code; + const auto symbol = id; + LOG(md, Diag, "Extracted security ID: {}, Symbol: {}", id, symbol); + + const std::string trimmed_id = TwseUtils::trim(id); + if (required_securities.find(trimmed_id) == + required_securities.end()) { + LOG(md, Diag, "Security {} not in required list. Skipping.", + trimmed_id); + continue; + } + + if (securities_set.find(trimmed_id) != securities_set.end()) { + LOG(md, Diag, "Security {} already processed. Skipping.", id); + } else { + securities_set.insert(trimmed_id); + + // Generate or retrieve SecurityID + bts::domain::SecurityID sec_id{sec_id_store->get_or_add( + api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, + id)}; + // LOG(md, Info, "SecurityID in store: {}", sec_id); + // Populate security details + bts::domain::Security sec; + sec.security_common.venue = bts::api::domain::VENUE_TWSE; + sec.security_common.id = sec_id; + sec.security_common.exchange_sec_id = id; + sec.security_common.desc = id; + sec.security_common.symbol = symbol; + sec.security_common.asset = id; + sec.security_common.group = id; + + // Helper function for minimum increment + auto get_min_inc = [](const std::string &id) { + // LOG(md, Info, "Getting minimum increment for ID: {}", id); + UNUSED(id); + return 0.1; + }; + // Populate other fields + sec.security_common.min_price_increment = get_min_inc(id); + sec.security_common.min_quantity = get_min_inc(id); + sec.security_common.min_quantity_increment = get_min_inc(id); + // LOG(md, Info, "Set minimum increments for security."); + + sec.security_common.cash_per_point = stod(format_1.trading_unit); + sec.security_common.currency = format_1.trading_currency_code; + // LOG(md, Info, "Set cash per point: {}, Currency: {}", + // sec.security_common.cash_per_point, + // sec.security_common.currency); + + // Serialize Security to JSON + nlohmann::json j; + to_json(j, sec); + std::string json_str = j.dump() + "\n"; + + // Write JSON to file + std::ofstream outfile(twse_securities_path, std::ios::app); + if (outfile.is_open()) { + outfile << json_str; + outfile.close(); + // LOG(md, Info, "Security data written to file."); + } else { + // LOG(md, Info, "Failed to open securities file for writing."); + } + + std::vector const_security_list; + const_security_list.push_back(&sec); + listener->on_securities(const_security_list); + } + position += Header.information_length - 10; + } else { + LOG(md, Diag, "Unknown Format"); + auto payload_length = Header.information_length; + position += payload_length - 10; + } + } + } + close(sock); + return 0; +} + +void loadSecurities( + const std::string &filePath, + std::set &securitiesSet, + std::shared_ptr sec_id_store, + std::vector> &securitiesVector, + std::shared_ptr listener) { + + std::ifstream infile(filePath); + if (!infile.is_open()) { + LOG(md, Error, "Failed to open securities file for reading: {}", + filePath); + return; + } + + std::string line; + while (std::getline(infile, line)) { + try { + nlohmann::json j = nlohmann::json::parse(line); + std::shared_ptr sec = + std::make_shared(); + from_json(j, *sec); + + const std::string id = sec->security_common.exchange_sec_id; + if (securitiesSet.find(id) == securitiesSet.end()) { + // Generate or retrieve SecurityID + bts::domain::SecurityID sec_id{sec_id_store->get_or_add( + api::domain::VENUE_TWSE, api::domain::SECURITY_TYPE_SPOT, id)}; + sec->security_common.id = sec_id; + securitiesSet.insert(id); + securitiesVector.push_back(sec); + LOG(md, Diag, "Loaded security: {}", id); + } else { + LOG(md, Diag, "Duplicate security ID: {}", id); + } + } catch (const nlohmann::json::parse_error &e) { + std::string err = e.what(); + LOG(md, Error, "JSON parse error: {}", err); + } catch (const std::exception &e) { + std::string err = e.what(); + LOG(md, Error, "Error loading security: {}", err); + } + } + infile.close(); + + // Notify listener about the loaded securities + if (!securitiesVector.empty()) { + std::vector securitiesToNotify; + for (const auto &sec : securitiesVector) { + securitiesToNotify.push_back(sec.get()); + } + listener->on_securities(securitiesToNotify); + } +} + +void bts::md::twse::TwseDataGateway::run( + [[maybe_unused]] const api::md::TwseMarketDataConfig &config, + [[maybe_unused]] std::shared_ptr listner, + [[maybe_unused]] std::shared_ptr sec_id_store) { + + LOG(md, Diag, "In run, twsedatagateway.cpp"); + std::string twse_securities_path = + "/home/bts/sparkdata/data/twse_load_securities.jsonl"; + std::string sec_path = "/home/bts/sparkdata/data/1st_ip_securities_20250116_4"; + std::string book_path = + "/home/bts/sparkdata/data/1st_ip_securities_20250116_27"; + std::set securities_set; + LOG(md, Diag, "securities_set size: {}", securities_set.size()); + + // const std::unordered_set required_securities = { + // "0050", "2317", "2454", "2603", "2615", "00631L", "00633L", + // "00634R", "00637L", "00638R", "00653L", "00654R", "00675L", "00676R", + // "00637L", "006204", "00929", "00919", "006208"}; + +const std::unordered_set required_securities = { + "0050", "2317", "2454", "2603", "2615", "00631L", "00633L", + "00634R", "00637L", "00638R", "00653L", "00654R", "00675L", "00676R", + "00637L", "006204", "00929", "00919", "006208",/* + "6239", "0050", "00703", "00706L", "00707R", "00961", "01001T", "2613", + "0056", "0055", "6239", "00703", "00706L", "00707R", "00961", "01001T", + "2613", "0056", "0055", "00657K", "00690", "00708L", "00712", "00715L", + "00919", "00934", "00938", "00943", "00952", "00685L",*/"054276", "054277", "054278" +}; + + std::vector> securities_vector; + loadSecurities( + twse_securities_path, securities_set, sec_id_store, securities_vector, + listner); + bool run_on_live = true; + if (run_on_live) { + runLive( + securities_set, required_securities, twse_securities_path, listner, + sec_id_store); + } else { + runOnBinaryFile( + sec_path, securities_set, required_securities, twse_securities_path, + listner, sec_id_store); + + runOnBinaryFile( + book_path, securities_set, required_securities, twse_securities_path, + listner, sec_id_store); + } +} + +} // namespace bts::md::twse diff --git a/src/server/sparkd/md/twse/TwseDataGateway.h b/src/server/sparkd/md/twse/TwseDataGateway.h new file mode 100644 index 000000000..4d34091f5 --- /dev/null +++ b/src/server/sparkd/md/twse/TwseDataGateway.h @@ -0,0 +1,27 @@ +#include "lib/api/md/TwseMarketDataConfig.pb.h" +#include // Add this line for std::shared_ptr + + +namespace bts::persistence { +class IDGenerator; +} + +namespace bts::md { +class IMarketDataListener; +} + +namespace bts::md::twse { + + +class TwseDataGateway { +public: + static void run(const api::md::TwseMarketDataConfig &config, + std::shared_ptr listner, + std::shared_ptr sec_id_store); + // (void)config; + // (void)listener; + // (void)sec_id_store; + // // Dummy implementation +}; + +} // namespace bts::md::twse diff --git a/src/server/sparkd/md/twse/TwseDataProcessors.h b/src/server/sparkd/md/twse/TwseDataProcessors.h new file mode 100644 index 000000000..ed3620578 --- /dev/null +++ b/src/server/sparkd/md/twse/TwseDataProcessors.h @@ -0,0 +1,37 @@ +// TwseDataProcessors.h +#ifndef TWSE_DATA_PROCESSORS_H +#define TWSE_DATA_PROCESSORS_H + +#include "TwseMessages.h" +#include "lib/domain/Security.h" +#include "lib/md/IMarketDataListener.h" + +namespace bts::md::twse { +class TwseDataProcessors { + public: + // TwseMessages::Format_1 format_1; + // static bool processBookMessage( + // const TwseMessages::Format_6 &format_6, + // const TwseMessages::Format_1 &format_1, + // std::shared_ptr listener) { + + // auto sec_id = format_1.sec_code; + // if (!sec_id) + // return false; + + // domain::Book book; + // auto &bids = book.bids(); + // auto &asks = book.asks(); + // book.set_security_id(sec_id.value()); + + // for(const auto "e : format_6.quotes){ + // bts::domain::Price price(quote.price_field); + // bts::domain::Quantity quantity(quote.volume_field); + // } + // listener -> on_book(book); + // return true; + // } +}; +} // namespace bts::md::twse + +#endif // TWSE_DATA_PROCESSORS_H diff --git a/src/server/sparkd/md/twse/TwseMessages.h b/src/server/sparkd/md/twse/TwseMessages.h new file mode 100644 index 000000000..8909a384e --- /dev/null +++ b/src/server/sparkd/md/twse/TwseMessages.h @@ -0,0 +1,430 @@ +// TwseMessages.h +#include "TwseUtils.h" +#include "lib/logging/logging.h" +#include +#include +#include +#include +#ifndef TWSE_MESSAGES_H +#define TWSE_MESSAGES_H + +namespace bts::md::twse { +class TwseMessages { + public: + struct Header_Fields { + size_t information_length; + size_t business_type; + size_t transmission_format_code; + size_t transmission_format_version; + size_t transmission_sn; + static const size_t HEADER_SIZE = + 9; // Total size of header including all fields + + Header_Fields(const std::vector &buffer, size_t &offset) { + information_length = + stoull(TwseUtils::decodePackBCD(&buffer[offset], 2)); + business_type = + stoull(TwseUtils::decodePackBCD(&buffer[offset + 2], 1)); + transmission_format_code = + stoull(TwseUtils::decodePackBCD(&buffer[offset + 3], 1)); + transmission_format_version = + stoull(TwseUtils::decodePackBCD(&buffer[offset + 4], 1)); + transmission_sn = + stoull(TwseUtils::decodePackBCD(&buffer[offset + 5], 4)); + + // move offset past header + offset += HEADER_SIZE; + } + friend std::ostream &operator<<( + std::ostream &os, const Header_Fields &hf) { + // std::cout<> bids; + std::vector> asks; + + struct RealTimeQuotes { + std::string price_field; + uint32_t volume_field; + + RealTimeQuotes( + const std::vector &buffer, + const size_t &offset = 0) { + + price_field = TwseUtils::decodePackBCD(&buffer[offset], 5); + volume_field = + stoul(TwseUtils::decodePackBCD(&buffer[offset + 5], 4)); + //offset = offset + 9; + } + + friend std::ostream &operator<<( + std::ostream &os, + const Format_6::RealTimeQuotes &real_time_quotes) { + os << "\n Price Field: " << real_time_quotes.price_field << "\n" + << "Volume Field: " << real_time_quotes.volume_field << "\n"; + return os; + } + }; + std::vector quotes; + // std::vector quotes; + friend std::ostream &operator<<( + std::ostream &os, const Format_6 &format) { + os << "\nStock Code: ." << format.stock_code << ".\n" + << "Matching Time: " << format.matching_time << "\n" + << "Disclosed Item Remarks (Binary): " + << std::bitset<8>(format.disclosed_item_remarks) + << " (Decimal: " << static_cast(format.disclosed_item_remarks) + << ")\n" + << "Rise/Fall Remarks (Binary): " + << std::bitset<8>(format.rise_fall_remarks) + << " (Decimal: " << static_cast(format.rise_fall_remarks) + << ")\n" + << "Status Remarks (Binary): " + << std::bitset<8>(format.status_remarks) + << " (Decimal: " << static_cast(format.status_remarks) << ")\n" + << "Accumulative Trading Volume: " + << format.accumalative_trading_volume << "\n"; + + os << "Real Time Quotes:\n"; + for (const auto "e : format.quotes) { + os << quote; + } + os << "Bids: \n"; + for (const auto &bid : format.bids) { + os << "Price = " << bid.first << " Quantity = " << bid.second + << "\n"; + } + + os << "\nAsks: \n"; + for (const auto &ask : format.asks) { + os << "Price = " << ask.first << " Quantity = " << ask.second + << "\n"; + } + // os << "Quotes:\n"; + // for (const auto &real_time_quotes : format.quotes) { + // os << real_time_quotes; + // } + return os; + } + + Format_6( + const std::vector &buffer, const size_t &offset = 0) { + size_t position = offset; + + stock_code = std::string( + buffer.begin() + position, buffer.begin() + position + 6); + stock_code = TwseUtils::trim(stock_code); + position += 6; + + matching_time = TwseUtils::decodePackBCD(&buffer[position], 6); + position += 6; + + disclosed_item_remarks = buffer[position]; + position += 1; + + rise_fall_remarks = buffer[position]; + position += 1; + + status_remarks = buffer[position]; + position += 1; + + accumalative_trading_volume = + stoul(TwseUtils::decodePackBCD(&buffer[position], 4)); + position += 4; + + if (disclosed_item_remarks & 0x80) { // If Bit 7 is 1 + LOG(md, Diag, "Price Field and Volume Field present"); + // std::cin.ignore(); + auto rtq = RealTimeQuotes(buffer, position); + quotes.push_back(rtq); + } else { + LOG(md, Diag, "No Price Field and Volume Field"); + } + LOG(md, Diag, "Purchasing Price and Volumes"); + + std::cout << std::dec; + // Read Purchase Price and Volume + uint8_t purchasing_count = + (disclosed_item_remarks >> 4) & 0x07; // Get bits 6-4 + for (uint8_t i = 0; i < purchasing_count; ++i) { + std::string purchase_price = + TwseUtils::decodePackBCD(&buffer[position], 5); // Price Field + double bid_price = + std::stod(TwseUtils::decodePackBCD(&buffer[position], 5)); + position += 5; // Move to the volume field + + std::string volume_field = + TwseUtils::decodePackBCD(&buffer[position], 4); // Volume Field + uint32_t bid_volume = + stoul(TwseUtils::decodePackBCD(&buffer[position], 4)); + position += 4; // Move to the next price field + LOG(md, Info, "Purchase Price {}: {} Volume {}: {}", i + 1, + purchase_price, i + 1, volume_field); + bids.emplace_back(bid_price, bid_volume); + } + + // Read Sell Price and Volume if applicable + uint8_t selling_count = + (disclosed_item_remarks >> 1) & 0x07; // Get bits 3-1 + for (uint8_t i = 0; i < selling_count; ++i) { + std::string sell_price = TwseUtils::decodePackBCD( + &buffer[position], 5); // Price Field: 5 bytes + double ask_price = + std::stod(sell_price); // Use the 5-byte decoded value + position += 5; // Move to the volume field + + std::string volume_field = TwseUtils::decodePackBCD( + &buffer[position], 4); // Volume Field: 4 bytes + uint32_t ask_volume = + stoul(volume_field); // Use the 4-byte decoded value + position += 4; // Move to the next price field + + LOG(md, Info, "Sell Price {}: {} Volume {}: {}", i + 1, sell_price, + i + 1, volume_field); + + asks.emplace_back(ask_price, ask_volume); + } + } + }; + + struct Format_1 { + std::string sec_code; + std::string stock_abbreviation; + std::string industry_category; + std::string stock_category; //can be blank + std::string stock_entries; //can be blank + std::string stock_anomaly_code; + std::string board_code; + std::string today_currency_price; + std::string trading_currency_code; + std::string rise_stop_price; + std::string fall_stop_price; + std::string non_10_face_value_indicator; + std::string abnormal_recommendation_indicator; + std::string abnormal_securities_indicator; + std::string day_trading_indicator; + std::string exemption_unchanged_market_margin_sale_indicator; + std::string exemption_unchanged_market_securities_lending_sale_indicator; + std::string matching_cycle_seconds; + std::string warrant_id; + std::string + exercise_strike_price; // This will need to be decoded from PACK BCD + std::string + prev_business_day_exercise_volume; // This will also need PACK BCD decoding + std::string prev_business_day_cancellation_volume; // PACK BCD format + std::string issuing_balance_volume; // PACK BCD format + std::string strike_ratio; // PACK BCD format + std::string upper_limit_price; // PACK BCD format + std::string lower_limit_price; // PACK BCD format + std::string maturity_date; + std::string foreign_stock_id; + std::string trading_unit; // PACK BCD format + std::string trading_currency_code_foreign_stock; + std::string market_information_note; + + Format_1( + const std::vector &buffer, const size_t &offset = 0) { + size_t position = offset; + + sec_code = std::string( + buffer.begin() + position, buffer.begin() + position + 6); + sec_code = TwseUtils::trim(sec_code); + position += 6; + + stock_abbreviation = std::string( + buffer.begin() + position, buffer.begin() + position + 16); + position += 16; + + industry_category = std::string( + buffer.begin() + position, buffer.begin() + position + 2); + + // std::cout << "Industry Category (Hex): "; + // for (char c : industry_category_str) { + // std::cout << std::setw(2) << std::setfill('0') << std::hex << + // (int)(unsigned char)c << " "; + // } + // std::cout << std::endl; + // Convert the ASCII string to an integer + position += 2; + + //hex values show pointing at the right place + stock_category = std::string( + buffer.begin() + position, buffer.begin() + position + 2); + // std::cout << "Stock Category (String): " << stock_category << std::endl; + // std::cout << "Stock Category (ASCII Values): "; + // for (char c : stock_category) { + // std::cout << (int)(unsigned char)c << " "; // Prints the ASCII value of each character + // } + // std::cout << std::endl; + + // std::cout << "Stock Category (Hex): "; + // for (char c : stock_category) { + // std::cout << std::setw(2) << std::setfill('0') << std::hex << + // (int)(unsigned char)c << " "; + // } + // std::cout << std::endl; + position += 2; + + stock_entries = std::string( + buffer.begin() + position, buffer.begin() + position + 2); + position += 2; + + stock_anomaly_code = TwseUtils::decodePackBCD(&buffer[position], 1); + position += 1; + + board_code = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + today_currency_price = TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + rise_stop_price = TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + fall_stop_price = TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + non_10_face_value_indicator = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + abnormal_recommendation_indicator = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + abnormal_securities_indicator = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + day_trading_indicator = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + exemption_unchanged_market_margin_sale_indicator = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + exemption_unchanged_market_securities_lending_sale_indicator = + std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + matching_cycle_seconds = + TwseUtils::decodePackBCD(&buffer[position], 3); + position += 3; + + warrant_id = std::string( + buffer.begin() + position, buffer.begin() + position + 1); + position += 1; + + exercise_strike_price = TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + prev_business_day_exercise_volume = + TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + prev_business_day_cancellation_volume = + TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + issuing_balance_volume = + TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + strike_ratio = TwseUtils::decodePackBCD(&buffer[position], 4); + position += 4; + + upper_limit_price = TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + lower_limit_price = TwseUtils::decodePackBCD(&buffer[position], 5); + position += 5; + + maturity_date = TwseUtils::decodePackBCD(&buffer[position], 4); + position += 4; + + foreign_stock_id = TwseUtils::decodePackBCD(&buffer[position], 1); + position += 1; + + trading_unit = TwseUtils::decodePackBCD(&buffer[position], 3); + position += 3; + + trading_currency_code = TwseUtils::decodePackBCD(&buffer[position], 3); + position += 3; + + market_information_note = + TwseUtils::decodePackBCD(&buffer[position], 1); + position += 1; + } + + friend std::ostream &operator<<( + std::ostream &os, const Format_1 &format) { + os << "\nSEC Code: " << format.sec_code << "\n" + << "Stock Abbreviation: " << format.stock_abbreviation << "\n" + << "Industry Category: " << format.industry_category << "\n" + << "Stock Category(can be blank): " << format.stock_category << "\n" + << "Stock Entries(can be blank): " << format.stock_entries << "\n" + << "Stock Anomaly code: " << format.stock_anomaly_code << "\n" + << "Board Code: " << format.board_code << "\n" + << "Today Currency Price: " << format.today_currency_price << "\n" + << "Rise Stop Price: " << format.rise_stop_price << "\n" + << "Fall Stop Price: " << format.fall_stop_price << "\n" + << "Non-$10 Face Value Indicator: " + << format.non_10_face_value_indicator << "\n" + << "Abnormal Recommendation Indicator: " + << format.abnormal_recommendation_indicator << "\n" + << "Abnormal Securities Indicator: " + << format.abnormal_securities_indicator << "\n" + << "Day Trading Indicator: " << format.day_trading_indicator << "\n" + << "Exemption of Unchanged Market Margin Sale Indicator: " + << format.exemption_unchanged_market_margin_sale_indicator << "\n" + << "Exemption of Unchanged Market Securities Lending Sale " + "Indicator: " + << format + .exemption_unchanged_market_securities_lending_sale_indicator + << "\n" + << "Matching Cycle Seconds: " << format.matching_cycle_seconds + << "\n" + << "Warrant ID: " << format.warrant_id << "\n" + << "Exercise (Strike) Price: " << format.exercise_strike_price + << "\n" + << "Previous Business Day Exercise Volume: " + << format.prev_business_day_exercise_volume << "\n" + << "Previous Business Day Cancellation Volume: " + << format.prev_business_day_cancellation_volume << "\n" + << "Issuing Balance Volume: " << format.issuing_balance_volume + << "\n" + << "Strike Ratio: " << format.strike_ratio << "\n" + << "Upper Limit Price: " << format.upper_limit_price << "\n" + << "Lower Limit Price: " << format.lower_limit_price << "\n" + << "Maturity Date: " << format.maturity_date << "\n" + << "Foreign Stock ID: " << format.foreign_stock_id << "\n" + << "Trading Unit: " << format.trading_unit << "\n" + << "Trading Currency Code: " << format.trading_currency_code << "\n" + << "Market Information Note: " << format.market_information_note + << "\n"; + return os; + } + }; +}; +} //namespace bts::md::twse + +#endif // TWSE_MESSAGES_H diff --git a/src/server/sparkd/md/twse/TwseSecurityDetailsMapper.h b/src/server/sparkd/md/twse/TwseSecurityDetailsMapper.h new file mode 100644 index 000000000..110b28f78 --- /dev/null +++ b/src/server/sparkd/md/twse/TwseSecurityDetailsMapper.h @@ -0,0 +1,13 @@ +// TwseSecurityDetailsMapper.h +#ifndef TWSE_SECURITY_DETAILS_MAPPER_H +#define TWSE_SECURITY_DETAILS_MAPPER_H + +namespace TWSE { + class TwseSecurityDetailsMapper { + public: + TwseSecurityDetailsMapper() = default; + ~TwseSecurityDetailsMapper() = default; + }; +} + +#endif // TWSE_SECURITY_DETAILS_MAPPER_H diff --git a/src/server/sparkd/md/twse/TwseUtils.h b/src/server/sparkd/md/twse/TwseUtils.h new file mode 100644 index 000000000..008e83386 --- /dev/null +++ b/src/server/sparkd/md/twse/TwseUtils.h @@ -0,0 +1,124 @@ +// TwseUtils.h +#include +#include +#include +#include +#ifndef TWSE_UTILS_H +#define TWSE_UTILS_H + +namespace bts::md::twse { +class TwseUtils { + public: + // Function to trim whitespace from a string + static std::string trim(const std::string &str) { + size_t first = str.find_first_not_of(' '); + if (std::string::npos == first) { + return str; + } + size_t last = str.find_last_not_of(' '); + return str.substr(first, (last - first + 1)); + } + static std::string getCurrentDate() { + auto now = std::chrono::system_clock::now(); + std::time_t now_c = std::chrono::system_clock::to_time_t(now); + std::tm local_tm = *std::localtime(&now_c); + + char buffer[11]; // Buffer to hold formatted date + std::strftime(buffer, sizeof(buffer), "%Y-%m-%d", &local_tm); + return std::string(buffer); + } + static std::string convertToISO( + const std::string &match_time, + const std::string &format = "datetime_no_timezone") { + // Ensure match_time is not longer than 15 characters + if (match_time.length() > 15) { + throw std::invalid_argument( + "in function convertToISO: match_time must not be longer than 15 " + "characters: -> " + + match_time); + } + + // If match_time is shorter than 12 characters, append '0's to the end (for microseconds) + std::string adjusted_match_time = match_time; + while (adjusted_match_time.length() < 12) { + adjusted_match_time += '0'; // Append '0' for microseconds + } + + // Remove non-numeric characters (e.g., ":" and ".") + adjusted_match_time.erase( + std::remove_if( + adjusted_match_time.begin(), adjusted_match_time.end(), + [](char c) { return !std::isdigit(c); }), + adjusted_match_time.end()); + + // Validate time components + try { + if (adjusted_match_time.length() != 12) { + throw std::invalid_argument( + "Invalid length for time format: " + match_time); + } + + std::string hours = adjusted_match_time.substr(0, 2); + std::string minutes = adjusted_match_time.substr(2, 2); + std::string seconds = adjusted_match_time.substr(4, 2); + std::string microseconds = + adjusted_match_time.substr(6, 6); // Ensure this is 6 digits + + // Now safely convert to integers + int h = std::abs(std::stoi(hours)) % 24; + int m = std::abs(std::stoi(minutes)) % 60; + int s = std::abs(std::stoi(seconds)) % 60; + + // Get the current date dynamically + std::string current_date = getCurrentDate(); + + // Construct the desired ISO format + std::ostringstream iso_stream; + if (format == "datetime_with_timezone") { + iso_stream << current_date << "T" << std::setfill('0') + << std::setw(2) << h << ":" << std::setfill('0') + << std::setw(2) << m << ":" << std::setfill('0') + << std::setw(2) << s << "." << microseconds + << "Z"; // Assuming UTC for now + } else if (format == "datetime_no_timezone") { + iso_stream << current_date << "T" << std::setfill('0') + << std::setw(2) << h << ":" << std::setfill('0') + << std::setw(2) << m << ":" << std::setfill('0') + << std::setw(2) << s << "." << microseconds; + } else if (format == "date_only") { + iso_stream << current_date; + } else { + throw std::invalid_argument( + "Unknown format specified for convertToISO: " + format); + } + + return iso_stream.str(); + } catch (const std::exception &e) { + std::cerr << "Error in convertToISO: " << e.what() << std::endl; + throw; + } + } + + static std::string decodePackBCD(const unsigned char *data, size_t length) { + std::stringstream ss; + for (size_t i = 0; i < length; ++i) { + // Extract the upper and lower nibble (4 bits each) + int upper = (data[i] >> 4) & 0x0F; + int lower = data[i] & 0x0F; + + // Append both nibbles to the stringstream + ss << upper << lower; + } + + std::string result = ss.str(); + + // Remove leading zeros + // result.erase(0, result.find_first_not_of('0')); + + // If the result is empty (all zeros), return "0" + return result.empty() ? "0" : result; + } +}; +} //namespace bts::md::twse + +#endif // TWSE_UTILS_H