Change: moved file paths to config for taifex file replay

This commit is contained in:
Mohammad Arham committed 2024-10-06 08:29:27 +00:00
1 parent 58c5be268f
commit a8f4ec07c8
6 files changed
+1196 -1164

No files matched your search

+3 -9
View File
@@ -10,14 +10,8 @@ option csharp_namespace = "bts.api.md";
message TaifexMarketDataConfig {
uint32 unique_check_value = 1;
bool read_from_file = 20;
//uint32 num_depth_levels = 1;
//uint32 depth_throttle_ms = 2;
string ref_data_file_name = 21;
string realtime_data_file_name = 22;
bts.api.domain.SecurityCriteria security_criteria = 3;
// Market data isn't subscribed for securities listed here (even if they are
// included by 'security_criteria'). There are some securities that don't
// fit into our system just yet, such as 'BTCBIDR', which trades in prices
// larger than our cpp 'domain::Price' class supports.
//repeated string excluded_securities = 4;
//string url_rest_base = 5;
//string url_ws_base = 6;
}
@@ -2,6 +2,7 @@
#include "TaifexMessages.h"
#include "TaifexSecurityDetailsMapper.h"
#include "lib/api/domain/Venue.pb.h"
#include "lib/api/md/TaifexMarketDataConfig.pb.h"
#include "lib/config/config.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/core/TimeStamp.h"
@@ -12,6 +13,7 @@
#include "lib/logging/logging.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/persistence/IDGenerator.h"
#include "lib/utils/bts_assert.h"
namespace bts::md::taifex {
const size_t FOOTER_SIZE = 3;
@@ -230,7 +232,6 @@ void handleParsedMessage(
TaifexUtils::remove_spaces(i083_message.prod_id));
auto dec_loc = security_details->get_dec_loc(
TaifexUtils::remove_spaces(i083_message.prod_id));
LOG(md, Error, "XSEC: {}", i083_message.prod_id);
if (!sec_id.has_value() || !dec_loc.has_value()) {
break;
}
@@ -293,8 +294,8 @@ bool readBinaryFile(
const size_t BUFFER_SIZE = 1024 * 1024;
std::ifstream file(file_name, std::ios::binary);
if (!file) {
std::cerr << "Failed to open file: " << file_name << std::endl;
assert(false);
LOG(md, Error, "File not found: {}",file_name);
bts_assert(false);
}
std::vector<unsigned char> buffer(
@@ -302,7 +303,6 @@ bool readBinaryFile(
std::vector<unsigned char> prev_data(1024); // Stores leftover data
bool start = true;
size_t prev_data_len = 0; // Length of leftover data
int counter = 0;
while (file.read(
reinterpret_cast<char *>(buffer.data() + prev_data_len), BUFFER_SIZE)) {
size_t bytes_read = file.gcount();
@@ -310,21 +310,15 @@ bool readBinaryFile(
break; // No more data to read
}
if (!start) {
counter++;
// Move data from prev_data to the start of the buffer
std::memcpy(buffer.data(), prev_data.data(), prev_data_len);
}
if (counter == 5)
std::cout << "awein";
size_t position = 0;
while (position < (BUFFER_SIZE + prev_data_len - HEADER_SIZE)) {
auto Header = TaifexMessages::Header_Fields(buffer, position);
size_t message_length = Header.Body_Length + HEADER_SIZE + FOOTER_SIZE;
if (position == 0 || position > (BUFFER_SIZE + prev_data_len - 2000)) {
std::cout << "awein";
}
// Check if the full message fits within the buffer
if ((BUFFER_SIZE + prev_data_len - position) >= message_length) {
// Handle the parsed message
@@ -351,14 +345,16 @@ bool readBinaryFile(
//const std::string &filename,
void bts::md::taifex::TaifexDatahandler::run(
bool use_file,
const api::md::TaifexMarketDataConfig &config,
std::shared_ptr<IMarketDataListener> listner,
std::shared_ptr<persistence::IDGenerator> sec_id_store) {
std::string reference_file_name = "reference_data_20240806_1";
std::string realtime_file_name = "real_time_data_20240806_274";
std::shared_ptr<TaifexSecurityDetailsMapper> sec_details =
std::make_shared<TaifexSecurityDetailsMapper>();
if (use_file) {
if (config.read_from_file()) {
bts_assert(!config.ref_data_file_name().empty());
bts_assert(!config.realtime_data_file_name().empty());
std::string reference_file_name = config.ref_data_file_name();//"reference_data_20240806_1";
std::string realtime_file_name = config.realtime_data_file_name();//"real_time_data_20240806_274";
std::string base_path =
config::get<bts::config::SystemConfig>().data_path();
readBinaryFile(
@@ -1,5 +1,6 @@
#pragma once
#include "lib/api/md/TaifexMarketDataConfig.pb.h"
#include <memory>
namespace bts::persistence {
@@ -15,7 +16,7 @@ namespace bts::md::taifex {
class TaifexDatahandler {
public:
static void run(
bool use_file,
const api::md::TaifexMarketDataConfig &config,
std::shared_ptr<IMarketDataListener> listner,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
};
File diff suppressed because it is too large. Load diff
@@ -1,7 +1,7 @@
#include "lib/domain/SecurityID.h"
#include <optional>
#include <string>
#include <sstream>
#include <string>
namespace bts::md::taifex {
class TaifexSecurityDetailsMapper {
struct security_details {
@@ -88,4 +88,4 @@ class TaifexSecurityDetailsMapper {
std::unordered_map<std::string, security_details> sec_deets_map;
};
}//bts::md::taifex
} // namespace bts::md::taifex
+266 -254
View File
@@ -9,300 +9,312 @@
namespace bts::md::taifex {
class TaifexUtils {
public:
public:
static void printBuffer(
const std::vector<unsigned char> &buffer,
size_t offset = 0,
size_t length = std::numeric_limits<size_t>::max()) {
// Adjust length if it exceeds buffer size
if (length == std::numeric_limits<size_t>::max() ||
offset + length > buffer.size()) {
length = buffer.size() - offset;
}
if (offset >= buffer.size())
offset = buffer.size() - 1;
for (size_t i = 0; i < length; ++i) {
printf(
" offset: %zu, i: %zu, value: %.2X|", offset, i,
(unsigned int)buffer[offset + i]);
}
// std::cout << std::dec << std::endl;
}
//Replace std::localtime with localtime_s (use localtime_r for POSIX)
static std::string getCurrentDate() {
auto now = std::chrono::system_clock::now();
std::time_t now_c = std::chrono::system_clock::to_time_t(now);
std::tm local_tm = *std::localtime(&now_c);
char buffer[11]; // Buffer to hold formatted date
std::strftime(buffer, sizeof(buffer), "%Y-%m-%d", &local_tm);
return std::string(buffer);
}
static boost::posix_time::ptime convertToPTime(const std::string &match_time) {
if (match_time.length() != 12) {
throw std::invalid_argument(
"match_time must be exactly 12 characters long");
const std::vector<unsigned char> &buffer,
size_t offset = 0,
size_t length = std::numeric_limits<size_t>::max()) {
// Adjust length if it exceeds buffer size
if (length == std::numeric_limits<size_t>::max() ||
offset + length > buffer.size()) {
length = buffer.size() - offset;
}
if (offset >= buffer.size())
offset = buffer.size() - 1;
for (size_t i = 0; i < length; ++i) {
printf(
" offset: %zu, i: %zu, value: %.2X|", offset, i,
(unsigned int)buffer[offset + i]);
}
// std::cout << std::dec << std::endl;
}
// Extract hours, minutes, seconds, and microseconds from match_time
std::string hours = match_time.substr(0, 2);
std::string minutes = match_time.substr(2, 2);
std::string seconds = match_time.substr(4, 2);
std::string microseconds = match_time.substr(6, 6);
//Replace std::localtime with localtime_s (use localtime_r for POSIX)
static std::string getCurrentDate() {
auto now = std::chrono::system_clock::now();
std::time_t now_c = std::chrono::system_clock::to_time_t(now);
std::tm local_tm = *std::localtime(&now_c);
// Get the current date
std::string date = getCurrentDate();
// Combine the date and time
std::string datetime = date + " " + hours + ":" + minutes + ":" + seconds;
// Convert to ptime
boost::posix_time::ptime pt = boost::posix_time::time_from_string(datetime);
// Add fractional seconds (microseconds)
boost::posix_time::ptime result =
pt + boost::posix_time::microseconds(std::stoi(microseconds));
return result;
}
static std::string convertToISO(
const std::string &match_time,
const std::string &format = "datetime_no_timezone") {
if (match_time.length() != 12) {
throw std::invalid_argument(
"in function convertToISO: match_time must be exactly 12 characters "
"long: -> " +
match_time);
char buffer[11]; // Buffer to hold formatted date
std::strftime(buffer, sizeof(buffer), "%Y-%m-%d", &local_tm);
return std::string(buffer);
}
try {
// Extract time components
static boost::posix_time::ptime convertToPTime(
const std::string &match_time) {
if (match_time.length() != 12) {
throw std::invalid_argument(
"match_time must be exactly 12 characters long");
}
// Extract hours, minutes, seconds, and microseconds from match_time
std::string hours = match_time.substr(0, 2);
std::string minutes = match_time.substr(2, 2);
std::string seconds = match_time.substr(4, 2);
std::string microseconds = match_time.substr(6, 6);
// Validate and sanitize time components
int h = std::abs(std::stoi(hours)) % 24;
int m = std::abs(std::stoi(minutes)) % 60;
int s = std::abs(std::stoi(seconds)) % 60;
// Get the current date
std::string date = getCurrentDate();
// Get the current date dynamically
std::string current_date = getCurrentDate();
// Combine the date and time
std::string datetime = date + " " + hours + ":" + minutes + ":" + seconds;
// Construct the desired ISO format
std::ostringstream iso_stream;
if (format == "datetime_with_timezone") {
// Full date-time with time zone: "YYYY-MM-DDTHH:MM:SS+02:00" (for simplicity using UTC "Z")
iso_stream << current_date << "T" << std::setfill('0') << std::setw(2)
<< h << ":" << std::setfill('0') << std::setw(2) << m << ":"
<< std::setfill('0') << std::setw(2) << s
<< "Z"; // Assuming UTC for now
} else if (format == "datetime_no_timezone") {
// Date-time without time zone: "YYYY-MM-DDTHH:MM:SS"
iso_stream << current_date << "T" << std::setfill('0') << std::setw(2)
<< h << ":" << std::setfill('0') << std::setw(2) << m << ":"
<< std::setfill('0') << std::setw(2) << s;
} else if (format == "date_only") {
// Date only: "YYYY-MM-DD"
iso_stream << current_date;
} else {
// Convert to ptime
boost::posix_time::ptime pt =
boost::posix_time::time_from_string(datetime);
// Add fractional seconds (microseconds)
boost::posix_time::ptime result =
pt + boost::posix_time::microseconds(std::stoi(microseconds));
return result;
}
static std::string convertToISO(
const std::string &match_time,
const std::string &format = "datetime_no_timezone") {
if (match_time.length() != 12) {
throw std::invalid_argument(
"Unknown format specified for convertToISO: " + format);
"in function convertToISO: match_time must be exactly 12 "
"characters "
"long: -> " +
match_time);
}
return iso_stream.str();
} catch (const std::exception &e) {
std::cerr << "Error in convertToISO: " << e.what() << std::endl;
throw;
}
}
try {
// Extract time components
std::string hours = match_time.substr(0, 2);
std::string minutes = match_time.substr(2, 2);
std::string seconds = match_time.substr(4, 2);
static std::string sanitizeISOString(const std::string &isoString) {
std::string result = isoString;
// Validate and sanitize time components
int h = std::abs(std::stoi(hours)) % 24;
int m = std::abs(std::stoi(minutes)) % 60;
int s = std::abs(std::stoi(seconds)) % 60;
// Remove any whitespace
result.erase(
std::remove_if(result.begin(), result.end(), ::isspace), result.end());
// Get the current date dynamically
std::string current_date = getCurrentDate();
// Ensure it's at least 20 characters long (e.g., YYYY-MM-DDTHH:MM:SS)
if (result.size() != 20) {
if (result.size() < 20) {
// If it's too short, pad with zeros at the end (assuming it's missing time components)
result.append(20 - result.size(), '0');
} else {
// If it's too long, truncate to 20 characters
result = result.substr(0, 20);
}
}
// Check for and correct any time zone issues or missing Z (UTC indicator)
size_t posT = result.find('T');
if (posT != std::string::npos) {
size_t posZ = result.find('Z', posT);
if (posZ == std::string::npos) {
size_t posMinus = result.find('-', posT);
size_t posPlus = result.find('+', posT);
if (posMinus != std::string::npos && posMinus > posT) {
// Check if it's "-0" or "-00" or "-000", etc.
if (std::all_of(
result.begin() + posMinus + 1, result.end(),
[](char c) { return c == '0'; })) {
result = result.substr(0, posMinus) + "Z";
}
} else if (posPlus != std::string::npos && posPlus > posT) {
// Check if it's "+0" or "+00" or "+000", etc.
if (std::all_of(
result.begin() + posPlus + 1, result.end(),
[](char c) { return c == '0'; })) {
result = result.substr(0, posPlus) + "Z";
}
// Construct the desired ISO format
std::ostringstream iso_stream;
if (format == "datetime_with_timezone") {
// Full date-time with time zone: "YYYY-MM-DDTHH:MM:SS+02:00" (for simplicity using UTC "Z")
iso_stream << current_date << "T" << std::setfill('0')
<< std::setw(2) << h << ":" << std::setfill('0')
<< std::setw(2) << m << ":" << std::setfill('0')
<< std::setw(2) << s << "Z"; // Assuming UTC for now
} else if (format == "datetime_no_timezone") {
// Date-time without time zone: "YYYY-MM-DDTHH:MM:SS"
iso_stream << current_date << "T" << std::setfill('0')
<< std::setw(2) << h << ":" << std::setfill('0')
<< std::setw(2) << m << ":" << std::setfill('0')
<< std::setw(2) << s;
} else if (format == "date_only") {
// Date only: "YYYY-MM-DD"
iso_stream << current_date;
} else {
// If there's no timezone indicator, assume UTC and add 'Z'
result += "Z";
throw std::invalid_argument(
"Unknown format specified for convertToISO: " + format);
}
return iso_stream.str();
} catch (const std::exception &e) {
std::cerr << "Error in convertToISO: " << e.what() << std::endl;
throw;
}
}
static std::string sanitizeISOString(const std::string &isoString) {
std::string result = isoString;
// Remove any whitespace
result.erase(
std::remove_if(result.begin(), result.end(), ::isspace),
result.end());
// Ensure it's at least 20 characters long (e.g., YYYY-MM-DDTHH:MM:SS)
if (result.size() != 20) {
if (result.size() < 20) {
// If it's too short, pad with zeros at the end (assuming it's missing time components)
result.append(20 - result.size(), '0');
} else {
// If it's too long, truncate to 20 characters
result = result.substr(0, 20);
}
}
}
return result;
}
static std::string time_decodePackBCD(const unsigned char *data, size_t length) {
std::stringstream ss;
for (size_t i = 0; i < length; ++i) {
int upper = (data[i] >> 4) & 0x0F;
int lower = data[i] & 0x0F;
// Check if the decoded values are valid digits
if (upper > 9 || lower > 9) {
// Instead of throwing an error, replace invalid digits with '0'
upper = upper > 9 ? 0 : upper;
lower = lower > 9 ? 0 : lower;
// Check for and correct any time zone issues or missing Z (UTC indicator)
size_t posT = result.find('T');
if (posT != std::string::npos) {
size_t posZ = result.find('Z', posT);
if (posZ == std::string::npos) {
size_t posMinus = result.find('-', posT);
size_t posPlus = result.find('+', posT);
if (posMinus != std::string::npos && posMinus > posT) {
// Check if it's "-0" or "-00" or "-000", etc.
if (std::all_of(
result.begin() + posMinus + 1, result.end(),
[](char c) { return c == '0'; })) {
result = result.substr(0, posMinus) + "Z";
}
} else if (posPlus != std::string::npos && posPlus > posT) {
// Check if it's "+0" or "+00" or "+000", etc.
if (std::all_of(
result.begin() + posPlus + 1, result.end(),
[](char c) { return c == '0'; })) {
result = result.substr(0, posPlus) + "Z";
}
} else {
// If there's no timezone indicator, assume UTC and add 'Z'
result += "Z";
}
}
}
ss << upper << lower;
return result;
}
std::string result = ss.str();
static std::string time_decodePackBCD(
const unsigned char *data, size_t length) {
std::stringstream ss;
for (size_t i = 0; i < length; ++i) {
int upper = (data[i] >> 4) & 0x0F;
int lower = data[i] & 0x0F;
// Ensure the result is exactly 12 characters long (hhmmssxxxxxx)
if (result.length() < 12) {
// If less than 12 characters, pad with zeros at the end
result = result + std::string(12 - result.length(), '0');
} else if (result.length() > 12) {
// If more than 12 characters, keep the first 12
result = result.substr(0, 12);
// Check if the decoded values are valid digits
if (upper > 9 || lower > 9) {
// Instead of throwing an error, replace invalid digits with '0'
upper = upper > 9 ? 0 : upper;
lower = lower > 9 ? 0 : lower;
}
ss << upper << lower;
}
std::string result = ss.str();
// Ensure the result is exactly 12 characters long (hhmmssxxxxxx)
if (result.length() < 12) {
// If less than 12 characters, pad with zeros at the end
result = result + std::string(12 - result.length(), '0');
} else if (result.length() > 12) {
// If more than 12 characters, keep the first 12
result = result.substr(0, 12);
}
// Ensure the time part (hhmmss) is valid
int hours = std::stoi(result.substr(0, 2));
int minutes = std::stoi(result.substr(2, 2));
int seconds = std::stoi(result.substr(4, 2));
hours = std::min(23, std::max(0, hours));
minutes = std::min(59, std::max(0, minutes));
seconds = std::min(59, std::max(0, seconds));
// Reconstruct the result with valid time
std::stringstream final_ss;
final_ss << std::setfill('0') << std::setw(2) << hours
<< std::setfill('0') << std::setw(2) << minutes
<< std::setfill('0') << std::setw(2) << seconds
<< result.substr(6); // Keep the last 6 characters (xxxxxx) as is
return final_ss.str();
}
// Ensure the time part (hhmmss) is valid
int hours = std::stoi(result.substr(0, 2));
int minutes = std::stoi(result.substr(2, 2));
int seconds = std::stoi(result.substr(4, 2));
// need to change parameters
static std::string decodePackBCD(const unsigned char *data, size_t length) {
std::stringstream ss;
for (size_t i = 0; i < length; ++i) {
// Extract the upper and lower nibble (4 bits each)
int upper = (data[i] >> 4) & 0x0F;
int lower = data[i] & 0x0F;
hours = std::min(23, std::max(0, hours));
minutes = std::min(59, std::max(0, minutes));
seconds = std::min(59, std::max(0, seconds));
// Append both nibbles to the stringstream
ss << upper << lower;
}
// Reconstruct the result with valid time
std::stringstream final_ss;
final_ss << std::setfill('0') << std::setw(2) << hours << std::setfill('0')
<< std::setw(2) << minutes << std::setfill('0') << std::setw(2)
<< seconds
<< result.substr(6); // Keep the last 6 characters (xxxxxx) as is
std::string result = ss.str();
return final_ss.str();
}
// Remove leading zeros
result.erase(0, result.find_first_not_of('0'));
// need to change parameters
static std::string decodePackBCD(const unsigned char *data, size_t length) {
std::stringstream ss;
for (size_t i = 0; i < length; ++i) {
// Extract the upper and lower nibble (4 bits each)
int upper = (data[i] >> 4) & 0x0F;
int lower = data[i] & 0x0F;
// If the result is empty (all zeros), return "0"
return result.empty() ? "0" : result;
}
static std::string extractString(
const std::vector<unsigned char> &buffer,
size_t &position,
size_t length) {
// Extract the substring from the buffer
std::string result(
buffer.begin() + position, buffer.begin() + position + length);
position += length;
// Append both nibbles to the stringstream
ss << upper << lower;
// Log the raw result
return result;
}
std::string result = ss.str();
// Remove leading zeros
result.erase(0, result.find_first_not_of('0'));
// If the result is empty (all zeros), return "0"
return result.empty() ? "0" : result;
}
static std::string extractString(
const std::vector<unsigned char> &buffer, size_t &position, size_t length) {
// Extract the substring from the buffer
std::string result(
buffer.begin() + position, buffer.begin() + position + length);
position += length;
// Log the raw result
return result;
}
// Helper function to extract a BCD value from the buffer and convert to
// unsigned long
static unsigned long extractBCD_STOUL(
const std::vector<unsigned char> &buffer, size_t &position, size_t length) {
std::string bcd_str = decodePackBCD(&buffer[position], length);
position += length;
return std::stoul(bcd_str);
}
static std::string extractBCD_STRING(
const std::vector<unsigned char> &buffer, size_t &position, size_t length) {
std::string bcd_str = decodePackBCD(&buffer[position], length);
position += length;
return bcd_str;
}
// Helper function to extract a single character and convert to bool
static bool extractBool(const std::vector<unsigned char> &buffer, size_t &position) {
bool result = (buffer[position] == 'Y');
position += 1;
return result;
}
static std::string extractBCDDate(
const std::vector<unsigned char> &buffer,
size_t &current_position,
size_t size) {
if (size != 4) {
throw std::invalid_argument("BCD date must be 4 bytes long");
// Helper function to extract a BCD value from the buffer and convert to
// unsigned long
static unsigned long extractBCD_STOUL(
const std::vector<unsigned char> &buffer,
size_t &position,
size_t length) {
std::string bcd_str = decodePackBCD(&buffer[position], length);
position += length;
return std::stoul(bcd_str);
}
static std::string extractBCD_STRING(
const std::vector<unsigned char> &buffer,
size_t &position,
size_t length) {
std::string bcd_str = decodePackBCD(&buffer[position], length);
position += length;
return bcd_str;
}
// Helper function to extract a single character and convert to bool
static bool extractBool(
const std::vector<unsigned char> &buffer, size_t &position) {
bool result = (buffer[position] == 'Y');
position += 1;
return result;
}
std::string date;
for (size_t i = 0; i < size; ++i) {
unsigned char byte = buffer[current_position++];
char high = (byte >> 4) + '0';
char low = (byte & 0x0F) + '0';
date.push_back(high);
date.push_back(low);
static std::string extractBCDDate(
const std::vector<unsigned char> &buffer,
size_t &current_position,
size_t size) {
if (size != 4) {
throw std::invalid_argument("BCD date must be 4 bytes long");
}
std::string date;
for (size_t i = 0; i < size; ++i) {
unsigned char byte = buffer[current_position++];
char high = (byte >> 4) + '0';
char low = (byte & 0x0F) + '0';
date.push_back(high);
date.push_back(low);
}
// Ensure the date is in YYYYMMDD format
if (date.length() != 8) {
throw std::runtime_error("Extracted date is not in YYYYMMDD format");
}
return date;
}
// Ensure the date is in YYYYMMDD format
if (date.length() != 8) {
throw std::runtime_error("Extracted date is not in YYYYMMDD format");
}
return date;
}
static auto remove_spaces(std::string &name) {
std::string removed = "";
const char space = ' ';
for (auto c : name) {
if (c != space) {
removed += c;
}
}
return removed;
};
static auto remove_spaces(std::string &name) {
std::string removed = "";
const char space = ' ';
for (auto c : name) {
if (c != space) {
removed += c;
}
}
return removed;
};
};
} //namespace bts::md::taifex