From a0d8dfd3de5b4eab66de90b71a24f276ff05686e Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 31 Jul 2024 15:38:58 +0000 Subject: [PATCH] build: Resolve app data dir and shared mem prefix at runtime We really can't define them at compile time anymore as we build multiple exes that depend on libspark in the same dir (e.g., spark api server, pubsub, ds-publisher, webapp, etc.). That means one libspark.so (or dll) must serve all exes. We should compile a consistent libspark to mitigate the risk of running an exe with the wrong version (of libspark). For example, running pubsub with a libspark configured for spark api server, leftover from an earlier build. --- CMakeLists.txt | 5 +- CMakePresets.json | 33 +++------- .../Bts.DataStreamWebApp.Server/Dockerfile | 2 +- .../DataStreamPublisher/Dockerfile | 2 +- src/DataStreamSuite/PubSubServer/Dockerfile | 2 +- src/lib/cmake/GetLinuxClientSources.cmake | 1 + src/lib/md/ShmSegmentName.cpp | 29 +++++++++ src/lib/md/ShmSegmentName.h | 13 ++-- src/lib/md/shared_memory_prefix.h | 5 ++ src/lib/os/app_data_dir.h | 9 +++ src/lib/os/os.cpp | 63 +++++++++++-------- src/lib/os/os.h | 3 + src/server/sparkd/Dockerfile | 2 +- 13 files changed, 107 insertions(+), 62 deletions(-) create mode 100644 src/lib/md/ShmSegmentName.cpp create mode 100644 src/lib/md/shared_memory_prefix.h create mode 100644 src/lib/os/app_data_dir.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ab6ca60c2..619fe0158 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,9 +37,8 @@ include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zli link_directories(${SPARK_LIB}) add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}") -add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}") -if(DEFINED APP_DATA_DIR_OVERRIDE) - add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}") +if(DEFINED APP_DATA_DIR) + add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}") endif() add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}") if(WIN32) diff --git a/CMakePresets.json b/CMakePresets.json index 25ab101b6..b6739df17 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -20,8 +20,6 @@ "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600" }, "cacheVariables": { - "APP_DATA_DIR": "${dollar}{APPDATA}/bts", - "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}", "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}", "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd", "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT", @@ -35,8 +33,7 @@ "EXPORT_MD_CONSUMER": { "type": "BOOL", "value": "ON" - }, - "SHARED_MEMORY_PREFIX": "spark" + } } }, @@ -52,8 +49,6 @@ "binaryDir": "${sourceDir}/build/${presetName}", "cacheVariables": { "platform": "", - "APP_DATA_DIR": "${dollar}{HOME}/sparkdata", - "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}", "CMAKE_BUILD_WITH_INSTALL_RPATH": { "type": "BOOL", "value": "ON" @@ -70,8 +65,7 @@ "BUILD_ADMIN_WEB_APP": { "type": "BOOL", "value": "ON" - }, - "SHARED_MEMORY_PREFIX": "sparkd" + } } }, @@ -152,11 +146,7 @@ { "name": "spark-api-server-windows", "inherits": "spark", - "displayName": "Spark Api Server (Windows)", - "cacheVariables": { - "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}", - "SHARED_MEMORY_PREFIX": "spark_api_server" - } + "displayName": "Spark Api Server (Windows)" }, { @@ -165,7 +155,6 @@ "displayName": "Spark Api Server (Linux)", "generator": "Ninja Multi-Config", "cacheVariables": { - "APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}", "BUILD_ADMIN_WEB_APP": { "type": "BOOL", "value": "OFF" @@ -178,9 +167,13 @@ "CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG", "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3", + "CMAKE_EXPORT_COMPILE_COMMANDS": { + "type": "BOOL", + "value": "ON" + }, "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", - "DOTNET_BINARY_DIR": "${sourceDir}/build/$/bin/net7.0", + "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$/bin/net7.0", "EXPORT_MD_CONSUMER": { "type": "BOOL", "value": "ON" @@ -218,11 +211,8 @@ "description": "Data Stream utils ported over from Edge", "binaryDir": "${sourceDir}/build/${presetName}", "cacheVariables": { - "APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream", - "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf", - "DOTNET_BINARY_DIR": "${sourceDir}/build/$/bin/net7.0;${sourceDir}/build/$/bin/net7.0-windows", - "SHARED_MEMORY_PREFIX": "data_stream" + "DOTNET_BINARY_DIR": "${sourceDir}/build/$/bin/net7.0;${sourceDir}/build/$/bin/net7.0-windows" } }, @@ -238,8 +228,6 @@ "inherits": "sparkd", "generator": "Ninja Multi-Config", "cacheVariables": { - "APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream", - "APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}", "BUILD_ADMIN_WEB_APP": { "type": "BOOL", "value": "OFF" @@ -262,8 +250,7 @@ "EXPORT_MD_CONSUMER": { "type": "BOOL", "value": "ON" - }, - "SHARED_MEMORY_PREFIX": "data_stream" + } } } ], diff --git a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile index 043e1786b..92786d929 100644 --- a/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile +++ b/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile @@ -7,5 +7,5 @@ ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" RUN dotnet dev-certs https --trust COPY --chown=bts:bts "${SOURCE_DIR}" "bin" COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot" -COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf" +COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf" WORKDIR "/home/bts/bin" diff --git a/src/DataStreamSuite/DataStreamPublisher/Dockerfile b/src/DataStreamSuite/DataStreamPublisher/Dockerfile index acb72c782..1a323f2e0 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Dockerfile +++ b/src/DataStreamSuite/DataStreamPublisher/Dockerfile @@ -12,5 +12,5 @@ RUN apt-get update \ USER bts COPY --chown=bts:bts "${SOURCE_DIR}" "bin" -COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf" +COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf" WORKDIR "/home/bts/bin" diff --git a/src/DataStreamSuite/PubSubServer/Dockerfile b/src/DataStreamSuite/PubSubServer/Dockerfile index 3c50dc5d6..b45188a37 100644 --- a/src/DataStreamSuite/PubSubServer/Dockerfile +++ b/src/DataStreamSuite/PubSubServer/Dockerfile @@ -4,5 +4,5 @@ ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish" ARG CONF_FILE="src/DataStreamSuite/PubSubServer/spark.conf" COPY --chown=bts:bts "${SOURCE_DIR}" "bin" -COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf" +COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/PubSubServer/conf/spark.conf" WORKDIR "/home/bts/bin" diff --git a/src/lib/cmake/GetLinuxClientSources.cmake b/src/lib/cmake/GetLinuxClientSources.cmake index f1ac13e50..f45392840 100644 --- a/src/lib/cmake/GetLinuxClientSources.cmake +++ b/src/lib/cmake/GetLinuxClientSources.cmake @@ -104,6 +104,7 @@ function(get_cpp_sources dest) md/SecurityOracle.cpp md/SecurityStore.cpp md/SettlementRecorder.cpp + md/ShmSegmentName.cpp md/Shm/FixedSecurity.cpp md/SyntheticSecurityService.cpp md/cme/MDP3Utils.cpp diff --git a/src/lib/md/ShmSegmentName.cpp b/src/lib/md/ShmSegmentName.cpp new file mode 100644 index 000000000..8c0eff658 --- /dev/null +++ b/src/lib/md/ShmSegmentName.cpp @@ -0,0 +1,29 @@ +#include "ShmSegmentName.h" +#include "lib/utils/bts_assert.h" +#include "lib/os/os.h" + +namespace bts::md { + +bool ShmSegmentName::prefix_initialized_ = false; +std::string ShmSegmentName::prefix_ = {}; +std::mutex ShmSegmentName::prefix_mx_ = {}; + +std::string ShmSegmentName::prefix() { + // Optimistically check if prefix has been initialized without using a lock + if (!prefix_initialized_) { + // It hasn't been initialized. Use a lock to initialize it + std::lock_guard lock{prefix_mx_}; + // Verify it hasn't been initialized again (since the initial check was done without a lock). + if (!prefix_initialized_) { + if (!os::try_expand_env(SHARED_MEMORY_PREFIX, prefix_)) { + prefix_ = os::process_path().filename().string(); + } + } + bts_assert(!prefix_.empty()); + prefix_initialized_ = true; + } + return prefix_ + "_" + + config::get().shared_memory_namespace(); +} + +} // namespace bts::md diff --git a/src/lib/md/ShmSegmentName.h b/src/lib/md/ShmSegmentName.h index e7680fb8c..e4b788fb5 100644 --- a/src/lib/md/ShmSegmentName.h +++ b/src/lib/md/ShmSegmentName.h @@ -2,8 +2,12 @@ #include "lib/config/config.h" #include "lib/config/protos/market_data.pb.h" #include "lib/domain_model/Security.pb.h" +#include "lib/os/os.h" #include "lib/md/protos/co2.pb.h" +#include "lib/md/shared_memory_prefix.h" + #include +#include #include namespace bts::md { @@ -27,17 +31,16 @@ class ShmSegmentName { api::SecurityType type, api::Venue exchange, const std::string &suffix); + + static bool prefix_initialized_; + static std::string prefix_; + static std::mutex prefix_mx_; }; ////////////////////////////////////////////////// // Definition ////////////////////////////////////////////////// -inline std::string ShmSegmentName::prefix() { - return std::string{SHARED_MEMORY_PREFIX} + "_" + - config::get().shared_memory_namespace(); -} - inline std::string ShmSegmentName::prefix( const std::string &symbol, api::SecurityType type, api::Venue exchange) { return prefix() + "_" + symbol + "_" + api::SecurityType_Name(type) + "_" + diff --git a/src/lib/md/shared_memory_prefix.h b/src/lib/md/shared_memory_prefix.h new file mode 100644 index 000000000..5c8ffeb8d --- /dev/null +++ b/src/lib/md/shared_memory_prefix.h @@ -0,0 +1,5 @@ +#pragma once + +#ifndef SHARED_MEMORY_PREFIX +#define SHARED_MEMORY_PREFIX "${BTS_SHARED_MEMORY_PREFIX}" +#endif diff --git a/src/lib/os/app_data_dir.h b/src/lib/os/app_data_dir.h new file mode 100644 index 000000000..43d8dd047 --- /dev/null +++ b/src/lib/os/app_data_dir.h @@ -0,0 +1,9 @@ +#pragma once + +#ifndef APP_DATA_DIR +#ifdef _WIN32 +#define APP_DATA_DIR "${APPDATA}" +#else +#define APP_DATA_DIR "${HOME}/.local/share" +#endif +#endif diff --git a/src/lib/os/os.cpp b/src/lib/os/os.cpp index c2833d425..df75a0a31 100644 --- a/src/lib/os/os.cpp +++ b/src/lib/os/os.cpp @@ -1,6 +1,7 @@ #include "os.h" #include "lib/config/config.h" #include "lib/config/protos/system_config.pb.h" +#include "lib/os/app_data_dir.h" #include "lib/utils/bts_array_size.h" #include "lib/utils/bts_assert.h" #include "lib/utils/bts_debugger.h" @@ -65,6 +66,8 @@ namespace bts { namespace os { +using Path = boost::filesystem::path; + unsigned int process_id() { #ifdef _WIN32 return GetCurrentProcessId(); @@ -73,6 +76,24 @@ unsigned int process_id() { #endif } +Path process_path() { +#ifdef _WIN32 + char process_name[MAX_PATH]; + if (GetModuleFileName(NULL, process_name, MAX_PATH)) { + return Path(process_name); + } +#else + char process_name[PATH_MAX]; + ssize_t count = readlink("/proc/self/exe", process_name, PATH_MAX); + if (count != -1) { + // Null terminate the string + process_name[count] = '\0'; + return Path(process_name); + } +#endif + bts_assert_msg(false, "Failed to get process name"); +} + bool is_running(unsigned int pid) { #ifdef _WIN32 HANDLE handle = OpenProcess(SYNCHRONIZE, false, pid); @@ -638,7 +659,7 @@ int num_cores() { #endif } -std::string expand_env(std::string name, bool assert_vars_exist = true) { +std::string expand_env(std::string name, bool assert_vars_exist) { static const std::regex env_re{R"--(\$\{([^}]+)\})--"}; std::smatch match; while (std::regex_search(name, match, env_re)) { @@ -657,23 +678,29 @@ std::string expand_env(std::string name, bool assert_vars_exist = true) { } std::optional try_expand_env(const std::string &name) { + std::string result; + if (try_expand_env(name, result)) { + return result; + } + return std::nullopt; +} + +bool try_expand_env(const std::string &name, std::string &result) { + result = name; static const std::regex env_re{R"--(\$\{([^}]+)\})--"}; std::smatch match; - std::string result{name}; while (std::regex_search(result, match, env_re)) { const auto from = match[0]; const auto var_name = match[1].str(); const auto var = std::getenv(var_name.c_str()); if (!var) { - return std::nullopt; + return false; } result.replace(from.first, from.second, var); } - return result; + return true; } -using Path = boost::filesystem::path; - Path app_data_path() { // Since the app data path is expanded from environment variables, we only // compute it once and save the result in a static singleton. This way, even @@ -689,27 +716,9 @@ Path app_data_path() { if (app_data_path_) { return *app_data_path_; } - // APP_DATA_DIR_OVERRIDE is the name of an environment variable which can be used - // to override the default app data dir (%APPDATA% on windows, $HOME on linux). - // The name itself is a variable because this library is built by multiple - // executables, which may run simultaneously on the same machine. For example, - // to run 'sparkd' and 'spark api server' on the same box with different data - // dirs, one might define the environment variables: - // SPARKD_DATA_DIR=~/sparkdata - // SPARK_API_SERVER_DATA_DIR=~/spark_api_data - // In this example, sparkd and the api server were built with (respectively): - // APP_DATA_DIR_OVERRIDE="${SPARKD_DATA_DIR}", - // APP_DATA_DIR_OVERRIDE="${SPARK_API_SERVER_DATA_DIR}", -#ifdef APP_DATA_DIR_OVERRIDE - const auto data_dir = try_expand_env(APP_DATA_DIR_OVERRIDE); - if (data_dir) { - app_data_path_ = Path{*data_dir}; - } -#endif - if (!app_data_path_) { - app_data_path_ = Path{expand_env(APP_DATA_DIR)}; - } - bts_assert(app_data_path_); + std::string base_path; + bts_assert(try_expand_env(APP_DATA_DIR, base_path)); + app_data_path_ = Path{base_path} / "bts" / process_path().filename(); app_data_path_ = app_data_path_->make_preferred(); return *app_data_path_; } diff --git a/src/lib/os/os.h b/src/lib/os/os.h index bbefe2535..bf9db9de3 100644 --- a/src/lib/os/os.h +++ b/src/lib/os/os.h @@ -33,6 +33,7 @@ inline bool is_windows() { } BTS_EXPORT char const *platform_name(); BTS_EXPORT unsigned int process_id(); +boost::filesystem::path process_path(); BTS_EXPORT bool is_running(unsigned int pid); struct ProcInfo { ProcInfo() : pid(0), exe(), args() {} @@ -69,6 +70,8 @@ BTS_EXPORT void enable_core_dumps(); extern "C" BTS_EXPORT int get_random_port(); extern "C" BTS_EXPORT int num_cores(); +std::string expand_env(std::string name, bool assert_vars_exist = true); +bool try_expand_env(const std::string &name, std::string &result); std::string sparkd_wd(); /** diff --git a/src/server/sparkd/Dockerfile b/src/server/sparkd/Dockerfile index a07e06515..785e34165 100644 --- a/src/server/sparkd/Dockerfile +++ b/src/server/sparkd/Dockerfile @@ -6,5 +6,5 @@ RUN apt-get update \ USER "bts" COPY --chown=bts:bts "install/Release/bin" "bin" -COPY --chown=bts:bts "src/server/sparkd/spark.current.conf" "sparkdata/conf/spark.conf" +COPY --chown=bts:bts "src/server/sparkd/spark.current.conf" ".local/share/bts/sparkd/conf/spark.conf" WORKDIR "/home/bts/bin"