From 98f85b90ee41d7bfb239bed3e2a5ea723b872466 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 12 Oct 2024 15:51:34 +0000 Subject: [PATCH] refactor: Replace ConfigDownloader w/ lib_http fns Essentially all it did was fetch config files via http GET. I ported some http fns over from the cme-websocket-connector repo, which uses curlpp instead of curl, and also supports async requests (coroutines) via the 'concurrencpp' https://github.com/David-Haim/concurrencpp --- externals | 2 +- src/app_sparkd/CMakeLists.txt | 2 - .../md/binance_us/BinanceUsSubscription.cpp | 4 +- .../BinanceUsdsFuturesSubscription.cpp | 4 +- src/lib_http/CMakeLists.txt | 14 +- src/lib_http/http/concurren_runtime.h | 32 ++++ src/lib_http/http/curl_handle.cpp | 22 --- src/lib_http/http/curl_handle.h | 21 --- src/lib_http/http/curl_session.h | 32 ++++ src/lib_http/http/http_client.cpp | 138 +++++++++++++----- src/lib_http/http/http_client.h | 98 ++++++++++--- src/lib_http/http/http_headers.cpp | 44 ------ src/lib_http/http/http_headers.h | 55 +++++-- src/lib_http/http/http_request.cpp | 17 --- src/lib_http/http/http_request.h | 28 ---- src/lib_http/http/http_response.cpp | 26 ---- src/lib_http/http/http_response.h | 91 +++++++++--- src/lib_http/http/i_http_response.h | 35 +++++ src/lib_http/http/query_params.cpp | 18 --- src/lib_http/http/query_params.h | 54 ++++++- .../cmake/GetLinuxServerLinkLibraries.cmake | 9 +- src/lib_spark/spark/config/config.cpp | 22 ++- .../spark/utils/crypto/PrivateKey.cpp | 11 +- .../system/config/config_downloader.cpp | 73 --------- .../system/config/config_downloader.h | 27 ---- .../system/config/configuration_state.cpp | 84 ++++++----- .../system/config/configuration_state.h | 15 +- vcpkg.json | 6 +- 28 files changed, 539 insertions(+), 445 deletions(-) create mode 100644 src/lib_http/http/concurren_runtime.h delete mode 100644 src/lib_http/http/curl_handle.cpp delete mode 100644 src/lib_http/http/curl_handle.h create mode 100644 src/lib_http/http/curl_session.h delete mode 100644 src/lib_http/http/http_headers.cpp delete mode 100644 src/lib_http/http/http_request.cpp delete mode 100644 src/lib_http/http/http_request.h delete mode 100644 src/lib_http/http/http_response.cpp create mode 100644 src/lib_http/http/i_http_response.h delete mode 100644 src/lib_http/http/query_params.cpp delete mode 100644 src/lib_system/system/config/config_downloader.cpp delete mode 100644 src/lib_system/system/config/config_downloader.h diff --git a/externals b/externals index c5861af79..4f5e38555 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit c5861af798dcb4c4f44cccffdd60e24e4bc2f47d +Subproject commit 4f5e3855567b8014c52570142bce0ac4cbe4ab82 diff --git a/src/app_sparkd/CMakeLists.txt b/src/app_sparkd/CMakeLists.txt index 7445f3438..3d16868d7 100644 --- a/src/app_sparkd/CMakeLists.txt +++ b/src/app_sparkd/CMakeLists.txt @@ -142,8 +142,6 @@ target_link_libraries( spark databento::databento ${emumba-deribit-connector-library_LIBRARIES} - ssl - cryptopp tbb onixs-cmemdh-all-4.8.0 OnixS.IceImpactMulticastPriceFeedHandlerCpp diff --git a/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp b/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp index 4fd8146c0..90c006b45 100644 --- a/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp +++ b/src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp @@ -75,7 +75,7 @@ static ccapi::SessionConfigs mk_session_cfg( static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) { // For details on supported opts, read the docs // https://binance-docs.github.io/apidocs/spot/en/#partial-book-depth-streams - QueryParams opts; + std::map opts; switch (cfg.num_depth_levels()) { case 5: case 10: @@ -108,7 +108,7 @@ static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) { break; } - return opts.to_string(); + return http::query_params_string(opts); } static ccapi::Subscription mk_depth_sub( diff --git a/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp b/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp index cb45e48e9..bc79bb3b8 100644 --- a/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp +++ b/src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp @@ -69,7 +69,7 @@ static ccapi::SessionConfigs mk_session_cfg( static string mk_depth_opts(const BinanceUsdsFuturesMarketDataConfig &cfg) { // For details on supported opts, read the docs // https://binance-docs.github.io/apidocs/futures/en/#partial-book-depth-streams - QueryParams opts; + std::map opts; switch (cfg.num_depth_levels()) { case 5: case 10: @@ -103,7 +103,7 @@ static string mk_depth_opts(const BinanceUsdsFuturesMarketDataConfig &cfg) { break; } - return opts.to_string(); + return http::query_params_string(opts); } static ccapi::Subscription mk_depth_sub( diff --git a/src/lib_http/CMakeLists.txt b/src/lib_http/CMakeLists.txt index 0d9b2c85b..0e3e9ce3c 100644 --- a/src/lib_http/CMakeLists.txt +++ b/src/lib_http/CMakeLists.txt @@ -6,12 +6,14 @@ init_library(STATIC) target_link_libraries( ${PROJECT_NAME} PUBLIC - boost_system - boost_filesystem - crypto - curl protobuf - uuid - ssl system ) + +# curlpp +find_package(unofficial-curlpp CONFIG REQUIRED) +target_link_libraries(${PROJECT_NAME} PUBLIC unofficial::curlpp::curlpp) + +# concurrencpp +find_package(concurrencpp CONFIG REQUIRED) +target_link_libraries(${PROJECT_NAME} PUBLIC concurrencpp::concurrencpp) diff --git a/src/lib_http/http/concurren_runtime.h b/src/lib_http/http/concurren_runtime.h new file mode 100644 index 000000000..ac065efb7 --- /dev/null +++ b/src/lib_http/http/concurren_runtime.h @@ -0,0 +1,32 @@ +#pragma once + +#include "system/logging/logging.h" + +#include + +namespace bts::http { + +/** + * Manages the (global) lifetime of curlpp for the program. An atomic reference + * counter is used to synchronize initialization, and eventual termination. + */ +struct ConcurrenRuntime { + explicit ConcurrenRuntime() { + if (reference_count_++ == 0) { + LOG(Gen, Info, "Initializing curl"); + curlpp::initialize(); + } + } + + ~ConcurrenRuntime() { + if (--reference_count_ == 0) { + LOG(Gen, Info, "Terminating curl"); + curlpp::terminate(); + } + } + + private: + static inline std::atomic reference_count_{0}; +}; + +} // namespace bts::http diff --git a/src/lib_http/http/curl_handle.cpp b/src/lib_http/http/curl_handle.cpp deleted file mode 100644 index b45c8e2f9..000000000 --- a/src/lib_http/http/curl_handle.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "http/curl_handle.h" - -#include -#include - -namespace bts::http { - -CurlHandle::CurlHandle() { - const auto eCode = curl_global_init(CURL_GLOBAL_ALL); - if (eCode != CURLE_OK) { - throw std::runtime_error{"Error initializing libCURL"}; - } -} - -CurlHandle::~CurlHandle() { curl_global_cleanup(); } - -CurlHandle &CurlHandle::instance() { - static CurlHandle inst{}; - return inst; -} - -} // namespace bts::http diff --git a/src/lib_http/http/curl_handle.h b/src/lib_http/http/curl_handle.h deleted file mode 100644 index eaef53d54..000000000 --- a/src/lib_http/http/curl_handle.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -namespace bts::http { - -class CurlHandle { - public: - static CurlHandle &instance(); - - CurlHandle(CurlHandle const &) = delete; - CurlHandle(CurlHandle &&) = delete; - - CurlHandle &operator=(CurlHandle const &) = delete; - CurlHandle &operator=(CurlHandle &&) = delete; - - ~CurlHandle(); - - private: - CurlHandle(); -}; - -} // namespace bts::http diff --git a/src/lib_http/http/curl_session.h b/src/lib_http/http/curl_session.h new file mode 100644 index 000000000..a9890f88b --- /dev/null +++ b/src/lib_http/http/curl_session.h @@ -0,0 +1,32 @@ +#pragma once + +#include "system/logging/logging.h" + +#include + +namespace bts::http { + +/** + * Manages the (global) lifetime of curlpp for the program. An atomic reference + * counter is used to synchronize initialization, and eventual termination. + */ +struct CurlSession { + explicit CurlSession() { + if (reference_count_++ == 0) { + LOG(Gen, Info, "Initializing curl"); + curlpp::initialize(); + } + } + + ~CurlSession() { + if (--reference_count_ == 0) { + LOG(Gen, Info, "Terminating curl"); + curlpp::terminate(); + } + } + + private: + static inline std::atomic reference_count_{0}; +}; + +} // namespace bts::http diff --git a/src/lib_http/http/http_client.cpp b/src/lib_http/http/http_client.cpp index 8c4c6c886..88991a396 100644 --- a/src/lib_http/http/http_client.cpp +++ b/src/lib_http/http/http_client.cpp @@ -1,56 +1,116 @@ #include "http_client.h" -#include "http/http_request.h" #include "http/http_response.h" -#include "http/url.h" +#include "http/query_params.h" +#include "system/json/json_dumper.h" #include "system/logging/logging.h" #include -#include +#include +#include +#include +#include +#include +#include namespace bts::http { -static size_t write_data_( - void *ptr, size_t size, size_t nmemb, std::string *data) { - data->append((char *)ptr, size * nmemb); - return size * nmemb; +#define _LOG_REQ_BEGIN(req_type, url, headers) \ + LOG(Gen, Info, "Sending http " #req_type "to {}", url) + +using bts::io::JsonDumper; +using nlohmann::json; +using std::int64_t; +using std::list; +using std::string; +using std::tuple; +using std::unordered_map; + +static void perform(curlpp::Easy &req, IHttpResponse *response) { + if (response != nullptr) { + req.setOpt( + [&response](char *cstr, std::size_t sz, std::size_t n) { + return response->write_headers(cstr, sz, n); + }); + req.setOpt( + [&response](char *cstr, std::size_t sz, std::size_t n) { + return response->write(cstr, sz, n); + }); + } + try { + req.perform(); + } catch (const curlpp::LogicError &ex) { + response->write_errors(ex.what()); + } catch (const curlpp::RuntimeError &ex) { + response->write_errors(ex.what()); + } catch (const std::exception &ex) { + // This isn't a routine http error (those are the 'curlpp' exceptions). + // Log it and rethrow, but don't pass it to the http response. + LOG(Gen, Info, "Exception while performing http request. {}", + std::string{ex.what()}); + throw; + } + response->write_code(curlpp::infos::ResponseCode::get(req)); } -HttpResponse HttpClient::get(const HttpRequest &request) { - auto *curl = curl_easy_init(); - auto url_str = request.url().to_string(); - LOG(md, Info, "do GET: {}", url_str); - curl_easy_setopt(curl, CURLOPT_URL, url_str.c_str()); - - struct curl_slist *request_headers = NULL; - for (const auto &[k, v] : request.headers()) { - request_headers = curl_slist_append( - request_headers, fmt::format("{}: {}", k, v).c_str()); +static void init_request_common( + curlpp::Easy &curl_req, + const string &url, + const unordered_map &headers) { + curl_req.setOpt(url); + if (!headers.empty()) { + list req_hdrs; + for (const auto &[k, v] : headers) { + req_hdrs.push_back(fmt::format("{}:{}", k, v)); + } + curl_req.setOpt(req_hdrs); } + curl_req.setOpt(true); +} - std::string response_string; - std::string header_string; - std::string response_header_string; - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, &header_string); +void HttpClient::post( + const string &url, + const string &body, + const unordered_map &headers, + IHttpResponse *response) { + _LOG_REQ_BEGIN(POST, url, headers); + curlpp::Easy curl_req; + init_request_common(curl_req, url, headers); + curl_req.setOpt(true); + curl_req.setOpt(body); + perform(curl_req, response); +} - char *url; - long response_code; - double elapsed; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); - curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &elapsed); - curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url); - // curl_easy_getinfo(curl, CURLINFO_HEADER_OUT, &response_header_string); +void HttpClient::get( + const string &url, + const unordered_map &headers, + IHttpResponse *response) { + _LOG_REQ_BEGIN(GET, url, headers); + curlpp::Easy curl_req; + init_request_common(curl_req, url, headers); + curl_req.setOpt(true); + perform(curl_req, response); +} - curl_easy_perform(curl); - curl_easy_cleanup(curl); - curl = NULL; - HttpResponse response; - response.headers() = HttpHeaders(response_header_string); - response.body() = std::move(response_string); - response.code() = response_code; - LOG(md, Info, "do GET: {}. Received response:\n{}", url_str, response); - return response; +tuple HttpClient::get_oauth_token( + const string &url, const string &client_id, const string &client_secret) { + LOG(Gen, Info, "Requesting oauth token from {}", url); + std::vector> query_params{ + {"grant_type", "client_credentials"}, + {"client_id", client_id}, + {"client_secret", client_secret}}; + HttpResponse res; + HttpClient::post(url, query_params_string(query_params), {}, &res); + LOG(Gen, Info, "Received oauth token:\n{}", JsonDumper::dump(res)); + try { + const auto j = json::parse(res.body); + return { + j.at("access_token").get(), j.at("token_type").get(), + j.at("expires_in").get()}; + } catch (const std::exception &e) { + LOG(Gen, Error, "Failed to parse oauth response json.{}\n{}", + string{e.what()}, res.body); + } + return {}; } } // namespace bts::http diff --git a/src/lib_http/http/http_client.h b/src/lib_http/http/http_client.h index 026bddf08..c1004e573 100644 --- a/src/lib_http/http/http_client.h +++ b/src/lib_http/http/http_client.h @@ -1,35 +1,85 @@ -#include -#include -#include // std::size_t -#include // snprintf -#include -#include // strerror, strlen, memcpy, strcpy -#include -#include -#include -#include -#include -#include // std::unique_ptr +#pragma once + +#include "http/http_response.h" +#include "http/i_http_response.h" + +#include +#include +#include +#include #include -#include -#include // va_start, etc. -#include -#include #include -#include -#include +#include #include -#include namespace bts::http { -class HttpRequest; -class HttpResponse; - class HttpClient { public: - // HttpClient(const std::string &url); - HttpResponse get(const HttpRequest &request); + /** + * Performs an http GET request + */ + static void get( + const std::string &url, + const std::unordered_map &headers, + IHttpResponse *response); + + static concurrencpp::result get_async( + const std::string &url, + const std::unordered_map &headers); + + static concurrencpp::result get_async( + const std::string &url, + const std::unordered_map &headers, + std::shared_ptr executor); + + /** + * Performs an http POST request + */ + static void post( + const std::string &url, + const std::string &body, + const std::unordered_map &headers, + IHttpResponse *response); + + /** + * Requests an oauth access token + * + * @return tuple of: access token, token type, and seconds until expiration + */ + static std::tuple get_oauth_token( + const std::string &url, + const std::string &client_id, + const std::string &client_secret); + + private: + static std::mutex global_runtime_mx_; + static std::optional global_runtime_; }; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline concurrencpp::result HttpClient::get_async( + const std::string &url, + const std::unordered_map &headers) { + if (!global_runtime_) { + std::lock_guard lock{global_runtime_mx_}; + global_runtime_.emplace(); + } + return get_async(url, headers, global_runtime_->thread_executor()); +} + +inline concurrencpp::result HttpClient::get_async( + const std::string &url, + const std::unordered_map &headers, + std::shared_ptr executor) { + co_return co_await executor->submit([&]() { + HttpResponse res; + get(url, headers, &res); + return res; + }); +} + } // namespace bts::http diff --git a/src/lib_http/http/http_headers.cpp b/src/lib_http/http/http_headers.cpp deleted file mode 100644 index 7e5afd3cc..000000000 --- a/src/lib_http/http/http_headers.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "http_headers.h" -#include "system/string_utils.h" - -#include - -namespace bts::http { - -using bts::system::string_utils::split; - -HttpHeaders::HttpHeaders(const std::string &str) { - auto item_strs = split(str, "\n"); - for (const auto &item_str : item_strs) { - auto kv = split(item_str, ":"); - if (kv.empty()) { - try_emplace("", ""); - continue; - } - if (kv.size() == 1) { - try_emplace(kv[0], ""); - continue; - } - try_emplace(kv[0], kv[1]); - } -} - -std::string HttpHeaders::to_string() const { - auto it = begin(); - if (it == end()) { - return ""; - } - std::string rv{it->first + ": " + it->second}; - while (++it != end()) { - rv.append("\n" + it->first + ": " + it->second); - } - return rv; -} - -std::ostream &operator<<( - std::ostream &out, const HttpHeaders &headers) noexcept { - out << headers.to_string(); - return out; -} - -} // namespace bts::http diff --git a/src/lib_http/http/http_headers.h b/src/lib_http/http/http_headers.h index 694575758..f4628e121 100644 --- a/src/lib_http/http/http_headers.h +++ b/src/lib_http/http/http_headers.h @@ -1,26 +1,51 @@ #pragma once -#include "system/collections/flat_unordered_map.h" +#include "system/string_utils.h" -#include +#include namespace bts::http { -class HttpHeaders - : public collections::FlatUnorderedMap { - public: - HttpHeaders() = default; - explicit HttpHeaders(const std::string &header_str); +std::string headers_to_string( + const std::unordered_map &headers); - HttpHeaders(const HttpHeaders &) noexcept = default; - HttpHeaders(HttpHeaders &&) noexcept = default; - HttpHeaders &operator=(const HttpHeaders &) noexcept = default; - HttpHeaders &operator=(HttpHeaders &&) noexcept = default; +std::unordered_map headers_from_string( + const std::string &headers_str); - std::string to_string() const; -}; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// -std::ostream &operator<<( - std::ostream &out, const HttpHeaders &headers) noexcept; +inline std::string headers_to_string( + const std::unordered_map &headers) { + auto it = headers.begin(); + if (it == headers.end()) { + return ""; + } + std::string rv{it->first + ":" + it->second}; + while (++it != headers.end()) { + rv.append("\n" + it->first + ":" + it->second); + } + return rv; +} + +inline std::unordered_map headers_from_string( + const std::string &str) { + std::unordered_map headers; + auto item_strs = system::string_utils::split(str, "\n"); + for (const auto &item_str : item_strs) { + auto kv = system::string_utils::split(item_str, ":"); + if (kv.empty()) { + headers.try_emplace("", ""); + continue; + } + if (kv.size() == 1) { + headers.try_emplace(kv[0], ""); + continue; + } + headers.try_emplace(kv[0], kv[1]); + } + return headers; +} } // namespace bts::http diff --git a/src/lib_http/http/http_request.cpp b/src/lib_http/http/http_request.cpp deleted file mode 100644 index a7cfe54a1..000000000 --- a/src/lib_http/http/http_request.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "http_request.h" - -namespace bts::http { - -Url &HttpRequest::url() { return url_; } - -const Url &HttpRequest::url() const { return url_; } - -HttpHeaders &HttpRequest::headers() { return headers_; } - -const HttpHeaders &HttpRequest::headers() const { return headers_; } - -std::string &HttpRequest::body() { return body_; } - -const std::string &HttpRequest::body() const { return body_; } - -} // namespace bts::http diff --git a/src/lib_http/http/http_request.h b/src/lib_http/http/http_request.h deleted file mode 100644 index c5856d535..000000000 --- a/src/lib_http/http/http_request.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "http/http_headers.h" -#include "http/url.h" - -#include -#include - -namespace bts::http { - -class HttpRequest { - public: - Url &url(); - const Url &url() const; - - HttpHeaders &headers(); - const HttpHeaders &headers() const; - - std::string &body(); - const std::string &body() const; - - private: - Url url_; - HttpHeaders headers_; - std::string body_; -}; - -} // namespace bts::http diff --git a/src/lib_http/http/http_response.cpp b/src/lib_http/http/http_response.cpp deleted file mode 100644 index e97f66321..000000000 --- a/src/lib_http/http/http_response.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "http_response.h" - -#include - -namespace bts::http { - -HttpHeaders &HttpResponse::headers() { return headers_; } - -const HttpHeaders &HttpResponse::headers() const { return headers_; } - -std::string &HttpResponse::body() { return body_; } - -const std::string &HttpResponse::body() const { return body_; } - -std::int64_t &HttpResponse::code() { return code_; } - -std::int64_t HttpResponse::code() const { return code_; } - -std::ostream &operator<<( - std::ostream &out, const HttpResponse &response) noexcept { - out << response.headers() << "\n" - << response.body() << "\nCode: " << response.code(); - return out; -} - -} // namespace bts::http diff --git a/src/lib_http/http/http_response.h b/src/lib_http/http/http_response.h index 1b407bb3f..d729a02e1 100644 --- a/src/lib_http/http/http_response.h +++ b/src/lib_http/http/http_response.h @@ -1,35 +1,88 @@ #pragma once -#include "http/url.h" -#include "http_headers.h" +#include "http/i_http_response.h" +#include "system/FMT_IMPL.h" +#include +#include #include -#include +#include namespace bts::http { -class HttpResponse { +/** + * Basic implementation of `IHttpResponse`, with some useful state. + */ +class HttpResponse : public IHttpResponse { public: - Url &url(); - const Url &url() const; + std::string url; + std::unordered_map headers; + std::string body; + std::int64_t code; + std::string errors; - HttpHeaders &headers(); - const HttpHeaders &headers() const; + std::size_t write(char *cstr, std::size_t sz, std::size_t n) override; + std::size_t write_headers( + char *cstr, std::size_t sz, std::size_t n) override; + void write_errors(std::string &&str) override; + void write_code(std::int64_t code) override; + constexpr bool ok() const; - std::string &body(); - const std::string &body() const; - - std::int64_t &code(); - std::int64_t code() const; - - private: - Url url_; - HttpHeaders headers_; - std::string body_; - std::int64_t code_; + void reset(); }; +void to_json(nlohmann::json &j, const HttpResponse &res); +void from_json(const nlohmann::json &j, HttpResponse &res); + std::ostream &operator<<( std::ostream &out, const HttpResponse &response) noexcept; +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::size_t HttpResponse::write( + char *cstr, std::size_t sz, std::size_t n) { + body.append(cstr, sz * n); + return sz * n; +} + +inline std::size_t HttpResponse::write_headers( + char *cstr, std::size_t sz, std::size_t n) { + std::string str{cstr, sz * n}; + const auto colon_idx = str.find(':'); + if (colon_idx >= str.size()) { + headers[str] = {}; + } else { + headers[str.substr(0, colon_idx)] = + str.substr(colon_idx + 1, str.size() - colon_idx - 1); + } + return sz * n; +} + +inline void HttpResponse::write_errors(std::string &&str) { + errors = std::move(str); +} + +inline void HttpResponse::write_code(std::int64_t code) { this->code = code; } + +constexpr bool HttpResponse::ok() const { return code >= 200 && code < 300; } + +inline void HttpResponse::reset() { + headers.clear(); + body.clear(); + errors.clear(); + code = 0; +} + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE( + HttpResponse, url, headers, body, code, errors); + +inline std::ostream &operator<<( + std::ostream &out, const HttpResponse &res) noexcept { + return out << nlohmann::json{res}.dump(); +} + } // namespace bts::http + +FMT_IMPL(bts::http::HttpResponse, r, "{}", nlohmann::json{r}.dump()); diff --git a/src/lib_http/http/i_http_response.h b/src/lib_http/http/i_http_response.h new file mode 100644 index 000000000..ae99eab84 --- /dev/null +++ b/src/lib_http/http/i_http_response.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace bts::http { + +/** + * Simple http response interface, modeled after cURL callbacks + */ +class IHttpResponse { + public: + virtual ~IHttpResponse() = default; + + virtual std::size_t write(char *cstr, std::size_t sz, std::size_t n) = 0; + virtual std::size_t write_headers(char *cstr, std::size_t sz, std::size_t n); + virtual void write_errors(std::string &&str); + virtual void write_code(std::int64_t code); +}; + +////////////////////////////////////////////////// +// Definition +////////////////////////////////////////////////// + +inline std::size_t IHttpResponse::write_headers( + char * /*cstr*/, std::size_t sz, std::size_t n) { + return sz * n; +} + +inline void IHttpResponse::write_errors(std::string && /*str*/) {} + +inline void IHttpResponse::write_code(std::int64_t /*code*/) {} + +} // namespace bts::http diff --git a/src/lib_http/http/query_params.cpp b/src/lib_http/http/query_params.cpp deleted file mode 100644 index d2d87062f..000000000 --- a/src/lib_http/http/query_params.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "http/query_params.h" - -namespace bts::http { - -std::string QueryParams::to_string() const { - std::string rv; - auto it = begin(); - if (it == end()) { - return ""; - } - rv.append(it->first + "=" + it->second); - while (++it != end()) { - rv.append("&" + it->first + "=" + it->second); - } - return rv; -} - -} // namespace bts::http diff --git a/src/lib_http/http/query_params.h b/src/lib_http/http/query_params.h index 3bc10539d..4c89011c0 100644 --- a/src/lib_http/http/query_params.h +++ b/src/lib_http/http/query_params.h @@ -1,15 +1,63 @@ #pragma once -#include "system/collections/flat_unordered_map.h" +#include "system/FMT_IMPL.h" +#include #include +#include namespace bts::http { -class QueryParams - : public collections::FlatUnorderedMap { +template +inline std::string query_params_string(Iter first, Iter last) { + std::string rv; + for (auto it = first; it != last; it++) { + if (it->first.empty()) { + continue; + } + if (!rv.empty()) { + rv.append("&"); + } + rv.append(it->first); + if (!it->second.empty()) { + rv.append("="); + rv.append(it->second); + } + } + return rv; +} + +template +inline std::string query_params_string(Range &&range) { + return query_params_string(range.begin(), range.end()); +} + +class QueryParams : public std::unordered_map { public: std::string to_string() const; }; +inline std::string QueryParams::to_string() const { + std::string rv; + auto it = begin(); + if (it == end()) { + return ""; + } + rv.append(it->first + "=" + it->second); + while (++it != end()) { + rv.append("&" + it->first + "=" + it->second); + } + return rv; +} + +inline bool operator==(const QueryParams &lhs, const QueryParams &rhs) { + return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} + +inline bool operator!=(const QueryParams &lhs, const QueryParams &rhs) { + return !(lhs == rhs); +} + } // namespace bts::http + +FMT_IMPL(bts::http::QueryParams, p, "{}", p.to_string()); diff --git a/src/lib_spark/cmake/GetLinuxServerLinkLibraries.cmake b/src/lib_spark/cmake/GetLinuxServerLinkLibraries.cmake index 0db30883e..48493be5f 100644 --- a/src/lib_spark/cmake/GetLinuxServerLinkLibraries.cmake +++ b/src/lib_spark/cmake/GetLinuxServerLinkLibraries.cmake @@ -1,5 +1,7 @@ function(get_link_libraries dest) find_package(fmt CONFIG REQUIRED) + find_package(OpenSSL REQUIRED) + find_package(cryptopp CONFIG REQUIRED) set(${dest} system domain @@ -15,13 +17,12 @@ function(get_link_libraries dest) http SQLiteCpp sqlite3 - curl uuid - ssl - crypto + OpenSSL::SSL + OpenSSL::Crypto fmt::fmt tbb - cryptopp + cryptopp::cryptopp pugixml lttng-ust Pcap++ diff --git a/src/lib_spark/spark/config/config.cpp b/src/lib_spark/spark/config/config.cpp index da845e3ba..c187bbfb2 100644 --- a/src/lib_spark/spark/config/config.cpp +++ b/src/lib_spark/spark/config/config.cpp @@ -2,10 +2,12 @@ #define CONFIG_H_ #include "config.h" +#include "http/http_client.h" +#include "http/http_response.h" +#include "spark/protobufs/config_defaults.h" +#include "spark/protobufs/config_proto_files.h" #include "system/bts_array_size.h" #include "system/config/configuration_state.h" -#include "spark/protobufs/config_proto_files.h" -#include "spark/protobufs/config_defaults.h" #include "system/os.h" #include @@ -43,7 +45,21 @@ internal::ConfigurationState &config_state() { defaults_str.append(line); defaults_str.append("\n"); } - config_state_.emplace(proto_files_vec, defaults_str); + config_state_.emplace( + proto_files_vec, defaults_str, + [](const std::string &url, std::stringstream &out) { + http::HttpResponse res; + http::HttpClient::get(url, {}, &res); + if (!res.ok()) { + return false; + } + // This isn't very efficient. Our HttpResponse populates a string, which is copied + // to a stringstream, which is likely (eventually) dumped back to a string by the + // original caller. + // TODO: avoid copying strings in this pipeline. + out << res.body; + return true; + }); } } return *config_state_; diff --git a/src/lib_spark/spark/utils/crypto/PrivateKey.cpp b/src/lib_spark/spark/utils/crypto/PrivateKey.cpp index 620c13781..0fd0dba07 100644 --- a/src/lib_spark/spark/utils/crypto/PrivateKey.cpp +++ b/src/lib_spark/spark/utils/crypto/PrivateKey.cpp @@ -23,6 +23,9 @@ namespace openssl { namespace bts { namespace crypto { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + struct PrivateKey::Pimp { BTS_NO_COPY(Pimp); @@ -206,7 +209,8 @@ int private_encrypt_func( return -1; } else { return RSA_private_encrypt( - inlen, const_cast(in), out, rsa, RSA_PKCS1_PADDING); + inlen, const_cast(in), out, + const_cast(rsa), RSA_PKCS1_PADDING); } } @@ -223,7 +227,8 @@ int public_decrypt_func( return -1; } else { int i = RSA_public_decrypt( - inlen, const_cast(in), out, rsa, RSA_PKCS1_PADDING); + inlen, const_cast(in), out, + const_cast(rsa), RSA_PKCS1_PADDING); out[i] = '\0'; *outlen = i; return i; @@ -293,6 +298,8 @@ void PrivateKey::write_stream( BIO_free(biostream_out.getBIO()); } +#pragma GCC diagnostic pop + } // namespace crypto } // namespace bts #endif diff --git a/src/lib_system/system/config/config_downloader.cpp b/src/lib_system/system/config/config_downloader.cpp deleted file mode 100644 index 6075fc7cd..000000000 --- a/src/lib_system/system/config/config_downloader.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "config_downloader.h" -#include "system/facades/bts_assert.h" - -#include -#include -#include - -using namespace std; - -namespace bts { -namespace config { -namespace internal { - -ConfigDownloader::ConfigDownloader() : _curl(NULL) { - curl_global_init(CURL_GLOBAL_DEFAULT); - _curl = curl_easy_init(); - bts_assert(_curl); -} - -ConfigDownloader::~ConfigDownloader() { curl_easy_cleanup(_curl); } - -size_t curl_write(char *buffer, size_t size, size_t count, void *stream) { - stringstream *out = static_cast(stream); - bts_assert(out); - size_t bytes = size * count; - out->write(buffer, bytes); - return count; -} - -bool is_http(const string &url) { return url.substr(0, 7) == "http://"; } - -int no_linger(void *, curl_socket_t socket, curlsocktype) { - linger l; - l.l_onoff = 1; - l.l_linger = 0; - setsockopt(socket, SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(l)); - return CURL_SOCKOPT_OK; -} - -ConfigDownloader::Result -ConfigDownloader::get(const string &url, stringstream &out_stream) { - curl_easy_setopt(_curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, curl_write); - curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream); - curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0); - curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 6); - curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger); - - CURLcode res = curl_easy_perform(_curl); - - long http_code = 0; - CURLcode r2 = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &http_code); - bts_assert(r2 == CURLE_OK); - - bool used_http = is_http(url); - if (res == CURLE_OK && (http_code == 200 || !used_http)) { - return SUCCESS; - } else if ( - res == CURLE_OPERATION_TIMEDOUT || res == CURLE_COULDNT_RESOLVE_HOST || - res == CURLE_COULDNT_RESOLVE_PROXY || res == CURLE_COULDNT_CONNECT || - res == CURLE_REMOTE_ACCESS_DENIED || res == CURLE_FTP_CANT_GET_HOST) { - // Cannot rely on logging before config is initialized - // LOG(Error, Config) << "Permanent failure loading '" << url << "' - CURL - // error: " << curl_easy_strerror(res); - return PERMANENT_FAILURE; - } - return FAILURE; -} - -} // namespace internal -} // namespace config -} // namespace bts diff --git a/src/lib_system/system/config/config_downloader.h b/src/lib_system/system/config/config_downloader.h deleted file mode 100644 index 552a5d679..000000000 --- a/src/lib_system/system/config/config_downloader.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include -#include - -namespace bts { -namespace config { -namespace internal { - -class ConfigDownloader { - public: - ConfigDownloader(); - ConfigDownloader(const ConfigDownloader &) = delete; - ConfigDownloader &operator=(const ConfigDownloader &) = delete; - ~ConfigDownloader(); - - enum Result { FAILURE, SUCCESS, PERMANENT_FAILURE }; - - Result get(const std::string &url, std::stringstream &out_stream); - - private: - CURL *_curl; -}; - -} // namespace internal -} // namespace config -} // namespace bts diff --git a/src/lib_system/system/config/configuration_state.cpp b/src/lib_system/system/config/configuration_state.cpp index 4bf8136fe..aadd06558 100644 --- a/src/lib_system/system/config/configuration_state.cpp +++ b/src/lib_system/system/config/configuration_state.cpp @@ -1,23 +1,25 @@ #include "configuration_state.h" // #include "spark/config/config.h" +#include "system/FMT.h" +#include "system/bts_array_size.h" #include "system/config/config_error_collector.h" #include "system/config/config_property_evaluator.h" #include "system/config/config_value_queue.h" #include "system/config/configuration_exception.h" #include "system/config/preprocess_input_stream.h" -#include "system/facades/bts_assert.h" #include "system/defer.h" -#include "system/FMT.h" -#include "system/bts_array_size.h" +#include "system/facades/bts_assert.h" #include "system/os.h" #include "system/protobufs/config/system_config.pb.h" #include #include // iequals #include +#include #include #include #include +#include #include #include #include @@ -68,13 +70,16 @@ const char *CONFIG_SERVER_ENV_VAR = "BTSConfig"; inline tuple hostname(const std::string &path); ConfigurationState::ConfigurationState( - const std::vector &proto_files, const std::string &defaults) + const std::vector &proto_files, + const std::string &defaults, + std::function fetch) : _is_initialized(false), _initialization_mutex(), _dynamic_pool(), _prototype_factory(), _prototype(nullptr), _config_chain(), _defaults(), _properties(new ConfigPropertyEvaluator(*this)), _rule_engine(_properties), _macros(), _deferred_queue(new ConfigValueQueue(*_properties.get())), _host(), - _port(80), proto_files_(proto_files), defaults_(defaults) {} + _port(80), fetch_(fetch), proto_files_(proto_files), defaults_(defaults) { +} ConfigurationState::~ConfigurationState(void) {} @@ -202,9 +207,7 @@ void ConfigurationState::initialize_builtins(ConfigLog *log) { sys.set_num_cores(os::num_cores()); if (log != nullptr) { - CFG_LOG( - (*log), Info, "Initialized SystemConfig:\n{}", - sys.DebugString()); + CFG_LOG((*log), Info, "Initialized SystemConfig:\n{}", sys.DebugString()); } // @@ -416,7 +419,8 @@ int ConfigurationState::load(const std::string &url, ConfigLog &log) { bool loaded = false; CFG_LOG(log, Info, "Loading '{}'", url); loaded = load_message( - url, *new_message.get(), log, *_deferred_queue.get(), proto_files_); + url, *new_message.get(), log, *_deferred_queue.get(), fetch_, + proto_files_); if (_host.empty() || _host == "localhost") { const auto old_host = _host; const auto old_port = _port; @@ -912,18 +916,16 @@ bool load_message( proto::Message &out_message, ConfigLog &log, ConfigValueQueue &lookup, + std::function fetch, const std::vector &proto_files) { - ConfigDownloader downloader; - bool result = false; switch (get_path_type(url)) { case PathType::directory: - result = load_directory( - url, downloader, out_message, log, lookup, proto_files) > 0; + result = + load_directory(url, fetch, out_message, log, lookup, proto_files) > 0; break; case PathType::file: - result = load_file(url, downloader, out_message, log, lookup) == - ConfigDownloader::SUCCESS; + result = load_file(url, fetch, out_message, log, lookup); break; default: std::ostringstream error; @@ -939,7 +941,7 @@ bool load_message( int load_dir( const std::string &url, - ConfigDownloader &downloader, + std::function fetch, proto::Message &out_message, ConfigLog &log, ConfigValueQueue &lookup, @@ -961,7 +963,7 @@ int load_dir( boost::filesystem::path p(config_file); config_file = p.filename().string(); if (load_file( - url + config_file, downloader, out_message, log, lookup, false)) { + url + config_file, fetch, out_message, log, lookup, false)) { load_count++; if (!edge_has_include) { bts_assert_msg( @@ -979,7 +981,7 @@ int load_dir( int load_directory( std::string url, - ConfigDownloader &downloader, + std::function fetch, proto::Message &out_message, ConfigLog &log, ConfigValueQueue &lookup, @@ -993,50 +995,52 @@ int load_directory( // If a file called "all" exists in the config directory we can short circuit // ~60 http requests - ConfigDownloader::Result all_result = - load_file(url + ".all", downloader, out_message, log, lookup, false); - if (all_result == ConfigDownloader::SUCCESS) { + bool success = + load_file(url + ".all", fetch, out_message, log, lookup, false); + if (success) { return 1; } // load spark.conf int load_count = 0; - ConfigDownloader::Result result = load_file( - url + "spark.conf", downloader, out_message, log, lookup, false); - if (result == ConfigDownloader::SUCCESS) { + success = + load_file(url + "spark.conf", fetch, out_message, log, lookup, false); + if (success) { load_count++; - } else if (result == ConfigDownloader::PERMANENT_FAILURE) { - // If the server is down it doesn't make sense to suffer though the - // timeouts for the other files - throw ServerUnreachableException(url.c_str()); } - load_count += - load_dir(url, downloader, out_message, log, lookup, proto_files); + load_count += load_dir(url, fetch, out_message, log, lookup, proto_files); return load_count; } -ConfigDownloader::Result load_file( +bool load_file( std::string path, - ConfigDownloader &downloader, + std::function fetch, proto::Message &out_message, ConfigLog &log, ConfigValueQueue &lookup, bool required) { + // first, try loading local file. + if (boost::filesystem::exists(path)) { + std::ifstream fs{path}; + proto::io::IstreamInputStream proto_stream(&fs); + parse_stream(path, proto_stream, out_message, log, lookup); + return true; + } + bool success = false; stringstream url_stream; string url = url_from_string(path, required).to_string(); - ConfigDownloader::Result result = downloader.get(url, url_stream); - if (!(result == ConfigDownloader::SUCCESS)) { + success = fetch(url, url_stream); + if (!success) { CFG_LOG(log, Info, "Failed to load config from '{}'", url); bts_assert_msg(!required || url_stream, (path + " not found.").c_str()); - return result; + } else { + CFG_LOG(log, Info, "Parsing config loaded from '{}'", url); + proto::io::IstreamInputStream proto_stream(&url_stream); + parse_stream(path, proto_stream, out_message, log, lookup); } - CFG_LOG(log, Info, "Parsing config loaded from '{}'", url); - proto::io::IstreamInputStream proto_stream(&url_stream); - parse_stream(path, proto_stream, out_message, log, lookup); - - return result; + return success; } void parse_stream( diff --git a/src/lib_system/system/config/configuration_state.h b/src/lib_system/system/config/configuration_state.h index 50ae6129d..0a54ec88c 100644 --- a/src/lib_system/system/config/configuration_state.h +++ b/src/lib_system/system/config/configuration_state.h @@ -1,9 +1,7 @@ #pragma once -#include #ifndef CONFIGURATIONSTATE_H_ #define CONFIGURATIONSTATE_H_ -#include "system/config/config_downloader.h" #include "system/config/config_log.h" #include "system/config/configuration_exception.h" #include "system/config/text_format/text_format.h" @@ -18,6 +16,7 @@ #include #include #include +#include #include namespace google { @@ -50,7 +49,9 @@ class ConfigurationState { public: ConfigurationState( const std::vector &proto_files, - const std::string &defaults); + const std::string &defaults, + std::function + fetch); ~ConfigurationState(void); void initialize_defaults(ConfigLog &log); @@ -139,6 +140,7 @@ class ConfigurationState { // ConfigLog _log; std::string _host; uint32_t _port; + std::function fetch_; std::vector proto_files_; std::string defaults_; }; @@ -158,17 +160,18 @@ bool load_message( proto::Message &out_message, ConfigLog &, ConfigValueQueue &, + std::function fetch, const std::vector &proto_files); int load_directory( std::string url, - ConfigDownloader &, + std::function fetch, proto::Message &out_message, ConfigLog &, ConfigValueQueue &, const std::vector &proto_files); -ConfigDownloader::Result load_file( +bool load_file( std::string url, - ConfigDownloader &, + std::function fetch, proto::Message &out_message, ConfigLog &, ConfigValueQueue &, diff --git a/vcpkg.json b/vcpkg.json index 376bc7d18..beadfcabe 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,11 @@ { "dependencies": [ "fmt", - "catch2" + "catch2", + "concurrencpp", + "curlpp", + "cryptopp", + "openssl" ], "builtin-baseline": "bb002e69deb36c00e9ce21810f69eff349a833f7", "overrides": [