refactor: Replace ConfigDownloader w/ lib_http fns

Essentially all it did was fetch config files via http GET. I ported some http
fns over from the cme-websocket-connector repo, which uses curlpp instead of
curl, and also supports async requests (coroutines) via the 'concurrencpp'

https://github.com/David-Haim/concurrencpp
This commit is contained in:
John Colvin committed 2024-10-14 15:51:44 +00:00
1 parent 752e2bac82
commit 98f85b90ee
28 files changed
+539 -445

No files matched your search

-2
View File
@@ -142,8 +142,6 @@ target_link_libraries(
spark
databento::databento
${emumba-deribit-connector-library_LIBRARIES}
ssl
cryptopp
tbb
onixs-cmemdh-all-4.8.0
OnixS.IceImpactMulticastPriceFeedHandlerCpp
@@ -75,7 +75,7 @@ static ccapi::SessionConfigs mk_session_cfg(
static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) {
// For details on supported opts, read the docs
// https://binance-docs.github.io/apidocs/spot/en/#partial-book-depth-streams
QueryParams opts;
std::map<std::string, std::string> opts;
switch (cfg.num_depth_levels()) {
case 5:
case 10:
@@ -108,7 +108,7 @@ static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) {
break;
}
return opts.to_string();
return http::query_params_string(opts);
}
static ccapi::Subscription mk_depth_sub(
@@ -69,7 +69,7 @@ static ccapi::SessionConfigs mk_session_cfg(
static string mk_depth_opts(const BinanceUsdsFuturesMarketDataConfig &cfg) {
// For details on supported opts, read the docs
// https://binance-docs.github.io/apidocs/futures/en/#partial-book-depth-streams
QueryParams opts;
std::map<std::string, std::string> opts;
switch (cfg.num_depth_levels()) {
case 5:
case 10:
@@ -103,7 +103,7 @@ static string mk_depth_opts(const BinanceUsdsFuturesMarketDataConfig &cfg) {
break;
}
return opts.to_string();
return http::query_params_string(opts);
}
static ccapi::Subscription mk_depth_sub(
+8 -6
View File
@@ -6,12 +6,14 @@ init_library(STATIC)
target_link_libraries(
${PROJECT_NAME}
PUBLIC
boost_system
boost_filesystem
crypto
curl
protobuf
uuid
ssl
system
)
# curlpp
find_package(unofficial-curlpp CONFIG REQUIRED)
target_link_libraries(${PROJECT_NAME} PUBLIC unofficial::curlpp::curlpp)
# concurrencpp
find_package(concurrencpp CONFIG REQUIRED)
target_link_libraries(${PROJECT_NAME} PUBLIC concurrencpp::concurrencpp)
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "system/logging/logging.h"
#include <curlpp/cURLpp.hpp>
namespace bts::http {
/**
* Manages the (global) lifetime of curlpp for the program. An atomic reference
* counter is used to synchronize initialization, and eventual termination.
*/
struct ConcurrenRuntime {
explicit ConcurrenRuntime() {
if (reference_count_++ == 0) {
LOG(Gen, Info, "Initializing curl");
curlpp::initialize();
}
}
~ConcurrenRuntime() {
if (--reference_count_ == 0) {
LOG(Gen, Info, "Terminating curl");
curlpp::terminate();
}
}
private:
static inline std::atomic<int> reference_count_{0};
};
} // namespace bts::http
-22
View File
@@ -1,22 +0,0 @@
#include "http/curl_handle.h"
#include <curl/curl.h>
#include <stdexcept>
namespace bts::http {
CurlHandle::CurlHandle() {
const auto eCode = curl_global_init(CURL_GLOBAL_ALL);
if (eCode != CURLE_OK) {
throw std::runtime_error{"Error initializing libCURL"};
}
}
CurlHandle::~CurlHandle() { curl_global_cleanup(); }
CurlHandle &CurlHandle::instance() {
static CurlHandle inst{};
return inst;
}
} // namespace bts::http
-21
View File
@@ -1,21 +0,0 @@
#pragma once
namespace bts::http {
class CurlHandle {
public:
static CurlHandle &instance();
CurlHandle(CurlHandle const &) = delete;
CurlHandle(CurlHandle &&) = delete;
CurlHandle &operator=(CurlHandle const &) = delete;
CurlHandle &operator=(CurlHandle &&) = delete;
~CurlHandle();
private:
CurlHandle();
};
} // namespace bts::http
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "system/logging/logging.h"
#include <curlpp/cURLpp.hpp>
namespace bts::http {
/**
* Manages the (global) lifetime of curlpp for the program. An atomic reference
* counter is used to synchronize initialization, and eventual termination.
*/
struct CurlSession {
explicit CurlSession() {
if (reference_count_++ == 0) {
LOG(Gen, Info, "Initializing curl");
curlpp::initialize();
}
}
~CurlSession() {
if (--reference_count_ == 0) {
LOG(Gen, Info, "Terminating curl");
curlpp::terminate();
}
}
private:
static inline std::atomic<int> reference_count_{0};
};
} // namespace bts::http
+99 -39
View File
@@ -1,56 +1,116 @@
#include "http_client.h"
#include "http/http_request.h"
#include "http/http_response.h"
#include "http/url.h"
#include "http/query_params.h"
#include "system/json/json_dumper.h"
#include "system/logging/logging.h"
#include <curl/curl.h>
#include <fmt/format.h>
#include <curlpp/Easy.hpp>
#include <curlpp/Exception.hpp>
#include <curlpp/Infos.hpp>
#include <curlpp/Options.hpp>
#include <curlpp/cURLpp.hpp>
#include <string>
namespace bts::http {
static size_t write_data_(
void *ptr, size_t size, size_t nmemb, std::string *data) {
data->append((char *)ptr, size * nmemb);
return size * nmemb;
#define _LOG_REQ_BEGIN(req_type, url, headers) \
LOG(Gen, Info, "Sending http " #req_type "to {}", url)
using bts::io::JsonDumper;
using nlohmann::json;
using std::int64_t;
using std::list;
using std::string;
using std::tuple;
using std::unordered_map;
static void perform(curlpp::Easy &req, IHttpResponse *response) {
if (response != nullptr) {
req.setOpt<curlpp::options::HeaderFunction>(
[&response](char *cstr, std::size_t sz, std::size_t n) {
return response->write_headers(cstr, sz, n);
});
req.setOpt<curlpp::options::WriteFunction>(
[&response](char *cstr, std::size_t sz, std::size_t n) {
return response->write(cstr, sz, n);
});
}
try {
req.perform();
} catch (const curlpp::LogicError &ex) {
response->write_errors(ex.what());
} catch (const curlpp::RuntimeError &ex) {
response->write_errors(ex.what());
} catch (const std::exception &ex) {
// This isn't a routine http error (those are the 'curlpp' exceptions).
// Log it and rethrow, but don't pass it to the http response.
LOG(Gen, Info, "Exception while performing http request. {}",
std::string{ex.what()});
throw;
}
response->write_code(curlpp::infos::ResponseCode::get(req));
}
HttpResponse HttpClient::get(const HttpRequest &request) {
auto *curl = curl_easy_init();
auto url_str = request.url().to_string();
LOG(md, Info, "do GET: {}", url_str);
curl_easy_setopt(curl, CURLOPT_URL, url_str.c_str());
struct curl_slist *request_headers = NULL;
for (const auto &[k, v] : request.headers()) {
request_headers = curl_slist_append(
request_headers, fmt::format("{}: {}", k, v).c_str());
static void init_request_common(
curlpp::Easy &curl_req,
const string &url,
const unordered_map<string, string> &headers) {
curl_req.setOpt<curlpp::options::Url>(url);
if (!headers.empty()) {
list<string> req_hdrs;
for (const auto &[k, v] : headers) {
req_hdrs.push_back(fmt::format("{}:{}", k, v));
}
curl_req.setOpt<curlpp::options::HttpHeader>(req_hdrs);
}
curl_req.setOpt<curlpp::options::FailOnError>(true);
}
std::string response_string;
std::string header_string;
std::string response_header_string;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &header_string);
void HttpClient::post(
const string &url,
const string &body,
const unordered_map<string, string> &headers,
IHttpResponse *response) {
_LOG_REQ_BEGIN(POST, url, headers);
curlpp::Easy curl_req;
init_request_common(curl_req, url, headers);
curl_req.setOpt<curlpp::options::Post>(true);
curl_req.setOpt<curlpp::options::PostFields>(body);
perform(curl_req, response);
}
char *url;
long response_code;
double elapsed;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &elapsed);
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
// curl_easy_getinfo(curl, CURLINFO_HEADER_OUT, &response_header_string);
void HttpClient::get(
const string &url,
const unordered_map<string, string> &headers,
IHttpResponse *response) {
_LOG_REQ_BEGIN(GET, url, headers);
curlpp::Easy curl_req;
init_request_common(curl_req, url, headers);
curl_req.setOpt<curlpp::options::HttpGet>(true);
perform(curl_req, response);
}
curl_easy_perform(curl);
curl_easy_cleanup(curl);
curl = NULL;
HttpResponse response;
response.headers() = HttpHeaders(response_header_string);
response.body() = std::move(response_string);
response.code() = response_code;
LOG(md, Info, "do GET: {}. Received response:\n{}", url_str, response);
return response;
tuple<string, string, int64_t> HttpClient::get_oauth_token(
const string &url, const string &client_id, const string &client_secret) {
LOG(Gen, Info, "Requesting oauth token from {}", url);
std::vector<std::pair<string, string>> query_params{
{"grant_type", "client_credentials"},
{"client_id", client_id},
{"client_secret", client_secret}};
HttpResponse res;
HttpClient::post(url, query_params_string(query_params), {}, &res);
LOG(Gen, Info, "Received oauth token:\n{}", JsonDumper::dump(res));
try {
const auto j = json::parse(res.body);
return {
j.at("access_token").get<string>(), j.at("token_type").get<string>(),
j.at("expires_in").get<int64_t>()};
} catch (const std::exception &e) {
LOG(Gen, Error, "Failed to parse oauth response json.{}\n{}",
string{e.what()}, res.body);
}
return {};
}
} // namespace bts::http
+74 -24
View File
@@ -1,35 +1,85 @@
#include <algorithm>
#include <atomic>
#include <cstddef> // std::size_t
#include <cstdio> // snprintf
#include <cstdlib>
#include <cstring> // strerror, strlen, memcpy, strcpy
#include <ctime>
#include <curl/curl.h>
#include <fstream>
#include <functional>
#include <iostream>
#include <memory> // std::unique_ptr
#pragma once
#include "http/http_response.h"
#include "http/i_http_response.h"
#include <concurrencpp/concurrencpp.h>
#include <concurrencpp/executors/executor.h>
#include <concurrencpp/runtime/runtime.h>
#include <cstdint>
#include <mutex>
#include <sstream>
#include <stdarg.h> // va_start, etc.
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <sys/stat.h>
#include <sys/types.h>
#include <tuple>
#include <unordered_map>
#include <vector>
namespace bts::http {
class HttpRequest;
class HttpResponse;
class HttpClient {
public:
// HttpClient(const std::string &url);
HttpResponse get(const HttpRequest &request);
/**
* Performs an http GET request
*/
static void get(
const std::string &url,
const std::unordered_map<std::string, std::string> &headers,
IHttpResponse *response);
static concurrencpp::result<HttpResponse> get_async(
const std::string &url,
const std::unordered_map<std::string, std::string> &headers);
static concurrencpp::result<HttpResponse> get_async(
const std::string &url,
const std::unordered_map<std::string, std::string> &headers,
std::shared_ptr<concurrencpp::executor> executor);
/**
* Performs an http POST request
*/
static void post(
const std::string &url,
const std::string &body,
const std::unordered_map<std::string, std::string> &headers,
IHttpResponse *response);
/**
* Requests an oauth access token
*
* @return tuple of: access token, token type, and seconds until expiration
*/
static std::tuple<std::string, std::string, std::int64_t> get_oauth_token(
const std::string &url,
const std::string &client_id,
const std::string &client_secret);
private:
static std::mutex global_runtime_mx_;
static std::optional<concurrencpp::runtime> global_runtime_;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline concurrencpp::result<HttpResponse> HttpClient::get_async(
const std::string &url,
const std::unordered_map<std::string, std::string> &headers) {
if (!global_runtime_) {
std::lock_guard<std::mutex> lock{global_runtime_mx_};
global_runtime_.emplace();
}
return get_async(url, headers, global_runtime_->thread_executor());
}
inline concurrencpp::result<HttpResponse> HttpClient::get_async(
const std::string &url,
const std::unordered_map<std::string, std::string> &headers,
std::shared_ptr<concurrencpp::executor> executor) {
co_return co_await executor->submit([&]() {
HttpResponse res;
get(url, headers, &res);
return res;
});
}
} // namespace bts::http
-44
View File
@@ -1,44 +0,0 @@
#include "http_headers.h"
#include "system/string_utils.h"
#include <string>
namespace bts::http {
using bts::system::string_utils::split;
HttpHeaders::HttpHeaders(const std::string &str) {
auto item_strs = split(str, "\n");
for (const auto &item_str : item_strs) {
auto kv = split(item_str, ":");
if (kv.empty()) {
try_emplace("", "");
continue;
}
if (kv.size() == 1) {
try_emplace(kv[0], "");
continue;
}
try_emplace(kv[0], kv[1]);
}
}
std::string HttpHeaders::to_string() const {
auto it = begin();
if (it == end()) {
return "";
}
std::string rv{it->first + ": " + it->second};
while (++it != end()) {
rv.append("\n" + it->first + ": " + it->second);
}
return rv;
}
std::ostream &operator<<(
std::ostream &out, const HttpHeaders &headers) noexcept {
out << headers.to_string();
return out;
}
} // namespace bts::http
+40 -15
View File
@@ -1,26 +1,51 @@
#pragma once
#include "system/collections/flat_unordered_map.h"
#include "system/string_utils.h"
#include <string>
#include <unordered_map>
namespace bts::http {
class HttpHeaders
: public collections::FlatUnorderedMap<std::string, std::string> {
public:
HttpHeaders() = default;
explicit HttpHeaders(const std::string &header_str);
std::string headers_to_string(
const std::unordered_map<std::string, std::string> &headers);
HttpHeaders(const HttpHeaders &) noexcept = default;
HttpHeaders(HttpHeaders &&) noexcept = default;
HttpHeaders &operator=(const HttpHeaders &) noexcept = default;
HttpHeaders &operator=(HttpHeaders &&) noexcept = default;
std::unordered_map<std::string, std::string> headers_from_string(
const std::string &headers_str);
std::string to_string() const;
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
std::ostream &operator<<(
std::ostream &out, const HttpHeaders &headers) noexcept;
inline std::string headers_to_string(
const std::unordered_map<std::string, std::string> &headers) {
auto it = headers.begin();
if (it == headers.end()) {
return "";
}
std::string rv{it->first + ":" + it->second};
while (++it != headers.end()) {
rv.append("\n" + it->first + ":" + it->second);
}
return rv;
}
inline std::unordered_map<std::string, std::string> headers_from_string(
const std::string &str) {
std::unordered_map<std::string, std::string> headers;
auto item_strs = system::string_utils::split(str, "\n");
for (const auto &item_str : item_strs) {
auto kv = system::string_utils::split(item_str, ":");
if (kv.empty()) {
headers.try_emplace("", "");
continue;
}
if (kv.size() == 1) {
headers.try_emplace(kv[0], "");
continue;
}
headers.try_emplace(kv[0], kv[1]);
}
return headers;
}
} // namespace bts::http
-17
View File
@@ -1,17 +0,0 @@
#include "http_request.h"
namespace bts::http {
Url &HttpRequest::url() { return url_; }
const Url &HttpRequest::url() const { return url_; }
HttpHeaders &HttpRequest::headers() { return headers_; }
const HttpHeaders &HttpRequest::headers() const { return headers_; }
std::string &HttpRequest::body() { return body_; }
const std::string &HttpRequest::body() const { return body_; }
} // namespace bts::http
-28
View File
@@ -1,28 +0,0 @@
#pragma once
#include "http/http_headers.h"
#include "http/url.h"
#include <string>
#include <vector>
namespace bts::http {
class HttpRequest {
public:
Url &url();
const Url &url() const;
HttpHeaders &headers();
const HttpHeaders &headers() const;
std::string &body();
const std::string &body() const;
private:
Url url_;
HttpHeaders headers_;
std::string body_;
};
} // namespace bts::http
-26
View File
@@ -1,26 +0,0 @@
#include "http_response.h"
#include <fmt/format.h>
namespace bts::http {
HttpHeaders &HttpResponse::headers() { return headers_; }
const HttpHeaders &HttpResponse::headers() const { return headers_; }
std::string &HttpResponse::body() { return body_; }
const std::string &HttpResponse::body() const { return body_; }
std::int64_t &HttpResponse::code() { return code_; }
std::int64_t HttpResponse::code() const { return code_; }
std::ostream &operator<<(
std::ostream &out, const HttpResponse &response) noexcept {
out << response.headers() << "\n"
<< response.body() << "\nCode: " << response.code();
return out;
}
} // namespace bts::http
+72 -19
View File
@@ -1,35 +1,88 @@
#pragma once
#include "http/url.h"
#include "http_headers.h"
#include "http/i_http_response.h"
#include "system/FMT_IMPL.h"
#include <cstddef>
#include <nlohmann/json.hpp>
#include <string>
#include <vector>
#include <unordered_map>
namespace bts::http {
class HttpResponse {
/**
* Basic implementation of `IHttpResponse`, with some useful state.
*/
class HttpResponse : public IHttpResponse {
public:
Url &url();
const Url &url() const;
std::string url;
std::unordered_map<std::string, std::string> headers;
std::string body;
std::int64_t code;
std::string errors;
HttpHeaders &headers();
const HttpHeaders &headers() const;
std::size_t write(char *cstr, std::size_t sz, std::size_t n) override;
std::size_t write_headers(
char *cstr, std::size_t sz, std::size_t n) override;
void write_errors(std::string &&str) override;
void write_code(std::int64_t code) override;
constexpr bool ok() const;
std::string &body();
const std::string &body() const;
std::int64_t &code();
std::int64_t code() const;
private:
Url url_;
HttpHeaders headers_;
std::string body_;
std::int64_t code_;
void reset();
};
void to_json(nlohmann::json &j, const HttpResponse &res);
void from_json(const nlohmann::json &j, HttpResponse &res);
std::ostream &operator<<(
std::ostream &out, const HttpResponse &response) noexcept;
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::size_t HttpResponse::write(
char *cstr, std::size_t sz, std::size_t n) {
body.append(cstr, sz * n);
return sz * n;
}
inline std::size_t HttpResponse::write_headers(
char *cstr, std::size_t sz, std::size_t n) {
std::string str{cstr, sz * n};
const auto colon_idx = str.find(':');
if (colon_idx >= str.size()) {
headers[str] = {};
} else {
headers[str.substr(0, colon_idx)] =
str.substr(colon_idx + 1, str.size() - colon_idx - 1);
}
return sz * n;
}
inline void HttpResponse::write_errors(std::string &&str) {
errors = std::move(str);
}
inline void HttpResponse::write_code(std::int64_t code) { this->code = code; }
constexpr bool HttpResponse::ok() const { return code >= 200 && code < 300; }
inline void HttpResponse::reset() {
headers.clear();
body.clear();
errors.clear();
code = 0;
}
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(
HttpResponse, url, headers, body, code, errors);
inline std::ostream &operator<<(
std::ostream &out, const HttpResponse &res) noexcept {
return out << nlohmann::json{res}.dump();
}
} // namespace bts::http
FMT_IMPL(bts::http::HttpResponse, r, "{}", nlohmann::json{r}.dump());
+35
View File
@@ -0,0 +1,35 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <string>
namespace bts::http {
/**
* Simple http response interface, modeled after cURL callbacks
*/
class IHttpResponse {
public:
virtual ~IHttpResponse() = default;
virtual std::size_t write(char *cstr, std::size_t sz, std::size_t n) = 0;
virtual std::size_t write_headers(char *cstr, std::size_t sz, std::size_t n);
virtual void write_errors(std::string &&str);
virtual void write_code(std::int64_t code);
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::size_t IHttpResponse::write_headers(
char * /*cstr*/, std::size_t sz, std::size_t n) {
return sz * n;
}
inline void IHttpResponse::write_errors(std::string && /*str*/) {}
inline void IHttpResponse::write_code(std::int64_t /*code*/) {}
} // namespace bts::http
-18
View File
@@ -1,18 +0,0 @@
#include "http/query_params.h"
namespace bts::http {
std::string QueryParams::to_string() const {
std::string rv;
auto it = begin();
if (it == end()) {
return "";
}
rv.append(it->first + "=" + it->second);
while (++it != end()) {
rv.append("&" + it->first + "=" + it->second);
}
return rv;
}
} // namespace bts::http
+51 -3
View File
@@ -1,15 +1,63 @@
#pragma once
#include "system/collections/flat_unordered_map.h"
#include "system/FMT_IMPL.h"
#include <algorithm>
#include <string>
#include <unordered_map>
namespace bts::http {
class QueryParams
: public collections::FlatUnorderedMap<std::string, std::string> {
template <class Iter>
inline std::string query_params_string(Iter first, Iter last) {
std::string rv;
for (auto it = first; it != last; it++) {
if (it->first.empty()) {
continue;
}
if (!rv.empty()) {
rv.append("&");
}
rv.append(it->first);
if (!it->second.empty()) {
rv.append("=");
rv.append(it->second);
}
}
return rv;
}
template <class Range>
inline std::string query_params_string(Range &&range) {
return query_params_string(range.begin(), range.end());
}
class QueryParams : public std::unordered_map<std::string, std::string> {
public:
std::string to_string() const;
};
inline std::string QueryParams::to_string() const {
std::string rv;
auto it = begin();
if (it == end()) {
return "";
}
rv.append(it->first + "=" + it->second);
while (++it != end()) {
rv.append("&" + it->first + "=" + it->second);
}
return rv;
}
inline bool operator==(const QueryParams &lhs, const QueryParams &rhs) {
return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
}
inline bool operator!=(const QueryParams &lhs, const QueryParams &rhs) {
return !(lhs == rhs);
}
} // namespace bts::http
FMT_IMPL(bts::http::QueryParams, p, "{}", p.to_string());
@@ -1,5 +1,7 @@
function(get_link_libraries dest)
find_package(fmt CONFIG REQUIRED)
find_package(OpenSSL REQUIRED)
find_package(cryptopp CONFIG REQUIRED)
set(${dest}
system
domain
@@ -15,13 +17,12 @@ function(get_link_libraries dest)
http
SQLiteCpp
sqlite3
curl
uuid
ssl
crypto
OpenSSL::SSL
OpenSSL::Crypto
fmt::fmt
tbb
cryptopp
cryptopp::cryptopp
pugixml
lttng-ust
Pcap++
+19 -3
View File
@@ -2,10 +2,12 @@
#define CONFIG_H_
#include "config.h"
#include "http/http_client.h"
#include "http/http_response.h"
#include "spark/protobufs/config_defaults.h"
#include "spark/protobufs/config_proto_files.h"
#include "system/bts_array_size.h"
#include "system/config/configuration_state.h"
#include "spark/protobufs/config_proto_files.h"
#include "spark/protobufs/config_defaults.h"
#include "system/os.h"
#include <google/protobuf/descriptor.h>
@@ -43,7 +45,21 @@ internal::ConfigurationState &config_state() {
defaults_str.append(line);
defaults_str.append("\n");
}
config_state_.emplace(proto_files_vec, defaults_str);
config_state_.emplace(
proto_files_vec, defaults_str,
[](const std::string &url, std::stringstream &out) {
http::HttpResponse res;
http::HttpClient::get(url, {}, &res);
if (!res.ok()) {
return false;
}
// This isn't very efficient. Our HttpResponse populates a string, which is copied
// to a stringstream, which is likely (eventually) dumped back to a string by the
// original caller.
// TODO: avoid copying strings in this pipeline.
out << res.body;
return true;
});
}
}
return *config_state_;
@@ -23,6 +23,9 @@ namespace openssl {
namespace bts {
namespace crypto {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
struct PrivateKey::Pimp {
BTS_NO_COPY(Pimp);
@@ -206,7 +209,8 @@ int private_encrypt_func(
return -1;
} else {
return RSA_private_encrypt(
inlen, const_cast<unsigned char *>(in), out, rsa, RSA_PKCS1_PADDING);
inlen, const_cast<unsigned char *>(in), out,
const_cast<openssl::RSA *>(rsa), RSA_PKCS1_PADDING);
}
}
@@ -223,7 +227,8 @@ int public_decrypt_func(
return -1;
} else {
int i = RSA_public_decrypt(
inlen, const_cast<unsigned char *>(in), out, rsa, RSA_PKCS1_PADDING);
inlen, const_cast<unsigned char *>(in), out,
const_cast<openssl::RSA *>(rsa), RSA_PKCS1_PADDING);
out[i] = '\0';
*outlen = i;
return i;
@@ -293,6 +298,8 @@ void PrivateKey::write_stream(
BIO_free(biostream_out.getBIO());
}
#pragma GCC diagnostic pop
} // namespace crypto
} // namespace bts
#endif
@@ -1,73 +0,0 @@
#include "config_downloader.h"
#include "system/facades/bts_assert.h"
#include <boost/algorithm/string.hpp>
#include <sstream>
#include <stdio.h>
using namespace std;
namespace bts {
namespace config {
namespace internal {
ConfigDownloader::ConfigDownloader() : _curl(NULL) {
curl_global_init(CURL_GLOBAL_DEFAULT);
_curl = curl_easy_init();
bts_assert(_curl);
}
ConfigDownloader::~ConfigDownloader() { curl_easy_cleanup(_curl); }
size_t curl_write(char *buffer, size_t size, size_t count, void *stream) {
stringstream *out = static_cast<stringstream *>(stream);
bts_assert(out);
size_t bytes = size * count;
out->write(buffer, bytes);
return count;
}
bool is_http(const string &url) { return url.substr(0, 7) == "http://"; }
int no_linger(void *, curl_socket_t socket, curlsocktype) {
linger l;
l.l_onoff = 1;
l.l_linger = 0;
setsockopt(socket, SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(l));
return CURL_SOCKOPT_OK;
}
ConfigDownloader::Result
ConfigDownloader::get(const string &url, stringstream &out_stream) {
curl_easy_setopt(_curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, curl_write);
curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream);
curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0);
curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1);
curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 6);
curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger);
CURLcode res = curl_easy_perform(_curl);
long http_code = 0;
CURLcode r2 = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &http_code);
bts_assert(r2 == CURLE_OK);
bool used_http = is_http(url);
if (res == CURLE_OK && (http_code == 200 || !used_http)) {
return SUCCESS;
} else if (
res == CURLE_OPERATION_TIMEDOUT || res == CURLE_COULDNT_RESOLVE_HOST ||
res == CURLE_COULDNT_RESOLVE_PROXY || res == CURLE_COULDNT_CONNECT ||
res == CURLE_REMOTE_ACCESS_DENIED || res == CURLE_FTP_CANT_GET_HOST) {
// Cannot rely on logging before config is initialized
// LOG(Error, Config) << "Permanent failure loading '" << url << "' - CURL
// error: " << curl_easy_strerror(res);
return PERMANENT_FAILURE;
}
return FAILURE;
}
} // namespace internal
} // namespace config
} // namespace bts
@@ -1,27 +0,0 @@
#pragma once
#include <curl/curl.h>
#include <string>
namespace bts {
namespace config {
namespace internal {
class ConfigDownloader {
public:
ConfigDownloader();
ConfigDownloader(const ConfigDownloader &) = delete;
ConfigDownloader &operator=(const ConfigDownloader &) = delete;
~ConfigDownloader();
enum Result { FAILURE, SUCCESS, PERMANENT_FAILURE };
Result get(const std::string &url, std::stringstream &out_stream);
private:
CURL *_curl;
};
} // namespace internal
} // namespace config
} // namespace bts
@@ -1,23 +1,25 @@
#include "configuration_state.h"
// #include "spark/config/config.h"
#include "system/FMT.h"
#include "system/bts_array_size.h"
#include "system/config/config_error_collector.h"
#include "system/config/config_property_evaluator.h"
#include "system/config/config_value_queue.h"
#include "system/config/configuration_exception.h"
#include "system/config/preprocess_input_stream.h"
#include "system/facades/bts_assert.h"
#include "system/defer.h"
#include "system/FMT.h"
#include "system/bts_array_size.h"
#include "system/facades/bts_assert.h"
#include "system/os.h"
#include "system/protobufs/config/system_config.pb.h"
#include <algorithm>
#include <boost/algorithm/string.hpp> // iequals
#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>
#include <fstream>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <iostream>
#include <map>
#include <regex>
@@ -68,13 +70,16 @@ const char *CONFIG_SERVER_ENV_VAR = "BTSConfig";
inline tuple<string, uint32_t> hostname(const std::string &path);
ConfigurationState::ConfigurationState(
const std::vector<std::string> &proto_files, const std::string &defaults)
const std::vector<std::string> &proto_files,
const std::string &defaults,
std::function<bool(const std::string &url, std::stringstream &out)> fetch)
: _is_initialized(false), _initialization_mutex(), _dynamic_pool(),
_prototype_factory(), _prototype(nullptr), _config_chain(), _defaults(),
_properties(new ConfigPropertyEvaluator(*this)),
_rule_engine(_properties), _macros(),
_deferred_queue(new ConfigValueQueue(*_properties.get())), _host(),
_port(80), proto_files_(proto_files), defaults_(defaults) {}
_port(80), fetch_(fetch), proto_files_(proto_files), defaults_(defaults) {
}
ConfigurationState::~ConfigurationState(void) {}
@@ -202,9 +207,7 @@ void ConfigurationState::initialize_builtins(ConfigLog *log) {
sys.set_num_cores(os::num_cores());
if (log != nullptr) {
CFG_LOG(
(*log), Info, "Initialized SystemConfig:\n{}",
sys.DebugString());
CFG_LOG((*log), Info, "Initialized SystemConfig:\n{}", sys.DebugString());
}
//
@@ -416,7 +419,8 @@ int ConfigurationState::load(const std::string &url, ConfigLog &log) {
bool loaded = false;
CFG_LOG(log, Info, "Loading '{}'", url);
loaded = load_message(
url, *new_message.get(), log, *_deferred_queue.get(), proto_files_);
url, *new_message.get(), log, *_deferred_queue.get(), fetch_,
proto_files_);
if (_host.empty() || _host == "localhost") {
const auto old_host = _host;
const auto old_port = _port;
@@ -912,18 +916,16 @@ bool load_message(
proto::Message &out_message,
ConfigLog &log,
ConfigValueQueue &lookup,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
const std::vector<std::string> &proto_files) {
ConfigDownloader downloader;
bool result = false;
switch (get_path_type(url)) {
case PathType::directory:
result = load_directory(
url, downloader, out_message, log, lookup, proto_files) > 0;
result =
load_directory(url, fetch, out_message, log, lookup, proto_files) > 0;
break;
case PathType::file:
result = load_file(url, downloader, out_message, log, lookup) ==
ConfigDownloader::SUCCESS;
result = load_file(url, fetch, out_message, log, lookup);
break;
default:
std::ostringstream error;
@@ -939,7 +941,7 @@ bool load_message(
int load_dir(
const std::string &url,
ConfigDownloader &downloader,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
proto::Message &out_message,
ConfigLog &log,
ConfigValueQueue &lookup,
@@ -961,7 +963,7 @@ int load_dir(
boost::filesystem::path p(config_file);
config_file = p.filename().string();
if (load_file(
url + config_file, downloader, out_message, log, lookup, false)) {
url + config_file, fetch, out_message, log, lookup, false)) {
load_count++;
if (!edge_has_include) {
bts_assert_msg(
@@ -979,7 +981,7 @@ int load_dir(
int load_directory(
std::string url,
ConfigDownloader &downloader,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
proto::Message &out_message,
ConfigLog &log,
ConfigValueQueue &lookup,
@@ -993,50 +995,52 @@ int load_directory(
// If a file called "all" exists in the config directory we can short circuit
// ~60 http requests
ConfigDownloader::Result all_result =
load_file(url + ".all", downloader, out_message, log, lookup, false);
if (all_result == ConfigDownloader::SUCCESS) {
bool success =
load_file(url + ".all", fetch, out_message, log, lookup, false);
if (success) {
return 1;
}
// load spark.conf
int load_count = 0;
ConfigDownloader::Result result = load_file(
url + "spark.conf", downloader, out_message, log, lookup, false);
if (result == ConfigDownloader::SUCCESS) {
success =
load_file(url + "spark.conf", fetch, out_message, log, lookup, false);
if (success) {
load_count++;
} else if (result == ConfigDownloader::PERMANENT_FAILURE) {
// If the server is down it doesn't make sense to suffer though the
// timeouts for the other files
throw ServerUnreachableException(url.c_str());
}
load_count +=
load_dir(url, downloader, out_message, log, lookup, proto_files);
load_count += load_dir(url, fetch, out_message, log, lookup, proto_files);
return load_count;
}
ConfigDownloader::Result load_file(
bool load_file(
std::string path,
ConfigDownloader &downloader,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
proto::Message &out_message,
ConfigLog &log,
ConfigValueQueue &lookup,
bool required) {
// first, try loading local file.
if (boost::filesystem::exists(path)) {
std::ifstream fs{path};
proto::io::IstreamInputStream proto_stream(&fs);
parse_stream(path, proto_stream, out_message, log, lookup);
return true;
}
bool success = false;
stringstream url_stream;
string url = url_from_string(path, required).to_string();
ConfigDownloader::Result result = downloader.get(url, url_stream);
if (!(result == ConfigDownloader::SUCCESS)) {
success = fetch(url, url_stream);
if (!success) {
CFG_LOG(log, Info, "Failed to load config from '{}'", url);
bts_assert_msg(!required || url_stream, (path + " not found.").c_str());
return result;
} else {
CFG_LOG(log, Info, "Parsing config loaded from '{}'", url);
proto::io::IstreamInputStream proto_stream(&url_stream);
parse_stream(path, proto_stream, out_message, log, lookup);
}
CFG_LOG(log, Info, "Parsing config loaded from '{}'", url);
proto::io::IstreamInputStream proto_stream(&url_stream);
parse_stream(path, proto_stream, out_message, log, lookup);
return result;
return success;
}
void parse_stream(
@@ -1,9 +1,7 @@
#pragma once
#include <string>
#ifndef CONFIGURATIONSTATE_H_
#define CONFIGURATIONSTATE_H_
#include "system/config/config_downloader.h"
#include "system/config/config_log.h"
#include "system/config/configuration_exception.h"
#include "system/config/text_format/text_format.h"
@@ -18,6 +16,7 @@
#include <boost/scoped_ptr.hpp>
#include <boost/utility.hpp>
#include <mutex>
#include <string>
#include <vector>
namespace google {
@@ -50,7 +49,9 @@ class ConfigurationState {
public:
ConfigurationState(
const std::vector<std::string> &proto_files,
const std::string &defaults);
const std::string &defaults,
std::function<bool(const std::string &url, std::stringstream &out)>
fetch);
~ConfigurationState(void);
void initialize_defaults(ConfigLog &log);
@@ -139,6 +140,7 @@ class ConfigurationState {
// ConfigLog _log;
std::string _host;
uint32_t _port;
std::function<bool(const std::string &url, std::stringstream &out)> fetch_;
std::vector<std::string> proto_files_;
std::string defaults_;
};
@@ -158,17 +160,18 @@ bool load_message(
proto::Message &out_message,
ConfigLog &,
ConfigValueQueue &,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
const std::vector<std::string> &proto_files);
int load_directory(
std::string url,
ConfigDownloader &,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
proto::Message &out_message,
ConfigLog &,
ConfigValueQueue &,
const std::vector<std::string> &proto_files);
ConfigDownloader::Result load_file(
bool load_file(
std::string url,
ConfigDownloader &,
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
proto::Message &out_message,
ConfigLog &,
ConfigValueQueue &,
+5 -1
View File
@@ -1,7 +1,11 @@
{
"dependencies": [
"fmt",
"catch2"
"catch2",
"concurrencpp",
"curlpp",
"cryptopp",
"openssl"
],
"builtin-baseline": "bb002e69deb36c00e9ce21810f69eff349a833f7",
"overrides": [