refactor: Replace ConfigDownloader w/ lib_http fns
Essentially all it did was fetch config files via http GET. I ported some http fns over from the cme-websocket-connector repo, which uses curlpp instead of curl, and also supports async requests (coroutines) via the 'concurrencpp' https://github.com/David-Haim/concurrencpp
This commit is contained in:
1 parent
752e2bac82
commit
98f85b90ee
28 files changed
+539
-445
No files matched your search
+1
-1
Submodule externals updated: c5861af798...4f5e385556.
@@ -142,8 +142,6 @@ target_link_libraries(
|
||||
spark
|
||||
databento::databento
|
||||
${emumba-deribit-connector-library_LIBRARIES}
|
||||
ssl
|
||||
cryptopp
|
||||
tbb
|
||||
onixs-cmemdh-all-4.8.0
|
||||
OnixS.IceImpactMulticastPriceFeedHandlerCpp
|
||||
|
||||
@@ -75,7 +75,7 @@ static ccapi::SessionConfigs mk_session_cfg(
|
||||
static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) {
|
||||
// For details on supported opts, read the docs
|
||||
// https://binance-docs.github.io/apidocs/spot/en/#partial-book-depth-streams
|
||||
QueryParams opts;
|
||||
std::map<std::string, std::string> opts;
|
||||
switch (cfg.num_depth_levels()) {
|
||||
case 5:
|
||||
case 10:
|
||||
@@ -108,7 +108,7 @@ static string mk_depth_opts(const BinanceUsMarketDataConfig &cfg) {
|
||||
break;
|
||||
}
|
||||
|
||||
return opts.to_string();
|
||||
return http::query_params_string(opts);
|
||||
}
|
||||
|
||||
static ccapi::Subscription mk_depth_sub(
|
||||
|
||||
@@ -69,7 +69,7 @@ static ccapi::SessionConfigs mk_session_cfg(
|
||||
static string mk_depth_opts(const BinanceUsdsFuturesMarketDataConfig &cfg) {
|
||||
// For details on supported opts, read the docs
|
||||
// https://binance-docs.github.io/apidocs/futures/en/#partial-book-depth-streams
|
||||
QueryParams opts;
|
||||
std::map<std::string, std::string> opts;
|
||||
switch (cfg.num_depth_levels()) {
|
||||
case 5:
|
||||
case 10:
|
||||
@@ -103,7 +103,7 @@ static string mk_depth_opts(const BinanceUsdsFuturesMarketDataConfig &cfg) {
|
||||
break;
|
||||
}
|
||||
|
||||
return opts.to_string();
|
||||
return http::query_params_string(opts);
|
||||
}
|
||||
|
||||
static ccapi::Subscription mk_depth_sub(
|
||||
|
||||
@@ -6,12 +6,14 @@ init_library(STATIC)
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
PUBLIC
|
||||
boost_system
|
||||
boost_filesystem
|
||||
crypto
|
||||
curl
|
||||
protobuf
|
||||
uuid
|
||||
ssl
|
||||
system
|
||||
)
|
||||
|
||||
# curlpp
|
||||
find_package(unofficial-curlpp CONFIG REQUIRED)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC unofficial::curlpp::curlpp)
|
||||
|
||||
# concurrencpp
|
||||
find_package(concurrencpp CONFIG REQUIRED)
|
||||
target_link_libraries(${PROJECT_NAME} PUBLIC concurrencpp::concurrencpp)
|
||||
@@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include "system/logging/logging.h"
|
||||
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
/**
|
||||
* Manages the (global) lifetime of curlpp for the program. An atomic reference
|
||||
* counter is used to synchronize initialization, and eventual termination.
|
||||
*/
|
||||
struct ConcurrenRuntime {
|
||||
explicit ConcurrenRuntime() {
|
||||
if (reference_count_++ == 0) {
|
||||
LOG(Gen, Info, "Initializing curl");
|
||||
curlpp::initialize();
|
||||
}
|
||||
}
|
||||
|
||||
~ConcurrenRuntime() {
|
||||
if (--reference_count_ == 0) {
|
||||
LOG(Gen, Info, "Terminating curl");
|
||||
curlpp::terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static inline std::atomic<int> reference_count_{0};
|
||||
};
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,22 +0,0 @@
|
||||
#include "http/curl_handle.h"
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
CurlHandle::CurlHandle() {
|
||||
const auto eCode = curl_global_init(CURL_GLOBAL_ALL);
|
||||
if (eCode != CURLE_OK) {
|
||||
throw std::runtime_error{"Error initializing libCURL"};
|
||||
}
|
||||
}
|
||||
|
||||
CurlHandle::~CurlHandle() { curl_global_cleanup(); }
|
||||
|
||||
CurlHandle &CurlHandle::instance() {
|
||||
static CurlHandle inst{};
|
||||
return inst;
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,21 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
class CurlHandle {
|
||||
public:
|
||||
static CurlHandle &instance();
|
||||
|
||||
CurlHandle(CurlHandle const &) = delete;
|
||||
CurlHandle(CurlHandle &&) = delete;
|
||||
|
||||
CurlHandle &operator=(CurlHandle const &) = delete;
|
||||
CurlHandle &operator=(CurlHandle &&) = delete;
|
||||
|
||||
~CurlHandle();
|
||||
|
||||
private:
|
||||
CurlHandle();
|
||||
};
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include "system/logging/logging.h"
|
||||
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
/**
|
||||
* Manages the (global) lifetime of curlpp for the program. An atomic reference
|
||||
* counter is used to synchronize initialization, and eventual termination.
|
||||
*/
|
||||
struct CurlSession {
|
||||
explicit CurlSession() {
|
||||
if (reference_count_++ == 0) {
|
||||
LOG(Gen, Info, "Initializing curl");
|
||||
curlpp::initialize();
|
||||
}
|
||||
}
|
||||
|
||||
~CurlSession() {
|
||||
if (--reference_count_ == 0) {
|
||||
LOG(Gen, Info, "Terminating curl");
|
||||
curlpp::terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static inline std::atomic<int> reference_count_{0};
|
||||
};
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,56 +1,116 @@
|
||||
#include "http_client.h"
|
||||
#include "http/http_request.h"
|
||||
#include "http/http_response.h"
|
||||
#include "http/url.h"
|
||||
#include "http/query_params.h"
|
||||
#include "system/json/json_dumper.h"
|
||||
#include "system/logging/logging.h"
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <fmt/format.h>
|
||||
#include <curlpp/Easy.hpp>
|
||||
#include <curlpp/Exception.hpp>
|
||||
#include <curlpp/Infos.hpp>
|
||||
#include <curlpp/Options.hpp>
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
#include <string>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
static size_t write_data_(
|
||||
void *ptr, size_t size, size_t nmemb, std::string *data) {
|
||||
data->append((char *)ptr, size * nmemb);
|
||||
return size * nmemb;
|
||||
#define _LOG_REQ_BEGIN(req_type, url, headers) \
|
||||
LOG(Gen, Info, "Sending http " #req_type "to {}", url)
|
||||
|
||||
using bts::io::JsonDumper;
|
||||
using nlohmann::json;
|
||||
using std::int64_t;
|
||||
using std::list;
|
||||
using std::string;
|
||||
using std::tuple;
|
||||
using std::unordered_map;
|
||||
|
||||
static void perform(curlpp::Easy &req, IHttpResponse *response) {
|
||||
if (response != nullptr) {
|
||||
req.setOpt<curlpp::options::HeaderFunction>(
|
||||
[&response](char *cstr, std::size_t sz, std::size_t n) {
|
||||
return response->write_headers(cstr, sz, n);
|
||||
});
|
||||
req.setOpt<curlpp::options::WriteFunction>(
|
||||
[&response](char *cstr, std::size_t sz, std::size_t n) {
|
||||
return response->write(cstr, sz, n);
|
||||
});
|
||||
}
|
||||
try {
|
||||
req.perform();
|
||||
} catch (const curlpp::LogicError &ex) {
|
||||
response->write_errors(ex.what());
|
||||
} catch (const curlpp::RuntimeError &ex) {
|
||||
response->write_errors(ex.what());
|
||||
} catch (const std::exception &ex) {
|
||||
// This isn't a routine http error (those are the 'curlpp' exceptions).
|
||||
// Log it and rethrow, but don't pass it to the http response.
|
||||
LOG(Gen, Info, "Exception while performing http request. {}",
|
||||
std::string{ex.what()});
|
||||
throw;
|
||||
}
|
||||
response->write_code(curlpp::infos::ResponseCode::get(req));
|
||||
}
|
||||
|
||||
HttpResponse HttpClient::get(const HttpRequest &request) {
|
||||
auto *curl = curl_easy_init();
|
||||
auto url_str = request.url().to_string();
|
||||
LOG(md, Info, "do GET: {}", url_str);
|
||||
curl_easy_setopt(curl, CURLOPT_URL, url_str.c_str());
|
||||
|
||||
struct curl_slist *request_headers = NULL;
|
||||
for (const auto &[k, v] : request.headers()) {
|
||||
request_headers = curl_slist_append(
|
||||
request_headers, fmt::format("{}: {}", k, v).c_str());
|
||||
static void init_request_common(
|
||||
curlpp::Easy &curl_req,
|
||||
const string &url,
|
||||
const unordered_map<string, string> &headers) {
|
||||
curl_req.setOpt<curlpp::options::Url>(url);
|
||||
if (!headers.empty()) {
|
||||
list<string> req_hdrs;
|
||||
for (const auto &[k, v] : headers) {
|
||||
req_hdrs.push_back(fmt::format("{}:{}", k, v));
|
||||
}
|
||||
curl_req.setOpt<curlpp::options::HttpHeader>(req_hdrs);
|
||||
}
|
||||
curl_req.setOpt<curlpp::options::FailOnError>(true);
|
||||
}
|
||||
|
||||
std::string response_string;
|
||||
std::string header_string;
|
||||
std::string response_header_string;
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string);
|
||||
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &header_string);
|
||||
void HttpClient::post(
|
||||
const string &url,
|
||||
const string &body,
|
||||
const unordered_map<string, string> &headers,
|
||||
IHttpResponse *response) {
|
||||
_LOG_REQ_BEGIN(POST, url, headers);
|
||||
curlpp::Easy curl_req;
|
||||
init_request_common(curl_req, url, headers);
|
||||
curl_req.setOpt<curlpp::options::Post>(true);
|
||||
curl_req.setOpt<curlpp::options::PostFields>(body);
|
||||
perform(curl_req, response);
|
||||
}
|
||||
|
||||
char *url;
|
||||
long response_code;
|
||||
double elapsed;
|
||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &elapsed);
|
||||
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
|
||||
// curl_easy_getinfo(curl, CURLINFO_HEADER_OUT, &response_header_string);
|
||||
void HttpClient::get(
|
||||
const string &url,
|
||||
const unordered_map<string, string> &headers,
|
||||
IHttpResponse *response) {
|
||||
_LOG_REQ_BEGIN(GET, url, headers);
|
||||
curlpp::Easy curl_req;
|
||||
init_request_common(curl_req, url, headers);
|
||||
curl_req.setOpt<curlpp::options::HttpGet>(true);
|
||||
perform(curl_req, response);
|
||||
}
|
||||
|
||||
curl_easy_perform(curl);
|
||||
curl_easy_cleanup(curl);
|
||||
curl = NULL;
|
||||
HttpResponse response;
|
||||
response.headers() = HttpHeaders(response_header_string);
|
||||
response.body() = std::move(response_string);
|
||||
response.code() = response_code;
|
||||
LOG(md, Info, "do GET: {}. Received response:\n{}", url_str, response);
|
||||
return response;
|
||||
tuple<string, string, int64_t> HttpClient::get_oauth_token(
|
||||
const string &url, const string &client_id, const string &client_secret) {
|
||||
LOG(Gen, Info, "Requesting oauth token from {}", url);
|
||||
std::vector<std::pair<string, string>> query_params{
|
||||
{"grant_type", "client_credentials"},
|
||||
{"client_id", client_id},
|
||||
{"client_secret", client_secret}};
|
||||
HttpResponse res;
|
||||
HttpClient::post(url, query_params_string(query_params), {}, &res);
|
||||
LOG(Gen, Info, "Received oauth token:\n{}", JsonDumper::dump(res));
|
||||
try {
|
||||
const auto j = json::parse(res.body);
|
||||
return {
|
||||
j.at("access_token").get<string>(), j.at("token_type").get<string>(),
|
||||
j.at("expires_in").get<int64_t>()};
|
||||
} catch (const std::exception &e) {
|
||||
LOG(Gen, Error, "Failed to parse oauth response json.{}\n{}",
|
||||
string{e.what()}, res.body);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,35 +1,85 @@
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstddef> // std::size_t
|
||||
#include <cstdio> // snprintf
|
||||
#include <cstdlib>
|
||||
#include <cstring> // strerror, strlen, memcpy, strcpy
|
||||
#include <ctime>
|
||||
#include <curl/curl.h>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <memory> // std::unique_ptr
|
||||
#pragma once
|
||||
|
||||
#include "http/http_response.h"
|
||||
#include "http/i_http_response.h"
|
||||
|
||||
#include <concurrencpp/concurrencpp.h>
|
||||
#include <concurrencpp/executors/executor.h>
|
||||
#include <concurrencpp/runtime/runtime.h>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <stdarg.h> // va_start, etc.
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <tuple>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
class HttpRequest;
|
||||
class HttpResponse;
|
||||
|
||||
class HttpClient {
|
||||
public:
|
||||
// HttpClient(const std::string &url);
|
||||
HttpResponse get(const HttpRequest &request);
|
||||
/**
|
||||
* Performs an http GET request
|
||||
*/
|
||||
static void get(
|
||||
const std::string &url,
|
||||
const std::unordered_map<std::string, std::string> &headers,
|
||||
IHttpResponse *response);
|
||||
|
||||
static concurrencpp::result<HttpResponse> get_async(
|
||||
const std::string &url,
|
||||
const std::unordered_map<std::string, std::string> &headers);
|
||||
|
||||
static concurrencpp::result<HttpResponse> get_async(
|
||||
const std::string &url,
|
||||
const std::unordered_map<std::string, std::string> &headers,
|
||||
std::shared_ptr<concurrencpp::executor> executor);
|
||||
|
||||
/**
|
||||
* Performs an http POST request
|
||||
*/
|
||||
static void post(
|
||||
const std::string &url,
|
||||
const std::string &body,
|
||||
const std::unordered_map<std::string, std::string> &headers,
|
||||
IHttpResponse *response);
|
||||
|
||||
/**
|
||||
* Requests an oauth access token
|
||||
*
|
||||
* @return tuple of: access token, token type, and seconds until expiration
|
||||
*/
|
||||
static std::tuple<std::string, std::string, std::int64_t> get_oauth_token(
|
||||
const std::string &url,
|
||||
const std::string &client_id,
|
||||
const std::string &client_secret);
|
||||
|
||||
private:
|
||||
static std::mutex global_runtime_mx_;
|
||||
static std::optional<concurrencpp::runtime> global_runtime_;
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline concurrencpp::result<HttpResponse> HttpClient::get_async(
|
||||
const std::string &url,
|
||||
const std::unordered_map<std::string, std::string> &headers) {
|
||||
if (!global_runtime_) {
|
||||
std::lock_guard<std::mutex> lock{global_runtime_mx_};
|
||||
global_runtime_.emplace();
|
||||
}
|
||||
return get_async(url, headers, global_runtime_->thread_executor());
|
||||
}
|
||||
|
||||
inline concurrencpp::result<HttpResponse> HttpClient::get_async(
|
||||
const std::string &url,
|
||||
const std::unordered_map<std::string, std::string> &headers,
|
||||
std::shared_ptr<concurrencpp::executor> executor) {
|
||||
co_return co_await executor->submit([&]() {
|
||||
HttpResponse res;
|
||||
get(url, headers, &res);
|
||||
return res;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,44 +0,0 @@
|
||||
#include "http_headers.h"
|
||||
#include "system/string_utils.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
using bts::system::string_utils::split;
|
||||
|
||||
HttpHeaders::HttpHeaders(const std::string &str) {
|
||||
auto item_strs = split(str, "\n");
|
||||
for (const auto &item_str : item_strs) {
|
||||
auto kv = split(item_str, ":");
|
||||
if (kv.empty()) {
|
||||
try_emplace("", "");
|
||||
continue;
|
||||
}
|
||||
if (kv.size() == 1) {
|
||||
try_emplace(kv[0], "");
|
||||
continue;
|
||||
}
|
||||
try_emplace(kv[0], kv[1]);
|
||||
}
|
||||
}
|
||||
|
||||
std::string HttpHeaders::to_string() const {
|
||||
auto it = begin();
|
||||
if (it == end()) {
|
||||
return "";
|
||||
}
|
||||
std::string rv{it->first + ": " + it->second};
|
||||
while (++it != end()) {
|
||||
rv.append("\n" + it->first + ": " + it->second);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
std::ostream &operator<<(
|
||||
std::ostream &out, const HttpHeaders &headers) noexcept {
|
||||
out << headers.to_string();
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,26 +1,51 @@
|
||||
#pragma once
|
||||
|
||||
#include "system/collections/flat_unordered_map.h"
|
||||
#include "system/string_utils.h"
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
class HttpHeaders
|
||||
: public collections::FlatUnorderedMap<std::string, std::string> {
|
||||
public:
|
||||
HttpHeaders() = default;
|
||||
explicit HttpHeaders(const std::string &header_str);
|
||||
std::string headers_to_string(
|
||||
const std::unordered_map<std::string, std::string> &headers);
|
||||
|
||||
HttpHeaders(const HttpHeaders &) noexcept = default;
|
||||
HttpHeaders(HttpHeaders &&) noexcept = default;
|
||||
HttpHeaders &operator=(const HttpHeaders &) noexcept = default;
|
||||
HttpHeaders &operator=(HttpHeaders &&) noexcept = default;
|
||||
std::unordered_map<std::string, std::string> headers_from_string(
|
||||
const std::string &headers_str);
|
||||
|
||||
std::string to_string() const;
|
||||
};
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
std::ostream &operator<<(
|
||||
std::ostream &out, const HttpHeaders &headers) noexcept;
|
||||
inline std::string headers_to_string(
|
||||
const std::unordered_map<std::string, std::string> &headers) {
|
||||
auto it = headers.begin();
|
||||
if (it == headers.end()) {
|
||||
return "";
|
||||
}
|
||||
std::string rv{it->first + ":" + it->second};
|
||||
while (++it != headers.end()) {
|
||||
rv.append("\n" + it->first + ":" + it->second);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
inline std::unordered_map<std::string, std::string> headers_from_string(
|
||||
const std::string &str) {
|
||||
std::unordered_map<std::string, std::string> headers;
|
||||
auto item_strs = system::string_utils::split(str, "\n");
|
||||
for (const auto &item_str : item_strs) {
|
||||
auto kv = system::string_utils::split(item_str, ":");
|
||||
if (kv.empty()) {
|
||||
headers.try_emplace("", "");
|
||||
continue;
|
||||
}
|
||||
if (kv.size() == 1) {
|
||||
headers.try_emplace(kv[0], "");
|
||||
continue;
|
||||
}
|
||||
headers.try_emplace(kv[0], kv[1]);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,17 +0,0 @@
|
||||
#include "http_request.h"
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
Url &HttpRequest::url() { return url_; }
|
||||
|
||||
const Url &HttpRequest::url() const { return url_; }
|
||||
|
||||
HttpHeaders &HttpRequest::headers() { return headers_; }
|
||||
|
||||
const HttpHeaders &HttpRequest::headers() const { return headers_; }
|
||||
|
||||
std::string &HttpRequest::body() { return body_; }
|
||||
|
||||
const std::string &HttpRequest::body() const { return body_; }
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,28 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "http/http_headers.h"
|
||||
#include "http/url.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
class HttpRequest {
|
||||
public:
|
||||
Url &url();
|
||||
const Url &url() const;
|
||||
|
||||
HttpHeaders &headers();
|
||||
const HttpHeaders &headers() const;
|
||||
|
||||
std::string &body();
|
||||
const std::string &body() const;
|
||||
|
||||
private:
|
||||
Url url_;
|
||||
HttpHeaders headers_;
|
||||
std::string body_;
|
||||
};
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,26 +0,0 @@
|
||||
#include "http_response.h"
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
HttpHeaders &HttpResponse::headers() { return headers_; }
|
||||
|
||||
const HttpHeaders &HttpResponse::headers() const { return headers_; }
|
||||
|
||||
std::string &HttpResponse::body() { return body_; }
|
||||
|
||||
const std::string &HttpResponse::body() const { return body_; }
|
||||
|
||||
std::int64_t &HttpResponse::code() { return code_; }
|
||||
|
||||
std::int64_t HttpResponse::code() const { return code_; }
|
||||
|
||||
std::ostream &operator<<(
|
||||
std::ostream &out, const HttpResponse &response) noexcept {
|
||||
out << response.headers() << "\n"
|
||||
<< response.body() << "\nCode: " << response.code();
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,35 +1,88 @@
|
||||
#pragma once
|
||||
|
||||
#include "http/url.h"
|
||||
#include "http_headers.h"
|
||||
#include "http/i_http_response.h"
|
||||
#include "system/FMT_IMPL.h"
|
||||
|
||||
#include <cstddef>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
class HttpResponse {
|
||||
/**
|
||||
* Basic implementation of `IHttpResponse`, with some useful state.
|
||||
*/
|
||||
class HttpResponse : public IHttpResponse {
|
||||
public:
|
||||
Url &url();
|
||||
const Url &url() const;
|
||||
std::string url;
|
||||
std::unordered_map<std::string, std::string> headers;
|
||||
std::string body;
|
||||
std::int64_t code;
|
||||
std::string errors;
|
||||
|
||||
HttpHeaders &headers();
|
||||
const HttpHeaders &headers() const;
|
||||
std::size_t write(char *cstr, std::size_t sz, std::size_t n) override;
|
||||
std::size_t write_headers(
|
||||
char *cstr, std::size_t sz, std::size_t n) override;
|
||||
void write_errors(std::string &&str) override;
|
||||
void write_code(std::int64_t code) override;
|
||||
constexpr bool ok() const;
|
||||
|
||||
std::string &body();
|
||||
const std::string &body() const;
|
||||
|
||||
std::int64_t &code();
|
||||
std::int64_t code() const;
|
||||
|
||||
private:
|
||||
Url url_;
|
||||
HttpHeaders headers_;
|
||||
std::string body_;
|
||||
std::int64_t code_;
|
||||
void reset();
|
||||
};
|
||||
|
||||
void to_json(nlohmann::json &j, const HttpResponse &res);
|
||||
void from_json(const nlohmann::json &j, HttpResponse &res);
|
||||
|
||||
std::ostream &operator<<(
|
||||
std::ostream &out, const HttpResponse &response) noexcept;
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline std::size_t HttpResponse::write(
|
||||
char *cstr, std::size_t sz, std::size_t n) {
|
||||
body.append(cstr, sz * n);
|
||||
return sz * n;
|
||||
}
|
||||
|
||||
inline std::size_t HttpResponse::write_headers(
|
||||
char *cstr, std::size_t sz, std::size_t n) {
|
||||
std::string str{cstr, sz * n};
|
||||
const auto colon_idx = str.find(':');
|
||||
if (colon_idx >= str.size()) {
|
||||
headers[str] = {};
|
||||
} else {
|
||||
headers[str.substr(0, colon_idx)] =
|
||||
str.substr(colon_idx + 1, str.size() - colon_idx - 1);
|
||||
}
|
||||
return sz * n;
|
||||
}
|
||||
|
||||
inline void HttpResponse::write_errors(std::string &&str) {
|
||||
errors = std::move(str);
|
||||
}
|
||||
|
||||
inline void HttpResponse::write_code(std::int64_t code) { this->code = code; }
|
||||
|
||||
constexpr bool HttpResponse::ok() const { return code >= 200 && code < 300; }
|
||||
|
||||
inline void HttpResponse::reset() {
|
||||
headers.clear();
|
||||
body.clear();
|
||||
errors.clear();
|
||||
code = 0;
|
||||
}
|
||||
|
||||
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(
|
||||
HttpResponse, url, headers, body, code, errors);
|
||||
|
||||
inline std::ostream &operator<<(
|
||||
std::ostream &out, const HttpResponse &res) noexcept {
|
||||
return out << nlohmann::json{res}.dump();
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
|
||||
FMT_IMPL(bts::http::HttpResponse, r, "{}", nlohmann::json{r}.dump());
|
||||
@@ -0,0 +1,35 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
/**
|
||||
* Simple http response interface, modeled after cURL callbacks
|
||||
*/
|
||||
class IHttpResponse {
|
||||
public:
|
||||
virtual ~IHttpResponse() = default;
|
||||
|
||||
virtual std::size_t write(char *cstr, std::size_t sz, std::size_t n) = 0;
|
||||
virtual std::size_t write_headers(char *cstr, std::size_t sz, std::size_t n);
|
||||
virtual void write_errors(std::string &&str);
|
||||
virtual void write_code(std::int64_t code);
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline std::size_t IHttpResponse::write_headers(
|
||||
char * /*cstr*/, std::size_t sz, std::size_t n) {
|
||||
return sz * n;
|
||||
}
|
||||
|
||||
inline void IHttpResponse::write_errors(std::string && /*str*/) {}
|
||||
|
||||
inline void IHttpResponse::write_code(std::int64_t /*code*/) {}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,18 +0,0 @@
|
||||
#include "http/query_params.h"
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
std::string QueryParams::to_string() const {
|
||||
std::string rv;
|
||||
auto it = begin();
|
||||
if (it == end()) {
|
||||
return "";
|
||||
}
|
||||
rv.append(it->first + "=" + it->second);
|
||||
while (++it != end()) {
|
||||
rv.append("&" + it->first + "=" + it->second);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
@@ -1,15 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#include "system/collections/flat_unordered_map.h"
|
||||
#include "system/FMT_IMPL.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace bts::http {
|
||||
|
||||
class QueryParams
|
||||
: public collections::FlatUnorderedMap<std::string, std::string> {
|
||||
template <class Iter>
|
||||
inline std::string query_params_string(Iter first, Iter last) {
|
||||
std::string rv;
|
||||
for (auto it = first; it != last; it++) {
|
||||
if (it->first.empty()) {
|
||||
continue;
|
||||
}
|
||||
if (!rv.empty()) {
|
||||
rv.append("&");
|
||||
}
|
||||
rv.append(it->first);
|
||||
if (!it->second.empty()) {
|
||||
rv.append("=");
|
||||
rv.append(it->second);
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
template <class Range>
|
||||
inline std::string query_params_string(Range &&range) {
|
||||
return query_params_string(range.begin(), range.end());
|
||||
}
|
||||
|
||||
class QueryParams : public std::unordered_map<std::string, std::string> {
|
||||
public:
|
||||
std::string to_string() const;
|
||||
};
|
||||
|
||||
inline std::string QueryParams::to_string() const {
|
||||
std::string rv;
|
||||
auto it = begin();
|
||||
if (it == end()) {
|
||||
return "";
|
||||
}
|
||||
rv.append(it->first + "=" + it->second);
|
||||
while (++it != end()) {
|
||||
rv.append("&" + it->first + "=" + it->second);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
inline bool operator==(const QueryParams &lhs, const QueryParams &rhs) {
|
||||
return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
|
||||
}
|
||||
|
||||
inline bool operator!=(const QueryParams &lhs, const QueryParams &rhs) {
|
||||
return !(lhs == rhs);
|
||||
}
|
||||
|
||||
} // namespace bts::http
|
||||
|
||||
FMT_IMPL(bts::http::QueryParams, p, "{}", p.to_string());
|
||||
@@ -1,5 +1,7 @@
|
||||
function(get_link_libraries dest)
|
||||
find_package(fmt CONFIG REQUIRED)
|
||||
find_package(OpenSSL REQUIRED)
|
||||
find_package(cryptopp CONFIG REQUIRED)
|
||||
set(${dest}
|
||||
system
|
||||
domain
|
||||
@@ -15,13 +17,12 @@ function(get_link_libraries dest)
|
||||
http
|
||||
SQLiteCpp
|
||||
sqlite3
|
||||
curl
|
||||
uuid
|
||||
ssl
|
||||
crypto
|
||||
OpenSSL::SSL
|
||||
OpenSSL::Crypto
|
||||
fmt::fmt
|
||||
tbb
|
||||
cryptopp
|
||||
cryptopp::cryptopp
|
||||
pugixml
|
||||
lttng-ust
|
||||
Pcap++
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
#define CONFIG_H_
|
||||
|
||||
#include "config.h"
|
||||
#include "http/http_client.h"
|
||||
#include "http/http_response.h"
|
||||
#include "spark/protobufs/config_defaults.h"
|
||||
#include "spark/protobufs/config_proto_files.h"
|
||||
#include "system/bts_array_size.h"
|
||||
#include "system/config/configuration_state.h"
|
||||
#include "spark/protobufs/config_proto_files.h"
|
||||
#include "spark/protobufs/config_defaults.h"
|
||||
#include "system/os.h"
|
||||
|
||||
#include <google/protobuf/descriptor.h>
|
||||
@@ -43,7 +45,21 @@ internal::ConfigurationState &config_state() {
|
||||
defaults_str.append(line);
|
||||
defaults_str.append("\n");
|
||||
}
|
||||
config_state_.emplace(proto_files_vec, defaults_str);
|
||||
config_state_.emplace(
|
||||
proto_files_vec, defaults_str,
|
||||
[](const std::string &url, std::stringstream &out) {
|
||||
http::HttpResponse res;
|
||||
http::HttpClient::get(url, {}, &res);
|
||||
if (!res.ok()) {
|
||||
return false;
|
||||
}
|
||||
// This isn't very efficient. Our HttpResponse populates a string, which is copied
|
||||
// to a stringstream, which is likely (eventually) dumped back to a string by the
|
||||
// original caller.
|
||||
// TODO: avoid copying strings in this pipeline.
|
||||
out << res.body;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
return *config_state_;
|
||||
|
||||
@@ -23,6 +23,9 @@ namespace openssl {
|
||||
namespace bts {
|
||||
namespace crypto {
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
||||
|
||||
struct PrivateKey::Pimp {
|
||||
BTS_NO_COPY(Pimp);
|
||||
|
||||
@@ -206,7 +209,8 @@ int private_encrypt_func(
|
||||
return -1;
|
||||
} else {
|
||||
return RSA_private_encrypt(
|
||||
inlen, const_cast<unsigned char *>(in), out, rsa, RSA_PKCS1_PADDING);
|
||||
inlen, const_cast<unsigned char *>(in), out,
|
||||
const_cast<openssl::RSA *>(rsa), RSA_PKCS1_PADDING);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,7 +227,8 @@ int public_decrypt_func(
|
||||
return -1;
|
||||
} else {
|
||||
int i = RSA_public_decrypt(
|
||||
inlen, const_cast<unsigned char *>(in), out, rsa, RSA_PKCS1_PADDING);
|
||||
inlen, const_cast<unsigned char *>(in), out,
|
||||
const_cast<openssl::RSA *>(rsa), RSA_PKCS1_PADDING);
|
||||
out[i] = '\0';
|
||||
*outlen = i;
|
||||
return i;
|
||||
@@ -293,6 +298,8 @@ void PrivateKey::write_stream(
|
||||
BIO_free(biostream_out.getBIO());
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
} // namespace crypto
|
||||
} // namespace bts
|
||||
#endif
|
||||
@@ -1,73 +0,0 @@
|
||||
#include "config_downloader.h"
|
||||
#include "system/facades/bts_assert.h"
|
||||
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <sstream>
|
||||
#include <stdio.h>
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace bts {
|
||||
namespace config {
|
||||
namespace internal {
|
||||
|
||||
ConfigDownloader::ConfigDownloader() : _curl(NULL) {
|
||||
curl_global_init(CURL_GLOBAL_DEFAULT);
|
||||
_curl = curl_easy_init();
|
||||
bts_assert(_curl);
|
||||
}
|
||||
|
||||
ConfigDownloader::~ConfigDownloader() { curl_easy_cleanup(_curl); }
|
||||
|
||||
size_t curl_write(char *buffer, size_t size, size_t count, void *stream) {
|
||||
stringstream *out = static_cast<stringstream *>(stream);
|
||||
bts_assert(out);
|
||||
size_t bytes = size * count;
|
||||
out->write(buffer, bytes);
|
||||
return count;
|
||||
}
|
||||
|
||||
bool is_http(const string &url) { return url.substr(0, 7) == "http://"; }
|
||||
|
||||
int no_linger(void *, curl_socket_t socket, curlsocktype) {
|
||||
linger l;
|
||||
l.l_onoff = 1;
|
||||
l.l_linger = 0;
|
||||
setsockopt(socket, SOL_SOCKET, SO_LINGER, (char *)&l, sizeof(l));
|
||||
return CURL_SOCKOPT_OK;
|
||||
}
|
||||
|
||||
ConfigDownloader::Result
|
||||
ConfigDownloader::get(const string &url, stringstream &out_stream) {
|
||||
curl_easy_setopt(_curl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, curl_write);
|
||||
curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream);
|
||||
curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0);
|
||||
curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1);
|
||||
curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 6);
|
||||
curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger);
|
||||
|
||||
CURLcode res = curl_easy_perform(_curl);
|
||||
|
||||
long http_code = 0;
|
||||
CURLcode r2 = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &http_code);
|
||||
bts_assert(r2 == CURLE_OK);
|
||||
|
||||
bool used_http = is_http(url);
|
||||
if (res == CURLE_OK && (http_code == 200 || !used_http)) {
|
||||
return SUCCESS;
|
||||
} else if (
|
||||
res == CURLE_OPERATION_TIMEDOUT || res == CURLE_COULDNT_RESOLVE_HOST ||
|
||||
res == CURLE_COULDNT_RESOLVE_PROXY || res == CURLE_COULDNT_CONNECT ||
|
||||
res == CURLE_REMOTE_ACCESS_DENIED || res == CURLE_FTP_CANT_GET_HOST) {
|
||||
// Cannot rely on logging before config is initialized
|
||||
// LOG(Error, Config) << "Permanent failure loading '" << url << "' - CURL
|
||||
// error: " << curl_easy_strerror(res);
|
||||
return PERMANENT_FAILURE;
|
||||
}
|
||||
return FAILURE;
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
} // namespace config
|
||||
} // namespace bts
|
||||
@@ -1,27 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <string>
|
||||
|
||||
namespace bts {
|
||||
namespace config {
|
||||
namespace internal {
|
||||
|
||||
class ConfigDownloader {
|
||||
public:
|
||||
ConfigDownloader();
|
||||
ConfigDownloader(const ConfigDownloader &) = delete;
|
||||
ConfigDownloader &operator=(const ConfigDownloader &) = delete;
|
||||
~ConfigDownloader();
|
||||
|
||||
enum Result { FAILURE, SUCCESS, PERMANENT_FAILURE };
|
||||
|
||||
Result get(const std::string &url, std::stringstream &out_stream);
|
||||
|
||||
private:
|
||||
CURL *_curl;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
} // namespace config
|
||||
} // namespace bts
|
||||
@@ -1,23 +1,25 @@
|
||||
#include "configuration_state.h"
|
||||
// #include "spark/config/config.h"
|
||||
#include "system/FMT.h"
|
||||
#include "system/bts_array_size.h"
|
||||
#include "system/config/config_error_collector.h"
|
||||
#include "system/config/config_property_evaluator.h"
|
||||
#include "system/config/config_value_queue.h"
|
||||
#include "system/config/configuration_exception.h"
|
||||
#include "system/config/preprocess_input_stream.h"
|
||||
#include "system/facades/bts_assert.h"
|
||||
#include "system/defer.h"
|
||||
#include "system/FMT.h"
|
||||
#include "system/bts_array_size.h"
|
||||
#include "system/facades/bts_assert.h"
|
||||
#include "system/os.h"
|
||||
#include "system/protobufs/config/system_config.pb.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <boost/algorithm/string.hpp> // iequals
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
#include <fstream>
|
||||
#include <google/protobuf/descriptor.h>
|
||||
#include <google/protobuf/descriptor.pb.h>
|
||||
#include <google/protobuf/io/zero_copy_stream_impl.h>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <regex>
|
||||
@@ -68,13 +70,16 @@ const char *CONFIG_SERVER_ENV_VAR = "BTSConfig";
|
||||
inline tuple<string, uint32_t> hostname(const std::string &path);
|
||||
|
||||
ConfigurationState::ConfigurationState(
|
||||
const std::vector<std::string> &proto_files, const std::string &defaults)
|
||||
const std::vector<std::string> &proto_files,
|
||||
const std::string &defaults,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch)
|
||||
: _is_initialized(false), _initialization_mutex(), _dynamic_pool(),
|
||||
_prototype_factory(), _prototype(nullptr), _config_chain(), _defaults(),
|
||||
_properties(new ConfigPropertyEvaluator(*this)),
|
||||
_rule_engine(_properties), _macros(),
|
||||
_deferred_queue(new ConfigValueQueue(*_properties.get())), _host(),
|
||||
_port(80), proto_files_(proto_files), defaults_(defaults) {}
|
||||
_port(80), fetch_(fetch), proto_files_(proto_files), defaults_(defaults) {
|
||||
}
|
||||
|
||||
ConfigurationState::~ConfigurationState(void) {}
|
||||
|
||||
@@ -202,9 +207,7 @@ void ConfigurationState::initialize_builtins(ConfigLog *log) {
|
||||
|
||||
sys.set_num_cores(os::num_cores());
|
||||
if (log != nullptr) {
|
||||
CFG_LOG(
|
||||
(*log), Info, "Initialized SystemConfig:\n{}",
|
||||
sys.DebugString());
|
||||
CFG_LOG((*log), Info, "Initialized SystemConfig:\n{}", sys.DebugString());
|
||||
}
|
||||
|
||||
//
|
||||
@@ -416,7 +419,8 @@ int ConfigurationState::load(const std::string &url, ConfigLog &log) {
|
||||
bool loaded = false;
|
||||
CFG_LOG(log, Info, "Loading '{}'", url);
|
||||
loaded = load_message(
|
||||
url, *new_message.get(), log, *_deferred_queue.get(), proto_files_);
|
||||
url, *new_message.get(), log, *_deferred_queue.get(), fetch_,
|
||||
proto_files_);
|
||||
if (_host.empty() || _host == "localhost") {
|
||||
const auto old_host = _host;
|
||||
const auto old_port = _port;
|
||||
@@ -912,18 +916,16 @@ bool load_message(
|
||||
proto::Message &out_message,
|
||||
ConfigLog &log,
|
||||
ConfigValueQueue &lookup,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
const std::vector<std::string> &proto_files) {
|
||||
ConfigDownloader downloader;
|
||||
|
||||
bool result = false;
|
||||
switch (get_path_type(url)) {
|
||||
case PathType::directory:
|
||||
result = load_directory(
|
||||
url, downloader, out_message, log, lookup, proto_files) > 0;
|
||||
result =
|
||||
load_directory(url, fetch, out_message, log, lookup, proto_files) > 0;
|
||||
break;
|
||||
case PathType::file:
|
||||
result = load_file(url, downloader, out_message, log, lookup) ==
|
||||
ConfigDownloader::SUCCESS;
|
||||
result = load_file(url, fetch, out_message, log, lookup);
|
||||
break;
|
||||
default:
|
||||
std::ostringstream error;
|
||||
@@ -939,7 +941,7 @@ bool load_message(
|
||||
|
||||
int load_dir(
|
||||
const std::string &url,
|
||||
ConfigDownloader &downloader,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
proto::Message &out_message,
|
||||
ConfigLog &log,
|
||||
ConfigValueQueue &lookup,
|
||||
@@ -961,7 +963,7 @@ int load_dir(
|
||||
boost::filesystem::path p(config_file);
|
||||
config_file = p.filename().string();
|
||||
if (load_file(
|
||||
url + config_file, downloader, out_message, log, lookup, false)) {
|
||||
url + config_file, fetch, out_message, log, lookup, false)) {
|
||||
load_count++;
|
||||
if (!edge_has_include) {
|
||||
bts_assert_msg(
|
||||
@@ -979,7 +981,7 @@ int load_dir(
|
||||
|
||||
int load_directory(
|
||||
std::string url,
|
||||
ConfigDownloader &downloader,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
proto::Message &out_message,
|
||||
ConfigLog &log,
|
||||
ConfigValueQueue &lookup,
|
||||
@@ -993,50 +995,52 @@ int load_directory(
|
||||
|
||||
// If a file called "all" exists in the config directory we can short circuit
|
||||
// ~60 http requests
|
||||
ConfigDownloader::Result all_result =
|
||||
load_file(url + ".all", downloader, out_message, log, lookup, false);
|
||||
if (all_result == ConfigDownloader::SUCCESS) {
|
||||
bool success =
|
||||
load_file(url + ".all", fetch, out_message, log, lookup, false);
|
||||
if (success) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// load spark.conf
|
||||
int load_count = 0;
|
||||
ConfigDownloader::Result result = load_file(
|
||||
url + "spark.conf", downloader, out_message, log, lookup, false);
|
||||
if (result == ConfigDownloader::SUCCESS) {
|
||||
success =
|
||||
load_file(url + "spark.conf", fetch, out_message, log, lookup, false);
|
||||
if (success) {
|
||||
load_count++;
|
||||
} else if (result == ConfigDownloader::PERMANENT_FAILURE) {
|
||||
// If the server is down it doesn't make sense to suffer though the
|
||||
// timeouts for the other files
|
||||
throw ServerUnreachableException(url.c_str());
|
||||
}
|
||||
|
||||
load_count +=
|
||||
load_dir(url, downloader, out_message, log, lookup, proto_files);
|
||||
load_count += load_dir(url, fetch, out_message, log, lookup, proto_files);
|
||||
return load_count;
|
||||
}
|
||||
|
||||
ConfigDownloader::Result load_file(
|
||||
bool load_file(
|
||||
std::string path,
|
||||
ConfigDownloader &downloader,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
proto::Message &out_message,
|
||||
ConfigLog &log,
|
||||
ConfigValueQueue &lookup,
|
||||
bool required) {
|
||||
// first, try loading local file.
|
||||
if (boost::filesystem::exists(path)) {
|
||||
std::ifstream fs{path};
|
||||
proto::io::IstreamInputStream proto_stream(&fs);
|
||||
parse_stream(path, proto_stream, out_message, log, lookup);
|
||||
return true;
|
||||
}
|
||||
bool success = false;
|
||||
stringstream url_stream;
|
||||
string url = url_from_string(path, required).to_string();
|
||||
ConfigDownloader::Result result = downloader.get(url, url_stream);
|
||||
if (!(result == ConfigDownloader::SUCCESS)) {
|
||||
success = fetch(url, url_stream);
|
||||
if (!success) {
|
||||
CFG_LOG(log, Info, "Failed to load config from '{}'", url);
|
||||
bts_assert_msg(!required || url_stream, (path + " not found.").c_str());
|
||||
return result;
|
||||
} else {
|
||||
CFG_LOG(log, Info, "Parsing config loaded from '{}'", url);
|
||||
proto::io::IstreamInputStream proto_stream(&url_stream);
|
||||
parse_stream(path, proto_stream, out_message, log, lookup);
|
||||
}
|
||||
|
||||
CFG_LOG(log, Info, "Parsing config loaded from '{}'", url);
|
||||
proto::io::IstreamInputStream proto_stream(&url_stream);
|
||||
parse_stream(path, proto_stream, out_message, log, lookup);
|
||||
|
||||
return result;
|
||||
return success;
|
||||
}
|
||||
|
||||
void parse_stream(
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
#pragma once
|
||||
#include <string>
|
||||
#ifndef CONFIGURATIONSTATE_H_
|
||||
#define CONFIGURATIONSTATE_H_
|
||||
|
||||
#include "system/config/config_downloader.h"
|
||||
#include "system/config/config_log.h"
|
||||
#include "system/config/configuration_exception.h"
|
||||
#include "system/config/text_format/text_format.h"
|
||||
@@ -18,6 +16,7 @@
|
||||
#include <boost/scoped_ptr.hpp>
|
||||
#include <boost/utility.hpp>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace google {
|
||||
@@ -50,7 +49,9 @@ class ConfigurationState {
|
||||
public:
|
||||
ConfigurationState(
|
||||
const std::vector<std::string> &proto_files,
|
||||
const std::string &defaults);
|
||||
const std::string &defaults,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)>
|
||||
fetch);
|
||||
~ConfigurationState(void);
|
||||
|
||||
void initialize_defaults(ConfigLog &log);
|
||||
@@ -139,6 +140,7 @@ class ConfigurationState {
|
||||
// ConfigLog _log;
|
||||
std::string _host;
|
||||
uint32_t _port;
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch_;
|
||||
std::vector<std::string> proto_files_;
|
||||
std::string defaults_;
|
||||
};
|
||||
@@ -158,17 +160,18 @@ bool load_message(
|
||||
proto::Message &out_message,
|
||||
ConfigLog &,
|
||||
ConfigValueQueue &,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
const std::vector<std::string> &proto_files);
|
||||
int load_directory(
|
||||
std::string url,
|
||||
ConfigDownloader &,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
proto::Message &out_message,
|
||||
ConfigLog &,
|
||||
ConfigValueQueue &,
|
||||
const std::vector<std::string> &proto_files);
|
||||
ConfigDownloader::Result load_file(
|
||||
bool load_file(
|
||||
std::string url,
|
||||
ConfigDownloader &,
|
||||
std::function<bool(const std::string &url, std::stringstream &out)> fetch,
|
||||
proto::Message &out_message,
|
||||
ConfigLog &,
|
||||
ConfigValueQueue &,
|
||||
|
||||
+5
-1
@@ -1,7 +1,11 @@
|
||||
{
|
||||
"dependencies": [
|
||||
"fmt",
|
||||
"catch2"
|
||||
"catch2",
|
||||
"concurrencpp",
|
||||
"curlpp",
|
||||
"cryptopp",
|
||||
"openssl"
|
||||
],
|
||||
"builtin-baseline": "bb002e69deb36c00e9ce21810f69eff349a833f7",
|
||||
"overrides": [
|
||||
|
||||
Reference in new issue
Block a user