Files
spark/src/lib/config/ConfigurationState.cpp
T
John Colvin 9d79490a20 fix build (client): Change 'WIN32' macros to '_WIN32'
Not sure exactly how this broke, but the WIN32 macro is no longer defined.
2022-10-12 20:13:18 -04:00

962 lines
32 KiB
C++

#include "ConfigurationState.h"
#include "ConfigErrorCollector.h"
#include "ConfigPropertyEvaluator.h"
#include "ConfigValueQueue.h"
#include "ConfigurationException.h"
#include "PreprocessInputStream.h"
#include "config.h"
#include "lib/os/os.h"
#include "lib/utils/bts_array_size.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/defer.h"
#include "lib/utils/is_unittest_environment.h"
#include "lib/config/protos/system.pb.h"
#include <boost/algorithm/string.hpp> // iequals
#include <boost/filesystem.hpp>
#include <fstream>
#include <iostream>
#include <map>
#include <tuple>
#include <vector>
#define URDL_HEADER_ONLY 1
#define URDL_DISALE_SSL 1
#include <urdl/url.hpp>
#include "lib/config_defaults.h"
#include "lib/config_proto_files.h"
#ifdef _WIN32
#include <Winsock2.h>
#include <d2d1.h>
#else
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/in.h>
#include <sys/socket.h>
#endif
#include "lib/os/os.h"
using namespace std;
namespace bts {
namespace config {
namespace internal {
const char *MESSAGE_STRING_RULE = "rule";
const char *MESSAGE_STRING_VALUE = "value";
const char *MESSAGE_STRING_INCLUDE = "include";
const char *MESSAGE_STRING_INCLUDE_BACKUP = "fallback";
const char *MESSAGE_STRING_MACRO_TYPE = "MacroType";
const char *MESSAGE_STRING_MACRO = "macro";
const char *CONFIG_SERVER_ENV_VAR = "BTSConfig";
inline tuple<string, uint32_t> hostname(const std::string &path);
ConfigurationState::ConfigurationState(void)
: _is_initialized(false), _initialization_mutex(), _dynamic_pool(),
_prototype_factory(), _prototype(nullptr), _config_chain(), _defaults(),
_properties(new ConfigPropertyEvaluator(*this)),
_rule_engine(_properties), _macros(),
_deferred_queue(new ConfigValueQueue(*_properties.get())), _host(),
_port(80) {}
ConfigurationState::~ConfigurationState(void) {}
std::string get_command_argument(const char *key) {
std::vector<std::string> cmd_line = os::get_command_line();
std::vector<std::string>::iterator begin = cmd_line.begin(),
end = cmd_line.end();
for (; begin != end; begin++) {
if (boost::iequals(*begin, key)) {
begin++;
if (begin == end) {
break;
} else {
os::setenv(CONFIG_SERVER_ENV_VAR, begin->c_str());
return *begin;
}
}
}
return "";
}
string ConfigurationState::settings_path() {
// Look for the -ConfigServer option on the command line
std::string cmd_arg = get_command_argument("-ConfigServer");
if (!cmd_arg.empty())
return cmd_arg;
// Then the BTSCONFIG environment variable
const char *env_server = getenv(CONFIG_SERVER_ENV_VAR);
if (env_server)
return env_server;
// then look in sparkdata
return os::sparkdata_path("conf");
}
void ConfigurationState::initialize_defaults(IConfigLog &log) {
initialize_defaults(
log,
{std::string("lib/config/protos/"), std::string("lib/api/domain/")});
}
void ConfigurationState::initialize_defaults(
IConfigLog &log, const std::vector<std::string> &proto_dirs) {
std::lock_guard<std::recursive_mutex> lock(_initialization_mutex);
if (_is_initialized)
return;
generate_all_config_proto(log, proto_dirs);
std::shared_ptr<proto::Message> new_message(_prototype->New());
ErrorsOnlyLog elog(log);
std::string defaults;
for (const char *str : config_static_defaults) {
defaults += str;
defaults += "\n";
}
proto::io::ArrayInputStream input_stream(
defaults.c_str(), (int)defaults.length());
parse_stream(
"static_defaults.conf", input_stream, *new_message.get(), elog,
*_deferred_queue.get());
_defaults = new_message;
// do not process 'include' for static defaults
_is_initialized = true;
}
void ConfigurationState::initialize_builtins() {
System sys;
sys.set_host_name(os::host_name());
sys.set_process_name(os::get_application_name());
sys.set_directory(os::get_install_path());
sys.set_platform(os::platform_name());
sys.set_qualifier(os::get_qualifier());
sys.set_unittest_environment(utils::internal::is_unittest_environment());
sys.set_data_path(os::bts_data_path());
sys.set_os_user(os::username());
#ifndef NDEBUG
sys.set_build_type("debug");
#endif
sys.set_config_host(_host);
sys.set_config_port(_port);
if (!_host.empty()) {
hostent *host_entry = gethostbyname(_host.c_str());
bts_assert_msg(
host_entry != NULL, "Unable to resolve host name " + _host);
in_addr *address = (in_addr *)host_entry->h_addr_list[0];
string ip_address = inet_ntoa(*address);
sys.set_config_ip(ip_address);
}
#ifdef _WIN32
sys.set_is_remote_app(GetSystemMetrics(SM_REMOTESESSION) == 1);
// Screen dpi, Assumes screen dpi is uniform in x and y
// Computes 2 different ratios depending - one will be right depending on
// whether the process calls SetDPIAware()
HDC screen = GetDC(0);
int log_pixels_x = GetDeviceCaps(screen, LOGPIXELSX);
auto ratio2 = log_pixels_x / 96.0;
int virtual_width = GetDeviceCaps(screen, HORZRES);
int physical_width = GetDeviceCaps(screen, DESKTOPHORZRES);
double ratio = (double)physical_width / (double)virtual_width;
sys.set_dpi(log_pixels_x * ratio);
sys.set_dpi_ratio(max(ratio, ratio2));
ReleaseDC(0, screen);
#endif
std::vector<std::string> args = os::get_command_line();
for (std::vector<std::string>::iterator iter = args.begin();
iter != args.end(); iter++) {
sys.add_cmd_args(*iter);
}
// Version string
std::string app_version_file = os::get_install_path() + "/spark.version";
if (boost::filesystem::exists(app_version_file)) {
std::ifstream input(app_version_file.c_str(), std::ifstream::in);
if (!input.fail()) {
// Read the last line of the file
std::string version;
while (!input.eof()) {
std::string temp;
std::getline(input, temp);
if (!temp.empty())
version = temp;
}
sys.set_version(version);
input.close();
}
}
sys.set_num_cores(os::num_cores());
//
// Add the System value to the defaults
const proto::FieldDescriptor *field_desc =
_defaults->GetDescriptor()->FindFieldByName("System");
const proto::Descriptor *message_desc = field_desc->message_type();
// Create a meta message
proto::Message *new_message =
_defaults->GetReflection()->AddMessage(_defaults.get(), field_desc);
const proto::FieldDescriptor *value_desc =
message_desc->FindFieldByName("value");
new_message->GetReflection()
->MutableMessage(new_message, value_desc)
->CopyFrom(sys);
initialize_introspection();
}
void ConfigurationState::initialize_introspection() {
// With this config can be self describing enough
// that another language can request "all" of config
ConfigIntrospection ci;
proto::DescriptorPool::generated_pool();
for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) {
std::string proto_path = "core/bts_core/config/";
proto_path += config_proto_files[x];
const proto::FileDescriptor *file_desc =
proto::DescriptorPool::generated_pool()->FindFileByName(proto_path);
if (file_desc != nullptr) {
ConfigIntrospection::ConfigProto *config_proto = ci.add_config();
bts_assert(config_proto);
config_proto->set_name(file_desc->name());
for (int y = 0; y < file_desc->message_type_count(); y++) {
const proto::Descriptor *message_desc = file_desc->message_type(y);
if (message_desc != nullptr) {
config_proto->add_type(message_desc->full_name());
}
}
}
}
// Add to defaults
const proto::FieldDescriptor *field_desc =
_defaults->GetDescriptor()->FindFieldByName("ConfigIntrospection");
const proto::Descriptor *message_desc = field_desc->message_type();
proto::Message *new_message =
_defaults->GetReflection()->AddMessage(_defaults.get(), field_desc);
const proto::FieldDescriptor *value_desc =
message_desc->FindFieldByName("value");
new_message->GetReflection()
->MutableMessage(new_message, value_desc)
->CopyFrom(ci);
}
void ConfigurationState::initialize_macros() {
std::vector<std::shared_ptr<proto::Message>>::iterator iter;
for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) {
const proto::Message *message = (*iter).get();
add_macro(message);
}
add_macro(_defaults.get());
}
void ConfigurationState::add_macro(const proto::Message *message) {
bts_assert(message);
const proto::FieldDescriptor *field_desc =
message->GetDescriptor()->FindFieldByName(MESSAGE_STRING_MACRO);
if (field_desc) {
int size = message->GetReflection()->FieldSize(*message, field_desc);
for (int index = 0; index < size; index++) {
// read "name" and "expression" fields
const proto::Message &meta_message =
message->GetReflection()->GetRepeatedMessage(
*message, field_desc, index);
const proto::FieldDescriptor *name_desc =
meta_message.GetDescriptor()->FindFieldByName("name");
const std::string name = '$' + meta_message.GetReflection()->GetString(
meta_message, name_desc);
const proto::FieldDescriptor *exp_desc =
meta_message.GetDescriptor()->FindFieldByName("expression");
const std::string expression =
meta_message.GetReflection()->GetString(meta_message, exp_desc);
if (_macros.find(name) == _macros.end()) {
// eval expression to Value
// expression::Value result =
// _rule_engine.eval_to_value(expression);
_macros[name] = expression;
}
}
}
}
expression::Value ConfigurationState::get_macro(const std::string &name) {
map<string, string>::iterator iter = _macros.find(name);
if (iter == _macros.end()) {
std::string error = std::string("Macro '") + name + "' is undefined";
throw ConfigurationException(error);
}
return _rule_engine.eval_to_value(iter->second);
}
std::string ConfigurationState::get_macro_string(const std::string &name) {
try {
expression::Value result = get_macro(name);
return expression::value_to_string(result);
} catch (const ConfigurationException &e) {
return e.what();
}
}
void check_rule(
proto::Message *message, expression::ExpressionEvaluator &rule_engine) {
const proto::Reflection *reflection = message->GetReflection();
const proto::Descriptor *descriptor = message->GetDescriptor();
for (int findex = 0; findex < descriptor->field_count(); ++findex) {
const proto::FieldDescriptor *field = descriptor->field(findex);
if (field->is_repeated()) {
int field_size = reflection->FieldSize(*message, field);
for (int index = 0; index < field_size; ++index) {
const proto::Message &field_message =
reflection->GetRepeatedMessage(*message, field, index);
const proto::FieldDescriptor *rule_descriptor =
field_message.GetDescriptor()->FindFieldByName(
MESSAGE_STRING_RULE);
if (rule_descriptor) { // Macros don't have rules
const std::string rule =
field_message.GetReflection()->GetString(
field_message, rule_descriptor);
rule_engine.eval(rule);
}
}
}
}
}
void ConfigurationState::check_rules() {
initialize();
std::vector<std::shared_ptr<proto::Message>>::iterator iter;
for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) {
proto::Message *message = (*iter).get();
check_rule(message, _rule_engine);
}
check_rule(_defaults.get(), _rule_engine);
}
void ConfigurationState::initialize_settings() {
std::lock_guard<std::recursive_mutex> lock(_initialization_mutex);
if (_is_initialized)
return;
ConfigLog log;
initialize_defaults(log);
std::string settings = settings_path();
std::string fallback = get_command_argument("-ConfigFallback");
DEFER( // Do this even if load() fails
initialize_builtins(); initialize_macros(); _deferred_queue->process(););
load(settings, fallback, log);
}
void ConfigurationState::reset() {
std::lock_guard<std::recursive_mutex> lock(_initialization_mutex);
_config_chain.clear();
_prototype = nullptr;
_defaults.reset();
_prototype_factory.reset();
_dynamic_pool.reset();
_macros.clear();
_deferred_queue->clear();
_is_initialized = false;
_host.clear();
_port = 80;
}
int ConfigurationState::load(
const std::string &url, const std::string &fallback_url, IConfigLog &log) {
std::shared_ptr<proto::Message> new_message(_prototype->New());
bool loaded = false;
try {
loaded =
load_message(url, *new_message.get(), log, *_deferred_queue.get());
} catch (ServerUnreachableException &) {
if (fallback_url.empty()) {
throw;
}
}
if (!loaded && !fallback_url.empty()) {
std::string message = "Failed to load: " + url + " using fallback.";
log.warning(message);
loaded = load_message(
fallback_url, *new_message.get(), log, *_deferred_queue.get());
if (_host.empty() || _host == "localhost") {
tie(_host, _port) = hostname(fallback_url);
}
}
if (_host.empty() || _host == "localhost") {
tie(_host, _port) = hostname(url);
}
if (!loaded)
return 0;
std::string include = get_include(*new_message.get());
std::string fallback = get_fallback(*new_message.get());
// insert at front?
_config_chain.push_back(new_message);
// Recursively load includes
if (!include.empty()) {
return 1 + load(include, fallback, log);
}
return 1;
}
int ConfigurationState::load_string(
const std::string &values, bool print_errors) {
ConfigLog log;
log.set_print_errors(print_errors);
initialize_defaults(log);
std::shared_ptr<proto::Message> new_message(_prototype->New());
proto::io::ArrayInputStream input_stream(
values.c_str(), (int)values.length());
parse_stream(
"test string", input_stream, *new_message.get(), log,
*_deferred_queue.get());
std::string include = get_include(*new_message.get());
std::string fallback = get_fallback(*new_message.get());
_config_chain.push_back(new_message);
int count = 0;
if (!include.empty()) {
count += load(include, fallback, log);
}
if (_host.empty()) {
_host = "localhost";
}
initialize_builtins();
initialize_macros();
_deferred_queue->process();
return 1 + count;
}
void ConfigurationState::add_metadata_field(
proto::FileDescriptorProto *file_dp,
proto::DescriptorProto *message_dp,
const proto::Descriptor *base_field,
int index) {
// Create the metadata message
proto::DescriptorProto *meta_message = file_dp->add_message_type();
std::string meta_name = base_field->name() + "meta";
meta_message->set_name(meta_name);
// Add 'rule'
proto::FieldDescriptorProto *rule_field = meta_message->add_field();
rule_field->set_name(MESSAGE_STRING_RULE);
rule_field->set_number(1);
rule_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
rule_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL);
rule_field->set_default_value("true");
// Add 'value'
proto::FieldDescriptorProto *value_field = meta_message->add_field();
value_field->set_name(MESSAGE_STRING_VALUE);
value_field->set_number(2);
value_field->set_type_name(base_field->full_name());
value_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED);
// Now add a field to message_dp of that type
proto::FieldDescriptorProto *field_desc = message_dp->add_field();
field_desc->set_name(base_field->name());
field_desc->set_number(index);
field_desc->set_type_name(meta_name);
field_desc->set_label(proto::FieldDescriptorProto_Label_LABEL_REPEATED);
}
// Construct a new Message that describes the save file format
// This can then be used with google::protobuf::TextFormat to load and save
// files
void ConfigurationState::generate_all_config_proto(
IConfigLog &log, const std::vector<std::string> &proto_dirs) {
const char *bts_config_classname = "__BTSConfig";
int field_number = 0;
proto::FileDescriptorProto bts_config_dp;
bts_config_dp.set_name("config.proto");
proto::DescriptorProto *config_message = bts_config_dp.add_message_type();
config_message->set_name(bts_config_classname);
// Add the 'include' field
proto::FieldDescriptorProto *include_field = config_message->add_field();
include_field->set_name(MESSAGE_STRING_INCLUDE);
include_field->set_number(++field_number);
include_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
include_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL);
// Add the 'fallback' field
proto::FieldDescriptorProto *backup_field = config_message->add_field();
backup_field->set_name(MESSAGE_STRING_INCLUDE_BACKUP);
backup_field->set_number(++field_number);
backup_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
backup_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL);
// Add 'macro' message type
proto::DescriptorProto *macro_message = bts_config_dp.add_message_type();
macro_message->set_name(MESSAGE_STRING_MACRO_TYPE);
proto::FieldDescriptorProto *name_field = macro_message->add_field();
name_field->set_name("name");
name_field->set_number(1);
name_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
name_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED);
proto::FieldDescriptorProto *expression_field = macro_message->add_field();
expression_field->set_name("expression");
expression_field->set_number(2);
expression_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
expression_field->set_label(
proto::FieldDescriptorProto_Label_LABEL_REQUIRED);
// add 'macro' message field
proto::FieldDescriptorProto *macro_field = config_message->add_field();
macro_field->set_name(MESSAGE_STRING_MACRO);
macro_field->set_number(++field_number);
macro_field->set_type(proto::FieldDescriptorProto_Type_TYPE_MESSAGE);
macro_field->set_type_name(MESSAGE_STRING_MACRO_TYPE);
macro_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REPEATED);
// add a meta type and field for each generated message
proto::DescriptorPool::generated_pool();
auto find_proto_file = [&proto_dirs](const std::string &name) -> const proto::FileDescriptor * {
for (const auto &proto_dir : proto_dirs) {
const auto * res = proto::DescriptorPool::generated_pool()->FindFileByName(
proto_dir + name);
if (res != nullptr) {
return res;
}
}
return nullptr;
};
for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) {
const auto *file_desc = find_proto_file(config_proto_files[x]);
if (file_desc != nullptr) {
bts_config_dp.add_dependency(file_desc->name());
for (int y = 0; y < file_desc->message_type_count(); y++) {
const proto::Descriptor *message_desc = file_desc->message_type(y);
add_metadata_field(
&bts_config_dp, config_message, message_desc, ++field_number);
}
} else {
#ifdef _WIN32
// on linux the linker might throw away config protos that
// are not used in sparkd. On windows we build a dll so that
// should not happen
throw InvalidDescriptorException(config_proto_files[x]);
#endif
}
}
// Generate a proper FileDesctiptor from the FileDescriptorProto
// The newly created descriptors live in the pool
_dynamic_pool.reset(
new proto::DescriptorPool(proto::DescriptorPool::generated_pool()));
ConfigErrorCollector errors(log, "");
const proto::FileDescriptor *file_desc =
_dynamic_pool->BuildFileCollectingErrors(bts_config_dp, &errors);
if (!file_desc) {
std::string error_message = errors.ToString();
if (!error_message.empty()) {
bts_assert_msg(file_desc, error_message);
} else {
bts_assert_msg(
false,
"Config: error building parser message. No errors logged by "
"protobuf library.\n This could mean you have defined 2 config "
"keys with the same name. Unfortunately that's all I know.");
}
}
const proto::Descriptor *bts_config_desc =
file_desc->FindMessageTypeByName(bts_config_classname);
_prototype_factory.reset(new proto::DynamicMessageFactory());
_prototype_factory->SetDelegateToGeneratedFactory(true);
_prototype = _prototype_factory->GetPrototype(bts_config_desc);
}
bool is_valid_key_descriptor(const proto::Descriptor *property_desc) {
std::string file_name = property_desc->file()->name();
size_t file_name_start = file_name.find_last_of('/') + 1;
file_name = file_name.substr(file_name_start);
for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) {
std::string proto_name = config_proto_files[x];
if (boost::iequals(file_name, proto_name)) {
return true;
}
}
return false;
}
const proto::Message *
ConfigurationState::try_get_message(const std::string &property_name) {
initialize_settings();
const proto::Message *message;
// Iterate through the loaded files
std::vector<std::shared_ptr<proto::Message>>::iterator iter;
for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) {
message = try_get_value(property_name, **iter);
if (message) {
return message;
}
}
// Fall back to defaults
message = try_get_value(property_name, *_defaults.get());
if (message) {
return message;
}
return nullptr;
}
const proto::Message *ConfigurationState::try_get_message_or_default(
const std::string &property_name) {
const auto pmsg = try_get_message(property_name);
if (pmsg) {
return pmsg;
}
// If we can't find a value we should return a default instance
const auto *descriptor = get_descriptor_by_name(property_name);
if (descriptor) {
return proto::MessageFactory::generated_factory()->GetPrototype(
descriptor);
}
return nullptr;
}
const proto::Message &
ConfigurationState::get_message(const proto::Descriptor *property_desc) {
if (!is_valid_key_descriptor(property_desc)) {
throw InvalidKeyException(property_desc->name().c_str());
}
const proto::Message *message = try_get_message(property_desc->name());
if (!message) {
// Get the message prototype - will only have compiled in defaults
message = proto::MessageFactory::generated_factory()->GetPrototype(
property_desc);
}
return *message;
}
const proto::Message *ConfigurationState::try_get_value(
const std::string &field_name, const proto::Message &message) const {
// 1) Get the metadata descriptor
const proto::Descriptor *message_desc = message.GetDescriptor();
const proto::FieldDescriptor *meta_desc =
message_desc->FindFieldByName(field_name);
if (meta_desc) {
// 2) Get the repeated metadata
int meta_size = message.GetReflection()->FieldSize(message, meta_desc);
for (int index = 0; index < meta_size; index++) {
// 3) Iterate over repeated field and check rule
const proto::Message &meta_message =
message.GetReflection()->GetRepeatedMessage(
message, meta_desc, index);
const proto::FieldDescriptor *rule_desc =
meta_message.GetDescriptor()->FindFieldByName(MESSAGE_STRING_RULE);
const std::string rule =
meta_message.GetReflection()->GetString(meta_message, rule_desc);
if (_rule_engine.eval(rule)) {
// 4) return value
const proto::FieldDescriptor *value_desc =
meta_message.GetDescriptor()->FindFieldByName(
MESSAGE_STRING_VALUE);
const proto::Message &message_value =
meta_message.GetReflection()->GetMessage(
meta_message, value_desc,
proto::MessageFactory::generated_factory());
return &message_value;
} else {
// TODO: Consider a rule result cache
// pro: faster lookups after first parse
// con: right now config is read only and perfectly parallel after
// load
}
}
}
return nullptr;
}
const google::protobuf::Descriptor *
ConfigurationState::get_descriptor_by_name(const std::string &name) {
const proto::Message *pmsg = _defaults.get();
if (pmsg) {
// Get the metadata descriptor (Meta = rule+value fields)
const proto::Descriptor *desc = pmsg->GetDescriptor();
const proto::FieldDescriptor *meta_desc = desc->FindFieldByName(name);
if (meta_desc) {
// Find the "value" field descriptor
const proto::Descriptor *pmsg_desc = meta_desc->message_type();
const proto::FieldDescriptor *value_desc =
pmsg_desc->FindFieldByName(MESSAGE_STRING_VALUE);
if (value_desc) {
// return it's type
return value_desc->message_type();
}
}
}
return nullptr;
}
PathType::type get_path_type(std::string path) {
if (path.find("://") != std::string::npos) {
return PathType::directory;
// const char last_char = path.at(path.length()-1);
// return last_char == '/' ? PathType::directory : PathType::file;
} else {
if (boost::filesystem::is_directory(path)) {
return PathType::directory;
} else if (boost::filesystem::exists(path)) {
return PathType::file;
} else {
return PathType::not_found;
}
}
}
urdl::url url_from_string(std::string &path, bool required) {
boost::system::error_code ec;
// attempt to parse the path as a URL - if that fails assume it is an
// absolute file path and append file:///
urdl::url url = urdl::url::from_string(path, ec);
if (ec) {
std::string file_prefix = "file:///";
std::string abs_path = boost::filesystem::absolute(path).string();
url = urdl::url::from_string(
file_prefix + os::normalize_path(abs_path), ec);
}
bts_assert_msg(!required || !ec, ec.message().c_str());
return url;
}
inline tuple<string, uint32_t> hostname(const std::string &path) {
boost::system::error_code ec;
string host;
uint32_t port = 80;
// Try to parse a url with urdl
urdl::url url = urdl::url::from_string(path, ec);
if (!ec) {
host = url.host();
port = url.port();
} else {
// Check for a UNC path
string norm_path = path;
os::normalize_path(norm_path);
if (norm_path[0] == '/' && norm_path[1] == '/') {
size_t host_len = norm_path.find_first_of('/', 2) - 2;
if (host_len > 0) {
host = norm_path.substr(2, host_len);
}
}
}
// If we are still empty return "localhost"
if (host.empty())
host = "localhost";
return std::make_tuple(host, port);
}
// Loads a single message file or directory
bool load_message(
std::string url,
proto::Message &out_message,
IConfigLog &log,
ConfigValueQueue &lookup) {
ConfigDownloader downloader;
bool result = false;
switch (get_path_type(url)) {
case PathType::directory:
result = load_directory(url, downloader, out_message, log, lookup) > 0;
break;
case PathType::file:
result = load_file(url, downloader, out_message, log, lookup) ==
ConfigDownloader::SUCCESS;
break;
default:
std::ostringstream error;
error << "Error '" << url << "' is not a valid configuration path";
std::string err_str = error.str();
log.error(err_str);
throw ConfigurationException(err_str.c_str());
break;
}
return result;
}
int load_directory(
std::string url,
ConfigDownloader &downloader,
proto::Message &out_message,
IConfigLog &log,
ConfigValueQueue &lookup) {
LogAggregator aggregate_log(log);
aggregate_log.info("URL: " + url);
const char last_char = url.at(url.length() - 1);
if (last_char != '/') {
url = url + '/';
}
// If a file called "all" exists in the config directory we can short circuit
// ~60 http requests
ConfigDownloader::Result all_result = load_file(
url + ".all", downloader, out_message, aggregate_log, lookup, false);
if (all_result == ConfigDownloader::SUCCESS) {
log.info(aggregate_log.text());
return 1;
}
// load spark.conf
int load_count = 0;
ConfigDownloader::Result result = load_file(
url + "spark.conf", downloader, out_message, aggregate_log, lookup,
false);
if (result == ConfigDownloader::SUCCESS) {
load_count++;
} else if (result == ConfigDownloader::PERMANENT_FAILURE) {
// If the server is down it doesn't make sense to suffer though the
// timeouts for the other files
throw ServerUnreachableException(url.c_str());
}
bool edge_has_include = !get_include(out_message).empty();
for (const char *proto_file : config_proto_files) {
std::string config_file(proto_file);
if (config_file == "system.proto")
continue;
if (config_file == "edge.proto")
continue;
const char proto_ext[] = ".proto";
const char conf_ext[] = ".conf";
size_t extension_pos = config_file.rfind(proto_ext);
// path_to_config_proto/foo.proto => foo.conf
config_file.replace(
extension_pos, sizeof(proto_ext), conf_ext, sizeof(conf_ext));
boost::filesystem::path p(config_file);
config_file = p.filename().string();
if (load_file(
url + config_file, downloader, out_message, aggregate_log, lookup,
false)) {
load_count++;
if (!edge_has_include) {
bts_assert_msg(
get_include(out_message).empty(),
"Includes may only be specified in spark.conf");
}
// NOTE: This code allows any config key to go in any .conf file
// If this becomse a problem we could iterate over the loaded messages
// and ensure they came from the correct location.
}
}
log.info(aggregate_log.text());
return load_count;
}
ConfigDownloader::Result load_file(
std::string path,
ConfigDownloader &downloader,
proto::Message &out_message,
IConfigLog &log,
ConfigValueQueue &lookup,
bool required) {
stringstream url_stream;
string url = url_from_string(path, required).to_string();
ConfigDownloader::Result result = downloader.get(url, url_stream);
if (!(result == ConfigDownloader::SUCCESS)) {
bts_assert_msg(!required || url_stream, (path + " not found.").c_str());
return result;
}
log.info("Loaded: " + path);
proto::io::IstreamInputStream proto_stream(&url_stream);
parse_stream(path, proto_stream, out_message, log, lookup);
return result;
}
void parse_stream(
const std::string &file,
proto::io::ZeroCopyInputStream &input_stream,
proto::Message &out_message,
IConfigLog &log,
ConfigValueQueue &lookup) {
ConfigErrorCollector error_collector(log, file.c_str());
bts::google::protobuf::TextFormat::Parser parser;
parser.RecordErrorsTo(&error_collector);
parser.SetValueLookup(&lookup);
lookup.set_filename(file);
parser.AllowPartialMessage(true);
bool success = parser.Parse(&input_stream, &out_message);
if (!success) {
std::string error_message = error_collector.ToString();
log.error(error_message);
throw ConfigurationException(error_message);
}
}
std::string get_field(const proto::Message &message, const char *name) {
const proto::Descriptor *message_descriptor = message.GetDescriptor();
const proto::FieldDescriptor *include_field =
message_descriptor->FindFieldByName(name);
const std::string include =
message.GetReflection()->GetString(message, include_field);
return include;
}
std::string get_include(const proto::Message &message) {
return get_field(message, MESSAGE_STRING_INCLUDE);
}
std::string get_fallback(const proto::Message &message) {
return get_field(message, MESSAGE_STRING_INCLUDE_BACKUP);
}
} // namespace internal
} // namespace config
} // namespace bts