So we can subscribe to depth for "FTX Options" or "FTX BTC Futures", rather than having to list each security explicitly. This commit also unhooks binance from MarketDataGateway. A new paradigm is emerging with the FTX impl, and it'll be easier to redo Binance than try to make the existing impl fit the new mold. Unfortunately, ccapi doesn't provide all the security metadata we need, such as the security type and expiration date/time, so we get the list of FTX securities directly from the exchange.
963 lines
32 KiB
C++
963 lines
32 KiB
C++
#include "ConfigurationState.h"
|
|
|
|
#include "ConfigErrorCollector.h"
|
|
#include "ConfigPropertyEvaluator.h"
|
|
#include "ConfigValueQueue.h"
|
|
#include "ConfigurationException.h"
|
|
#include "PreprocessInputStream.h"
|
|
#include "config.h"
|
|
#include "lib/os/os.h"
|
|
#include "lib/utils/bts_array_size.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
#include "lib/utils/defer.h"
|
|
#include "lib/utils/is_unittest_environment.h"
|
|
|
|
#include "lib/config/protos/system.pb.h"
|
|
|
|
#include <boost/algorithm/string.hpp> // iequals
|
|
#include <boost/filesystem.hpp>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <tuple>
|
|
#include <vector>
|
|
|
|
#define URDL_HEADER_ONLY 1
|
|
#define URDL_DISALE_SSL 1
|
|
#include <urdl/url.hpp>
|
|
|
|
#include "lib/config_defaults.h"
|
|
#include "lib/config_proto_files.h"
|
|
|
|
#ifdef _WIN32
|
|
#include <Winsock2.h>
|
|
#include <d2d1.h>
|
|
#else
|
|
#include <arpa/inet.h>
|
|
#include <netdb.h>
|
|
#include <netinet/in.h>
|
|
#include <sys/socket.h>
|
|
#endif
|
|
|
|
#include "lib/os/os.h"
|
|
|
|
using namespace std;
|
|
|
|
namespace bts {
|
|
namespace config {
|
|
namespace internal {
|
|
|
|
const char *MESSAGE_STRING_RULE = "rule";
|
|
const char *MESSAGE_STRING_VALUE = "value";
|
|
const char *MESSAGE_STRING_INCLUDE = "include";
|
|
const char *MESSAGE_STRING_INCLUDE_BACKUP = "fallback";
|
|
const char *MESSAGE_STRING_MACRO_TYPE = "MacroType";
|
|
const char *MESSAGE_STRING_MACRO = "macro";
|
|
|
|
const char *CONFIG_SERVER_ENV_VAR = "BTSConfig";
|
|
|
|
inline tuple<string, uint32_t> hostname(const std::string &path);
|
|
|
|
ConfigurationState::ConfigurationState(void)
|
|
: _is_initialized(false), _initialization_mutex(), _dynamic_pool(),
|
|
_prototype_factory(), _prototype(nullptr), _config_chain(), _defaults(),
|
|
_properties(new ConfigPropertyEvaluator(*this)),
|
|
_rule_engine(_properties), _macros(),
|
|
_deferred_queue(new ConfigValueQueue(*_properties.get())), _host(),
|
|
_port(80) {}
|
|
|
|
ConfigurationState::~ConfigurationState(void) {}
|
|
|
|
std::string get_command_argument(const char *key) {
|
|
std::vector<std::string> cmd_line = os::get_command_line();
|
|
std::vector<std::string>::iterator begin = cmd_line.begin(),
|
|
end = cmd_line.end();
|
|
for (; begin != end; begin++) {
|
|
if (boost::iequals(*begin, key)) {
|
|
begin++;
|
|
if (begin == end) {
|
|
break;
|
|
} else {
|
|
os::setenv(CONFIG_SERVER_ENV_VAR, begin->c_str());
|
|
return *begin;
|
|
}
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
string ConfigurationState::settings_path() {
|
|
// Look for the -ConfigServer option on the command line
|
|
std::string cmd_arg = get_command_argument("-ConfigServer");
|
|
if (!cmd_arg.empty())
|
|
return cmd_arg;
|
|
|
|
// Then the BTSCONFIG environment variable
|
|
const char *env_server = getenv(CONFIG_SERVER_ENV_VAR);
|
|
if (env_server)
|
|
return env_server;
|
|
|
|
// then look in sparkdata
|
|
return os::sparkdata_path("conf");
|
|
}
|
|
|
|
void ConfigurationState::initialize_defaults(IConfigLog &log) {
|
|
initialize_defaults(
|
|
log, {"lib/config/protos/", "lib/api/domain/", "lib/api/md/"});
|
|
}
|
|
|
|
void ConfigurationState::initialize_defaults(
|
|
IConfigLog &log, const std::vector<std::string> &proto_dirs) {
|
|
std::lock_guard<std::recursive_mutex> lock(_initialization_mutex);
|
|
if (_is_initialized)
|
|
return;
|
|
|
|
generate_all_config_proto(log, proto_dirs);
|
|
|
|
std::shared_ptr<proto::Message> new_message(_prototype->New());
|
|
ErrorsOnlyLog elog(log);
|
|
|
|
std::string defaults;
|
|
for (const char *str : config_static_defaults) {
|
|
defaults += str;
|
|
defaults += "\n";
|
|
}
|
|
|
|
proto::io::ArrayInputStream input_stream(
|
|
defaults.c_str(), (int)defaults.length());
|
|
parse_stream(
|
|
"static_defaults.conf", input_stream, *new_message.get(), elog,
|
|
*_deferred_queue.get());
|
|
_defaults = new_message;
|
|
|
|
// do not process 'include' for static defaults
|
|
|
|
_is_initialized = true;
|
|
}
|
|
|
|
void ConfigurationState::initialize_builtins() {
|
|
System sys;
|
|
|
|
sys.set_host_name(os::host_name());
|
|
sys.set_process_name(os::get_application_name());
|
|
sys.set_directory(os::get_install_path());
|
|
sys.set_platform(os::platform_name());
|
|
sys.set_qualifier(os::get_qualifier());
|
|
sys.set_unittest_environment(utils::internal::is_unittest_environment());
|
|
sys.set_data_path(os::bts_data_path());
|
|
sys.set_os_user(os::username());
|
|
#ifndef NDEBUG
|
|
sys.set_build_type("debug");
|
|
#endif
|
|
sys.set_config_host(_host);
|
|
sys.set_config_port(_port);
|
|
if (!_host.empty()) {
|
|
hostent *host_entry = gethostbyname(_host.c_str());
|
|
bts_assert_msg(
|
|
host_entry != NULL, "Unable to resolve host name " + _host);
|
|
in_addr *address = (in_addr *)host_entry->h_addr_list[0];
|
|
string ip_address = inet_ntoa(*address);
|
|
sys.set_config_ip(ip_address);
|
|
}
|
|
#ifdef _WIN32
|
|
sys.set_is_remote_app(GetSystemMetrics(SM_REMOTESESSION) == 1);
|
|
|
|
// Screen dpi, Assumes screen dpi is uniform in x and y
|
|
// Computes 2 different ratios depending - one will be right depending on
|
|
// whether the process calls SetDPIAware()
|
|
HDC screen = GetDC(0);
|
|
int log_pixels_x = GetDeviceCaps(screen, LOGPIXELSX);
|
|
auto ratio2 = log_pixels_x / 96.0;
|
|
int virtual_width = GetDeviceCaps(screen, HORZRES);
|
|
int physical_width = GetDeviceCaps(screen, DESKTOPHORZRES);
|
|
double ratio = (double)physical_width / (double)virtual_width;
|
|
sys.set_dpi(log_pixels_x * ratio);
|
|
sys.set_dpi_ratio(max(ratio, ratio2));
|
|
ReleaseDC(0, screen);
|
|
|
|
#endif
|
|
|
|
std::vector<std::string> args = os::get_command_line();
|
|
for (std::vector<std::string>::iterator iter = args.begin();
|
|
iter != args.end(); iter++) {
|
|
sys.add_cmd_args(*iter);
|
|
}
|
|
|
|
// Version string
|
|
std::string app_version_file = os::get_install_path() + "/spark.version";
|
|
if (boost::filesystem::exists(app_version_file)) {
|
|
std::ifstream input(app_version_file.c_str(), std::ifstream::in);
|
|
if (!input.fail()) {
|
|
// Read the last line of the file
|
|
std::string version;
|
|
while (!input.eof()) {
|
|
std::string temp;
|
|
std::getline(input, temp);
|
|
if (!temp.empty())
|
|
version = temp;
|
|
}
|
|
|
|
sys.set_version(version);
|
|
input.close();
|
|
}
|
|
}
|
|
|
|
sys.set_num_cores(os::num_cores());
|
|
|
|
//
|
|
// Add the System value to the defaults
|
|
const proto::FieldDescriptor *field_desc =
|
|
_defaults->GetDescriptor()->FindFieldByName("System");
|
|
const proto::Descriptor *message_desc = field_desc->message_type();
|
|
|
|
// Create a meta message
|
|
proto::Message *new_message =
|
|
_defaults->GetReflection()->AddMessage(_defaults.get(), field_desc);
|
|
const proto::FieldDescriptor *value_desc =
|
|
message_desc->FindFieldByName("value");
|
|
new_message->GetReflection()
|
|
->MutableMessage(new_message, value_desc)
|
|
->CopyFrom(sys);
|
|
|
|
initialize_introspection();
|
|
}
|
|
|
|
void ConfigurationState::initialize_introspection() {
|
|
// With this config can be self describing enough
|
|
// that another language can request "all" of config
|
|
ConfigIntrospection ci;
|
|
proto::DescriptorPool::generated_pool();
|
|
for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) {
|
|
std::string proto_path = "core/bts_core/config/";
|
|
proto_path += config_proto_files[x];
|
|
const proto::FileDescriptor *file_desc =
|
|
proto::DescriptorPool::generated_pool()->FindFileByName(proto_path);
|
|
if (file_desc != nullptr) {
|
|
ConfigIntrospection::ConfigProto *config_proto = ci.add_config();
|
|
bts_assert(config_proto);
|
|
config_proto->set_name(file_desc->name());
|
|
for (int y = 0; y < file_desc->message_type_count(); y++) {
|
|
const proto::Descriptor *message_desc = file_desc->message_type(y);
|
|
if (message_desc != nullptr) {
|
|
config_proto->add_type(message_desc->full_name());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add to defaults
|
|
const proto::FieldDescriptor *field_desc =
|
|
_defaults->GetDescriptor()->FindFieldByName("ConfigIntrospection");
|
|
const proto::Descriptor *message_desc = field_desc->message_type();
|
|
proto::Message *new_message =
|
|
_defaults->GetReflection()->AddMessage(_defaults.get(), field_desc);
|
|
const proto::FieldDescriptor *value_desc =
|
|
message_desc->FindFieldByName("value");
|
|
new_message->GetReflection()
|
|
->MutableMessage(new_message, value_desc)
|
|
->CopyFrom(ci);
|
|
}
|
|
|
|
void ConfigurationState::initialize_macros() {
|
|
std::vector<std::shared_ptr<proto::Message>>::iterator iter;
|
|
for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) {
|
|
const proto::Message *message = (*iter).get();
|
|
add_macro(message);
|
|
}
|
|
add_macro(_defaults.get());
|
|
}
|
|
|
|
void ConfigurationState::add_macro(const proto::Message *message) {
|
|
bts_assert(message);
|
|
const proto::FieldDescriptor *field_desc =
|
|
message->GetDescriptor()->FindFieldByName(MESSAGE_STRING_MACRO);
|
|
if (field_desc) {
|
|
int size = message->GetReflection()->FieldSize(*message, field_desc);
|
|
for (int index = 0; index < size; index++) {
|
|
// read "name" and "expression" fields
|
|
const proto::Message &meta_message =
|
|
message->GetReflection()->GetRepeatedMessage(
|
|
*message, field_desc, index);
|
|
const proto::FieldDescriptor *name_desc =
|
|
meta_message.GetDescriptor()->FindFieldByName("name");
|
|
const std::string name = '$' + meta_message.GetReflection()->GetString(
|
|
meta_message, name_desc);
|
|
|
|
const proto::FieldDescriptor *exp_desc =
|
|
meta_message.GetDescriptor()->FindFieldByName("expression");
|
|
const std::string expression =
|
|
meta_message.GetReflection()->GetString(meta_message, exp_desc);
|
|
|
|
if (_macros.find(name) == _macros.end()) {
|
|
// eval expression to Value
|
|
// expression::Value result =
|
|
// _rule_engine.eval_to_value(expression);
|
|
_macros[name] = expression;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
expression::Value ConfigurationState::get_macro(const std::string &name) {
|
|
map<string, string>::iterator iter = _macros.find(name);
|
|
if (iter == _macros.end()) {
|
|
std::string error = std::string("Macro '") + name + "' is undefined";
|
|
throw ConfigurationException(error);
|
|
}
|
|
return _rule_engine.eval_to_value(iter->second);
|
|
}
|
|
|
|
std::string ConfigurationState::get_macro_string(const std::string &name) {
|
|
try {
|
|
expression::Value result = get_macro(name);
|
|
return expression::value_to_string(result);
|
|
} catch (const ConfigurationException &e) {
|
|
return e.what();
|
|
}
|
|
}
|
|
|
|
void check_rule(
|
|
proto::Message *message, expression::ExpressionEvaluator &rule_engine) {
|
|
const proto::Reflection *reflection = message->GetReflection();
|
|
const proto::Descriptor *descriptor = message->GetDescriptor();
|
|
for (int findex = 0; findex < descriptor->field_count(); ++findex) {
|
|
const proto::FieldDescriptor *field = descriptor->field(findex);
|
|
if (field->is_repeated()) {
|
|
int field_size = reflection->FieldSize(*message, field);
|
|
for (int index = 0; index < field_size; ++index) {
|
|
const proto::Message &field_message =
|
|
reflection->GetRepeatedMessage(*message, field, index);
|
|
const proto::FieldDescriptor *rule_descriptor =
|
|
field_message.GetDescriptor()->FindFieldByName(
|
|
MESSAGE_STRING_RULE);
|
|
if (rule_descriptor) { // Macros don't have rules
|
|
const std::string rule =
|
|
field_message.GetReflection()->GetString(
|
|
field_message, rule_descriptor);
|
|
rule_engine.eval(rule);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void ConfigurationState::check_rules() {
|
|
initialize();
|
|
std::vector<std::shared_ptr<proto::Message>>::iterator iter;
|
|
for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) {
|
|
proto::Message *message = (*iter).get();
|
|
check_rule(message, _rule_engine);
|
|
}
|
|
check_rule(_defaults.get(), _rule_engine);
|
|
}
|
|
|
|
void ConfigurationState::initialize_settings() {
|
|
std::lock_guard<std::recursive_mutex> lock(_initialization_mutex);
|
|
if (_is_initialized)
|
|
return;
|
|
|
|
ConfigLog log;
|
|
initialize_defaults(log);
|
|
|
|
std::string settings = settings_path();
|
|
std::string fallback = get_command_argument("-ConfigFallback");
|
|
|
|
DEFER( // Do this even if load() fails
|
|
initialize_builtins(); initialize_macros(); _deferred_queue->process(););
|
|
load(settings, fallback, log);
|
|
}
|
|
|
|
void ConfigurationState::reset() {
|
|
std::lock_guard<std::recursive_mutex> lock(_initialization_mutex);
|
|
|
|
_config_chain.clear();
|
|
_prototype = nullptr;
|
|
_defaults.reset();
|
|
_prototype_factory.reset();
|
|
_dynamic_pool.reset();
|
|
_macros.clear();
|
|
_deferred_queue->clear();
|
|
_is_initialized = false;
|
|
_host.clear();
|
|
_port = 80;
|
|
}
|
|
|
|
int ConfigurationState::load(
|
|
const std::string &url, const std::string &fallback_url, IConfigLog &log) {
|
|
std::shared_ptr<proto::Message> new_message(_prototype->New());
|
|
|
|
bool loaded = false;
|
|
try {
|
|
loaded =
|
|
load_message(url, *new_message.get(), log, *_deferred_queue.get());
|
|
} catch (ServerUnreachableException &) {
|
|
if (fallback_url.empty()) {
|
|
throw;
|
|
}
|
|
}
|
|
|
|
if (!loaded && !fallback_url.empty()) {
|
|
std::string message = "Failed to load: " + url + " using fallback.";
|
|
log.warning(message);
|
|
loaded = load_message(
|
|
fallback_url, *new_message.get(), log, *_deferred_queue.get());
|
|
if (_host.empty() || _host == "localhost") {
|
|
tie(_host, _port) = hostname(fallback_url);
|
|
}
|
|
}
|
|
if (_host.empty() || _host == "localhost") {
|
|
tie(_host, _port) = hostname(url);
|
|
}
|
|
if (!loaded)
|
|
return 0;
|
|
|
|
std::string include = get_include(*new_message.get());
|
|
std::string fallback = get_fallback(*new_message.get());
|
|
|
|
// insert at front?
|
|
_config_chain.push_back(new_message);
|
|
|
|
// Recursively load includes
|
|
if (!include.empty()) {
|
|
return 1 + load(include, fallback, log);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
int ConfigurationState::load_string(
|
|
const std::string &values, bool print_errors) {
|
|
ConfigLog log;
|
|
log.set_print_errors(print_errors);
|
|
initialize_defaults(log);
|
|
|
|
std::shared_ptr<proto::Message> new_message(_prototype->New());
|
|
|
|
proto::io::ArrayInputStream input_stream(
|
|
values.c_str(), (int)values.length());
|
|
parse_stream(
|
|
"test string", input_stream, *new_message.get(), log,
|
|
*_deferred_queue.get());
|
|
|
|
std::string include = get_include(*new_message.get());
|
|
std::string fallback = get_fallback(*new_message.get());
|
|
|
|
_config_chain.push_back(new_message);
|
|
|
|
int count = 0;
|
|
if (!include.empty()) {
|
|
count += load(include, fallback, log);
|
|
}
|
|
|
|
if (_host.empty()) {
|
|
_host = "localhost";
|
|
}
|
|
|
|
initialize_builtins();
|
|
initialize_macros();
|
|
_deferred_queue->process();
|
|
return 1 + count;
|
|
}
|
|
|
|
void ConfigurationState::add_metadata_field(
|
|
proto::FileDescriptorProto *file_dp,
|
|
proto::DescriptorProto *message_dp,
|
|
const proto::Descriptor *base_field,
|
|
int index) {
|
|
// Create the metadata message
|
|
proto::DescriptorProto *meta_message = file_dp->add_message_type();
|
|
std::string meta_name = base_field->name() + "meta";
|
|
meta_message->set_name(meta_name);
|
|
|
|
// Add 'rule'
|
|
proto::FieldDescriptorProto *rule_field = meta_message->add_field();
|
|
rule_field->set_name(MESSAGE_STRING_RULE);
|
|
rule_field->set_number(1);
|
|
rule_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
|
|
rule_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL);
|
|
rule_field->set_default_value("true");
|
|
|
|
// Add 'value'
|
|
proto::FieldDescriptorProto *value_field = meta_message->add_field();
|
|
value_field->set_name(MESSAGE_STRING_VALUE);
|
|
value_field->set_number(2);
|
|
value_field->set_type_name(base_field->full_name());
|
|
value_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED);
|
|
|
|
// Now add a field to message_dp of that type
|
|
proto::FieldDescriptorProto *field_desc = message_dp->add_field();
|
|
field_desc->set_name(base_field->name());
|
|
field_desc->set_number(index);
|
|
field_desc->set_type_name(meta_name);
|
|
field_desc->set_label(proto::FieldDescriptorProto_Label_LABEL_REPEATED);
|
|
}
|
|
|
|
// Construct a new Message that describes the save file format
|
|
// This can then be used with google::protobuf::TextFormat to load and save
|
|
// files
|
|
void ConfigurationState::generate_all_config_proto(
|
|
IConfigLog &log, const std::vector<std::string> &proto_dirs) {
|
|
const char *bts_config_classname = "__BTSConfig";
|
|
int field_number = 0;
|
|
|
|
proto::FileDescriptorProto bts_config_dp;
|
|
bts_config_dp.set_name("config.proto");
|
|
proto::DescriptorProto *config_message = bts_config_dp.add_message_type();
|
|
config_message->set_name(bts_config_classname);
|
|
|
|
// Add the 'include' field
|
|
proto::FieldDescriptorProto *include_field = config_message->add_field();
|
|
include_field->set_name(MESSAGE_STRING_INCLUDE);
|
|
include_field->set_number(++field_number);
|
|
include_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
|
|
include_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL);
|
|
|
|
// Add the 'fallback' field
|
|
proto::FieldDescriptorProto *backup_field = config_message->add_field();
|
|
backup_field->set_name(MESSAGE_STRING_INCLUDE_BACKUP);
|
|
backup_field->set_number(++field_number);
|
|
backup_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
|
|
backup_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL);
|
|
|
|
// Add 'macro' message type
|
|
proto::DescriptorProto *macro_message = bts_config_dp.add_message_type();
|
|
macro_message->set_name(MESSAGE_STRING_MACRO_TYPE);
|
|
proto::FieldDescriptorProto *name_field = macro_message->add_field();
|
|
name_field->set_name("name");
|
|
name_field->set_number(1);
|
|
name_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
|
|
name_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED);
|
|
proto::FieldDescriptorProto *expression_field = macro_message->add_field();
|
|
expression_field->set_name("expression");
|
|
expression_field->set_number(2);
|
|
expression_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING);
|
|
expression_field->set_label(
|
|
proto::FieldDescriptorProto_Label_LABEL_REQUIRED);
|
|
|
|
// add 'macro' message field
|
|
proto::FieldDescriptorProto *macro_field = config_message->add_field();
|
|
macro_field->set_name(MESSAGE_STRING_MACRO);
|
|
macro_field->set_number(++field_number);
|
|
macro_field->set_type(proto::FieldDescriptorProto_Type_TYPE_MESSAGE);
|
|
macro_field->set_type_name(MESSAGE_STRING_MACRO_TYPE);
|
|
macro_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REPEATED);
|
|
|
|
// add a meta type and field for each generated message
|
|
proto::DescriptorPool::generated_pool();
|
|
|
|
auto find_proto_file =
|
|
[&proto_dirs](const std::string &name) -> const proto::FileDescriptor * {
|
|
for (const auto &proto_dir : proto_dirs) {
|
|
const auto *res =
|
|
proto::DescriptorPool::generated_pool()->FindFileByName(
|
|
proto_dir + name);
|
|
if (res != nullptr) {
|
|
return res;
|
|
}
|
|
}
|
|
return nullptr;
|
|
};
|
|
|
|
for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) {
|
|
const auto *file_desc = find_proto_file(config_proto_files[x]);
|
|
if (file_desc != nullptr) {
|
|
bts_config_dp.add_dependency(file_desc->name());
|
|
for (int y = 0; y < file_desc->message_type_count(); y++) {
|
|
const proto::Descriptor *message_desc = file_desc->message_type(y);
|
|
add_metadata_field(
|
|
&bts_config_dp, config_message, message_desc, ++field_number);
|
|
}
|
|
} else {
|
|
#ifdef _WIN32
|
|
// on linux the linker might throw away config protos that
|
|
// are not used in sparkd. On windows we build a dll so that
|
|
// should not happen
|
|
throw InvalidDescriptorException(config_proto_files[x]);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
// Generate a proper FileDesctiptor from the FileDescriptorProto
|
|
// The newly created descriptors live in the pool
|
|
_dynamic_pool.reset(
|
|
new proto::DescriptorPool(proto::DescriptorPool::generated_pool()));
|
|
ConfigErrorCollector errors(log, "");
|
|
const proto::FileDescriptor *file_desc =
|
|
_dynamic_pool->BuildFileCollectingErrors(bts_config_dp, &errors);
|
|
if (!file_desc) {
|
|
std::string error_message = errors.ToString();
|
|
if (!error_message.empty()) {
|
|
bts_assert_msg(file_desc, error_message);
|
|
} else {
|
|
bts_assert_msg(
|
|
false,
|
|
"Config: error building parser message. No errors logged by "
|
|
"protobuf library.\n This could mean you have defined 2 config "
|
|
"keys with the same name. Unfortunately that's all I know.");
|
|
}
|
|
}
|
|
const proto::Descriptor *bts_config_desc =
|
|
file_desc->FindMessageTypeByName(bts_config_classname);
|
|
|
|
_prototype_factory.reset(new proto::DynamicMessageFactory());
|
|
_prototype_factory->SetDelegateToGeneratedFactory(true);
|
|
_prototype = _prototype_factory->GetPrototype(bts_config_desc);
|
|
}
|
|
|
|
bool is_valid_key_descriptor(const proto::Descriptor *property_desc) {
|
|
std::string file_name = property_desc->file()->name();
|
|
size_t file_name_start = file_name.find_last_of('/') + 1;
|
|
file_name = file_name.substr(file_name_start);
|
|
for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) {
|
|
std::string proto_name = config_proto_files[x];
|
|
if (boost::iequals(file_name, proto_name)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
const proto::Message *ConfigurationState::try_get_message(
|
|
const std::string &property_name) {
|
|
initialize_settings();
|
|
|
|
const proto::Message *message;
|
|
|
|
// Iterate through the loaded files
|
|
|
|
std::vector<std::shared_ptr<proto::Message>>::iterator iter;
|
|
for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) {
|
|
message = try_get_value(property_name, **iter);
|
|
if (message) {
|
|
return message;
|
|
}
|
|
}
|
|
|
|
// Fall back to defaults
|
|
message = try_get_value(property_name, *_defaults.get());
|
|
if (message) {
|
|
return message;
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
const proto::Message *ConfigurationState::try_get_message_or_default(
|
|
const std::string &property_name) {
|
|
const auto pmsg = try_get_message(property_name);
|
|
if (pmsg) {
|
|
return pmsg;
|
|
}
|
|
// If we can't find a value we should return a default instance
|
|
const auto *descriptor = get_descriptor_by_name(property_name);
|
|
if (descriptor) {
|
|
return proto::MessageFactory::generated_factory()->GetPrototype(
|
|
descriptor);
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
const proto::Message &ConfigurationState::get_message(
|
|
const proto::Descriptor *property_desc) {
|
|
if (!is_valid_key_descriptor(property_desc)) {
|
|
throw InvalidKeyException(property_desc->name().c_str());
|
|
}
|
|
|
|
const proto::Message *message = try_get_message(property_desc->name());
|
|
if (!message) {
|
|
// Get the message prototype - will only have compiled in defaults
|
|
message = proto::MessageFactory::generated_factory()->GetPrototype(
|
|
property_desc);
|
|
}
|
|
|
|
return *message;
|
|
}
|
|
|
|
const proto::Message *ConfigurationState::try_get_value(
|
|
const std::string &field_name, const proto::Message &message) const {
|
|
// 1) Get the metadata descriptor
|
|
const proto::Descriptor *message_desc = message.GetDescriptor();
|
|
const proto::FieldDescriptor *meta_desc =
|
|
message_desc->FindFieldByName(field_name);
|
|
if (meta_desc) {
|
|
// 2) Get the repeated metadata
|
|
int meta_size = message.GetReflection()->FieldSize(message, meta_desc);
|
|
for (int index = 0; index < meta_size; index++) {
|
|
// 3) Iterate over repeated field and check rule
|
|
const proto::Message &meta_message =
|
|
message.GetReflection()->GetRepeatedMessage(
|
|
message, meta_desc, index);
|
|
const proto::FieldDescriptor *rule_desc =
|
|
meta_message.GetDescriptor()->FindFieldByName(MESSAGE_STRING_RULE);
|
|
const std::string rule =
|
|
meta_message.GetReflection()->GetString(meta_message, rule_desc);
|
|
|
|
if (_rule_engine.eval(rule)) {
|
|
// 4) return value
|
|
const proto::FieldDescriptor *value_desc =
|
|
meta_message.GetDescriptor()->FindFieldByName(
|
|
MESSAGE_STRING_VALUE);
|
|
const proto::Message &message_value =
|
|
meta_message.GetReflection()->GetMessage(
|
|
meta_message, value_desc,
|
|
proto::MessageFactory::generated_factory());
|
|
return &message_value;
|
|
} else {
|
|
// TODO: Consider a rule result cache
|
|
// pro: faster lookups after first parse
|
|
// con: right now config is read only and perfectly parallel after
|
|
// load
|
|
}
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
const google::protobuf::Descriptor *ConfigurationState::get_descriptor_by_name(
|
|
const std::string &name) {
|
|
const proto::Message *pmsg = _defaults.get();
|
|
if (pmsg) {
|
|
// Get the metadata descriptor (Meta = rule+value fields)
|
|
const proto::Descriptor *desc = pmsg->GetDescriptor();
|
|
const proto::FieldDescriptor *meta_desc = desc->FindFieldByName(name);
|
|
if (meta_desc) {
|
|
// Find the "value" field descriptor
|
|
const proto::Descriptor *pmsg_desc = meta_desc->message_type();
|
|
const proto::FieldDescriptor *value_desc =
|
|
pmsg_desc->FindFieldByName(MESSAGE_STRING_VALUE);
|
|
if (value_desc) {
|
|
// return it's type
|
|
return value_desc->message_type();
|
|
}
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
PathType::type get_path_type(std::string path) {
|
|
if (path.find("://") != std::string::npos) {
|
|
return PathType::directory;
|
|
// const char last_char = path.at(path.length()-1);
|
|
// return last_char == '/' ? PathType::directory : PathType::file;
|
|
} else {
|
|
if (boost::filesystem::is_directory(path)) {
|
|
return PathType::directory;
|
|
} else if (boost::filesystem::exists(path)) {
|
|
return PathType::file;
|
|
} else {
|
|
return PathType::not_found;
|
|
}
|
|
}
|
|
}
|
|
|
|
urdl::url url_from_string(std::string &path, bool required) {
|
|
boost::system::error_code ec;
|
|
// attempt to parse the path as a URL - if that fails assume it is an
|
|
// absolute file path and append file:///
|
|
urdl::url url = urdl::url::from_string(path, ec);
|
|
if (ec) {
|
|
std::string file_prefix = "file:///";
|
|
std::string abs_path = boost::filesystem::absolute(path).string();
|
|
url = urdl::url::from_string(
|
|
file_prefix + os::normalize_path(abs_path), ec);
|
|
}
|
|
bts_assert_msg(!required || !ec, ec.message().c_str());
|
|
|
|
return url;
|
|
}
|
|
|
|
inline tuple<string, uint32_t> hostname(const std::string &path) {
|
|
boost::system::error_code ec;
|
|
string host;
|
|
uint32_t port = 80;
|
|
// Try to parse a url with urdl
|
|
urdl::url url = urdl::url::from_string(path, ec);
|
|
if (!ec) {
|
|
host = url.host();
|
|
port = url.port();
|
|
} else {
|
|
// Check for a UNC path
|
|
string norm_path = path;
|
|
os::normalize_path(norm_path);
|
|
if (norm_path[0] == '/' && norm_path[1] == '/') {
|
|
size_t host_len = norm_path.find_first_of('/', 2) - 2;
|
|
if (host_len > 0) {
|
|
host = norm_path.substr(2, host_len);
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we are still empty return "localhost"
|
|
if (host.empty())
|
|
host = "localhost";
|
|
|
|
return std::make_tuple(host, port);
|
|
}
|
|
|
|
// Loads a single message file or directory
|
|
bool load_message(
|
|
std::string url,
|
|
proto::Message &out_message,
|
|
IConfigLog &log,
|
|
ConfigValueQueue &lookup) {
|
|
ConfigDownloader downloader;
|
|
|
|
bool result = false;
|
|
switch (get_path_type(url)) {
|
|
case PathType::directory:
|
|
result = load_directory(url, downloader, out_message, log, lookup) > 0;
|
|
break;
|
|
case PathType::file:
|
|
result = load_file(url, downloader, out_message, log, lookup) ==
|
|
ConfigDownloader::SUCCESS;
|
|
break;
|
|
default:
|
|
std::ostringstream error;
|
|
error << "Error '" << url << "' is not a valid configuration path";
|
|
std::string err_str = error.str();
|
|
log.error(err_str);
|
|
throw ConfigurationException(err_str.c_str());
|
|
break;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
int load_directory(
|
|
std::string url,
|
|
ConfigDownloader &downloader,
|
|
proto::Message &out_message,
|
|
IConfigLog &log,
|
|
ConfigValueQueue &lookup) {
|
|
LogAggregator aggregate_log(log);
|
|
aggregate_log.info("URL: " + url);
|
|
|
|
const char last_char = url.at(url.length() - 1);
|
|
if (last_char != '/') {
|
|
url = url + '/';
|
|
}
|
|
|
|
// If a file called "all" exists in the config directory we can short circuit
|
|
// ~60 http requests
|
|
ConfigDownloader::Result all_result = load_file(
|
|
url + ".all", downloader, out_message, aggregate_log, lookup, false);
|
|
if (all_result == ConfigDownloader::SUCCESS) {
|
|
log.info(aggregate_log.text());
|
|
return 1;
|
|
}
|
|
|
|
// load spark.conf
|
|
int load_count = 0;
|
|
ConfigDownloader::Result result = load_file(
|
|
url + "spark.conf", downloader, out_message, aggregate_log, lookup,
|
|
false);
|
|
if (result == ConfigDownloader::SUCCESS) {
|
|
load_count++;
|
|
} else if (result == ConfigDownloader::PERMANENT_FAILURE) {
|
|
// If the server is down it doesn't make sense to suffer though the
|
|
// timeouts for the other files
|
|
throw ServerUnreachableException(url.c_str());
|
|
}
|
|
|
|
bool edge_has_include = !get_include(out_message).empty();
|
|
|
|
for (const char *proto_file : config_proto_files) {
|
|
std::string config_file(proto_file);
|
|
if (config_file == "system.proto")
|
|
continue;
|
|
if (config_file == "edge.proto")
|
|
continue;
|
|
|
|
const char proto_ext[] = ".proto";
|
|
const char conf_ext[] = ".conf";
|
|
size_t extension_pos = config_file.rfind(proto_ext);
|
|
// path_to_config_proto/foo.proto => foo.conf
|
|
config_file.replace(
|
|
extension_pos, sizeof(proto_ext), conf_ext, sizeof(conf_ext));
|
|
boost::filesystem::path p(config_file);
|
|
config_file = p.filename().string();
|
|
if (load_file(
|
|
url + config_file, downloader, out_message, aggregate_log, lookup,
|
|
false)) {
|
|
load_count++;
|
|
if (!edge_has_include) {
|
|
bts_assert_msg(
|
|
get_include(out_message).empty(),
|
|
"Includes may only be specified in spark.conf");
|
|
}
|
|
|
|
// NOTE: This code allows any config key to go in any .conf file
|
|
// If this becomse a problem we could iterate over the loaded messages
|
|
// and ensure they came from the correct location.
|
|
}
|
|
}
|
|
|
|
log.info(aggregate_log.text());
|
|
|
|
return load_count;
|
|
}
|
|
|
|
ConfigDownloader::Result load_file(
|
|
std::string path,
|
|
ConfigDownloader &downloader,
|
|
proto::Message &out_message,
|
|
IConfigLog &log,
|
|
ConfigValueQueue &lookup,
|
|
bool required) {
|
|
stringstream url_stream;
|
|
string url = url_from_string(path, required).to_string();
|
|
ConfigDownloader::Result result = downloader.get(url, url_stream);
|
|
if (!(result == ConfigDownloader::SUCCESS)) {
|
|
bts_assert_msg(!required || url_stream, (path + " not found.").c_str());
|
|
return result;
|
|
}
|
|
|
|
log.info("Loaded: " + path);
|
|
|
|
proto::io::IstreamInputStream proto_stream(&url_stream);
|
|
parse_stream(path, proto_stream, out_message, log, lookup);
|
|
|
|
return result;
|
|
}
|
|
|
|
void parse_stream(
|
|
const std::string &file,
|
|
proto::io::ZeroCopyInputStream &input_stream,
|
|
proto::Message &out_message,
|
|
IConfigLog &log,
|
|
ConfigValueQueue &lookup) {
|
|
ConfigErrorCollector error_collector(log, file.c_str());
|
|
bts::google::protobuf::TextFormat::Parser parser;
|
|
parser.RecordErrorsTo(&error_collector);
|
|
parser.SetValueLookup(&lookup);
|
|
lookup.set_filename(file);
|
|
parser.AllowPartialMessage(true);
|
|
bool success = parser.Parse(&input_stream, &out_message);
|
|
if (!success) {
|
|
std::string error_message = error_collector.ToString();
|
|
log.error(error_message);
|
|
throw ConfigurationException(error_message);
|
|
}
|
|
}
|
|
|
|
std::string get_field(const proto::Message &message, const char *name) {
|
|
const proto::Descriptor *message_descriptor = message.GetDescriptor();
|
|
const proto::FieldDescriptor *include_field =
|
|
message_descriptor->FindFieldByName(name);
|
|
const std::string include =
|
|
message.GetReflection()->GetString(message, include_field);
|
|
|
|
return include;
|
|
}
|
|
|
|
std::string get_include(const proto::Message &message) {
|
|
return get_field(message, MESSAGE_STRING_INCLUDE);
|
|
}
|
|
|
|
std::string get_fallback(const proto::Message &message) {
|
|
return get_field(message, MESSAGE_STRING_INCLUDE_BACKUP);
|
|
}
|
|
|
|
} // namespace internal
|
|
} // namespace config
|
|
} // namespace bts
|