fix build: Copy a handful of files over from spark_crypto.dev

This fixes the last of the boost-related errors
This commit is contained in:
John Colvin committed 2022-09-28 19:49:17 -04:00
1 parent c9b5ff844f
commit a82932652b
26 files changed
+2445 -2392

No files matched your search

+35 -41
View File
@@ -9,33 +9,28 @@ namespace config {
namespace internal {
HTTPServer::HTTPServer()
: _io_service(),
_acceptor(_io_service, tcp::endpoint(boost::asio::ip::address_v4::loopback(), 0)),
_cur_socket(),
_request_response_pairs(),
_thread(),
_done(false)
{
}
: _io_service(),
_acceptor(
_io_service,
tcp::endpoint(boost::asio::ip::address_v4::loopback(), 0)),
_cur_socket(), _request_response_pairs(), _thread(), _done(false) {}
unsigned int HTTPServer::port() const
{
unsigned int HTTPServer::port() const {
return _acceptor.local_endpoint().port();
}
void HTTPServer::add_response(const std::string& path, const std::string& response, const std::string& content)
{
void HTTPServer::add_response(
const std::string &path,
const std::string &response,
const std::string &content) {
_request_response_pairs[path] = response + content;
}
void HTTPServer::start()
{
void HTTPServer::start() {
_thread.reset(new boost::thread(boost::bind(&HTTPServer::worker, this)));
}
void HTTPServer::stop()
{
void HTTPServer::stop() {
_done = true;
_io_service.stop();
@@ -45,55 +40,52 @@ void HTTPServer::stop()
}
}
void HTTPServer::worker()
{
void HTTPServer::worker() {
start_accept();
_io_service.run();
}
void HTTPServer::start_accept()
{
void HTTPServer::start_accept() {
_cur_socket.reset(new boost::asio::ip::tcp::socket(_io_service));
_acceptor.async_accept(*_cur_socket.get(), boost::bind(&HTTPServer::handle_accept, this, _cur_socket.get(), boost::asio::placeholders::error));
_acceptor.async_accept(
*_cur_socket.get(),
boost::bind(
&HTTPServer::handle_accept, this, _cur_socket.get(),
boost::asio::placeholders::error));
}
void HTTPServer::handle_accept(boost::asio::ip::tcp::socket* socket, const boost::system::error_code& error)
{
if (!error)
{
void HTTPServer::handle_accept(
boost::asio::ip::tcp::socket *socket,
const boost::system::error_code &error) {
if (!error) {
handle_connect(socket);
}
start_accept();
}
void HTTPServer::handle_connect(boost::asio::ip::tcp::socket* socket)
{
void HTTPServer::handle_connect(boost::asio::ip::tcp::socket *socket) {
boost::asio::streambuf buffer;
size_t size = boost::asio::read_until(*socket, buffer, "\r\n\r\n");
std::string request(size, 0);
buffer.sgetn(&request[0], size);
size_t start = request.find("GET ") + 4;
size_t end = request.find(" HTTP/1.1\r\n");
size_t url_args = request.find("?");
if (url_args != std::string::npos)
end = std::min(end, url_args);
std::string request_path = request.substr(start, end-start);
std::string request_path = request.substr(start, end - start);
std::map<std::string, std::string>::iterator request_match =
_request_response_pairs.find(request_path);
if (request_match == _request_response_pairs.end())
{
std::map<std::string, std::string>::iterator request_match =
_request_response_pairs.find(request_path);
if (request_match == _request_response_pairs.end()) {
std::string response = "HTTP/1.0 404 NOT FOUND\r\n"
"Content-Type: text/plain\r\n\r\n"
"not found\r\n";
boost::asio::write(*socket, boost::asio::buffer(response));
}
else
{
} else {
boost::asio::write(*socket, boost::asio::buffer(request_match->second));
}
@@ -102,4 +94,6 @@ void HTTPServer::handle_connect(boost::asio::ip::tcp::socket* socket)
socket->close(ec);
}
}}}
} // namespace internal
} // namespace config
} // namespace bts
+15 -10
View File
@@ -8,6 +8,7 @@
#include <boost/asio/read_until.hpp>
#include <boost/asio/streambuf.hpp>
#include <boost/asio/write.hpp>
#define BOOST_BIND_GLOBAL_PLACEHOLDERS
#include <boost/bind.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/scoped_ptr.hpp>
@@ -17,28 +18,31 @@
#include <boost/asio/connect.hpp>
#include <boost/scoped_ptr.hpp>
#include <string>
#include <map>
#include <string>
namespace bts {
namespace config {
namespace internal {
class HTTPServer
{
public:
class HTTPServer {
public:
HTTPServer();
unsigned int port() const;
void start();
void add_response(const std::string& path, const std::string& response, const std::string& content);
void add_response(
const std::string &path,
const std::string &response,
const std::string &content);
void stop();
private:
private:
void worker();
void start_accept();
void handle_accept(boost::asio::ip::tcp::socket*, const boost::system::error_code& error);
void handle_connect(boost::asio::ip::tcp::socket*);
void handle_accept(
boost::asio::ip::tcp::socket *, const boost::system::error_code &error);
void handle_connect(boost::asio::ip::tcp::socket *);
boost::asio::io_service _io_service;
boost::asio::ip::tcp::acceptor _acceptor;
@@ -46,8 +50,9 @@ private:
std::map<std::string, std::string> _request_response_pairs;
boost::scoped_ptr<boost::thread> _thread;
bool _done;
};
}}} // end namespace
} // namespace internal
} // namespace config
} // namespace bts
#endif
File diff suppressed because it is too large. Load diff
+311 -360
View File
@@ -1,8 +1,8 @@
#include "lib/utils/TempFile.h"
#include "../config.h"
#include "../ConfigurationState.h"
#include "../ConfigurationException.h"
#include "../ConfigurationState.h"
#include "../config.h"
#include "lib/expression/ParserExceptions.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/is_unittest_environment.h"
@@ -19,9 +19,9 @@
#define URDL_HEADER_ONLY 1
#define URDL_DISALE_SSL 1
#include <urdl/url.hpp>
#include <urdl/test/http_server.hpp>
#include <catch/catch.hpp>
#include <urdl/test/http_server.hpp>
#include <urdl/url.hpp>
using namespace bts::config;
using namespace bts::config::internal;
@@ -31,124 +31,107 @@ using bts::config::test::Foo;
using bts::config::test::FooGroup;
using bts::config::test::StringTable;
namespace bts {
int config_tests = 0;
TEST_CASE("Configuration Tests","[config]") {
TEST_CASE("Configuration Tests", "[config]") {
bts::utils::internal::force_unittest_environment();
SECTION("basic_get_value_test")
{
initialize_for_test(
"Foo { \n"
" rule: \"true\"\n"
" value {\n"
" name: \"Name A\"\n"
" id: 65\n"
" email: \"test@example.com\"\n"
" }\n"
"}\n");
SECTION("basic_get_value_test") {
initialize_for_test("Foo { \n"
" rule: \"true\"\n"
" value {\n"
" name: \"Name A\"\n"
" id: 65\n"
" email: \"test@example.com\"\n"
" }\n"
"}\n");
const Foo& foo = get<Foo>();
const Foo &foo = get<Foo>();
REQUIRE(foo.name() == "Name A");
REQUIRE(foo.id() == 65);
REQUIRE(foo.email().Get(0) == "test@example.com");
}
SECTION("default_value_test")
{
SECTION("default_value_test") {
initialize_for_test(""); // Only loads defaults.conf
// Should load a Foo from testkeys.conf
const Foo& foo = get<Foo>();
const Foo &foo = get<Foo>();
REQUIRE(foo.name() == "Blue Foo"); // from testkeys.proto
REQUIRE(foo.id() == 1234); // from testkeys.proto
REQUIRE(foo.id() == 1234); // from testkeys.proto
REQUIRE(foo.email().size() > 0);
REQUIRE(foo.email().Get(0) == "default@example.com"); // from testkeys.conf
REQUIRE(
foo.email().Get(0) == "default@example.com"); // from testkeys.conf
}
SECTION("default_include_test")
{
SECTION("default_include_test") {
// test defaults.conf with an include
}
SECTION("invalid_key_type_test")
{
SECTION("invalid_key_type_test") {
// Load an initialize with no files - OK all values default
const Foo& foo = get<Foo>(); // OK
const Foo &foo = get<Foo>(); // OK
REQUIRE(foo.name() == "Blue Foo");
bool error = false;
try
{
try {
get<config::test::InvalidMessageType>(); // BAD
}
catch(...)
{
} catch (...) {
error = true;
}
REQUIRE(error == true);
}
SECTION("invalid_config_test2")
{
SECTION("invalid_config_test2") {
utils::TempFile config_b;
*config_b.out() <<
"Foo {\n"
" value { \n"
" name: \"bob\"\n"
" id: 7319 \n"
" }";
*config_b.out() << "Foo {\n"
" value { \n"
" name: \"bob\"\n"
" id: 7319 \n"
" }";
bool error = false;
try
{
try {
initialize_for_test("include: \"" + config_b.path() + "\"", false);
}
catch(const ConfigurationException& err)
{
} catch (const ConfigurationException &err) {
std::string what = err.what();
bts_assert(what.find("Expected identifier.") != std::string::npos);
error = true;
}
REQUIRE(error == true);
}
// Tests that we can load using a File:/// url
// Tests complex value load
SECTION("file_url_test")
{
SECTION("file_url_test") {
utils::TempFile config_a;
utils::TempFile config_b;
*config_a.out() << "include: \"" << config_b.path() << "\"" << std::endl <<
"Foo { \n"
" rule: \"false\"\n"
" value {\n"
" name: \"Name A\"\n"
" id: 65\n"
" email: \"A@bluecg.com\"\n"
" }\n"
"}\n";
*config_a.out() << "include: \"" << config_b.path() << "\"" << std::endl
<< "Foo { \n"
" rule: \"false\"\n"
" value {\n"
" name: \"Name A\"\n"
" id: 65\n"
" email: \"A@bluecg.com\"\n"
" }\n"
"}\n";
*config_b.out() <<
"Foo {\n"
" rule: \"true\"\n"
" value {\n"
" name: \"Name B\"\n"
" id: 66 "
" email: \"A@bluecg.com\" \n"
" }}";
*config_b.out() << "Foo {\n"
" rule: \"true\"\n"
" value {\n"
" name: \"Name B\"\n"
" id: 66 "
" email: \"A@bluecg.com\" \n"
" }}";
// Get key Foo without loading any config files - this will show the default value
// Get key Foo without loading any config files - this will show the
// default value
Foo foo = bts::config::get<Foo>();
REQUIRE(foo.name() == "Blue Foo");
@@ -163,23 +146,20 @@ TEST_CASE("Configuration Tests","[config]") {
REQUIRE(foo.id() == 66);
}
SECTION("http_test")
{
SECTION("http_test") {
HTTPServer server;
std::string port = boost::lexical_cast<std::string>(server.port());
std::string path = "/spark.conf";
std::string response =
"HTTP/1.0 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n";
std::string content =
"FooGroup {\n"
" rule: \"true\"\n"
" value {\n"
" group: \"http_group\"\n"
" foo {\n"
" name: \"http\"\n"
" id: 80\n"
"}}}\n";
std::string response = "HTTP/1.0 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n";
std::string content = "FooGroup {\n"
" rule: \"true\"\n"
" value {\n"
" group: \"http_group\"\n"
" foo {\n"
" name: \"http\"\n"
" id: 80\n"
"}}}\n";
server.add_response(path, response, content);
server.start();
@@ -190,7 +170,7 @@ TEST_CASE("Configuration Tests","[config]") {
server.stop();
const FooGroup& foogrp = get<FooGroup>();
const FooGroup &foogrp = get<FooGroup>();
REQUIRE(foogrp.group() == "http_group");
REQUIRE(foogrp.foo().name() == "http");
@@ -198,41 +178,41 @@ TEST_CASE("Configuration Tests","[config]") {
REQUIRE(get<System>().config_host() == "127.0.0.1");
}
SECTION("http_folder_test")
{
SECTION("http_folder_test") {
HTTPServer server;
std::string port = boost::lexical_cast<std::string>(server.port());
std::string config_url = "http://localhost:" + port + "/";
std::string response =
"HTTP/1.0 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n";
std::string response = "HTTP/1.0 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n";
server.add_response("/spark.conf", response, "include: \"" + config_url + "other/\"");
server.add_response("/testkeys.conf", response,
"FooGroup {\n"
" rule: \"true\"\n"
" value {\n"
" group: \"http_group\"\n"
" foo {\n"
" name: \"http\"\n"
" id: 80\n"
"}}}\n");
server.add_response("/other/spark.conf", response,
"Foo {\n"
" rule: \"true\"\n"
" value: {\n"
" name: \"included_value\"\n"
" id: 4096\n"
" }}\n");
server.add_response(
"/spark.conf", response, "include: \"" + config_url + "other/\"");
server.add_response(
"/testkeys.conf", response,
"FooGroup {\n"
" rule: \"true\"\n"
" value {\n"
" group: \"http_group\"\n"
" foo {\n"
" name: \"http\"\n"
" id: 80\n"
"}}}\n");
server.add_response(
"/other/spark.conf", response,
"Foo {\n"
" rule: \"true\"\n"
" value: {\n"
" name: \"included_value\"\n"
" id: 4096\n"
" }}\n");
server.start();
initialize_for_test("include: \"" + config_url + "\"");
server.stop();
const FooGroup& foogrp = get<FooGroup>();
const FooGroup &foogrp = get<FooGroup>();
REQUIRE(foogrp.group() == "http_group");
REQUIRE(foogrp.foo().name() == "http");
@@ -242,44 +222,39 @@ TEST_CASE("Configuration Tests","[config]") {
REQUIRE(get<Foo>().id() == 4096);
}
SECTION("test_rules")
{
initialize_for_test(
"StringTable {\n"
" value { \n"
" entries: [\"foo\", \"bar\", \"baz\"] \n"
" }\n"
" }\n"
"RuleRef { \n"
" # use this rule of StringTable has a cow \n"
" rule: \"'cow' in StringTable.entries\" \n"
" value { \n"
" a_string: \"AAA\"\n"
" a_int: -1\n"
" }\n"
"}\n"
"RuleRef { \n"
" rule: \"'baz' in StringTable.entries\" \n"
" value { \n"
" a_string: \"BBB\"\n"
" a_int: 99\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.a_int == 789\"\n"
" value {\n"
" name: \"A\"\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.a_string == 'BBB'\"\n"
" value {\n"
" name: \"B\"\n"
" }\n"
"}\n"
);
SECTION("test_rules") {
initialize_for_test("StringTable {\n"
" value { \n"
" entries: [\"foo\", \"bar\", \"baz\"] \n"
" }\n"
" }\n"
"RuleRef { \n"
" # use this rule of StringTable has a cow \n"
" rule: \"'cow' in StringTable.entries\" \n"
" value { \n"
" a_string: \"AAA\"\n"
" a_int: -1\n"
" }\n"
"}\n"
"RuleRef { \n"
" rule: \"'baz' in StringTable.entries\" \n"
" value { \n"
" a_string: \"BBB\"\n"
" a_int: 99\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.a_int == 789\"\n"
" value {\n"
" name: \"A\"\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.a_string == 'BBB'\"\n"
" value {\n"
" name: \"B\"\n"
" }\n"
"}\n");
// What should happen:
// Foo A
@@ -297,286 +272,262 @@ TEST_CASE("Configuration Tests","[config]") {
// success a_string == 'BBB'
// return Foo B
const Foo& foo = get<Foo>();
const Foo &foo = get<Foo>();
REQUIRE(foo.name() == "B");
}
SECTION("test_recursive_rules") {
initialize_for_test("StringTable {\n"
" rule: \"Foo.name == 'foo'\"\n"
" value { \n"
" entries: [\"foo\", \"bar\", \"baz\"] \n"
" }\n"
" }\n"
"RuleRef { \n"
" rule: \"'foo' in StringTable.entries\" \n"
" value { \n"
" a_string: \"AAA\"\n"
" a_int: -1\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.a_string == 'BBB'\"\n"
" value {\n"
" name: \"foo\"\n"
" }\n"
"}\n");
SECTION("test_recursive_rules")
{
initialize_for_test(
"StringTable {\n"
" rule: \"Foo.name == 'foo'\"\n"
" value { \n"
" entries: [\"foo\", \"bar\", \"baz\"] \n"
" }\n"
" }\n"
"RuleRef { \n"
" rule: \"'foo' in StringTable.entries\" \n"
" value { \n"
" a_string: \"AAA\"\n"
" a_int: -1\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.a_string == 'BBB'\"\n"
" value {\n"
" name: \"foo\"\n"
" }\n"
"}\n"
);
// Foo cannot be queried because RuleRef's rule requests StringTable which requests RuleRef
// Foo cannot be queried because RuleRef's rule requests StringTable which
// requests RuleRef
bool error = false;
try
{
try {
get<Foo>();
}
catch(const expression::RecursivePropertyGetException&)
{
} catch (const expression::RecursivePropertyGetException &) {
error = true;
}
REQUIRE(error);
}
SECTION("test_nested_properties")
{
initialize_for_test(
"RuleRef { \n"
" value { \n"
" nested {\n"
" list_int: [7,8,9] \n"
" nested {\n"
" a_string: 'ABC' \n"
" }\n"
" }\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.nested.nested.a_string == 'ABC' and 9 in RuleRef.nested.list_int\"\n"
" value {\n"
" name: \"foo\"\n"
" }\n"
"}\n"
);
SECTION("test_nested_properties") {
initialize_for_test("RuleRef { \n"
" value { \n"
" nested {\n"
" list_int: [7,8,9] \n"
" nested {\n"
" a_string: 'ABC' \n"
" }\n"
" }\n"
" }\n"
"}\n"
"Foo { \n"
" rule: \"RuleRef.nested.nested.a_string == 'ABC' "
"and 9 in RuleRef.nested.list_int\"\n"
" value {\n"
" name: \"foo\"\n"
" }\n"
"}\n");
REQUIRE(get<Foo>().name() == "foo");
}
SECTION("built_in_properties_test")
{
initialize_for_test(
"Foo { \n"
" rule: \"System.unittest_environment == true \"\n"
" value {\n"
" name: \"unit tests\"\n"
" }\n"
"}\n"
);
SECTION("built_in_properties_test") {
initialize_for_test("Foo { \n"
" rule: \"System.unittest_environment == true \"\n"
" value {\n"
" name: \"unit tests\"\n"
" }\n"
"}\n");
REQUIRE(get<Foo>().name() == "unit tests");
}
SECTION("nonbreaking_space_test")
{
SECTION("nonbreaking_space_test") {
bool parse_error = false;
try
{
try {
initialize_for_test(
"Foo { "
" value {\xA0\xA0 "
" name: \"valid name\""
" \xA0 }"
"} ", false);
}
catch(const bts::config::ConfigurationException)
{
"Foo { "
" value {\xA0\xA0 "
" name: \"valid name\""
" \xA0 }"
"} ",
false);
} catch (const bts::config::ConfigurationException &) {
parse_error = true;
}
REQUIRE(parse_error);
}
SECTION("macro_bool_test")
{
initialize_for_test(
"macro"
"{"
" name: \"bool\""
" expression: \"'c' in ['a', 'b', 'c', 'd']\""
"}"
""
"Foo"
"{"
" rule: \"$bool\""
" value {"
" name: 'good'"
" }"
"}"
""
"Foo"
"{"
" value {"
" name: 'bad'"
" }"
"}"
);
SECTION("macro_bool_test") {
initialize_for_test("macro"
"{"
" name: \"bool\""
" expression: \"'c' in ['a', 'b', 'c', 'd']\""
"}"
""
"Foo"
"{"
" rule: \"$bool\""
" value {"
" name: 'good'"
" }"
"}"
""
"Foo"
"{"
" value {"
" name: 'bad'"
" }"
"}");
REQUIRE(config::get<Foo>().name() == "good");
}
SECTION("macro_list_test")
{
initialize_for_test(
"macro"
"{"
" name: \"list\""
" expression: \"['a', 'b', 'c', 'd']\""
"}"
""
"Foo"
"{"
" rule: \"'a' in $list\""
" value {"
" name: 'good'"
" }"
"}"
""
"Foo"
"{"
" value {"
" name: 'bad'"
" }"
"}"
);
SECTION("macro_list_test") {
initialize_for_test("macro"
"{"
" name: \"list\""
" expression: \"['a', 'b', 'c', 'd']\""
"}"
""
"Foo"
"{"
" rule: \"'a' in $list\""
" value {"
" name: 'good'"
" }"
"}"
""
"Foo"
"{"
" value {"
" name: 'bad'"
" }"
"}");
REQUIRE(config::get<Foo>().name() == "good");
}
SECTION("fallback_test") {
//HTTPServer main;
// HTTPServer main;
HTTPServer fallback;
//std::string main_url = "http://localhost:" + boost::lexical_cast<std::string>(main.port()) + "/";
// std::string main_url = "http://localhost:" +
// boost::lexical_cast<std::string>(main.port()) + "/";
std::string main_url = "http://does_not_exist:12345/";
//main.stop();
std::string fallback_url = "http://127.0.0.1:" + boost::lexical_cast<std::string>(fallback.port()) + "/";
std::string response =
"HTTP/1.0 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n";
// main.stop();
std::string fallback_url =
"http://127.0.0.1:" +
boost::lexical_cast<std::string>(fallback.port()) + "/";
std::string response = "HTTP/1.0 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n";
std::string content =
"FooGroup {\n"
" value {\n"
" group: \"http_group\"\n"
" foo {\n"
" name: \"http\"\n"
" id: 80\n"
"}}}\n";
std::string content = "FooGroup {\n"
" value {\n"
" group: \"http_group\"\n"
" foo {\n"
" name: \"http\"\n"
" id: 80\n"
"}}}\n";
fallback.add_response("/spark.conf", response, content);
fallback.start();
initialize_for_test(
"include: '" + main_url + "'\n"
"fallback: '" + fallback_url + "'\n",false
);
"include: '" + main_url +
"'\n"
"fallback: '" +
fallback_url + "'\n",
false);
fallback.stop();
REQUIRE(get<FooGroup>().foo().name() == "http");
REQUIRE(get<FooGroup>().foo().id() == 80);
}
SECTION("macro_definition_order_test") {
// The $is_crusible test
// o2_cme_enabled depends on Foo which has a rule which depends on is_crucible.
// Eager evaluation caused a bug which would cause loading o2_cme_enabled to set is_crucible false. Case 3993
initialize_for_test(
"Foo {"
" rule: '$is_crucible'"
" value { name: 'cme_1u7_cert' }"
"}"
""
"Foo {"
" value { name: '' }"
"}"
""
"macro {"
" name: 'o2_cme_enabled'"
" expression: \"Foo.name != ''\" "
"}"
""
"macro {"
" name: 'is_crucible'"
" expression: 'true'"
"}"
);
// o2_cme_enabled depends on Foo which has a rule which depends on
// is_crucible. Eager evaluation caused a bug which would cause loading
// o2_cme_enabled to set is_crucible false. Case 3993
initialize_for_test("Foo {"
" rule: '$is_crucible'"
" value { name: 'cme_1u7_cert' }"
"}"
""
"Foo {"
" value { name: '' }"
"}"
""
"macro {"
" name: 'o2_cme_enabled'"
" expression: \"Foo.name != ''\" "
"}"
""
"macro {"
" name: 'is_crucible'"
" expression: 'true'"
"}");
REQUIRE(get<Foo>().name() == "cme_1u7_cert");
}
SECTION("repeated_value_order_preserved") {
initialize_for_test(
"Foo {"
" value { name: 'edge-eharding' id: 80 }"
"}"
"StringTable {"
" value {"
" entries: 'http://'"
" entries: @Foo.name"
" entries: ':'"
" entries: @Foo.id"
" entries: '/risk'"
" }"
"}");
initialize_for_test("Foo {"
" value { name: 'edge-eharding' id: 80 }"
"}"
"StringTable {"
" value {"
" entries: 'http://'"
" entries: @Foo.name"
" entries: ':'"
" entries: @Foo.id"
" entries: '/risk'"
" }"
"}");
std::string result;
for (const auto& part : get<StringTable>().entries()) {
for (const auto &part : get<StringTable>().entries()) {
result += part;
}
REQUIRE(result == "http://edge-eharding:80/risk");
}
SECTION("string_concat_test") {
initialize_for_test(
"Ports {\n"
" value { \n"
" config_port: 80 \n"
" }\n"
"}\n"
"RuleRef { \n"
" value { \n"
" a_string: \"localhost\"\n"
" }\n"
"}\n"
"macro{\n"
" name: 'list'\n"
" expression: \"['http://', RuleRef.a_string, ':', Ports.config_port, '/risk']\"\n"
"}"
"StringTable {\n"
"value {\n"
" entries: 'http://'\n"
" entries: @RuleRef.a_string\n"
" entries: ':'\n"
" entries: @Ports.config_port\n"
" entries: '/risk'\n"
"}}\n"
"Foo { \n"
" value {\n"
" name: $list\n"
" url: @StringTable.entries \n"
" }\n"
"}"
);
initialize_for_test("Ports {\n"
" value { \n"
" config_port: 80 \n"
" }\n"
"}\n"
"RuleRef { \n"
" value { \n"
" a_string: \"localhost\"\n"
" }\n"
"}\n"
"macro{\n"
" name: 'list'\n"
" expression: \"['http://', RuleRef.a_string, ':', "
"Ports.config_port, '/risk']\"\n"
"}"
"StringTable {\n"
"value {\n"
" entries: 'http://'\n"
" entries: @RuleRef.a_string\n"
" entries: ':'\n"
" entries: @Ports.config_port\n"
" entries: '/risk'\n"
"}}\n"
"Foo { \n"
" value {\n"
" name: $list\n"
" url: @StringTable.entries \n"
" }\n"
"}");
auto foo = config::get<Foo>();
auto expected = "http://localhost:80/risk";
REQUIRE(foo.name() == expected);
REQUIRE(foo.url() == expected);
}
reset();
}
}
} // namespace bts
+37 -30
View File
@@ -1,6 +1,6 @@
#include "SparkServices.h"
#include "lib/orderman/order_factory/DynamicOrderRegistry.h"
#include "lib/logging/logging.h"
#include "lib/orderman/order_factory/DynamicOrderRegistry.h"
namespace bts {
@@ -12,43 +12,50 @@ struct SparkServices::PImpl {
};
SparkServices::SparkServices(orderman::DynamicOrderRegistry &r)
: me(new PImpl(r)) {}
: me(new PImpl(r)) {}
SparkServices::~SparkServices() {}
void SparkServices::register_dynamic_order(
api::orderman::DynamicOrderDescriptor const &desc,
std::function<std::unique_ptr<Order>()> const &builder) {
api::orderman::DynamicOrderDescriptor const &desc,
std::function<std::unique_ptr<Order>()> const &builder) {
me->dyn_order_registry.register_dynamic_order(desc, builder);
}
bool validate_param(const api::orderman::DynamicOrderParamDescriptor& param);
bool validate_type(const std::string name, const api::orderman::DynamicOrderTypeDescriptor& type) {
switch(type.descriptor_case()) {
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kPrimitive: break;
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kArray:
if (!type.array().has_element_type()) {
LOG(OrderMan, Error, "Array parameter {} missing element type", name);
bool validate_param(const api::orderman::DynamicOrderParamDescriptor &param);
bool validate_type(
const std::string name,
const api::orderman::DynamicOrderTypeDescriptor &type) {
switch (type.descriptor_case()) {
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kPrimitive:
break;
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kArray:
if (!type.array().has_element_type()) {
LOG(OrderMan, Error, "Array parameter {} missing element type", name);
return false;
}
if (!validate_type(name + "::array", type.array().element_type()))
return false;
break;
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kStructure:
for (const auto &field : type.structure().field()) {
if (!validate_param(field))
return false;
}
if (!validate_type(name + "::array", type.array().element_type())) return false;
break;
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kStructure:
for(const auto& field : type.structure().field()) {
if (!validate_param(field)) return false;
}
break;
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kEnumerated:
if (type.enumerated().element_size() == 0) {
LOG(OrderMan, Error, "Enumerated parameter {} missing options list", name);
return false;
}
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::DESCRIPTOR_NOT_SET:
}
break;
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::kEnumerated:
if (type.enumerated().element_size() == 0) {
LOG(OrderMan, Error, "Enumerated parameter {} missing options list",
name);
return false;
}
case api::orderman::DynamicOrderTypeDescriptor::DescriptorCase::
DESCRIPTOR_NOT_SET:
break;
}
return true;
}
bool validate_param(const api::orderman::DynamicOrderParamDescriptor& param) {
bool validate_param(const api::orderman::DynamicOrderParamDescriptor &param) {
if (param.param_name().empty()) {
LOG(OrderMan, Error, "Parameter missing param_name");
return false;
@@ -61,12 +68,12 @@ bool validate_param(const api::orderman::DynamicOrderParamDescriptor& param) {
return true;
}
bool validate_descriptor(const api::orderman::DynamicOrderDescriptor& desc) {
bool validate_descriptor(const api::orderman::DynamicOrderDescriptor &desc) {
if (desc.dynamic_order_type().empty()) {
LOG(OrderMan, Error, "Algo missing 'dynamic_order_type'");
return false;
}
for (const auto& param : desc.param_desc()) {
for (const auto &param : desc.param_desc()) {
if (!validate_param(param)) {
return false;
}
@@ -75,8 +82,8 @@ bool validate_descriptor(const api::orderman::DynamicOrderDescriptor& desc) {
}
AlgoRegistrationResult SparkServices::register_dynamic_algo_order(
api::orderman::DynamicOrderDescriptor const &desc,
std::function<std::unique_ptr<orderman::IAlgoOrder>()> const &builder) {
api::orderman::DynamicOrderDescriptor const &desc,
std::function<std::unique_ptr<orderman::IAlgoOrder>()> const &builder) {
if (validate_descriptor(desc)) {
me->dyn_order_registry.register_dynamic_algo_order(desc, builder);
+2 -4
View File
@@ -13,10 +13,7 @@ class IAlgoOrder;
class DynamicOrderRegistry;
} // namespace orderman
enum class AlgoRegistrationResult {
SUCCESS,
ERROR
};
enum class AlgoRegistrationResult { SUCCESS, ERROR };
class SparkServices {
public:
@@ -30,6 +27,7 @@ class SparkServices {
AlgoRegistrationResult register_dynamic_algo_order(
api::orderman::DynamicOrderDescriptor const &desc,
std::function<std::unique_ptr<orderman::IAlgoOrder>()> const &builder);
private:
struct PImpl;
std::unique_ptr<PImpl> me;
+8 -7
View File
@@ -1,9 +1,9 @@
#include "lib/config/config.h"
#include "lib/config/protos/system.pb.h"
#include "lib/logging/logging.h"
#include "lib/orderman/actors/order_manager/OrderBuilder.h"
#include "lib/plugin/SparkServices.h"
#include "lib/utils/PBUtils.h"
#include "lib/config/config.h"
#include "lib/config/protos/system.pb.h"
#include <algorithm>
#include <boost/dll.hpp>
@@ -16,10 +16,9 @@ namespace plugin {
using namespace bts::orderman;
void init_plugins(SparkServices &services) {
static std::vector<decltype(boost::dll::import<void(SparkServices &)>(
static std::vector<decltype(boost::dll::import_symbol<void(SparkServices &)>(
boost::filesystem::path(), ""))>
keep_alive;
@@ -44,16 +43,18 @@ void init_plugins(SparkServices &services) {
if (pl == "order_plugin_example.so" &&
config::get<config::DeploymentType>().type() != "dev") {
LOG(Plugin, Info,
"Skipping order_plugin_example.so because we are not running in dev mode",
"Skipping order_plugin_example.so because we are not running in "
"dev mode",
plugin_path);
continue;
}
if (pl == "libspark.so") continue;
if (pl == "libspark.so")
continue;
LOG(Plugin, Info, "Attempting to initialize plugin {}", pl);
try {
auto init_plugin = boost::dll::import<void(SparkServices &)>(
auto init_plugin = boost::dll::import_symbol<void(SparkServices &)>(
f.path(), "init_spark_plugin");
init_plugin(services);
keep_alive.push_back(init_plugin);
+2 -1
View File
@@ -4,4 +4,5 @@ namespace bts {
class SparkServices;
namespace plugin {
void init_plugins(SparkServices &);
}}
}
} // namespace bts
+195 -193
View File
@@ -1,17 +1,17 @@
#include "QBConnection.h"
#include "lib/logging/logging.h"
#include "lib/os/os.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/datetime.h"
#include "lib/logging/logging.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/datetime.h"
#include "lib/quickbus/routing/Router.h"
#include "lib/quickbus/network/endpoint_description.h"
#include "lib/quickbus/core/Advertisement.h"
#include "lib/quickbus/network/endpoint_description.h"
#include "lib/quickbus/routing/Router.h"
#include "lib/quickbus/network/quickbus_msgs.pb.h"
#include "lib/config/protos/quickbus_cfg.pb.h"
#include "lib/quickbus/network/quickbus_msgs.pb.h"
#include <boost/algorithm/string/join.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
@@ -22,28 +22,24 @@ namespace qbus {
using boost::asio::ip::tcp;
using namespace boost;
struct QBConnection::PImpl
{
PImpl(tcp::socket &s,Router &router,config::QuickbusNodeCfg const&cfg)
: socket(std::move(s)),
router(router),
pending_send(),
write_buffer(),
write_buffer_size(),
read_buffer(),
read_buffer_size(),
has_intro(false),
intro(),
my_id(),
my_desc(local_endpoint_description(socket)),
remote_desc(remote_endpoint_description(socket)),
cfg(cfg),
last_received_ts(datetime::now()),
last_sent_ts(datetime::now()),
hb_timer(socket.get_io_service())
{
write_buffer.reserve(100*1024);
read_buffer.reserve(100*1024);
#if BOOST_VERSION >= 107000
#define GET_IO_SERVICE(s) \
((boost::asio::io_context &)(s).get_executor().context())
#else
#define GET_IO_SERVICE(s) ((s).get_io_service())
#endif
struct QBConnection::PImpl {
PImpl(tcp::socket &s, Router &router, config::QuickbusNodeCfg const &cfg)
: socket(std::move(s)), router(router), pending_send(), write_buffer(),
write_buffer_size(), read_buffer(), read_buffer_size(),
has_intro(false), intro(), my_id(),
my_desc(local_endpoint_description(socket)),
remote_desc(remote_endpoint_description(socket)), cfg(cfg),
last_received_ts(datetime::now()), last_sent_ts(datetime::now()),
hb_timer(GET_IO_SERVICE(socket)) {
write_buffer.reserve(100 * 1024);
read_buffer.reserve(100 * 1024);
}
tcp::socket socket;
@@ -69,141 +65,132 @@ struct QBConnection::PImpl
boost::asio::deadline_timer hb_timer;
};
QBConnection::QBConnection (tcp::socket s,Router &router, config::QuickbusNodeCfg const&cfg)
:
me(new PImpl(s,router,cfg))
{}
QBConnection::QBConnection(
tcp::socket s, Router &router, config::QuickbusNodeCfg const &cfg)
: me(new PImpl(s, router, cfg)) {}
QBConnection::~QBConnection()
{}
QBConnection::~QBConnection() {}
void QBConnection::init()
{
void QBConnection::init() {
send_intro();
recv_intro();
check_hbs();
}
bool QBConnection::connected() const
{return me->socket.is_open();}
void QBConnection::send_intro()
{
LOG(QBus,Info,"Sending intro from {} to {}", me->my_desc , me->remote_desc );
bool QBConnection::connected() const { return me->socket.is_open(); }
void QBConnection::send_intro() {
LOG(QBus, Info, "Sending intro from {} to {}", me->my_desc,
me->remote_desc);
bts_assert(!is_send_in_flight());
auto &intro = *me->pending_send.add_op()->mutable_intro();
intro.set_host_name(os::host_name());
intro.set_app_cmd_line(boost::algorithm::join(os::get_command_line()," "));
intro.set_app_cmd_line(boost::algorithm::join(os::get_command_line(), " "));
intro.set_node_id(me->router.id().value());
intro.set_pid(os::process_id());
try_send_pending();
}
void QBConnection::recv_intro()
{
recv_msg_header();
}
void QBConnection::recv_intro() { recv_msg_header(); }
void QBConnection::check_hbs()
{
void QBConnection::check_hbs() {
auto self = shared_from_this();
me->hb_timer.expires_from_now(boost::posix_time::milliseconds(me->cfg.hb_period_msecs()));
me->hb_timer.async_wait([self,this](const boost::system::error_code& ec) {
me->hb_timer.expires_from_now(
boost::posix_time::milliseconds(me->cfg.hb_period_msecs()));
me->hb_timer.async_wait([self, this](const boost::system::error_code &ec) {
if (!ec && connected()) {
if (datetime::is_at_least_msecs_old(
me->last_received_ts,
me->cfg.hb_period_msecs() * me->cfg.max_missed_hbs()))
close_connection("No heartbeats received");
else {
if (!is_send_in_flight() &&
datetime::is_at_least_msecs_old(
me->last_sent_ts, me->cfg.hb_period_msecs())) {
if (!ec && connected()) {
if (datetime::is_at_least_msecs_old (me->last_received_ts,
me->cfg.hb_period_msecs() *
me->cfg.max_missed_hbs()))
close_connection("No heartbeats received");
else {
if (!is_send_in_flight() &&
datetime::is_at_least_msecs_old (me->last_sent_ts, me->cfg.hb_period_msecs())) {
LOG(QBus,Diag,"{} sending HB to {}", me->my_desc ,me->remote_desc );
me->pending_send.add_op()->mutable_heartbeat();
try_send_pending();
}
check_hbs();
LOG(QBus, Diag, "{} sending HB to {}", me->my_desc,
me->remote_desc);
me->pending_send.add_op()->mutable_heartbeat();
try_send_pending();
}
check_hbs();
}
});
}
});
}
void QBConnection::recv_msg_header()
{
void QBConnection::recv_msg_header() {
auto self = shared_from_this();
asio::async_read (me->socket,
asio::buffer (&me->read_buffer_size, sizeof(me->read_buffer_size)),
[this, self](system::error_code error, std::size_t ) {
if (!error)
recv_msg_body();
else
close_connection(error.message());
});
asio::async_read(
me->socket,
asio::buffer(&me->read_buffer_size, sizeof(me->read_buffer_size)),
[this, self](system::error_code error, std::size_t) {
if (!error)
recv_msg_body();
else
close_connection(error.message());
});
}
void QBConnection::recv_msg_body()
{
void QBConnection::recv_msg_body() {
auto self = shared_from_this();
me->read_buffer.resize(me->read_buffer_size);
asio::async_read (me->socket,
asio::buffer (&me->read_buffer[0],me->read_buffer_size),
[this, self](system::error_code error, std::size_t ) {
if (!error) {
msgs::QBMsg m;
auto parse_ok = m.ParseFromArray (&(me->read_buffer[0]),
me->read_buffer.size ());
if (!parse_ok)
close_connection("Corrupt message");
else
handle_msg(&m);
}
else
close_connection(error.message());
});
asio::async_read(
me->socket, asio::buffer(&me->read_buffer[0], me->read_buffer_size),
[this, self](system::error_code error, std::size_t) {
if (!error) {
msgs::QBMsg m;
auto parse_ok = m.ParseFromArray(
&(me->read_buffer[0]), me->read_buffer.size());
if (!parse_ok)
close_connection("Corrupt message");
else
handle_msg(&m);
} else
close_connection(error.message());
});
}
void QBConnection::handle_msg(msgs::QBMsg *msgs)
{
void QBConnection::handle_msg(msgs::QBMsg *msgs) {
me->last_received_ts = datetime::now();
for (auto const&op : msgs->op()) {
for (auto const &op : msgs->op()) {
if (op.op_case() == msgs::QBOp::kIntro) {
if (me->has_intro) {
close_connection("Received two intro messages");
return;
} else {
LOG(QBus,Info,
"Received intro from {} for {} intro: {} connection is owned by node: {}"
LOG(QBus, Info,
"Received intro from {} for {} intro: {} connection is owned "
"by node: {}"
" tcp connection is local {} to remote {}",
me->remote_desc ,me->my_desc ,
utils::pb::to_string(op.intro()), me->router.id().value(),
me->my_desc, me->remote_desc );
me->remote_desc, me->my_desc, utils::pb::to_string(op.intro()),
me->router.id().value(), me->my_desc, me->remote_desc);
me->has_intro = true;
me->intro=op.intro();
me->intro = op.intro();
me->my_id = NodeID(me->intro.node_id());
if (me->router.is_neighbour(me->my_id)) {
LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" ,
me->my_desc , me->remote_desc , me->my_id.value());
if (me->socket.is_open())
LOG(QBus, Warn,
"Refusing connection from {} to {} reason: {} is already "
"connected",
me->my_desc, me->remote_desc, me->my_id.value());
if (me->socket.is_open())
me->socket.close();
return;
}
else {
LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." ,
me->my_desc , me->remote_desc , me->my_id.value());
me->router.add_neighbour(me->my_id,shared_from_this());
} else {
LOG(QBus, Warn,
"Accepted connection from {} to {}. {} is now connected.",
me->my_desc, me->remote_desc, me->my_id.value());
me->router.add_neighbour(me->my_id, shared_from_this());
}
}
break;
@@ -213,7 +200,7 @@ void QBConnection::handle_msg(msgs::QBMsg *msgs)
close_connection("Expected intro");
return;
}
switch (op.op_case()) {
case msgs::QBOp::kIntro:
@@ -221,23 +208,23 @@ void QBConnection::handle_msg(msgs::QBMsg *msgs)
break;
case msgs::QBOp::kNotification: {
LOG(QBus,Diag,
LOG(QBus, Diag,
"{} received notification from {} for table {} sid: {} size: {}",
me->my_desc , me->remote_desc , op.notification().table_id(),
me->my_desc, me->remote_desc, op.notification().table_id(),
op.notification().subscription_id(),
op.notification().update().size());
auto &source_update = op.notification().update();
Update target_update;
target_update.resize(source_update.size());
for (int i = 0; i < source_update.size(); i++) {
auto &target_change = target_update[i];
auto &source_change = op.notification().update(i);
auto &source_change = op.notification().update(i);
target_change.key = RecordKey(source_change.key());
target_change.value = std::make_shared<Record>(std::move(source_change.value()));
target_change.value =
std::make_shared<Record>(std::move(source_change.value()));
switch (source_change.type()) {
case msgs::Change::REMOVE:
@@ -254,53 +241,66 @@ void QBConnection::handle_msg(msgs::QBMsg *msgs)
break;
}
}
me->router.notify(me->my_id,
TableID(op.notification().table_id()),
SubscriptionID(op.notification().subscription_id()),target_update);
me->router.notify(
me->my_id, TableID(op.notification().table_id()),
SubscriptionID(op.notification().subscription_id()),
target_update);
break;
}
case msgs::QBOp::kAdvertisement: {
LOG(QBus,Info, "{} received advertisement from {} adv: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.advertisement()));
Advertisement ad(Topic(op.advertisement().topic()),op.advertisement().maintain_synopsis());
me->router.advertise(me->my_id, TableID(op.advertisement().table_id()),ad);
case msgs::QBOp::kAdvertisement: {
LOG(QBus, Info, "{} received advertisement from {} adv: {}",
me->my_desc, me->remote_desc,
utils::pb::to_string(op.advertisement()));
Advertisement ad(
Topic(op.advertisement().topic()),
op.advertisement().maintain_synopsis());
me->router.advertise(
me->my_id, TableID(op.advertisement().table_id()), ad);
break;
}
case msgs::QBOp::kUnadvertisement:
LOG(QBus,Info, "{} received unadvertisement from {} unadv: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.unadvertisement()));
LOG(QBus, Info, "{} received unadvertisement from {} unadv: {}",
me->my_desc, me->remote_desc,
utils::pb::to_string(op.unadvertisement()));
me->router.unadvertise(me->my_id, TableID(op.unadvertisement().table_id()));
me->router.unadvertise(
me->my_id, TableID(op.unadvertisement().table_id()));
break;
case msgs::QBOp::kSubscription:
LOG(QBus,Info, "{} received subscription from {} sub: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.subscription()));
LOG(QBus, Info, "{} received subscription from {} sub: {}",
me->my_desc, me->remote_desc,
utils::pb::to_string(op.subscription()));
me->router.subscribe(me->my_id, TableID(op.subscription().table_id()),
SubscriptionID(op.subscription().subscription_id()));
me->router.subscribe(
me->my_id, TableID(op.subscription().table_id()),
SubscriptionID(op.subscription().subscription_id()));
break;
case msgs::QBOp::kUnsubscription:
LOG(QBus,Info, "{} received unsubscription from {} unsub: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.unsubscription()));
LOG(QBus, Info, "{} received unsubscription from {} unsub: {}",
me->my_desc, me->remote_desc,
utils::pb::to_string(op.unsubscription()));
me->router.unsubscribe(me->my_id, TableID(op.unsubscription().table_id()));
me->router.unsubscribe(
me->my_id, TableID(op.unsubscription().table_id()));
break;
case msgs::QBOp::kHeartbeat:
LOG(QBus,Diag,"{} received heartbeat from {}", me->my_desc ,me->remote_desc) ;
LOG(QBus, Diag, "{} received heartbeat from {}", me->my_desc,
me->remote_desc);
break;
case msgs::QBOp::OP_NOT_SET:
LOG(QBus,Error,"{} received corrupted message from {}", me->my_desc ,me->remote_desc) ;
LOG(QBus, Error, "{} received corrupted message from {}", me->my_desc,
me->remote_desc);
close_connection("Corrupted message");
return;
} // switch
} // for
} // for
recv_msg_header();
}
void QBConnection::advertise(NodeID const& sender, TableID const&id,Advertisement const&ad)
{
void QBConnection::advertise(
NodeID const &sender, TableID const &id, Advertisement const &ad) {
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
@@ -309,18 +309,22 @@ void QBConnection::advertise(NodeID const& sender, TableID const&id,Advertisemen
target_ad.set_table_id(id.value());
target_ad.set_maintain_synopsis(ad.maintain_synopsis);
LOG(QBus,Info, "{} is sending an advertisement to {} ad: {}" ,
me->my_desc , me->remote_desc,utils::pb::to_string(target_ad));
LOG(QBus, Info, "{} is sending an advertisement to {} ad: {}", me->my_desc,
me->remote_desc, utils::pb::to_string(target_ad));
try_send_pending();
}
void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID const&sid, Update const&u)
{
void QBConnection::notify(
NodeID const &sender,
TableID const &id,
SubscriptionID const &sid,
Update const &u) {
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_notification = *me->pending_send.add_op()->mutable_notification();
auto &target_notification =
*me->pending_send.add_op()->mutable_notification();
target_notification.set_table_id(id.value());
target_notification.set_subscription_id(sid.value());
@@ -333,7 +337,7 @@ void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID co
auto *target_change = target_update.Add();
target_change->set_key(ch.key.value());
switch(ch.type) {
switch (ch.type) {
case Change::REMOVE:
target_change->set_type(msgs::Change::REMOVE);
break;
@@ -348,27 +352,27 @@ void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID co
}
}
LOG(QBus,Diag,"{} is sending a notification to {} table id: {}",
me->my_desc , me->remote_desc , id.value() ) ;
LOG(QBus, Diag, "{} is sending a notification to {} table id: {}",
me->my_desc, me->remote_desc, id.value());
try_send_pending();
}
void QBConnection::unadvertise(NodeID const&sender,TableID const&id)
{
void QBConnection::unadvertise(NodeID const &sender, TableID const &id) {
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_unadvertise = *me->pending_send.add_op()->mutable_unadvertisement();
auto &target_unadvertise =
*me->pending_send.add_op()->mutable_unadvertisement();
target_unadvertise.set_table_id(id.value());
LOG(QBus,Info,"{} is sending a unadvertisement to {} table id: {}",
me->my_desc , me->remote_desc , id.value()) ;
LOG(QBus, Info, "{} is sending a unadvertisement to {} table id: {}",
me->my_desc, me->remote_desc, id.value());
try_send_pending();
}
void QBConnection::subscribe(NodeID const&sender, TableID const&id,SubscriptionID const&sid)
{
void QBConnection::subscribe(
NodeID const &sender, TableID const &id, SubscriptionID const &sid) {
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
@@ -377,53 +381,52 @@ void QBConnection::subscribe(NodeID const&sender, TableID const&id,SubscriptionI
target_subscribe.set_table_id(id.value());
target_subscribe.set_subscription_id(sid.value());
LOG(QBus,Info,"{} is sending a subscription to {} sub: {}",
me->my_desc , me->remote_desc , utils::pb::to_string(target_subscribe));
LOG(QBus, Info, "{} is sending a subscription to {} sub: {}", me->my_desc,
me->remote_desc, utils::pb::to_string(target_subscribe));
try_send_pending();
}
void QBConnection::unsubscribe (NodeID const&sender, TableID const&id)
{
void QBConnection::unsubscribe(NodeID const &sender, TableID const &id) {
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_unsubscribe = *me->pending_send.add_op()->mutable_unsubscription();
auto &target_unsubscribe =
*me->pending_send.add_op()->mutable_unsubscription();
target_unsubscribe.set_table_id(id.value());
LOG(QBus,Info,"{} is sending a unsubscription to {} table id: {}",
me->my_desc , me->remote_desc , id.value());
LOG(QBus, Info, "{} is sending a unsubscription to {} table id: {}",
me->my_desc, me->remote_desc, id.value());
try_send_pending();
}
void QBConnection::close_connection(std::string const&reason)
{
void QBConnection::close_connection(std::string const &reason) {
if (me->socket.is_open()) {
LOG(QBus,Warn, "{} is closing its connection to {} reason: {}" ,
me->my_desc , me->remote_desc , reason);
LOG(QBus, Warn, "{} is closing its connection to {} reason: {}",
me->my_desc, me->remote_desc, reason);
me->socket.close();
if (me->has_intro)
me->router.remove_neighbour(me->my_id);
}
}
bool QBConnection::is_send_in_flight() const
{return me->write_buffer.size() != 0;}
bool QBConnection::is_send_in_flight() const {
return me->write_buffer.size() != 0;
}
bool QBConnection::can_send_now() const
{return !is_send_in_flight() && me->pending_send.op_size();}
bool QBConnection::can_send_now() const {
return !is_send_in_flight() && me->pending_send.op_size();
}
void QBConnection::try_send_pending()
{
void QBConnection::try_send_pending() {
if (can_send_now()) {
send(me->pending_send);
me->pending_send.Clear();
}
}
void QBConnection::send(msgs::QBMsg const&msg)
{
void QBConnection::send(msgs::QBMsg const &msg) {
auto self = shared_from_this();
me->last_sent_ts = datetime::now();
@@ -431,25 +434,24 @@ void QBConnection::send(msgs::QBMsg const&msg)
bts_assert(me->write_buffer.size());
me->write_buffer_size = me->write_buffer.size();
std::array<asio::mutable_buffer,2> to_send;
to_send[0] = asio::buffer (&(me->write_buffer_size), sizeof(me->write_buffer_size));
to_send[1] = asio::buffer (&(me->write_buffer[0]), me->write_buffer.size());
std::array<asio::mutable_buffer, 2> to_send;
to_send[0] =
asio::buffer(&(me->write_buffer_size), sizeof(me->write_buffer_size));
to_send[1] = asio::buffer(&(me->write_buffer[0]), me->write_buffer.size());
LOG(QBus,Diag,
"Sending message from {} to {} size: {}",
me->my_desc , me->remote_desc , me->write_buffer_size);
LOG(QBus, Diag, "Sending message from {} to {} size: {}", me->my_desc,
me->remote_desc, me->write_buffer_size);
asio::async_write (me->socket, to_send,
[this, self](system::error_code error, std::size_t) {
if (!error) {
me->write_buffer.clear();
try_send_pending();
}
else
close_connection(error.message());
});
asio::async_write(
me->socket, to_send,
[this, self](system::error_code error, std::size_t) {
if (!error) {
me->write_buffer.clear();
try_send_pending();
} else
close_connection(error.message());
});
}
}}
} // namespace qbus
} // namespace bts
+49 -22
View File
@@ -3,14 +3,28 @@
#include "lib/quickbus/routing/INode.h"
#include "lib/utils/warn_ctrl.h"
#include <memory>
#include <boost/asio.hpp>
#include <functional>
#include <memory>
// fwds
namespace bts {namespace qbus {namespace msgs {class QBMsg;}}}
namespace bts {namespace qbus {class Router;}}
namespace bts {namespace config {class QuickbusNodeCfg;}}
namespace bts {
namespace qbus {
namespace msgs {
class QBMsg;
}
} // namespace qbus
} // namespace bts
namespace bts {
namespace qbus {
class Router;
}
} // namespace bts
namespace bts {
namespace config {
class QuickbusNodeCfg;
}
} // namespace bts
// end fwds
namespace bts {
@@ -18,23 +32,36 @@ namespace qbus {
GCC_DIAG_OFF("-Wnon-virtual-dtor")
class QBConnection : public std::enable_shared_from_this<QBConnection>, public INode
{
public:
QBConnection (boost::asio::ip::tcp::socket socket,Router &router,config::QuickbusNodeCfg const&);
class QBConnection : public std::enable_shared_from_this<QBConnection>,
public INode {
public:
QBConnection(
boost::asio::ip::tcp::socket socket,
Router &router,
config::QuickbusNodeCfg const &);
void init();
bool connected() const;
~QBConnection();
// from INode
void advertise(NodeID const& sender, TableID const&id,Advertisement const&adv) override;
void unadvertise(NodeID const& sender, TableID const&tid) override;
void subscribe (NodeID const&sender, TableID const&tid,SubscriptionID const&sid) override;
void notify(NodeID const&nid,TableID const&tid,SubscriptionID const&sid,Update const&u) override;
void unsubscribe (NodeID const&sender, TableID const&tid) override;
// from INode
void advertise(
NodeID const &sender,
TableID const &id,
Advertisement const &adv) override;
void unadvertise(NodeID const &sender, TableID const &tid) override;
void subscribe(
NodeID const &sender,
TableID const &tid,
SubscriptionID const &sid) override;
void notify(
NodeID const &nid,
TableID const &tid,
SubscriptionID const &sid,
Update const &u) override;
void unsubscribe(NodeID const &sender, TableID const &tid) override;
// end INode
private:
private:
struct PImpl;
std::unique_ptr<PImpl> me;
@@ -42,17 +69,17 @@ private:
bool can_send_now() const;
void try_send_pending();
void send_intro();
void recv_intro();
void check_hbs();
void send(msgs::QBMsg const&m);
void send(msgs::QBMsg const &m);
void recv_msg_header();
void recv_msg_body();
void handle_msg(msgs::QBMsg *m);
void close_connection(std::string const&reason);
void close_connection(std::string const &reason);
};
GCC_DIAG_RESTORE()
}}
} // namespace qbus
} // namespace bts
@@ -1,9 +1,9 @@
#include "QBConnectionManager.h"
#include "lib/quickbus/routing/Router.h"
#include "lib/quickbus/network/QBConnection.h"
#include "lib/logging/logging.h"
#include "lib/quickbus/network/QBConnection.h"
#include "lib/quickbus/network/endpoint_description.h"
#include "lib/quickbus/routing/Router.h"
#include "lib/utils/bts_assert.h"
#include "lib/config/protos/quickbus_cfg.pb.h"
@@ -17,19 +17,15 @@ using boost::asio::ip::tcp;
namespace bts {
namespace qbus {
struct QBConnectionManager::PImpl
{
PImpl(boost::asio::io_service &service,
Router &router, config::QuickbusNodeCfg const&cfg) :
router(router),
cfg(cfg),
io_service(service),
acceptor(io_service,tcp::endpoint(tcp::v4(),cfg.my_port())),
acceptor_socket(io_service),
connector_socket(io_service),
reconnect_timer(io_service),
parent_connection()
{}
struct QBConnectionManager::PImpl {
PImpl(
boost::asio::io_service &service,
Router &router,
config::QuickbusNodeCfg const &cfg)
: router(router), cfg(cfg), io_service(service),
acceptor(io_service, tcp::endpoint(tcp::v4(), cfg.my_port())),
acceptor_socket(io_service), connector_socket(io_service),
reconnect_timer(io_service), parent_connection() {}
Router &router;
const config::QuickbusNodeCfg cfg;
@@ -41,87 +37,80 @@ struct QBConnectionManager::PImpl
std::shared_ptr<QBConnection> parent_connection;
};
QBConnectionManager::QBConnectionManager (boost::asio::io_service &service,
Router &router,
config::QuickbusNodeCfg const&cfg):
me (new PImpl(service,router,cfg))
{
QBConnectionManager::QBConnectionManager(
boost::asio::io_service &service,
Router &router,
config::QuickbusNodeCfg const &cfg)
: me(new PImpl(service, router, cfg)) {
check_parent_connection();
if (me->cfg.my_port())
accept();
}
QBConnectionManager::~QBConnectionManager ()
{
LOG(QBus,Info, "Destroyed QBConnectionManager");
QBConnectionManager::~QBConnectionManager() {
LOG(QBus, Info, "Destroyed QBConnectionManager");
}
void QBConnectionManager::accept()
{
LOG(QBus,Info,"Accepting new connections on port {}", me->cfg.my_port());
void QBConnectionManager::accept() {
LOG(QBus, Info, "Accepting new connections on port {}", me->cfg.my_port());
me->acceptor.async_accept(
me->acceptor_socket,
[this](const boost::system::error_code ec) {
if (!ec) {
LOG(QBus,Info,"{} accepted connection from {}",
local_endpoint_description(me->acceptor_socket),
remote_endpoint_description(me->acceptor_socket)) ;
me->acceptor_socket, [this](const boost::system::error_code ec) {
if (!ec) {
LOG(QBus, Info, "{} accepted connection from {}",
local_endpoint_description(me->acceptor_socket),
remote_endpoint_description(me->acceptor_socket));
auto c = std::make_shared<QBConnection>(std::move(me->acceptor_socket), me->router,me->cfg);
c->init();
}
else
LOG(QBus,Info,"Accept failed on port {} msg: {} code: {}",
me->cfg.my_port(), ec.message(), ec.value());
accept();
});
auto c = std::make_shared<QBConnection>(
std::move(me->acceptor_socket), me->router, me->cfg);
c->init();
} else
LOG(QBus, Info, "Accept failed on port {} msg: {} code: {}",
me->cfg.my_port(), ec.message(), ec.value());
accept();
});
}
void QBConnectionManager::connect()
{
void QBConnectionManager::connect() {
auto addr = boost::asio::ip::address_v4::from_string(me->cfg.parent_host());
tcp::endpoint ep (addr,me->cfg.parent_port());
tcp::endpoint ep(addr, me->cfg.parent_port());
auto desc = endpoint_description(ep);
LOG(QBus,Info,"Initiating connection to {}", endpoint_description(ep));
LOG(QBus, Info, "Initiating connection to {}", endpoint_description(ep));
auto &socket = me->connector_socket;
socket.async_connect(ep,
[this,desc] (const boost::system::error_code ec) {
if (!ec) {
LOG(QBus,Info,"{} connected to {}",
local_endpoint_description(me->connector_socket),
remote_endpoint_description(me->connector_socket)) ;
bts_assert (!me->parent_connection);
me->parent_connection =
std::make_shared<QBConnection> (std::move(me->connector_socket),me->router,me->cfg);
me->parent_connection->init();
}
else
LOG(QBus,Info,"Failed to connect to {} error: {} code:{}",
desc ,ec.message(),ec.value());
});
socket.async_connect(ep, [this, desc](const boost::system::error_code ec) {
if (!ec) {
LOG(QBus, Info, "{} connected to {}",
local_endpoint_description(me->connector_socket),
remote_endpoint_description(me->connector_socket));
bts_assert(!me->parent_connection);
me->parent_connection = std::make_shared<QBConnection>(
std::move(me->connector_socket), me->router, me->cfg);
me->parent_connection->init();
} else
LOG(QBus, Info, "Failed to connect to {} error: {} code:{}", desc,
ec.message(), ec.value());
});
}
void QBConnectionManager::check_parent_connection()
{
if (!me->parent_connection || !me->parent_connection->connected())
void QBConnectionManager::check_parent_connection() {
if (!me->parent_connection || !me->parent_connection->connected())
if (me->cfg.parent_host().size()) {
me->parent_connection.reset();
connect();
}
me->reconnect_timer.expires_from_now(boost::posix_time::seconds(me->cfg.reconnect_period_secs()));
me->reconnect_timer.async_wait([this](const boost::system::error_code& ec) {
if (!ec)
check_parent_connection();
else
LOG(QBus,Error,"Reconnect timer failure. error: {} code: {}",
ec.message(), ec.value());
});
me->reconnect_timer.expires_from_now(
boost::posix_time::seconds(me->cfg.reconnect_period_secs()));
me->reconnect_timer.async_wait([this](const boost::system::error_code &ec) {
if (!ec)
check_parent_connection();
else
LOG(QBus, Error, "Reconnect timer failure. error: {} code: {}",
ec.message(), ec.value());
});
}
}}
} // namespace qbus
} // namespace bts
+19 -11
View File
@@ -1,14 +1,19 @@
#pragma once
#include <memory>
#include <boost/asio/io_service.hpp>
#include <functional>
#include <memory>
#include <string>
// fwds
namespace bts {
namespace qbus {class Router;}
namespace config {class QuickbusNodeCfg;}}
namespace boost {namespace asio {class io_service;}}
namespace qbus {
class Router;
}
namespace config {
class QuickbusNodeCfg;
}
} // namespace bts
// end fwds
namespace bts {
@@ -16,18 +21,21 @@ namespace qbus {
class INode;
class QBConnectionManager
{
public:
QBConnectionManager (boost::asio::io_service &, bts::qbus::Router &,config::QuickbusNodeCfg const&);
class QBConnectionManager {
public:
QBConnectionManager(
boost::asio::io_service &,
bts::qbus::Router &,
config::QuickbusNodeCfg const &);
~QBConnectionManager();
private:
private:
struct PImpl;
std::unique_ptr<PImpl> me;
void accept();
void connect();
void check_parent_connection();
};
}}
} // namespace qbus
} // namespace bts
@@ -6,17 +6,16 @@
namespace bts {
namespace qbus {
std::string endpoint_description(boost::asio::ip::tcp::endpoint const&ep)
{
std::string endpoint_description(boost::asio::ip::tcp::endpoint const &ep) {
try {
return (boost::format ("%s:%d") % ep.address ().to_string () % ep.port()).str();
return (boost::format("%s:%d") % ep.address().to_string() % ep.port())
.str();
} catch (boost::system::system_error &) {
return "<unknown>";
}
}
std::string local_endpoint_description(boost::asio::ip::tcp::socket const&s)
{
std::string local_endpoint_description(boost::asio::ip::tcp::socket const &s) {
try {
return endpoint_description(s.local_endpoint());
} catch (boost::system::system_error &) {
@@ -24,8 +23,7 @@ std::string local_endpoint_description(boost::asio::ip::tcp::socket const&s)
}
}
std::string remote_endpoint_description(boost::asio::ip::tcp::socket const&s)
{
std::string remote_endpoint_description(boost::asio::ip::tcp::socket const &s) {
try {
return endpoint_description(s.remote_endpoint());
} catch (boost::system::system_error &) {
@@ -33,4 +31,5 @@ std::string remote_endpoint_description(boost::asio::ip::tcp::socket const&s)
}
}
}}
} // namespace qbus
} // namespace bts
@@ -4,7 +4,8 @@
namespace bts {
namespace qbus {
std::string endpoint_description(boost::asio::ip::tcp::endpoint const&ep);
std::string local_endpoint_description(boost::asio::ip::tcp::socket const&s);
std::string remote_endpoint_description(boost::asio::ip::tcp::socket const&s);
}}
std::string endpoint_description(boost::asio::ip::tcp::endpoint const &ep);
std::string local_endpoint_description(boost::asio::ip::tcp::socket const &s);
std::string remote_endpoint_description(boost::asio::ip::tcp::socket const &s);
} // namespace qbus
} // namespace bts
+11 -11
View File
@@ -60,8 +60,8 @@ Dispatcher::~Dispatcher() {
}
void Dispatcher::run(size_t threads) {
bts_assert_msg(_me && _me->work == nullptr,
"Error dispatcher is already running.");
bts_assert_msg(
_me && _me->work == nullptr, "Error dispatcher is already running.");
_me->service.reset();
_me->work.reset(new io_service::work(_me->service));
DEFER(_me->work.reset());
@@ -148,14 +148,14 @@ Dispatcher &Dispatcher::create_new(const char *name) {
auto msg = fmt::format(
"Exception on dispatcher thread. What: {} Exception type: {}",
ex.what(), utils::type_name(typeid(ex)));
bts_assert_msg(false,msg);
bts_assert_msg(false, msg);
}
});
t.detach();
auto result =
c.wait_for(lock, chrono::seconds(5)); // releases the lock temporarily
bts_assert_msg(result == cv_status::no_timeout,
"Dispatcher failed to start.");
auto result = c.wait_for(
lock, boost::asio::chrono::seconds(5)); // releases the lock temporarily
bts_assert_msg(
result == cv_status::no_timeout, "Dispatcher failed to start.");
bts_assert(new_dispatcher); // Could happen to to spurious wake
return *new_dispatcher;
}
@@ -227,8 +227,8 @@ struct DispatcherTimerImpl {
void schedule() {
if (running) {
timer->expires_from_now(boost::posix_time::milliseconds(delay));
timer->async_wait(std::bind(&DispatcherTimerImpl::execute, this,
std::placeholders::_1));
timer->async_wait(std::bind(
&DispatcherTimerImpl::execute, this, std::placeholders::_1));
}
}
@@ -245,8 +245,8 @@ struct DispatcherTimerImpl {
unique_ptr<deadline_timer> timer;
};
DispatcherTimer::DispatcherTimer(Dispatcher &dispatcher, int64_t time_ms,
Action cb)
DispatcherTimer::DispatcherTimer(
Dispatcher &dispatcher, int64_t time_ms, Action cb)
: _me(new DispatcherTimerImpl(dispatcher, time_ms, cb)) {}
DispatcherTimer::DispatcherTimer(DispatcherTimer &&other)
+53 -56
View File
@@ -1,59 +1,59 @@
#ifndef NO_CPP0X
#pragma once
#include "lib/utils/bts_no_copy.h"
#include "lib/utils/bts_export.h"
#include "lib/utils/bts_no_copy.h"
#include <boost/asio/io_service.hpp>
#include <cstdint>
#include <functional>
#include <memory>
#include <cstdint>
/*
* Dispatcher is a simple delegate based event queue built on boost::asio.
*
* Each dispatcher is owned by a single thread which is responsible for calling run() to dispatch events.
* The run call will not return until terminate() is called. It is safe to call terminate() from the same thread
*
* Each dispatcher is owned by a single thread which is responsible for calling
* run() to dispatch events. The run call will not return until terminate() is
* called. It is safe to call terminate() from the same thread
*
* Sample use:
* Create a thread and get Dispatcher::current() for that thread. Pass that Dispatcher& to some other code.
* If that other code makes calls through the dispatcher post() (asyncronous) or invoke() (syncronous) the callback
* is guarunteed to happen on the Dispatcher's thread.
*
* Create a thread and get Dispatcher::current() for that thread. Pass that
* Dispatcher& to some other code. If that other code makes calls through the
* dispatcher post() (asyncronous) or invoke() (syncronous) the callback is
* guarunteed to happen on the Dispatcher's thread.
*
*
* Some use cases:
* 1) Short asyncronous work
* You can use Dispatcher::background() to get a handle to a common background thread (created on demand).
* This is useful for short tasks like asyncronous logging or timer events where you do not need to monopolize a thread
* of your own. You can also use post() with a delay to set up one time callbacks after a short time.
* Ex: Simple async logging
* #define ALOG(x) Dispatcher::background().post([=]() { LOG(Info, Something) << x; });
* You can use Dispatcher::background() to get a handle to a common background
* thread (created on demand). This is useful for short tasks like asyncronous
* logging or timer events where you do not need to monopolize a thread of your
* own. You can also use post() with a delay to set up one time callbacks after
* a short time. Ex: Simple async logging #define ALOG(x)
* Dispatcher::background().post([=]() { LOG(Info, Something) << x; });
*
* 2) Use a DispatcherTimer to drive a WorkScheduler without tying up a thread during the idle times.
* Ex:
* Dispatcher& d = Dispatcher::background();
* 2) Use a DispatcherTimer to drive a WorkScheduler without tying up a thread
* during the idle times. Ex: Dispatcher& d = Dispatcher::background();
* WorkScheduler* s = new WorkScheduler(...);
* DispatcherTimer t(d, 100, [=]() { s->dispatch(); });
* t.start();
*
* Now the WorkScheduler will be invoked every 100ms but you are still free to use
* d.post([](){ ... }); to do asyncronous work between timer ticks. If your async task takes longer
* than the timer interval the timer callback will fire immediately when it is done. Simlilarly if
* the timer callback is still working when you call post() your call will be queued and fire when the
* timer routine finishes.
* Now the WorkScheduler will be invoked every 100ms but you are still free to
* use d.post([](){ ... }); to do asyncronous work between timer ticks. If your
* async task takes longer than the timer interval the timer callback will fire
* immediately when it is done. Simlilarly if the timer callback is still
* working when you call post() your call will be queued and fire when the timer
* routine finishes.
*
* 3) Controlling access to non-thread safe resources.
* If you have an object which is not thread safe you can give it a Dispatcher "owner".
* A thread can test object.owner().check_access() to see if it is the owning thread and if not
* serialize access by using object.owner().invoke([&](){ object.unsafe_method(); };
* The unsafe_method might assert that access is only allowed from its owning thread.
*
* If you have an object which is not thread safe you can give it a Dispatcher
* "owner". A thread can test object.owner().check_access() to see if it is the
* owning thread and if not serialize access by using
* object.owner().invoke([&](){ object.unsafe_method(); }; The unsafe_method
* might assert that access is only allowed from its owning thread.
*
*/
namespace boost {
namespace asio {
class io_service;
}}
namespace bts {
namespace utils {
@@ -63,13 +63,12 @@ class DispatcherTimer;
typedef std::function<void()> Action;
class BTS_EXPORT Dispatcher
{
public:
class BTS_EXPORT Dispatcher {
public:
Dispatcher();
~Dispatcher();
void run(size_t threads=1);
void run(size_t threads = 1);
void add_thread();
bool check_access();
void terminate();
@@ -77,48 +76,46 @@ public:
bool is_threadpool();
// Get/create the dispatcher for the current thread
static Dispatcher& current();
static Dispatcher &current();
// Get a default background thread to dispatch work to
static Dispatcher& background();
static Dispatcher &background();
// Create a new background dispatcher thread and return the dispatcher for it
static Dispatcher& get_named(const char* name);
static Dispatcher& create_new(const char* name);
static Dispatcher &get_named(const char *name);
static Dispatcher &create_new(const char *name);
// Perform an action on the dispatcher thread. Block until the action is complete.
// Perform an action on the dispatcher thread. Block until the action is
// complete.
void invoke(Action action);
// Queue an action on the dispatcher thread and return immediately.
void post(Action&& action, int64_t delay_ms=0);
void post_copy(Action action, int64_t delay_ms=0);
void post(Action &&action, int64_t delay_ms = 0);
void post_copy(Action action, int64_t delay_ms = 0);
boost::asio::io_service* io_service();
boost::asio::io_service *io_service();
private:
private:
BTS_NO_COPY(Dispatcher);
friend struct DispatcherTimerImpl;
std::unique_ptr<DispatcherImpl> _me;
};
class BTS_EXPORT DispatcherTimer
{
public:
DispatcherTimer(Dispatcher& dispatcher, int64_t time_ms, Action cb);
DispatcherTimer(DispatcherTimer&& other);
class BTS_EXPORT DispatcherTimer {
public:
DispatcherTimer(Dispatcher &dispatcher, int64_t time_ms, Action cb);
DispatcherTimer(DispatcherTimer &&other);
~DispatcherTimer();
void start();
void stop();
bool running();
void set_interval(int64_t time_ms);
void set_callback(Action cb);
private:
private:
BTS_NO_COPY(DispatcherTimer);
std::unique_ptr<DispatcherTimerImpl> _me;
};
}}//namespace
} // namespace utils
} // namespace bts
#endif
+1
View File
@@ -2,6 +2,7 @@
#include "lib/os/os.h"
#include <sstream>
#include <string>
#include <boost/filesystem.hpp>
#include <stdlib.h>
+265 -267
View File
@@ -1,17 +1,16 @@
#include "datetime.h"
#include "bts_assert.h"
#include "TimeStamp.h"
#include "bts_assert.h"
#include <sstream>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <sstream>
namespace bts {
namespace datetime {
const boost::gregorian::date anchor (1970, boost::gregorian::Jan, 1);
const boost::gregorian::date anchor(1970, boost::gregorian::Jan, 1);
const uint64_t anchor_ticks = 621355968000000000;
const uint64_t micros_per_day = 86400000000;
const uint64_t ticks_per_micro = 10;
@@ -25,304 +24,283 @@ const uint64_t KindLocal = 0x8000000000000000;
unsigned long long ptime_to_ll(PTime const &t, uint64_t const &kind) {
uint64_t days = (t.date() - anchor).days();
uint64_t usecs = t.time_of_day().total_microseconds();
uint64_t ticks = anchor_ticks + usecs * ticks_per_micro + days * ticks_per_day;
return (ticks & ticks_mask) | kind;
uint64_t ticks =
anchor_ticks + usecs * ticks_per_micro + days * ticks_per_day;
return (ticks & ticks_mask) | kind;
}
unsigned long long ptime_to_ll(PTime const &t) {
return ptime_to_ll(t, KindUnspecified);
return ptime_to_ll(t, KindUnspecified);
}
LLDateTime ll_now() {
return ptime_to_ll(boost::posix_time::microsec_clock::universal_time(), KindUtc);
return ptime_to_ll(
boost::posix_time::microsec_clock::universal_time(), KindUtc);
}
LLDateTime ll_local_now() {
return ptime_to_ll(boost::posix_time::microsec_clock::local_time (), KindLocal);
return ptime_to_ll(
boost::posix_time::microsec_clock::local_time(), KindLocal);
}
long unix_now() {
return (boost::posix_time::microsec_clock::universal_time() - PTime(anchor)).total_microseconds() / (long)1E6;
}
#ifndef _WIN32
uint64_t unix_nano_now() {
struct timespec ts;
// CLOCK_REALTIME is already anchored at the unix epoch
clock_gettime(CLOCK_REALTIME, &ts);
return
static_cast<int64_t>(1000000000UL)*
static_cast<int64_t>(ts.tv_sec) +
static_cast<int64_t>(ts.tv_nsec);
}
#endif
std::chrono::system_clock::time_point utc_ll_to_timepoint(unsigned long long ll) {
auto us_since_epoch = ((ll & ticks_mask) - anchor_ticks) / ticks_per_micro;
return std::chrono::system_clock::time_point (std::chrono::microseconds(us_since_epoch));
return (boost::posix_time::microsec_clock::universal_time() - PTime(anchor))
.total_microseconds() /
(long)1E6;
}
long long ll_diff_in_us (unsigned long long ll1, unsigned long long ll2)
{
std::chrono::system_clock::time_point utc_ll_to_timepoint(
unsigned long long ll) {
auto us_since_epoch = ((ll & ticks_mask) - anchor_ticks) / ticks_per_micro;
return std::chrono::system_clock::time_point(
std::chrono::microseconds(us_since_epoch));
}
long long ll_diff_in_us(unsigned long long ll1, unsigned long long ll2) {
#ifndef NO_CPP0X
auto p1 = ll_to_ptime (ll1);
auto p2 = ll_to_ptime (ll2); // NOLINT
return (p2 - p1).total_microseconds ();
auto p1 = ll_to_ptime(ll1);
auto p2 = ll_to_ptime(ll2); // NOLINT
return (p2 - p1).total_microseconds();
#else
return 0;
#endif
}
long long ll_diff_in_seconds(unsigned long long ll1, unsigned long long ll2)
{
long long ll_diff_in_seconds(unsigned long long ll1, unsigned long long ll2) {
#ifndef NO_CPP0X
auto p1 = ll_to_ptime (ll1);
auto p2 = ll_to_ptime (ll2); // NOLINT
auto p1 = ll_to_ptime(ll1);
auto p2 = ll_to_ptime(ll2); // NOLINT
return (p2 - p1).total_seconds();
#else
return 0;
#endif
}
long long ptime_diff_in_us (PTime const &t1, PTime const &t2)
{
return (t1 - t2).total_microseconds ();
long long ptime_diff_in_us(PTime const &t1, PTime const &t2) {
return (t1 - t2).total_microseconds();
}
long long ptime_diff_in_ms (PTime const &t1, PTime const &t2)
{
return (t1 - t2).total_milliseconds ();
long long ptime_diff_in_ms(PTime const &t1, PTime const &t2) {
return (t1 - t2).total_milliseconds();
}
long long ptime_diff_in_secs (PTime const &t1, PTime const &t2)
{
return (t1 - t2).total_seconds ();
long long ptime_diff_in_secs(PTime const &t1, PTime const &t2) {
return (t1 - t2).total_seconds();
}
PTime timestamp_to_ptime(utils::TimeStamp const&t)
{
PTime timestamp_to_ptime(utils::TimeStamp const &t) {
auto now_ts = utils::TimeStamp::now();
auto now = datetime::local_now();
auto delta = (now_ts - t).milliseconds();
return now - boost::posix_time::milliseconds(delta);
return now - boost::posix_time::milliseconds(delta);
}
utils::TimeStamp ptime_to_timestamp(PTime const &t)
{
utils::TimeStamp ptime_to_timestamp(PTime const &t) {
auto now_ts = utils::TimeStamp::now();
auto now = datetime::now();
auto delta = (now - t).total_milliseconds();
return now_ts.shift_ms (delta);
return now_ts.shift_ms(delta);
}
long long usecs_since (PTime const &t1)
{
return (boost::posix_time::microsec_clock::universal_time () - t1).total_microseconds ();
long long usecs_since(PTime const &t1) {
return (boost::posix_time::microsec_clock::universal_time() - t1)
.total_microseconds();
}
long long msecs_since (PTime const &t1)
{
return (boost::posix_time::microsec_clock::universal_time () - t1).total_milliseconds ();
long long msecs_since(PTime const &t1) {
return (boost::posix_time::microsec_clock::universal_time() - t1)
.total_milliseconds();
}
PTime now ()
{
return boost::posix_time::microsec_clock::universal_time ();
}
PTime now() { return boost::posix_time::microsec_clock::universal_time(); }
PTime local_now ()
{
return boost::posix_time::microsec_clock::local_time ();
}
PTime local_now() { return boost::posix_time::microsec_clock::local_time(); }
PTime make_ptime (int year, int month, int day)
{
PTime make_ptime(int year, int month, int day) {
// January=1
bts_assert_d(month > 0);
boost::gregorian::date dt (year, boost::gregorian::greg_month (month),day);
return PTime (dt);
boost::gregorian::date dt(year, boost::gregorian::greg_month(month), day);
return PTime(dt);
}
PTime make_ptime (int year, int month, int day, int hour, int minute, int second, int millisecond)
{
boost::gregorian::date dt (year, boost::gregorian::greg_month (month),day);
return PTime (dt, boost::posix_time::time_duration(hour, minute, second) + boost::posix_time::milliseconds(millisecond));
PTime make_ptime(
int year,
int month,
int day,
int hour,
int minute,
int second,
int millisecond) {
boost::gregorian::date dt(year, boost::gregorian::greg_month(month), day);
return PTime(
dt, boost::posix_time::time_duration(hour, minute, second) +
boost::posix_time::milliseconds(millisecond));
}
PTime ll_to_ptime (LLDateTime ll) {
ll &= ticks_mask;
uint64_t ticks_part = ll % ticks_per_day;
uint64_t days_part = (ll - anchor_ticks) / ticks_per_day;
return PTime(anchor + boost::gregorian::date_duration(days_part),
boost::posix_time::time_duration(boost::posix_time::microseconds(ticks_part / ticks_per_micro)));
PTime ll_to_ptime(LLDateTime ll) {
ll &= ticks_mask;
uint64_t ticks_part = ll % ticks_per_day;
uint64_t days_part = (ll - anchor_ticks) / ticks_per_day;
return PTime(
anchor + boost::gregorian::date_duration(days_part),
boost::posix_time::time_duration(
boost::posix_time::microseconds(ticks_part / ticks_per_micro)));
}
extern "C" BTS_EXPORT int ll_to_cstr (LLDateTime ll, char*buffer, int buffer_sz)
{
const int str_sz = sizeof ("20121204-14:44:02.000");
bts_assert (buffer_sz >= str_sz);
auto str = FIX_timestamp (ll_to_ptime (ll));
bts_assert (str.size () == str_sz-1);
memcpy (buffer,&str[0],str.size ());
buffer[str.size ()] = '\0';
return str.size ();
extern "C" BTS_EXPORT int ll_to_cstr(
LLDateTime ll, char *buffer, int buffer_sz) {
const int str_sz = sizeof("20121204-14:44:02.000");
bts_assert(buffer_sz >= str_sz);
auto str = FIX_timestamp(ll_to_ptime(ll));
bts_assert(str.size() == str_sz - 1);
memcpy(buffer, &str[0], str.size());
buffer[str.size()] = '\0';
return str.size();
}
std::string time_string (PTime const &t)
{
int hours = t.time_of_day ().hours ();
int minutes = t.time_of_day ().minutes ();
int seconds = t.time_of_day ().seconds ();
std::string time_string(PTime const &t) {
int hours = t.time_of_day().hours();
int minutes = t.time_of_day().minutes();
int seconds = t.time_of_day().seconds();
int whole_ms = seconds * 1000 + minutes * 1000 * 60 + hours * 1000 * 60 * 60;
int frac_ms = t.time_of_day ().total_milliseconds () - whole_ms;
int frac_ms = t.time_of_day().total_milliseconds() - whole_ms;
std::ostringstream out;
out << std::setw( 2 ) << std::setfill( '0' ) << hours << ":"
<< std::setw( 2 ) << std::setfill( '0' ) << minutes << ":"
<< std::setw( 2 ) << std::setfill( '0' ) << seconds << "."
<< std::setw( 3 ) << std::setfill( '0' ) << frac_ms;
return out.str ();
out << std::setw(2) << std::setfill('0') << hours << ":" << std::setw(2)
<< std::setfill('0') << minutes << ":" << std::setw(2)
<< std::setfill('0') << seconds << "." << std::setw(3)
<< std::setfill('0') << frac_ms;
return out.str();
}
std::string date_string (PTime const &t)
{
std::string result (10,' ');
sprintf (&result[0], "%02d/%02d/%04d",
t.date ().month ().as_number (),
t.date ().day ().as_number (),
(unsigned short) t.date ().year ());
std::string date_string(PTime const &t) {
std::string result(10, ' ');
sprintf(
&result[0], "%02d/%02d/%04d", t.date().month().as_number(),
t.date().day().as_number(), (unsigned short)t.date().year());
return result;
}
std::string iso_date_time_string (PTime const &t)
{
std::string iso_date_time_string(PTime const &t) {
return boost::posix_time::to_iso_extended_string(t);
}
PTime parse_iso_date_time_string (std::string const &t)
{
if(t.find('T') == t.npos)
PTime parse_iso_date_time_string(std::string const &t) {
if (t.find('T') == t.npos)
return boost::posix_time::from_iso_extended_string(t + "T00:00:00");
else
return boost::posix_time::from_iso_extended_string(t);
}
std::string simple_date_time_string (PTime const &t)
{
std::string simple_date_time_string(PTime const &t) {
return boost::posix_time::to_simple_string(t);
}
PTime parse_simple_date_time_string (std::string const &t)
{
if(t.find(' ') == t.npos)
PTime parse_simple_date_time_string(std::string const &t) {
if (t.find(' ') == t.npos)
return boost::posix_time::time_from_string(t + " 00:00:00");
else
return boost::posix_time::time_from_string(t);
}
std::string FIX_date_time_string (PTime const &t)
{
std::string result (strlen ("20121204-14:44:02"),' ');
sprintf (&result[0],"%d%02d%02d-%02d:%02d:%02d",
(unsigned short) t.date ().year (),
t.date ().month ().as_number (),
t.date ().day ().as_number (),
t.time_of_day ().hours (),
t.time_of_day ().minutes (),
t.time_of_day ().seconds ());
PDuration parse_duration(std::string const &time_str) {
try {
return boost::posix_time::duration_from_string(time_str);
} catch (boost::bad_lexical_cast &) {
bts_assert(false);
return boost::posix_time::time_duration();
}
}
std::string FIX_date_time_string(PTime const &t) {
std::string result(strlen("20121204-14:44:02"), ' ');
sprintf(
&result[0], "%d%02d%02d-%02d:%02d:%02d", (unsigned short)t.date().year(),
t.date().month().as_number(), t.date().day().as_number(),
(int)t.time_of_day().hours(), (int)t.time_of_day().minutes(),
(int)t.time_of_day().seconds());
return result;
}
void FIX_timestamp (PTime const &t, std::string *out)
{
int hours = t.time_of_day ().hours ();
int minutes = t.time_of_day ().minutes ();
int seconds = t.time_of_day ().seconds ();
void FIX_timestamp(PTime const &t, std::string *out) {
int hours = t.time_of_day().hours();
int minutes = t.time_of_day().minutes();
int seconds = t.time_of_day().seconds();
int whole_ms = seconds * 1000 + minutes * 1000 * 60 + hours * 1000 * 60 * 60;
int frac_ms = t.time_of_day ().total_milliseconds () - whole_ms;
out->resize(sizeof ("20121204-14:44:02.000"));
int frac_ms = t.time_of_day().total_milliseconds() - whole_ms;
out->resize(sizeof("20121204-14:44:02.000"));
#ifdef WIN32
_snprintf (&(*out)[0],out->size (),
"%04d%02d%02d-%02d:%02d:%02d.%03d",
(int) t.date ().year (),
t.date ().month ().as_number (),
t.date ().day ().as_number (),
hours,
minutes,
seconds,
frac_ms);
_snprintf(
&(*out)[0], out->size(), "%04d%02d%02d-%02d:%02d:%02d.%03d",
(int)t.date().year(), t.date().month().as_number(),
t.date().day().as_number(), hours, minutes, seconds, frac_ms);
#else
snprintf (&(*out)[0],out->size (),
"%04d%02d%02d-%02d:%02d:%02d.%03d",
(int) t.date ().year (),
t.date ().month ().as_number (),
t.date ().day ().as_number (),
hours,
minutes,
seconds,
frac_ms);
snprintf(
&(*out)[0], out->size(), "%04d%02d%02d-%02d:%02d:%02d.%03d",
(int)t.date().year(), t.date().month().as_number(),
t.date().day().as_number(), hours, minutes, seconds, frac_ms);
#endif
out->resize (out->size ()-1);
out->resize(out->size() - 1);
}
std::string FIX_timestamp (PTime const &t)
{
std::string FIX_timestamp(PTime const &t) {
std::string result;
FIX_timestamp(t,&result);
return result;
FIX_timestamp(t, &result);
return result;
}
bool is_from_today(PTime const &time)
{
return (time.date() == boost::posix_time::microsec_clock::universal_time().date());
bool is_from_today(PTime const &time) {
return (
time.date() ==
boost::posix_time::microsec_clock::universal_time().date());
}
bool is_at_least_msecs_old (PTime const &time, uint64_t ms)
{
PTime now = boost::posix_time::microsec_clock::universal_time ();
return (now - time).total_milliseconds () >= (int64_t) ms;
bool is_at_least_msecs_old(PTime const &time, uint64_t ms) {
PTime now = boost::posix_time::microsec_clock::universal_time();
return (now - time).total_milliseconds() >= (int64_t)ms;
}
bool is_at_least_secs_old (PTime const &time, uint64_t sec)
{
PTime now = boost::posix_time::microsec_clock::universal_time ();
return (now - time).total_seconds () >= (int64_t) sec;
bool is_at_least_secs_old(PTime const &time, uint64_t sec) {
PTime now = boost::posix_time::microsec_clock::universal_time();
return (now - time).total_seconds() >= (int64_t)sec;
}
bool is_at_least_mins_old (PTime const &time, uint64_t mins)
{
PTime now = boost::posix_time::microsec_clock::universal_time ();
return ((now - time).total_seconds ()/60) >= (int64_t) mins;
bool is_at_least_mins_old(PTime const &time, uint64_t mins) {
PTime now = boost::posix_time::microsec_clock::universal_time();
return ((now - time).total_seconds() / 60) >= (int64_t)mins;
}
BTS_EXPORT bool is_at_least_hours_old (PTime const &time, uint64_t hours)
{
PTime now = boost::posix_time::microsec_clock::universal_time ();
int64_t secs = (int64_t) ((now - time).total_seconds ());
return (secs/60/60) >= (int64_t) hours;
BTS_EXPORT bool is_at_least_hours_old(PTime const &time, uint64_t hours) {
PTime now = boost::posix_time::microsec_clock::universal_time();
int64_t secs = (int64_t)((now - time).total_seconds());
return (secs / 60 / 60) >= (int64_t)hours;
}
BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days)
{
return is_at_least_hours_old (time, days *24);
BTS_EXPORT bool is_at_least_days_old(PTime const &time, uint64_t days) {
return is_at_least_hours_old(time, days * 24);
}
BTS_EXPORT LLDateTime ll_from_unix_epoch(long seconds)
{
BTS_EXPORT LLDateTime ll_from_unix_epoch(long seconds) {
PTime result(anchor, boost::posix_time::seconds(seconds));
return ptime_to_ll(result, KindUnspecified);
}
BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long milliseconds)
{
BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long milliseconds) {
PTime result(anchor, boost::posix_time::millisec(milliseconds));
return ptime_to_ll(result, KindUnspecified);
}
BTS_EXPORT void ll_to_parts(LLDateTime ll_time, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond)
{
BTS_EXPORT void ll_to_parts(
LLDateTime ll_time,
int &year,
int &month,
int &day,
int &hour,
int &minute,
int &second,
int &millisecond) {
PTime t = ll_to_ptime(ll_time);
year = t.date().year();
month = t.date().month();
@@ -334,12 +312,19 @@ BTS_EXPORT void ll_to_parts(LLDateTime ll_time, int& year, int& month, int& day,
millisecond = t.time_of_day().total_milliseconds() - whole_ms;
}
BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond)
{
return ptime_to_ll(make_ptime(year, month, day, hour, minute, second, millisecond)); // NOLINT
BTS_EXPORT LLDateTime ll_from_parts(
int year,
int month,
int day,
int hour,
int minute,
int second,
int millisecond) {
return ptime_to_ll(make_ptime(
year, month, day, hour, minute, second, millisecond)); // NOLINT
}
BTS_EXPORT LLDateTime ll_from_date(const Date& date) {
BTS_EXPORT LLDateTime ll_from_date(const Date &date) {
if (date.year == 0 && date.day == 0 && date.month == 0) {
return 0;
}
@@ -356,7 +341,6 @@ Date ll_to_date(LLDateTime time) {
return date;
}
bool is_local_time_sunday() {
auto now = local_now();
return now.date().day_of_week() == boost::date_time::Sunday;
@@ -369,7 +353,7 @@ bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end) {
return (now >= start && now <= end);
}
struct UtcTimeStamp {
struct UtcTimeStamp {
int hour;
int min;
int sec;
@@ -379,102 +363,114 @@ struct UtcTimeStamp {
int year;
};
struct UtcTimeStampConvertor
{
static UtcTimeStamp convert( const std::string& value)
{
bool haveMilliseconds = false;
struct UtcTimeStampConvertor {
static UtcTimeStamp convert(const std::string &value) {
bool haveMilliseconds = false;
switch( value.size() )
{
case 21: haveMilliseconds = true;
case 17: break;
default: bts_assert(false);
}
switch (value.size()) {
case 21:
haveMilliseconds = true;
case 17:
break;
default:
bts_assert(false);
}
int i = 0;
int c = 0;
for( c = 0; c < 8; ++c )
if( !isdigit(value[i++]) ) bts_assert(false);
if (value[i++] != '-') bts_assert(false);
for( c = 0; c < 2; ++c )
if( !isdigit(value[i++]) ) bts_assert(false);
if( value[i++] != ':' ) bts_assert(false);
for( c = 0; c < 2; ++c )
if( !isdigit(value[i++]) ) bts_assert(false);
if( value[i++] != ':' ) bts_assert(false);
for( c = 0; c < 2; ++c )
if( !isdigit(value[i++]) ) bts_assert(false);
int i = 0;
int c = 0;
for (c = 0; c < 8; ++c)
if (!isdigit(value[i++]))
bts_assert(false);
if (value[i++] != '-')
bts_assert(false);
for (c = 0; c < 2; ++c)
if (!isdigit(value[i++]))
bts_assert(false);
if (value[i++] != ':')
bts_assert(false);
for (c = 0; c < 2; ++c)
if (!isdigit(value[i++]))
bts_assert(false);
if (value[i++] != ':')
bts_assert(false);
for (c = 0; c < 2; ++c)
if (!isdigit(value[i++]))
bts_assert(false);
if( haveMilliseconds )
{
if( value[i++] != '.' ) bts_assert(false);
for( c = 0; c < 3; ++c )
if( !isdigit(value[i++]) ) bts_assert(false);
}
if (haveMilliseconds) {
if (value[i++] != '.')
bts_assert(false);
for (c = 0; c < 3; ++c)
if (!isdigit(value[i++]))
bts_assert(false);
}
int year, mon, mday, hour, min, sec, millis;
int year, mon, mday, hour, min, sec, millis;
i = 0;
i = 0;
year = value[i++] - '0';
year = 10 * year + value[i++] - '0';
year = 10 * year + value[i++] - '0';
year = 10 * year + value[i++] - '0';
year = value[i++] - '0';
year = 10 * year + value[i++] - '0';
year = 10 * year + value[i++] - '0';
year = 10 * year + value[i++] - '0';
mon = value[i++] - '0';
mon = 10 * mon + value[i++] - '0';
if( mon < 1 || 12 < mon ) bts_assert(false);
mon = value[i++] - '0';
mon = 10 * mon + value[i++] - '0';
if (mon < 1 || 12 < mon)
bts_assert(false);
mday = value[i++] - '0';
mday = 10 * mday + value[i++] - '0';
if( mday < 1 || 31 < mday ) bts_assert(false);
mday = value[i++] - '0';
mday = 10 * mday + value[i++] - '0';
if (mday < 1 || 31 < mday)
bts_assert(false);
++i; // skip '-'
++i; // skip '-'
hour = value[i++] - '0';
hour = 10 * hour + value[i++] - '0';
// No check for >= 0 as no '-' are converted here
if( 23 < hour ) bts_assert(false);
hour = value[i++] - '0';
hour = 10 * hour + value[i++] - '0';
// No check for >= 0 as no '-' are converted here
if (23 < hour)
bts_assert(false);
++i; // skip ':'
++i; // skip ':'
min = value[i++] - '0';
min = 10 * min + value[i++] - '0';
// No check for >= 0 as no '-' are converted here
if( 59 < min ) bts_assert(false);
min = value[i++] - '0';
min = 10 * min + value[i++] - '0';
// No check for >= 0 as no '-' are converted here
if (59 < min)
bts_assert(false);
++i; // skip ':'
++i; // skip ':'
sec = value[i++] - '0';
sec = 10 * sec + value[i++] - '0';
sec = value[i++] - '0';
sec = 10 * sec + value[i++] - '0';
// No check for >= 0 as no '-' are converted here
if( 60 < sec ) bts_assert(false);
// No check for >= 0 as no '-' are converted here
if (60 < sec)
bts_assert(false);
if( haveMilliseconds )
{
millis = (100 * (value[i+1] - '0')
+ 10 * (value[i+2] - '0')
+ (value[i+3] - '0'));
}
else
millis = 0;
if (haveMilliseconds) {
millis =
(100 * (value[i + 1] - '0') + 10 * (value[i + 2] - '0') +
(value[i + 3] - '0'));
} else
millis = 0;
return UtcTimeStamp {hour, min, sec, millis, mday, mon, year};
}
return UtcTimeStamp{hour, min, sec, millis, mday, mon, year};
}
};
PTime parse_FIX_timestamp (const std::string &ts_str)
{
auto ts = UtcTimeStampConvertor::convert (ts_str);
PTime parse_FIX_timestamp(const std::string &ts_str) {
auto ts = UtcTimeStampConvertor::convert(ts_str);
boost::gregorian::date dt(ts.year, ts.month, ts.day);
// fucking boost. unbelieavable.
auto ticks_per_second = boost::posix_time::time_duration::rep_type::res_adjust();
boost::posix_time::time_duration td (ts.hour, ts.min, ts.sec, ts.msec * (ticks_per_second / 1000));
auto ticks_per_second =
boost::posix_time::time_duration::rep_type::res_adjust();
boost::posix_time::time_duration td(
ts.hour, ts.min, ts.sec, ts.msec * (ticks_per_second / 1000));
return PTime(dt, td);
}
#ifndef NO_CPP0X
extern "C" BTS_EXPORT LLDateTime ll_today() {
return datetime::ll_from_date(today());
@@ -497,7 +493,9 @@ BTS_EXPORT Date add_days(Date indate, int days) {
extern "C" BTS_EXPORT LLDateTime ll_yesterday() {
auto now = datetime::now() - boost::gregorian::days(1);
return datetime::ll_from_parts(now.date().year(), now.date().month(), now.date().day(), 0, 0, 0, 0);
return datetime::ll_from_parts(
now.date().year(), now.date().month(), now.date().day(), 0, 0, 0, 0);
}
#endif
}}
} // namespace datetime
} // namespace bts
+116 -58
View File
@@ -1,90 +1,147 @@
#pragma once
#include "bts_export.h"
#include <string>
#include <chrono>
#include <string>
#include "windows_wrapper.h" // tick_count.h includes windows.h
namespace boost {namespace posix_time {class ptime;}}
namespace bts {namespace utils {class TimeStamp;}}
namespace boost {
namespace posix_time {
class ptime;
}
} // namespace boost
namespace boost {
namespace posix_time {
class time_duration;
}
} // namespace boost
namespace bts {
namespace utils {
class TimeStamp;
}
} // namespace bts
namespace bts {
namespace datetime {
typedef boost::posix_time::ptime PTime;
typedef boost::posix_time::time_duration PDuration;
typedef unsigned long long LLDateTime;
// Simple Date structure
struct Date {
struct Date {
Date() : year(0), month(0), day(0) {}
Date(uint16_t y, uint8_t m, uint8_t d) : year(y), month(m), day(d) {}
Date(uint16_t date) : year((date >> 9) + 1970), month(date >> 5 & 15), day(date&31) {}
Date(uint16_t date)
: year((date >> 9) + 1970), month(date >> 5 & 15), day(date & 31) {}
uint16_t year;
uint8_t month;
uint8_t day;
bool operator<(const Date& other) const {
if (year < other.year) return true;
if (year == other.year && month < other.month) return true;
if (year == other.year && month == other.month && day < other.day) return true;
bool operator<(const Date &other) const {
if (year < other.year)
return true;
if (year == other.year && month < other.month)
return true;
if (year == other.year && month == other.month && day < other.day)
return true;
return false;
}
bool operator>(const Date& other) const { return !(*this == other) && !(*this < other); }
bool operator==(const Date& other) const {
bool operator>(const Date &other) const {
return !(*this == other) && !(*this < other);
}
bool operator==(const Date &other) const {
return day == other.day && month == other.month && year == other.year;
}
bool operator!=(const Date& other) const { return !(*this == other); }
uint16_t to_short() { return ((year - 1970) << 9) + (month << 5) + day; } // NOLINT
bool operator!=(const Date &other) const { return !(*this == other); }
uint16_t to_short() {
return ((year - 1970) << 9) + (month << 5) + day;
} // NOLINT
};
extern "C" BTS_EXPORT LLDateTime ll_now ();
extern "C" BTS_EXPORT LLDateTime ll_local_now ();
extern "C" BTS_EXPORT double OA_date_from_ll (LLDateTime ll);
extern "C" BTS_EXPORT LLDateTime ll_from_OA_date (double oa_date);
extern "C" BTS_EXPORT LLDateTime ll_now();
extern "C" BTS_EXPORT LLDateTime ll_local_now();
extern "C" BTS_EXPORT double OA_date_from_ll(LLDateTime ll);
extern "C" BTS_EXPORT LLDateTime ll_from_OA_date(double oa_date);
extern "C" BTS_EXPORT long unix_now();
uint64_t unix_nano_now();
BTS_EXPORT std::chrono::system_clock::time_point utc_ll_to_timepoint(unsigned long long ll) ;
BTS_EXPORT PTime now ();
BTS_EXPORT PTime local_now ();
BTS_EXPORT PTime make_ptime (int year, int month, int day);
BTS_EXPORT PTime make_ptime (int year, int month, int day, int hour, int minute, int second, int millisecond);
BTS_EXPORT LLDateTime ptime_to_ll (PTime const &t);
BTS_EXPORT PTime ll_to_ptime (LLDateTime ll);
extern "C" BTS_EXPORT int ll_to_cstr (LLDateTime ll, char*buffer, int buffer_sz);
BTS_EXPORT long long ll_diff_in_us (unsigned long long start, unsigned long long stop);
BTS_EXPORT long long ll_diff_in_seconds(unsigned long long start, unsigned long long stop);
BTS_EXPORT long long ptime_diff_in_us (PTime const &t1, PTime const &t2);
BTS_EXPORT long long ptime_diff_in_ms (PTime const &t1, PTime const &t2);
BTS_EXPORT long long ptime_diff_in_secs (PTime const &t1, PTime const &t2);
BTS_EXPORT PTime timestamp_to_ptime(utils::TimeStamp const&t);
#ifndef _WIN32
inline uint64_t unix_nano_now() {
struct timespec ts;
// CLOCK_REALTIME is already anchored at the unix epoch
clock_gettime(CLOCK_REALTIME, &ts);
return static_cast<int64_t>(1000000000UL) * static_cast<int64_t>(ts.tv_sec) +
static_cast<int64_t>(ts.tv_nsec);
}
#endif
BTS_EXPORT std::chrono::system_clock::time_point
utc_ll_to_timepoint(unsigned long long ll);
BTS_EXPORT PTime now();
BTS_EXPORT PTime local_now();
BTS_EXPORT PTime make_ptime(int year, int month, int day);
BTS_EXPORT PTime make_ptime(
int year,
int month,
int day,
int hour,
int minute,
int second,
int millisecond);
BTS_EXPORT LLDateTime ptime_to_ll(PTime const &t);
BTS_EXPORT PTime ll_to_ptime(LLDateTime ll);
extern "C" BTS_EXPORT int
ll_to_cstr(LLDateTime ll, char *buffer, int buffer_sz);
BTS_EXPORT long long
ll_diff_in_us(unsigned long long start, unsigned long long stop);
BTS_EXPORT long long
ll_diff_in_seconds(unsigned long long start, unsigned long long stop);
BTS_EXPORT long long ptime_diff_in_us(PTime const &t1, PTime const &t2);
BTS_EXPORT long long ptime_diff_in_ms(PTime const &t1, PTime const &t2);
BTS_EXPORT long long ptime_diff_in_secs(PTime const &t1, PTime const &t2);
BTS_EXPORT PTime timestamp_to_ptime(utils::TimeStamp const &t);
utils::TimeStamp ptime_to_timestamp(PTime const &t);
BTS_EXPORT long long usecs_since (PTime const &t1);
BTS_EXPORT long long msecs_since (PTime const &t1);
BTS_EXPORT std::string time_string (PTime const &t);
BTS_EXPORT std::string date_string (PTime const &t);
BTS_EXPORT std::string iso_date_time_string (PTime const &t);
BTS_EXPORT PTime parse_iso_date_time_string (std::string const &t);
BTS_EXPORT std::string simple_date_time_string (PTime const &t);
BTS_EXPORT PTime parse_simple_date_time_string (std::string const &t);
BTS_EXPORT std::string FIX_date_time_string (PTime const &t);
BTS_EXPORT PTime parse_FIX_timestamp (const std::string &ts);
BTS_EXPORT std::string FIX_timestamp (PTime const &t);
BTS_EXPORT void FIX_timestamp (PTime const &t,std::string *out);
BTS_EXPORT long long usecs_since(PTime const &t1);
BTS_EXPORT long long msecs_since(PTime const &t1);
BTS_EXPORT std::string time_string(PTime const &t);
BTS_EXPORT std::string date_string(PTime const &t);
BTS_EXPORT std::string iso_date_time_string(PTime const &t);
BTS_EXPORT PTime parse_iso_date_time_string(std::string const &t);
BTS_EXPORT std::string simple_date_time_string(PTime const &t);
BTS_EXPORT PTime parse_simple_date_time_string(std::string const &t);
BTS_EXPORT PDuration parse_duration(std::string const &time_str);
BTS_EXPORT std::string FIX_date_time_string(PTime const &t);
BTS_EXPORT PTime parse_FIX_timestamp(const std::string &ts);
BTS_EXPORT std::string FIX_timestamp(PTime const &t);
BTS_EXPORT void FIX_timestamp(PTime const &t, std::string *out);
BTS_EXPORT bool is_from_today(PTime const &time);
BTS_EXPORT bool is_at_least_msecs_old (PTime const &time, uint64_t ms);
BTS_EXPORT bool is_at_least_secs_old (PTime const &time, uint64_t sec);
BTS_EXPORT bool is_at_least_mins_old (PTime const &time, uint64_t min);
BTS_EXPORT bool is_at_least_hours_old (PTime const &time, uint64_t hour);
BTS_EXPORT bool is_at_least_hours_old (PTime const &time, uint64_t hour);
BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days);
BTS_EXPORT bool is_at_least_msecs_old(PTime const &time, uint64_t ms);
BTS_EXPORT bool is_at_least_secs_old(PTime const &time, uint64_t sec);
BTS_EXPORT bool is_at_least_mins_old(PTime const &time, uint64_t min);
BTS_EXPORT bool is_at_least_hours_old(PTime const &time, uint64_t hour);
BTS_EXPORT bool is_at_least_hours_old(PTime const &time, uint64_t hour);
BTS_EXPORT bool is_at_least_days_old(PTime const &time, uint64_t days);
BTS_EXPORT LLDateTime ll_from_unix_epoch(long);
BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long);
extern "C" BTS_EXPORT void ll_to_parts(LLDateTime, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond);
extern "C" BTS_EXPORT void ll_to_parts(
LLDateTime,
int &year,
int &month,
int &day,
int &hour,
int &minute,
int &second,
int &millisecond);
BTS_EXPORT Date ll_to_date(LLDateTime);
extern "C" BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond);
BTS_EXPORT LLDateTime ll_from_date(const Date& date);
extern "C" BTS_EXPORT LLDateTime ll_from_parts(
int year,
int month,
int day,
int hour,
int minute,
int second,
int millisecond);
BTS_EXPORT LLDateTime ll_from_date(const Date &date);
bool is_local_time_sunday();
bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end);
@@ -96,13 +153,14 @@ BTS_EXPORT Date add_days(Date date, int days);
extern "C" BTS_EXPORT LLDateTime ll_yesterday();
#endif
}}
} // namespace datetime
} // namespace bts
// namespace std {
// template <>
// struct numeric_limits<bts::datetime::Date> {
// static bts::datetime::Date min() { return bts::datetime::Date(1970,1,1); }
// static bts::datetime::Date max() { return bts::datetime::Date(2097,12,31); }
// static bts::datetime::Date min() { return
// bts::datetime::Date(1970,1,1); } static bts::datetime::Date max() {
// return bts::datetime::Date(2097,12,31); }
// };
// }
+148 -127
View File
@@ -1,15 +1,14 @@
#include "ApiGateway.h"
#include "lib/logging/logging.h"
#include "lib/api/public/ApiUtils.h"
#include "lib/api/core/ApiClientInterop.h"
#include "lib/api/services/AuthService.client.pb.h"
#include "lib/api/public/ApiUtils.h"
#include "lib/api/services/AdminService.client.pb.h"
#include "lib/api/services/AuthService.client.pb.h"
#include "lib/logging/logging.h"
#include "lib/utils/bts_assert.h"
#include <vector>
#include <unordered_map>
#include <mutex>
#include <unordered_map>
#include <vector>
using namespace bts::api::gateway;
@@ -24,84 +23,90 @@ struct SubscriptionContext {
struct ApiGatewayImpl {
std::mutex _m;
const api::ServiceContext& _sc;
const api::ServiceContext &_sc;
api::user::AuthService _logon;
std::unordered_map<std::string, api::ApiClient*> _apiclient;
std::map<int32_t, SubscriptionContext*> _subscriptions;
std::unordered_map<std::string, api::ApiClient *> _apiclient;
std::map<int32_t, SubscriptionContext *> _subscriptions;
ApiGatewayImpl(const api::ServiceContext &ctx)
: _m(), _sc(ctx), _logon(ctx.locate<api::user::AuthService>()),
_apiclient(), _subscriptions() {}
ApiGatewayImpl(const api::ServiceContext& ctx) :
_m(),
_sc(ctx),
_logon(ctx.locate<api::user::AuthService>()),
_apiclient(),
_subscriptions()
{ }
void handle_login(const GatewayLogin& login, const Responder& respond) {
LOG(Api,Info, "Logon request for {}", login.username());
void handle_login(const GatewayLogin &login, const Responder &respond) {
LOG(Api, Info, "Logon request for {}", login.username());
api::user::AuthenticateLoginRequest alr;
alr.set_login(login.username());
alr.set_password(login.password());
_logon.authenticate_login(alr, [respond](const api::Result<api::user::AuthenticateLoginResponse>& logon_response) {
GatewayResponse resp;
resp.mutable_logon()->set_is_valid(false);
if (logon_response) {
if (logon_response.value()->is_valid()) {
resp.mutable_logon()->set_is_valid(true);
resp.mutable_logon()->mutable_token()->CopyFrom(logon_response.value()->token());
resp.mutable_logon()->set_can_create(logon_response.value()->can_create());
// Do NOT log in _ctx as it is shared across all web clients
}
else {
resp.mutable_logon()->set_reason(logon_response.value()->reason());
}
}
else {
resp.mutable_logon()->set_reason(logon_response.error_message());
}
respond(resp);
});
_logon.authenticate_login(
alr, [respond](const api::Result<api::user::AuthenticateLoginResponse>
&logon_response) {
GatewayResponse resp;
resp.mutable_logon()->set_is_valid(false);
if (logon_response) {
if (logon_response.value()->is_valid()) {
resp.mutable_logon()->set_is_valid(true);
resp.mutable_logon()->mutable_token()->CopyFrom(
logon_response.value()->token());
resp.mutable_logon()->set_can_create(
logon_response.value()->can_create());
// Do NOT log in _ctx as it is shared across all web clients
} else {
resp.mutable_logon()->set_reason(
logon_response.value()->reason());
}
} else {
resp.mutable_logon()->set_reason(
logon_response.error_message());
}
respond(resp);
});
}
void error(const Responder& r, const std::string msg) {
void error(const Responder &r, const std::string msg) {
GatewayResponse response;
response.set_error(msg);
r(response);
}
// Create or return the client proxy
api::ApiClient* get_api_client(const std::string& service_type, const std::string& qualifier, bool create=true) {
api::ApiClient *get_api_client(
const std::string &service_type,
const std::string &qualifier,
bool create = true) {
auto key = service_type + qualifier;
auto iter = _apiclient.find(key);
if (iter == _apiclient.end()) {
if (create) {
auto* pclient = api::api_client_create(&_sc.bus(), service_type.c_str(), qualifier.c_str());
auto *pclient = api::api_client_create(
&_sc.bus(), service_type.c_str(), qualifier.c_str());
_apiclient.insert(std::make_pair(key, pclient));
return pclient;
}
else return nullptr;
}
else {
} else
return nullptr;
} else {
return iter->second;
}
}
void handle_call(const GatewayCallHeader& header, const std::string& body, const Responder& reply) {
void handle_call(
const GatewayCallHeader &header,
const std::string &body,
const Responder &reply) {
std::lock_guard<std::mutex> lock(_m);
if (header.service_type().empty()) {
error(reply, "Invalid service name");
return;
}
api::ApiClient* pclient = get_api_client(header.service_type(), header.service_qualifier());
api::ApiClient *pclient =
get_api_client(header.service_type(), header.service_qualifier());
// Form the api request buffer
api::RequestHeader api_header;
api_header.set_method(header.method());
api_header.set_request_id(header.request_id());
api_header.mutable_token()->CopyFrom(header.token());
int header_size = 1 + api_header.ByteSize();
int header_size = 1 + api_header.ByteSizeLong();
bts_assert(header_size < 128);
std::string buffer;
buffer.reserve(header_size + body.size());
@@ -109,114 +114,124 @@ struct ApiGatewayImpl {
api_header.AppendToString(&buffer);
buffer += body;
// call send
Responder* presponder = new Responder(move(reply));
api_client_send(pclient, buffer.c_str(), buffer.size(), [](const char* buffer, size_t length, void* extra) {
// buffer = [size][ResponseHeader][Payload].
GatewayResponse response;
response.set_call_result(buffer, length);
LOG(Api,Diag, "Gateway Result: {}", api::toJSON(response));
// in response send to client
const Responder* r = static_cast<const Responder*>(extra);
if (r != nullptr) {
(*r)(response);
delete r;
}
}, presponder);
Responder *presponder = new Responder(move(reply));
api_client_send(
pclient, buffer.c_str(), buffer.size(),
[](const char *buffer, size_t length, void *extra) {
// buffer = [size][ResponseHeader][Payload].
GatewayResponse response;
response.set_call_result(buffer, length);
LOG(Api, Diag, "Gateway Result: {}", api::toJSON(response));
// in response send to client
const Responder *r = static_cast<const Responder *>(extra);
if (r != nullptr) {
(*r)(response);
delete r;
}
},
presponder);
}
void handle_subscribe(const GatewaySubscribe& request, const Responder& reply) {
void
handle_subscribe(const GatewaySubscribe &request, const Responder &reply) {
std::lock_guard<std::mutex> lock(_m);
api::ApiClient* pclient = get_api_client(request.service_type(), request.service_qualifier());
api::ApiClient *pclient =
get_api_client(request.service_type(), request.service_qualifier());
if (pclient == nullptr) {
error(reply, "Invalid service");
return;
}
// call api_client_subscribe() store the TableSubscription and the id and the reply channel
// When a tableupate fires attach the id to the reply and send it over the channel
// Responder* preply = new Responder(move(reply));
// call api_client_subscribe() store the TableSubscription and the id and
// the reply channel When a tableupate fires attach the id to the reply
// and send it over the channel Responder* preply = new
// Responder(move(reply));
auto id = request.request_id();
SubscriptionContext* pcontext = new SubscriptionContext { id, move(reply), nullptr };
SubscriptionContext *pcontext =
new SubscriptionContext{id, move(reply), nullptr};
auto table = api_client_subscribe(pclient, request.method_name().c_str(), request.stream_qualifier().c_str(),
[](api::EventType type, const api::StreamUpdate* changes, int count, void* extra) {
SubscriptionContext* pcontext = (SubscriptionContext*)extra;
GatewayResponse update;
update.mutable_tick()->set_id(pcontext->id);
if (type == api::EventType::TABLE_UPDATED) {
for (int x=0; x < count; ++x) {
const api::StreamUpdate& current = changes[x];
auto* row = update.mutable_tick()->add_updates();
row->set_update_type((StreamGWUpdate::UpdateType)current.type);
row->set_key(current.key);
if (current.type == api::StreamUpdate::CREATE || current.type == api::StreamUpdate::REPLACE) {
row->set_buffer(current.buffer, current.buffer_len);
}
}
}
else {
update.mutable_tick()->add_updates()->set_update_type(StreamGWUpdate::RESET);
}
pcontext->send(update);
}, pcontext);
auto table = api_client_subscribe(
pclient, request.method_name().c_str(),
request.stream_qualifier().c_str(),
[](api::EventType type, const api::StreamUpdate *changes, int count,
void *extra) {
SubscriptionContext *pcontext = (SubscriptionContext *)extra;
GatewayResponse update;
update.mutable_tick()->set_id(pcontext->id);
if (type == api::EventType::TABLE_UPDATED) {
for (int x = 0; x < count; ++x) {
const api::StreamUpdate &current = changes[x];
auto *row = update.mutable_tick()->add_updates();
row->set_update_type(
(StreamGWUpdate::UpdateType)current.type);
row->set_key(current.key);
if (current.type == api::StreamUpdate::CREATE ||
current.type == api::StreamUpdate::REPLACE) {
row->set_buffer(current.buffer, current.buffer_len);
}
}
} else {
update.mutable_tick()->add_updates()->set_update_type(
StreamGWUpdate::RESET);
}
pcontext->send(update);
},
pcontext);
pcontext->table = table;
_subscriptions[id] = pcontext;
// reply with empty response so we don't time out
reply({});
}
void handle_unsubscribe(const GatewayUnsubscribe& request, const Responder& reply) {
void handle_unsubscribe(
const GatewayUnsubscribe &request, const Responder &reply) {
auto id = request.subscription_id();
api::ApiClient* pclient = get_api_client(request.service_type(), request.service_qualifier(), false);
api::ApiClient *pclient = get_api_client(
request.service_type(), request.service_qualifier(), false);
bts_assert_d(pclient);
auto iter = _subscriptions.find(id);
if (iter != _subscriptions.end()) {
SubscriptionContext* pcontext = iter->second;
SubscriptionContext *pcontext = iter->second;
_subscriptions.erase(iter);
if(pcontext && pclient) {
if (pcontext && pclient) {
api_client_unsubscribe(pclient, pcontext->table);
}
// Note: not safe to delete context yet
api::api_post(_sc.bus(), [pcontext]() {
delete pcontext;
}, 5000);
api::api_post(
_sc.bus(), [pcontext]() { delete pcontext; }, 5000);
}
// empty response makes quickbus happy
reply({});
}
};
ApiGatewaySession::ApiGatewaySession() : _me() {}
ApiGatewaySession::ApiGatewaySession(const api::ServiceContext& ctx) :
_me(std::make_unique<ApiGatewayImpl>(ctx))
{}
ApiGatewaySession::~ApiGatewaySession(){}
ApiGatewaySession::ApiGatewaySession(const api::ServiceContext &ctx)
: _me(std::make_unique<ApiGatewayImpl>(ctx)) {}
ApiGatewaySession::~ApiGatewaySession() {}
void ApiGatewaySession::handle(const api::gateway::GatewayRequest& request, const Responder& reply) {
LOG(Api,Diag, "Gateway Request: {}", api::toJSON(request));
switch(request.request_case()) {
case GatewayRequest::kLogin:
_me->handle_login(request.login(), reply);
break;
case GatewayRequest::kCall:
_me->handle_call(request.call(), request.addendum(), reply);
break;
case GatewayRequest::kSubscribe:
_me->handle_subscribe(request.subscribe(), reply);
break;
case GatewayRequest::kUnsubscribe:
_me->handle_unsubscribe(request.unsubscribe(), reply);
break;
case GatewayRequest::REQUEST_NOT_SET: {
_me->error(reply, "Request type not set");
}
void ApiGatewaySession::handle(
const api::gateway::GatewayRequest &request, const Responder &reply) {
LOG(Api, Diag, "Gateway Request: {}", api::toJSON(request));
switch (request.request_case()) {
case GatewayRequest::kLogin:
_me->handle_login(request.login(), reply);
break;
case GatewayRequest::kCall:
_me->handle_call(request.call(), request.addendum(), reply);
break;
case GatewayRequest::kSubscribe:
_me->handle_subscribe(request.subscribe(), reply);
break;
case GatewayRequest::kUnsubscribe:
_me->handle_unsubscribe(request.unsubscribe(), reply);
break;
case GatewayRequest::REQUEST_NOT_SET: {
_me->error(reply, "Request type not set");
} break;
}
}
std::string get_server_info(const api::ServiceContext& ctx) {
std::string get_server_info(const api::ServiceContext &ctx) {
using namespace api::admin;
using std::endl;
api::VoidMessage vod;
@@ -229,15 +244,20 @@ std::string get_server_info(const api::ServiceContext& ctx) {
if (response.success()) {
ss << " \"users\": [" << endl;
bool first = true;
for (const auto& user : response.value()->users()) {
for (const auto &user : response.value()->users()) {
if (user.is_present()) {
ss << " ";
if (first) { first = false; }
else { ss << ","; }
if (first) {
first = false;
} else {
ss << ",";
}
ss << "{ ";
ss << "\"name\": \"" << user.name() << "\", ";
ss << "\"operator_id\": \"" << user.user_data().exchange_user_data().cme_data().operator_id() << "\"";
ss << "}" << endl;
ss << "\"operator_id\": \""
<< user.user_data().exchange_user_data().cme_data().operator_id()
<< "\"";
ss << "}" << endl;
}
}
ss << " ]" << endl;
@@ -248,4 +268,5 @@ std::string get_server_info(const api::ServiceContext& ctx) {
return ss.str();
}
}}
} // namespace http
} // namespace bts
+11 -12
View File
@@ -1,29 +1,28 @@
#pragma once
#include "lib/api/public/ServiceContext.h"
#include "lib/api/apigateway.pb.h"
#include <memory>
#include "lib/api/public/ServiceContext.h"
#include <functional>
#include <memory>
namespace bts {
namespace http {
typedef std::function<void(const api::gateway::GatewayResponse&)> Responder;
typedef std::function<void(const api::gateway::GatewayResponse &)> Responder;
struct ApiGatewayImpl;
class ApiGatewaySession {
std::shared_ptr<ApiGatewayImpl> _me;
public:
ApiGatewaySession();
ApiGatewaySession(const api::ServiceContext& ctx);
~ApiGatewaySession();
void handle(const api::gateway::GatewayRequest& request, const Responder&);
public:
ApiGatewaySession();
ApiGatewaySession(const api::ServiceContext &ctx);
~ApiGatewaySession();
void handle(const api::gateway::GatewayRequest &request, const Responder &);
};
std::string get_server_info(const api::ServiceContext&);
std::string get_server_info(const api::ServiceContext &);
}}
} // namespace http
} // namespace bts
+33 -26
View File
@@ -1,62 +1,68 @@
#include "MultipartFormDecoder.h"
#include "MultipartFormDecoder.h"
#include "lib/logging/logging.h"
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/iter_find.hpp>
namespace bts {
namespace http {
namespace bts {
namespace http {
using namespace std;
vector<string> split(const string& contents, const string& deliniator) {
vector<string> split(const string &contents, const string &deliniator) {
vector<string> result;
boost::iter_split(result, contents, boost::first_finder(deliniator));
return result;
}
std::tuple<std::string, std::map<std::string,std::string>> decode_header(const std::string& header) {
std::tuple<std::string, std::map<std::string, std::string>>
decode_header(const std::string &header) {
auto tokens = split(header, ";");
std::string header_name = *tokens.begin();
map<string,string> props;
for (const auto& token : tokens) {
map<string, string> props;
for (const auto &token : tokens) {
auto equals = token.find('=');
if (equals != std::string::npos) {
auto key = token.substr(1,equals-1);
auto value = token.substr(equals+1);
boost::algorithm::trim_if(value,
[](char c) { return c == ' ' || c == '"' || c == '\r' || c == '\n'; });
auto key = token.substr(1, equals - 1);
auto value = token.substr(equals + 1);
boost::algorithm::trim_if(value, [](char c) {
return c == ' ' || c == '"' || c == '\r' || c == '\n';
});
props[key] = value;
}
}
return { header_name, props };
return {header_name, props};
}
std::optional<MultiPartFormData> decode_form_data(const string content_type, const string& body) {
auto [_,props] = decode_header(content_type);
std::optional<MultiPartFormData>
decode_form_data(const string content_type, const string &body) {
auto [_, props] = decode_header(content_type);
(void)_;
auto boundary = props["boundary"];
if (boundary.empty()) {
LOG(Http,Warn,"Failed to decode boundary in content type '{}'", content_type);
LOG(Http, Warn, "Failed to decode boundary in content type '{}'",
content_type);
return std::nullopt;
}
MultiPartFormData result;
auto parts = split(body, "--" + boundary);
for (const auto& part : parts) {
for (const auto &part : parts) {
auto end_headers = part.find("\r\n\r\n");
if (end_headers == std::string::npos) continue;
if (part == "--\r\n") continue;
auto headers = split(part.substr(0,end_headers), "\r\n");
if (end_headers == std::string::npos)
continue;
if (part == "--\r\n")
continue;
auto headers = split(part.substr(0, end_headers), "\r\n");
FormPart form;
form.content_type = part.substr(end_headers+2);
for (const auto& header : headers) {
const char* content_type = "Content-Type: ";
form.content_type = part.substr(end_headers + 2);
for (const auto &header : headers) {
const char *content_type = "Content-Type: ";
if (boost::starts_with(header, content_type)) {
form.content_type = header.substr(sizeof(content_type));
}
else {
auto [_, props] = decode_header(header); (void)_;
} else {
auto [_, props] = decode_header(header);
(void)_;
if (!props["filename"].empty()) {
form.filename = props["filename"];
}
@@ -68,4 +74,5 @@ std::optional<MultiPartFormData> decode_form_data(const string content_type, con
return {result};
}
}} //namespace
} // namespace http
} // namespace bts
+11 -8
View File
@@ -1,11 +1,11 @@
#pragma once
#include <string>
#include <vector>
#pragma once
#include <map>
#include <optional>
#include <string>
#include <vector>
namespace bts {
namespace http {
namespace bts {
namespace http {
struct FormPart {
std::string content_type;
@@ -16,7 +16,10 @@ struct MultiPartFormData {
std::vector<FormPart> parts;
};
std::tuple<std::string, std::map<std::string,std::string>> decode_header(const std::string& header);
std::optional<MultiPartFormData> decode_form_data(const std::string content_type, const std::string& body);
std::tuple<std::string, std::map<std::string, std::string>>
decode_header(const std::string &header);
std::optional<MultiPartFormData>
decode_form_data(const std::string content_type, const std::string &body);
}} //namespace
} // namespace http
} // namespace bts
+372 -357
View File
@@ -1,13 +1,15 @@
// #define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT 1
#include "WebServer.h"
#include "ApiGateway.h"
#include "MultipartFormDecoder.h"
#include "lib/utils/Dispatcher.h"
#include "lib/api/core/ApiClientInterop.h"
#include "lib/api/apigateway.pb.h"
#include "lib/api/core/ApiClientInterop.h"
#include "lib/logging/logging.h"
#include "lib/os/os.h"
#include "lib/utils/TimeStamp.h"
#include "lib/tracing/PerfLog.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/TimeStamp.h"
#include <websocketpp/config/asio_no_tls.hpp>
#include <websocketpp/server.hpp>
@@ -16,26 +18,26 @@
#include <boost/filesystem.hpp>
#include <boost/tokenizer.hpp>
#include <vector>
#include <unordered_map>
#include <map>
#include <tuple>
#include <mutex>
#include <fstream>
#include <string>
#include <map>
#include <mutex>
#include <regex>
#include <string>
#include <tuple>
#include <unordered_map>
#include <vector>
typedef websocketpp::server<websocketpp::config::asio> WsServer;
typedef WsServer::endpoint::connection_ptr connection_ptr;
using websocketpp::connection_hdl;
using websocketpp::connection_hdl;
using websocketpp::lib::bind;
using websocketpp::lib::placeholders::_1;
using websocketpp::lib::placeholders::_2;
using websocketpp::lib::bind;
using namespace bts::api::gateway;
std::string read_file(const std::string& path) {
std::ifstream f{ path };
std::string read_file(const std::string &path) {
std::ifstream f{path};
auto sz = boost::filesystem::file_size(path);
std::string contents(sz, ' ');
f.read(&contents[0], sz);
@@ -45,364 +47,377 @@ std::string read_file(const std::string& path) {
namespace bts {
namespace http {
struct ServerState {
std::mutex _m;
utils::Dispatcher& _dispatcher;
std::map<connection_hdl, ApiGatewaySession, std::owner_less<connection_hdl>> _session;
WsServer _server;
int _port;
std::string _www;
std::string _conf;
const api::ServiceContext& _ctx;
metrics::MetricsContainer& _metrics;
std::shared_ptr<metrics::HistogramMetric<4>> _mtimer;
trace::PerfLog &_perf_log;
ServerState(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, trace::PerfLog &l,int port) :
_m(),
_dispatcher(utils::Dispatcher::get_named("Webserver")),
_session(),
_server(),
_port(port),
_www(),
_ctx(ctx),
struct ServerState {
std::mutex _m;
utils::Dispatcher &_dispatcher;
std::map<connection_hdl, ApiGatewaySession, std::owner_less<connection_hdl>>
_session;
WsServer _server;
int _port;
std::string _www;
std::string _conf;
const api::ServiceContext &_ctx;
metrics::MetricsContainer &_metrics;
std::shared_ptr<metrics::HistogramMetric<4>> _mtimer;
trace::PerfLog &_perf_log;
ServerState(
const api::ServiceContext &ctx,
metrics::MetricsContainer &metrics,
trace::PerfLog &l,
int port)
: _m(), _dispatcher(utils::Dispatcher::get_named("Webserver")),
_session(), _server(), _port(port), _www(), _ctx(ctx),
_metrics(metrics),
_mtimer(std::make_shared<metrics::HistogramMetric<4>>("spark_metrics_query_duration_ms",
std::array<double,4>({ 0.5, 1.0, 5.0, 10.0 }))),
_perf_log(l)
{
_www = os::sparkdata_path("www");
_conf = os::sparkdata_path("conf");
_mtimer(std::make_shared<metrics::HistogramMetric<4>>(
"spark_metrics_query_duration_ms",
std::array<double, 4>({0.5, 1.0, 5.0, 10.0}))),
_perf_log(l) {
_www = os::sparkdata_path("www");
_conf = os::sparkdata_path("conf");
_metrics.publish(_mtimer);
_metrics.publish(_mtimer);
_server.init_asio(_dispatcher.io_service());
_server.set_open_handler(bind(&ServerState::on_open, this, ::_1));
_server.set_close_handler(bind(&ServerState::on_close, this, ::_1));
_server.set_message_handler(bind(&ServerState::on_message, this, ::_1, ::_2));
_server.set_http_handler(bind(&ServerState::on_http, this, ::_1));
// Turn off logging
_server.clear_access_channels(websocketpp::log::alevel::all);
_server.clear_error_channels(websocketpp::log::alevel::all);
_server.set_open_handshake_timeout(0);
run();
}
_server.init_asio(_dispatcher.io_service());
_server.set_open_handler(bind(&ServerState::on_open, this, ::_1));
_server.set_close_handler(bind(&ServerState::on_close, this, ::_1));
_server.set_message_handler(
bind(&ServerState::on_message, this, ::_1, ::_2));
_server.set_http_handler(bind(&ServerState::on_http, this, ::_1));
// Turn off logging
_server.clear_access_channels(websocketpp::log::alevel::all);
_server.clear_error_channels(websocketpp::log::alevel::all);
_server.set_open_handshake_timeout(0);
run();
}
// starts listening
void run() {
_server.set_reuse_addr(true);
_server.listen(_port);
_server.start_accept();
}
// starts listening
void run() {
_server.set_reuse_addr(true);
_server.listen(_port);
_server.start_accept();
}
void on_open(connection_hdl hdl) {
auto con = _server.get_con_from_hdl(hdl);
if (con) {
LOG(Http,Info, "Web client connection from {}", con->get_remote_endpoint());
std::lock_guard<std::mutex> lock(_m);
_session[hdl] = ApiGatewaySession(_ctx);
}
}
void on_close(connection_hdl hdl) {
auto con = _server.get_con_from_hdl(hdl);
if (con) {
LOG(Http,Info, "Web client disconnected: {}", con->get_remote_endpoint());
std::lock_guard<std::mutex> lock(_m);
_session.erase(hdl);
}
}
void on_message(connection_hdl hdl, WsServer::message_ptr msg) {
void on_open(connection_hdl hdl) {
auto con = _server.get_con_from_hdl(hdl);
if (con) {
LOG(Http, Info, "Web client connection from {}",
con->get_remote_endpoint());
std::lock_guard<std::mutex> lock(_m);
auto iter = _session.find(hdl);
if (iter != _session.end()) {
GatewayRequest request;
if(request.ParseFromString(msg->get_payload())) {
iter->second.handle(request, [this,hdl](const GatewayResponse& response) {
websocketpp::lib::error_code ec;
_server.send(hdl, response.SerializeAsString(), websocketpp::frame::opcode::binary, ec);
if (ec) {
LOG(Http,Warn, "Error sending from websocket: '{}'", ec.message());
}
});
}
else {
LOG(Http,Info, "Could not parse request as GatewayRequest: {}", msg->get_payload());
_session[hdl] = ApiGatewaySession(_ctx);
}
}
void on_close(connection_hdl hdl) {
auto con = _server.get_con_from_hdl(hdl);
if (con) {
LOG(Http, Info, "Web client disconnected: {}",
con->get_remote_endpoint());
std::lock_guard<std::mutex> lock(_m);
_session.erase(hdl);
}
}
void on_message(connection_hdl hdl, WsServer::message_ptr msg) {
std::lock_guard<std::mutex> lock(_m);
auto iter = _session.find(hdl);
if (iter != _session.end()) {
GatewayRequest request;
if (request.ParseFromString(msg->get_payload())) {
iter->second.handle(
request, [this, hdl](const GatewayResponse &response) {
websocketpp::lib::error_code ec;
_server.send(
hdl, response.SerializeAsString(),
websocketpp::frame::opcode::binary, ec);
if (ec) {
LOG(Http, Warn, "Error sending from websocket: '{}'",
ec.message());
}
});
} else {
LOG(Http, Info, "Could not parse request as GatewayRequest: {}",
msg->get_payload());
}
}
}
void return_dotall(connection_ptr &con) {
std::vector<boost::filesystem::path> files;
auto iter = boost::filesystem::directory_iterator(_conf);
auto end = boost::filesystem::directory_iterator();
// Collect all .conf file
for (; iter != end; ++iter) {
auto ext = iter->path().filename().string().substr(
iter->path().filename().stem().size());
if (boost::iequals(ext, ".conf")) {
files.push_back(iter->path());
}
}
// Sort them so we at least it's stable
std::sort(files.begin(), files.end());
// Concatanate
std::string body;
for (const auto &p : files) {
body += "##########################\n";
body += "## " + p.filename().string();
body += "\n##########################\n";
body += read_file(p.string());
}
con->set_body(body);
con->set_status(websocketpp::http::status_code::ok);
}
void return_metrics(connection_ptr &con) {
utils::TimeStamp start = utils::TimeStamp::now();
std::stringstream ss;
_metrics.print(ss);
con->set_body(ss.str());
con->set_status(websocketpp::http::status_code::ok);
_mtimer->observe(
(utils::TimeStamp::now() - start).microseconds() / 1000.0);
}
void return_perf_trace_data(connection_ptr &con, size_t max_rows) {
utils::TimeStamp start = utils::TimeStamp::now();
std::stringstream ss;
_perf_log.print_latest_traces(ss, max_rows);
con->set_body(ss.str());
con->set_status(websocketpp::http::status_code::ok);
_mtimer->observe(
(utils::TimeStamp::now() - start).microseconds() / 1000.0);
}
void bad_request(connection_ptr &con, const std::string &message) {
con->set_body(message);
con->set_status(websocketpp::http::status_code::bad_request);
}
std::vector<std::string>
split_string(std::string body, std::string &deliniator) {
std::vector<std::string> result;
size_t i = body.find(deliniator);
while (i != std::string::npos) {
result.push_back(body.substr(0, i));
body = body.substr(i + deliniator.size());
i = body.find(deliniator);
}
return result;
}
void handle_file_upload(connection_ptr &con) {
auto content_type = con->get_request_header("Content-Type");
auto data = decode_form_data(content_type, con->get_request_body());
if (data) {
for (const auto &part : data.value().parts) {
if (!part.filename.empty()) {
LOG(Http, Info, "Received file upload: {}", part.filename);
}
}
} else {
LOG(Http, Warn,
"Failed to decode form data. Ensure you are using "
"'enctype=\"multipart/form-data\"'");
}
con->set_body("got it");
con->set_status(websocketpp::http::status_code::ok);
}
bool require_authentication(
connection_ptr &con,
const std::string &username,
const std::string &password) {
auto auth = con->get_request_header("Authorization");
if (!auth.empty() && auth.size() >= 7 &&
boost::iequals(auth.substr(0, 6), "Basic ")) {
auto b64 = auth.substr(6);
auto decoded = websocketpp::base64_decode(b64);
auto colon = decoded.find(':');
if (colon != std::string::npos) {
auto user = decoded.substr(0, colon);
auto pass = decoded.substr(colon + 1);
if (user == username && pass == password)
return true;
}
}
void return_dotall(connection_ptr& con) {
std::vector<boost::filesystem::path> files;
auto iter = boost::filesystem::directory_iterator(_conf);
auto end = boost::filesystem::directory_iterator();
// Collect all .conf file
for (; iter != end; ++iter) {
auto ext = iter->path().filename().string().substr(iter->path().filename().stem().size());
if(boost::iequals(ext, ".conf")) {
files.push_back(iter->path());
}
}
// Sort them so we at least it's stable
std::sort(files.begin(), files.end());
// Concatanate
std::string body;
for (const auto& p : files) {
body += "##########################\n";
body += "## " + p.filename().string();
body += "\n##########################\n";
body += read_file(p.string());
}
// Failed
con->append_header("WWW-Authenticate", "Basic realm=\"BTS\"");
con->set_status(websocketpp::http::status_code::unauthorized);
con->set_body("Not Authorized");
return false;
}
void check_auth_and_return_info(connection_ptr &con) {
if (require_authentication(
con, "bts", "bbb835af-b1ea-42d3-8be1-fd9d5555530d")) {
auto horrible_json = get_server_info(_ctx);
con->set_body(horrible_json);
con->set_status(websocketpp::http::status_code::ok);
}
}
void directory_index(
connection_ptr &con,
const std::string &resource,
const boost::filesystem::path &path) {
// directory listing
std::string body = "<!doctype html>\n";
body += "<head>";
body += "<script defer "
"src=\"https://use.fontawesome.com/releases/v5.0.10/js/all.js\" "
"integrity=\"sha384-"
"slN8GvtUJGnv6ca26v8EzVaR9DC58QEwsIk9q1QXdCU8Yu8ck/tL/"
"5szYlBbqmS+\" crossorigin=\"anonymous\"></script>\n";
body += "</head>\n";
body += "<body>\n";
body += "<h1>Index of " + resource + "</h1>\n";
auto iter = boost::filesystem::directory_iterator(path);
auto end = boost::filesystem::directory_iterator();
body += "<ul style=\"list-style-type: none;\">\n";
for (; iter != end; ++iter) {
auto x = iter->path();
body += "<li>";
if (boost::filesystem::is_regular_file(x)) {
body += "<i class=\"far fa-file\"></i> ";
} else if (boost::filesystem::is_directory(x)) {
body += "<i class=\"far fa-folder\"></i> ";
}
if (resource == "/") {
body += "<a href=" + x.filename().string() + ">";
} else {
body += "<a href=" + resource + "/" + x.filename().string() + ">";
}
body += x.filename().string();
body += "</a></li>\n";
}
body += "</ul>\n";
body += "</body></html>\n";
con->set_status(websocketpp::http::status_code::ok);
con->set_body(body);
}
boost::filesystem::path web_ui_path(const boost::filesystem::path &root) {
auto reldir = root / "web/ui";
if (boost::filesystem::exists(reldir))
return reldir;
reldir = root / "ui";
if (boost::filesystem::exists(reldir))
return reldir;
if (root.has_parent_path())
return web_ui_path(root.parent_path());
return _www;
}
boost::filesystem::path map_path_to_filesystem(const std::string &resource) {
std::string config = "/config";
std::string files = "/files";
std::string perf_trace = "/perf_trace";
std::string docs = "/docs";
if (resource.compare(0, config.size(), config) == 0) {
return _conf + resource.substr(config.size());
} else if (resource.compare(0, files.size(), files) == 0) {
return _www + resource.substr(files.size());
} else if (resource.compare(0, perf_trace.size(), perf_trace) == 0) {
return os::sparkd_wd() + "/data_files" + resource;
} else if (resource.compare(0, docs.size(), docs) == 0) {
return os::sparkd_wd() + "/web" + resource;
}
return web_ui_path(os::get_install_path()) / resource;
}
void return_file(connection_ptr &con, const boost::filesystem::path &path) {
con->set_body(read_file(path.string()));
con->set_status(websocketpp::http::status_code::ok);
}
void return_bts_installer(connection_ptr &con) {
std::string installer_bits = read_file(_www + "/bts_installer.exe");
const std::string installer_config = _www + "install_config.json";
if (boost::filesystem::is_regular_file(installer_config)) {
auto conf = read_file(installer_config);
std::string needle = "$HOST";
std::size_t pos;
do {
pos = conf.find(needle, 0);
if (pos != std::string::npos) {
auto host = con->get_uri()->get_host();
conf.replace(pos, needle.size(), host);
}
} while (pos != std::string::npos);
installer_bits += conf;
}
con->set_body(installer_bits);
con->set_status(websocketpp::http::status_code::ok);
}
std::string strip_query_str(const std::string &request) {
auto position = request.find("?");
if (position != std::string::npos) {
return request.substr(0, position);
}
return request;
}
void on_http(connection_hdl hdl) {
auto con = _server.get_con_from_hdl(hdl);
auto resource = strip_query_str(con->get_resource());
LOG(Http, Diag, "HTTP GET: {}", resource);
if (boost::iequals(resource, "/config/.all")) {
return_dotall(con);
} else if (boost::iequals(resource, "/metrics")) {
return_metrics(con);
} else if (boost::iequals(resource, "/perf_trace_data")) {
return_perf_trace_data(con, 1000);
} else if (std::regex_match(
resource, std::regex("/perf_trace_data/(\\d++)"))) {
std::regex rx = std::regex("/perf_trace_data/(\\d++)");
std::smatch matches;
if (std::regex_match(resource, matches, rx)) {
auto max_rows = std::atoi(matches[1].str().c_str());
return_perf_trace_data(con, max_rows);
} else
return_perf_trace_data(con, 1000);
} else if (boost::iequals(resource, "/server_info")) {
check_auth_and_return_info(con);
} else if (
boost::iequals(resource, "/bts_installer.exe") &&
boost::filesystem::is_regular_file(_www + "/bts_installer.exe")) {
return_bts_installer(con);
} else if (boost::iequals(resource, "/robots.txt")) {
con->set_status(websocketpp::http::status_code::ok);
std::string body = "User-agent: *\nDisallow: /";
con->set_body(body);
con->set_status(websocketpp::http::status_code::ok);
}
void return_metrics(connection_ptr& con) {
utils::TimeStamp start = utils::TimeStamp::now();
std::stringstream ss;
_metrics.print(ss);
con->set_body(ss.str());
con->set_status(websocketpp::http::status_code::ok);
_mtimer->observe((utils::TimeStamp::now() - start).microseconds() / 1000.0);
}
void return_perf_trace_data(connection_ptr& con,size_t max_rows) {
utils::TimeStamp start = utils::TimeStamp::now();
std::stringstream ss;
_perf_log.print_latest_traces(ss,max_rows);
con->set_body(ss.str());
con->set_status(websocketpp::http::status_code::ok);
_mtimer->observe((utils::TimeStamp::now() - start).microseconds() / 1000.0);
}
void bad_request(connection_ptr& con, const std::string& message) {
con->set_body(message);
con->set_status(websocketpp::http::status_code::bad_request);
}
std::vector<std::string> split_string(std::string body, std::string& deliniator) {
std::vector<std::string> result;
size_t i = body.find(deliniator);
while(i != std::string::npos) {
result.push_back(body.substr(0,i));
body = body.substr(i + deliniator.size());
i = body.find(deliniator);
}
return result;
}
void handle_file_upload(connection_ptr& con) {
auto content_type = con->get_request_header("Content-Type");
auto data = decode_form_data(content_type, con->get_request_body());
if (data) {
for(const auto& part : data.value().parts) {
if (!part.filename.empty()) {
LOG(Http,Info,"Received file upload: {}", part.filename);
}
} else if (boost::iequals(resource, "/upload")) {
handle_file_upload(con);
} else {
auto path = map_path_to_filesystem(resource);
if (boost::filesystem::is_directory(path)) {
auto index = path / "index.html";
if (boost::filesystem::is_regular_file(index)) {
return_file(con, index);
} else {
directory_index(con, resource, path);
}
}
else {
LOG(Http,Warn, "Failed to decode form data. Ensure you are using 'enctype=\"multipart/form-data\"'");
}
con->set_body("got it");
con->set_status(websocketpp::http::status_code::ok);
}
bool require_authentication(connection_ptr& con, const std::string& username, const std::string& password) {
auto auth = con->get_request_header("Authorization");
if (!auth.empty() && auth.size() >= 7 && boost::iequals(auth.substr(0, 6), "Basic ")) {
auto b64 = auth.substr(6);
auto decoded = websocketpp::base64_decode(b64);
auto colon = decoded.find(':');
if (colon != std::string::npos) {
auto user = decoded.substr(0, colon);
auto pass = decoded.substr(colon+1);
if (user == username && pass == password) return true;
}
}
// Failed
con->append_header("WWW-Authenticate", "Basic realm=\"BTS\"");
con->set_status(websocketpp::http::status_code::unauthorized);
con->set_body("Not Authorized");
return false;
}
void check_auth_and_return_info(connection_ptr& con) {
if (require_authentication(con, "bts", "bbb835af-b1ea-42d3-8be1-fd9d5555530d")) {
auto horrible_json = get_server_info(_ctx);
con->set_body(horrible_json);
con->set_status(websocketpp::http::status_code::ok);
} else if (boost::filesystem::is_regular_file(path)) {
return_file(con, path);
} else {
con->set_status(websocketpp::http::status_code::not_found);
con->set_body("Cannot find " + resource);
}
}
}
};
void directory_index(connection_ptr& con, const std::string& resource, const boost::filesystem::path& path) {
// directory listing
std::string body = "<!doctype html>\n";
body += "<head>";
body += "<script defer src=\"https://use.fontawesome.com/releases/v5.0.10/js/all.js\" integrity=\"sha384-slN8GvtUJGnv6ca26v8EzVaR9DC58QEwsIk9q1QXdCU8Yu8ck/tL/5szYlBbqmS+\" crossorigin=\"anonymous\"></script>\n";
body += "</head>\n";
body += "<body>\n";
body += "<h1>Index of " + resource + "</h1>\n";
auto iter = boost::filesystem::directory_iterator(path);
auto end = boost::filesystem::directory_iterator();
body += "<ul style=\"list-style-type: none;\">\n";
for (; iter != end; ++iter) {
auto x = iter->path();
body += "<li>";
if (boost::filesystem::is_regular_file(x)) {
body += "<i class=\"far fa-file\"></i> ";
}
else if (boost::filesystem::is_directory(x)) {
body += "<i class=\"far fa-folder\"></i> ";
}
if (resource == "/") {
body += "<a href=" + x.filename().string() + ">";
}
else {
body += "<a href=" + resource + "/" + x.filename().string() + ">";
}
body += x.filename().string();
body += "</a></li>\n";
}
body += "</ul>\n";
body += "</body></html>\n";
con->set_status(websocketpp::http::status_code::ok);
con->set_body(body);
}
SparkWebServer::SparkWebServer(
const api::ServiceContext &ctx,
metrics::MetricsContainer &metrics,
trace::PerfLog &l,
int port)
: _me(std::make_shared<ServerState>(ctx, metrics, l, port)) {}
boost::filesystem::path web_ui_path(const boost::filesystem::path& root) {
auto reldir = root / "web/ui";
if (boost::filesystem::exists(reldir)) return reldir;
reldir = root / "ui";
if (boost::filesystem::exists(reldir)) return reldir;
if (root.has_parent_path()) return web_ui_path(root.parent_path());
return _www;
}
boost::filesystem::path map_path_to_filesystem(const std::string& resource) {
std::string config = "/config";
std::string files = "/files";
std::string perf_trace = "/perf_trace";
std::string docs = "/docs";
if (resource.compare(0, config.size(), config) == 0) {
return _conf + resource.substr(config.size());
}
else if (resource.compare(0, files.size(), files) == 0) {
return _www + resource.substr(files.size());
}
else if (resource.compare(0, perf_trace.size(), perf_trace) == 0) {
return os::sparkd_wd() + "/data_files" + resource;
}
else if (resource.compare(0, docs.size(), docs) == 0) {
return os::sparkd_wd() + "/web" + resource;
}
return web_ui_path(os::get_install_path()) / resource;
}
void return_file(connection_ptr& con, const boost::filesystem::path& path) {
con->set_body(read_file(path.string()));
con->set_status(websocketpp::http::status_code::ok);
}
void return_bts_installer(connection_ptr& con) {
std::string installer_bits = read_file(_www+"/bts_installer.exe");
const std::string installer_config = _www+"install_config.json";
if (boost::filesystem::is_regular_file(installer_config)) {
auto conf = read_file(installer_config);
std::string needle = "$HOST";
std::size_t pos;
do {
pos = conf.find(needle,0);
if (pos != std::string::npos) {
auto host = con->get_uri()->get_host();
conf.replace(pos, needle.size(), host);
}
} while(pos != std::string::npos);
installer_bits += conf;
}
con->set_body(installer_bits);
con->set_status(websocketpp::http::status_code::ok);
}
std::string strip_query_str(const std::string& request) {
auto position = request.find("?");
if (position != std::string::npos) {
return request.substr(0, position);
}
return request;
}
void on_http(connection_hdl hdl) {
auto con = _server.get_con_from_hdl(hdl);
auto resource = strip_query_str(con->get_resource());
LOG(Http,Diag, "HTTP GET: {}", resource);
if (boost::iequals(resource, "/config/.all")) {
return_dotall(con);
}
else if (boost::iequals(resource, "/metrics")) {
return_metrics(con);
}
else if (boost::iequals(resource, "/perf_trace_data")) {
return_perf_trace_data(con,1000);
}
else if (std::regex_match(resource, std::regex("/perf_trace_data/(\\d++)"))) {
std::regex rx = std::regex("/perf_trace_data/(\\d++)");
std::smatch matches;
if (std::regex_match(resource, matches, rx)) {
auto max_rows = std::atoi(matches[1].str().c_str());
return_perf_trace_data(con,max_rows);
}
else
return_perf_trace_data(con,1000);
}
else if (boost::iequals(resource, "/server_info")) {
check_auth_and_return_info(con);
}
else if (boost::iequals(resource, "/bts_installer.exe") && boost::filesystem::is_regular_file(_www+"/bts_installer.exe")) {
return_bts_installer(con);
}
else if (boost::iequals(resource, "/robots.txt")) {
con->set_status(websocketpp::http::status_code::ok);
std::string body = "User-agent: *\nDisallow: /";
con->set_body(body);
}
else if (boost::iequals(resource, "/upload")) {
handle_file_upload(con);
}
else {
auto path = map_path_to_filesystem(resource);
if (boost::filesystem::is_directory(path)) {
auto index = path / "index.html";
if (boost::filesystem::is_regular_file(index)) {
return_file(con, index);
}
else {
directory_index(con, resource, path);
}
}
else if (boost::filesystem::is_regular_file(path)) {
return_file(con, path);
}
else {
con->set_status(websocketpp::http::status_code::not_found);
con->set_body("Cannot find " + resource);
}
}
}
};
SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, trace::PerfLog &l,int port) :
_me(std::make_shared<ServerState>(ctx, metrics, l,port))
{
}
}}
} // namespace http
} // namespace bts
+16 -8
View File
@@ -4,15 +4,23 @@
#include <memory>
namespace bts {
namespace trace {class PerfLog;}
namespace trace {
class PerfLog;
}
namespace http {
struct ServerState;
struct ServerState;
class SparkWebServer {
std::shared_ptr<ServerState> _me;
public:
SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, trace::PerfLog &,int port);
};
class SparkWebServer {
std::shared_ptr<ServerState> _me;
}}
public:
SparkWebServer(
const api::ServiceContext &ctx,
metrics::MetricsContainer &metrics,
trace::PerfLog &,
int port);
};
} // namespace http
} // namespace bts
@@ -1,5 +1,5 @@
#include "catch/catch.hpp"
#include "../MultipartFormDecoder.h"
#include "catch/catch.hpp"
#include <string>
@@ -8,38 +8,45 @@ namespace http {
namespace test {
TEST_CASE("Form header decoder", "[http]") {
auto header1 = decode_header("multipart/form-data; boundary=---------------------------123631036720284");
auto header1 =
decode_header("multipart/form-data; "
"boundary=---------------------------123631036720284");
REQUIRE(std::get<std::string>(header1) == "multipart/form-data");
REQUIRE(std::get<1>(header1)["boundary"] == "---------------------------123631036720284");
auto header2 = decode_header( "Content-Disposition: form-data; name=\"file\"; filename=\"file_a.txt\"\n");
REQUIRE(
std::get<1>(header1)["boundary"] ==
"---------------------------123631036720284");
auto header2 = decode_header("Content-Disposition: form-data; "
"name=\"file\"; filename=\"file_a.txt\"\n");
REQUIRE(std::get<std::string>(header2) == "Content-Disposition: form-data");
REQUIRE(std::get<1>(header2)["name"] == "file");
REQUIRE(std::get<1>(header2)["filename"] == "file_a.txt");
}
TEST_CASE("MultiPart form decoder", "[http]") {
std::string header = "multipart/form-data; boundary=---------------------------123631036720284";
std::string body =
"-----------------------------123631036720284\r\n"
"Content-Disposition: form-data; name=\"file\"; filename=\"file_a.txt\"\r\n"
"Content-Type: text/plain\r\n"
"\r\n"
"\r\n"
" a\r\n"
" aaa\r\n"
" aaaaa\r\n"
"aaaaaaaa\r\n"
"\r\n"
"\r\n"
"-----------------------------123631036720284--\r\n";
std::string header = "multipart/form-data; "
"boundary=---------------------------123631036720284";
std::string body = "-----------------------------123631036720284\r\n"
"Content-Disposition: form-data; name=\"file\"; "
"filename=\"file_a.txt\"\r\n"
"Content-Type: text/plain\r\n"
"\r\n"
"\r\n"
" a\r\n"
" aaa\r\n"
" aaaaa\r\n"
"aaaaaaaa\r\n"
"\r\n"
"\r\n"
"-----------------------------123631036720284--\r\n";
auto maybe_data = decode_form_data(header, body);
REQUIRE((bool)maybe_data);
const auto& data = maybe_data.value();
const auto &data = maybe_data.value();
REQUIRE(data.parts.size() == 1);
auto& part = data.parts[0];
auto &part = data.parts[0];
REQUIRE(part.filename == "file_a.txt");
}
}}} // ns
} // namespace test
} // namespace http
} // namespace bts