456 lines
14 KiB
C++
456 lines
14 KiB
C++
#include "QBConnection.h"
|
|
|
|
#include "lib/os/os.h"
|
|
#include "lib/utils/PBUtils.h"
|
|
#include "lib/utils/datetime.h"
|
|
#include "lib/logging/logging.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
|
|
#include "lib/quickbus/routing/Router.h"
|
|
#include "lib/quickbus/network/endpoint_description.h"
|
|
#include "lib/quickbus/core/Advertisement.h"
|
|
|
|
#include "lib/quickbus/network/quickbus_msgs.pb.h"
|
|
#include "lib/config/protos/quickbus_cfg.pb.h"
|
|
|
|
#include <boost/algorithm/string/join.hpp>
|
|
#include <boost/date_time/posix_time/posix_time.hpp>
|
|
|
|
namespace bts {
|
|
namespace qbus {
|
|
|
|
using boost::asio::ip::tcp;
|
|
using namespace boost;
|
|
|
|
struct QBConnection::PImpl
|
|
{
|
|
PImpl(tcp::socket &s,Router &router,config::QuickbusNodeCfg const&cfg)
|
|
: socket(std::move(s)),
|
|
router(router),
|
|
pending_send(),
|
|
write_buffer(),
|
|
write_buffer_size(),
|
|
read_buffer(),
|
|
read_buffer_size(),
|
|
has_intro(false),
|
|
intro(),
|
|
my_id(),
|
|
my_desc(local_endpoint_description(socket)),
|
|
remote_desc(remote_endpoint_description(socket)),
|
|
cfg(cfg),
|
|
last_received_ts(datetime::now()),
|
|
last_sent_ts(datetime::now()),
|
|
hb_timer(socket.get_io_service())
|
|
{
|
|
write_buffer.reserve(100*1024);
|
|
read_buffer.reserve(100*1024);
|
|
}
|
|
|
|
tcp::socket socket;
|
|
Router &router;
|
|
|
|
msgs::QBMsg pending_send;
|
|
std::string write_buffer;
|
|
int32_t write_buffer_size;
|
|
|
|
std::string read_buffer;
|
|
int32_t read_buffer_size;
|
|
|
|
bool has_intro;
|
|
msgs::Intro intro;
|
|
NodeID my_id;
|
|
|
|
const std::string my_desc;
|
|
const std::string remote_desc;
|
|
const config::QuickbusNodeCfg cfg;
|
|
|
|
datetime::PTime last_received_ts;
|
|
datetime::PTime last_sent_ts;
|
|
boost::asio::deadline_timer hb_timer;
|
|
};
|
|
|
|
QBConnection::QBConnection (tcp::socket s,Router &router, config::QuickbusNodeCfg const&cfg)
|
|
:
|
|
me(new PImpl(s,router,cfg))
|
|
{}
|
|
|
|
QBConnection::~QBConnection()
|
|
{}
|
|
|
|
void QBConnection::init()
|
|
{
|
|
send_intro();
|
|
recv_intro();
|
|
|
|
check_hbs();
|
|
}
|
|
|
|
bool QBConnection::connected() const
|
|
{return me->socket.is_open();}
|
|
|
|
void QBConnection::send_intro()
|
|
{
|
|
LOG(QBus,Info,"Sending intro from {} to {}", me->my_desc , me->remote_desc );
|
|
|
|
bts_assert(!is_send_in_flight());
|
|
|
|
auto &intro = *me->pending_send.add_op()->mutable_intro();
|
|
intro.set_host_name(os::host_name());
|
|
intro.set_app_cmd_line(boost::algorithm::join(os::get_command_line()," "));
|
|
intro.set_node_id(me->router.id().value());
|
|
intro.set_pid(os::process_id());
|
|
|
|
try_send_pending();
|
|
}
|
|
|
|
void QBConnection::recv_intro()
|
|
{
|
|
recv_msg_header();
|
|
}
|
|
|
|
void QBConnection::check_hbs()
|
|
{
|
|
auto self = shared_from_this();
|
|
|
|
me->hb_timer.expires_from_now(boost::posix_time::milliseconds(me->cfg.hb_period_msecs()));
|
|
me->hb_timer.async_wait([self,this](const boost::system::error_code& ec) {
|
|
|
|
if (!ec && connected()) {
|
|
if (datetime::is_at_least_msecs_old (me->last_received_ts,
|
|
me->cfg.hb_period_msecs() *
|
|
me->cfg.max_missed_hbs()))
|
|
close_connection("No heartbeats received");
|
|
else {
|
|
if (!is_send_in_flight() &&
|
|
datetime::is_at_least_msecs_old (me->last_sent_ts, me->cfg.hb_period_msecs())) {
|
|
|
|
LOG(QBus,Diag,"{} sending HB to {}", me->my_desc ,me->remote_desc );
|
|
me->pending_send.add_op()->mutable_heartbeat();
|
|
try_send_pending();
|
|
}
|
|
check_hbs();
|
|
}
|
|
}
|
|
});
|
|
|
|
}
|
|
|
|
void QBConnection::recv_msg_header()
|
|
{
|
|
auto self = shared_from_this();
|
|
|
|
asio::async_read (me->socket,
|
|
asio::buffer (&me->read_buffer_size, sizeof(me->read_buffer_size)),
|
|
[this, self](system::error_code error, std::size_t ) {
|
|
if (!error)
|
|
recv_msg_body();
|
|
else
|
|
close_connection(error.message());
|
|
});
|
|
}
|
|
|
|
void QBConnection::recv_msg_body()
|
|
{
|
|
auto self = shared_from_this();
|
|
|
|
me->read_buffer.resize(me->read_buffer_size);
|
|
asio::async_read (me->socket,
|
|
asio::buffer (&me->read_buffer[0],me->read_buffer_size),
|
|
[this, self](system::error_code error, std::size_t ) {
|
|
if (!error) {
|
|
msgs::QBMsg m;
|
|
auto parse_ok = m.ParseFromArray (&(me->read_buffer[0]),
|
|
me->read_buffer.size ());
|
|
if (!parse_ok)
|
|
close_connection("Corrupt message");
|
|
else
|
|
handle_msg(&m);
|
|
}
|
|
else
|
|
recv_msg_body();
|
|
|
|
});
|
|
}
|
|
|
|
void QBConnection::handle_msg(msgs::QBMsg *msgs)
|
|
{
|
|
me->last_received_ts = datetime::now();
|
|
for (auto const&op : msgs->op()) {
|
|
|
|
if (op.op_case() == msgs::QBOp::kIntro) {
|
|
if (me->has_intro) {
|
|
close_connection("Received two intro messages");
|
|
return;
|
|
} else {
|
|
LOG(QBus,Info,
|
|
"Received intro from {} for {} intro: {} connection is owned by node: {}"
|
|
" tcp connection is local {} to remote {}",
|
|
me->remote_desc ,me->my_desc ,
|
|
utils::pb::to_string(op.intro()), me->router.id().value(),
|
|
me->my_desc, me->remote_desc );
|
|
|
|
me->has_intro = true;
|
|
me->intro=op.intro();
|
|
me->my_id = NodeID(me->intro.node_id());
|
|
|
|
if (me->router.is_neighbour(me->my_id)) {
|
|
LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" ,
|
|
me->my_desc , me->remote_desc , me->my_id.value());
|
|
if (me->socket.is_open())
|
|
me->socket.close();
|
|
return;
|
|
}
|
|
else {
|
|
LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." ,
|
|
me->my_desc , me->remote_desc , me->my_id.value());
|
|
me->router.add_neighbour(me->my_id,shared_from_this());
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (!me->has_intro) {
|
|
close_connection("Expected intro");
|
|
return;
|
|
}
|
|
|
|
switch (op.op_case()) {
|
|
|
|
case msgs::QBOp::kIntro:
|
|
bts_assert(false);
|
|
break;
|
|
|
|
case msgs::QBOp::kNotification: {
|
|
LOG(QBus,Diag,
|
|
"{} received notification from {} for table {} sid: {} size: {}",
|
|
me->my_desc , me->remote_desc , op.notification().table_id(),
|
|
op.notification().subscription_id(),
|
|
op.notification().update().size());
|
|
|
|
|
|
auto &source_update = op.notification().update();
|
|
Update target_update;
|
|
target_update.resize(source_update.size());
|
|
|
|
for (int i = 0; i < source_update.size(); i++) {
|
|
auto &target_change = target_update[i];
|
|
auto &source_change = op.notification().update(i);
|
|
|
|
target_change.key = RecordKey(source_change.key());
|
|
target_change.value = std::make_shared<Record>(std::move(source_change.value()));
|
|
|
|
switch (source_change.type()) {
|
|
case msgs::Change::REMOVE:
|
|
target_change.type = Change::REMOVE;
|
|
break;
|
|
case msgs::Change::REPLACE:
|
|
target_change.type = Change::REPLACE;
|
|
break;
|
|
case msgs::Change::CREATE:
|
|
target_change.type = Change::CREATE;
|
|
break;
|
|
default:
|
|
bts_assert(false);
|
|
break;
|
|
}
|
|
}
|
|
me->router.notify(me->my_id,
|
|
TableID(op.notification().table_id()),
|
|
SubscriptionID(op.notification().subscription_id()),target_update);
|
|
break;
|
|
}
|
|
|
|
case msgs::QBOp::kAdvertisement: {
|
|
LOG(QBus,Info, "{} received advertisement from {} adv: {}" ,
|
|
me->my_desc , me->remote_desc , utils::pb::to_string(op.advertisement()));
|
|
|
|
Advertisement ad(Topic(op.advertisement().topic()),op.advertisement().maintain_synopsis());
|
|
me->router.advertise(me->my_id, TableID(op.advertisement().table_id()),ad);
|
|
break;
|
|
}
|
|
case msgs::QBOp::kUnadvertisement:
|
|
LOG(QBus,Info, "{} received unadvertisement from {} unadv: {}" ,
|
|
me->my_desc , me->remote_desc , utils::pb::to_string(op.unadvertisement()));
|
|
|
|
me->router.unadvertise(me->my_id, TableID(op.unadvertisement().table_id()));
|
|
break;
|
|
case msgs::QBOp::kSubscription:
|
|
LOG(QBus,Info, "{} received subscription from {} sub: {}" ,
|
|
me->my_desc , me->remote_desc , utils::pb::to_string(op.subscription()));
|
|
|
|
me->router.subscribe(me->my_id, TableID(op.subscription().table_id()),
|
|
SubscriptionID(op.subscription().subscription_id()));
|
|
break;
|
|
case msgs::QBOp::kUnsubscription:
|
|
LOG(QBus,Info, "{} received unsubscription from {} unsub: {}" ,
|
|
me->my_desc , me->remote_desc , utils::pb::to_string(op.unsubscription()));
|
|
|
|
me->router.unsubscribe(me->my_id, TableID(op.unsubscription().table_id()));
|
|
break;
|
|
case msgs::QBOp::kHeartbeat:
|
|
LOG(QBus,Diag,"{} received heartbeat from {}", me->my_desc ,me->remote_desc) ;
|
|
break;
|
|
case msgs::QBOp::OP_NOT_SET:
|
|
LOG(QBus,Error,"{} received corrupted message from {}", me->my_desc ,me->remote_desc) ;
|
|
close_connection("Corrupted message");
|
|
return;
|
|
} // switch
|
|
} // for
|
|
recv_msg_header();
|
|
}
|
|
|
|
void QBConnection::advertise(NodeID const& sender, TableID const&id,Advertisement const&ad)
|
|
{
|
|
bts_assert(me->router.id() == sender);
|
|
bts_assert(me->has_intro);
|
|
|
|
auto &target_ad = *me->pending_send.add_op()->mutable_advertisement();
|
|
target_ad.set_topic(ad.topic.value());
|
|
target_ad.set_table_id(id.value());
|
|
target_ad.set_maintain_synopsis(ad.maintain_synopsis);
|
|
|
|
LOG(QBus,Info, "{} is sending an advertisement to {} ad: {}" ,
|
|
me->my_desc , me->remote_desc,utils::pb::to_string(target_ad));
|
|
|
|
try_send_pending();
|
|
}
|
|
|
|
void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID const&sid, Update const&u)
|
|
{
|
|
bts_assert(me->router.id() == sender);
|
|
bts_assert(me->has_intro);
|
|
|
|
auto &target_notification = *me->pending_send.add_op()->mutable_notification();
|
|
|
|
target_notification.set_table_id(id.value());
|
|
target_notification.set_subscription_id(sid.value());
|
|
|
|
auto &target_update = *target_notification.mutable_update();
|
|
target_update.Reserve(u.size());
|
|
|
|
for (auto &ch : u) {
|
|
|
|
auto *target_change = target_update.Add();
|
|
target_change->set_key(ch.key.value());
|
|
|
|
switch(ch.type) {
|
|
case Change::REMOVE:
|
|
target_change->set_type(msgs::Change::REMOVE);
|
|
break;
|
|
case Change::REPLACE:
|
|
*target_change->mutable_value() = ch.value->value();
|
|
target_change->set_type(msgs::Change::REPLACE);
|
|
break;
|
|
case Change::CREATE:
|
|
*target_change->mutable_value() = ch.value->value();
|
|
target_change->set_type(msgs::Change::CREATE);
|
|
break;
|
|
}
|
|
}
|
|
|
|
LOG(QBus,Diag,"{} is sending a notification to {} table id: {}",
|
|
me->my_desc , me->remote_desc , id.value() ) ;
|
|
try_send_pending();
|
|
}
|
|
|
|
void QBConnection::unadvertise(NodeID const&sender,TableID const&id)
|
|
{
|
|
bts_assert(me->router.id() == sender);
|
|
bts_assert(me->has_intro);
|
|
|
|
auto &target_unadvertise = *me->pending_send.add_op()->mutable_unadvertisement();
|
|
target_unadvertise.set_table_id(id.value());
|
|
|
|
LOG(QBus,Info,"{} is sending a unadvertisement to {} table id: {}",
|
|
me->my_desc , me->remote_desc , id.value()) ;
|
|
|
|
try_send_pending();
|
|
}
|
|
|
|
void QBConnection::subscribe(NodeID const&sender, TableID const&id,SubscriptionID const&sid)
|
|
{
|
|
bts_assert(me->router.id() == sender);
|
|
bts_assert(me->has_intro);
|
|
|
|
auto &target_subscribe = *me->pending_send.add_op()->mutable_subscription();
|
|
|
|
target_subscribe.set_table_id(id.value());
|
|
target_subscribe.set_subscription_id(sid.value());
|
|
|
|
LOG(QBus,Info,"{} is sending a subscription to {} sub: {}",
|
|
me->my_desc , me->remote_desc , utils::pb::to_string(target_subscribe));
|
|
|
|
try_send_pending();
|
|
}
|
|
|
|
void QBConnection::unsubscribe (NodeID const&sender, TableID const&id)
|
|
{
|
|
bts_assert(me->router.id() == sender);
|
|
bts_assert(me->has_intro);
|
|
|
|
auto &target_unsubscribe = *me->pending_send.add_op()->mutable_unsubscription();
|
|
target_unsubscribe.set_table_id(id.value());
|
|
|
|
LOG(QBus,Info,"{} is sending a unsubscription to {} table id: {}",
|
|
me->my_desc , me->remote_desc , id.value());
|
|
|
|
try_send_pending();
|
|
}
|
|
|
|
void QBConnection::close_connection(std::string const&reason)
|
|
{
|
|
if (me->socket.is_open()) {
|
|
LOG(QBus,Warn, "{} is closing its connection to {} reason: {}" ,
|
|
me->my_desc , me->remote_desc , reason);
|
|
me->socket.close();
|
|
if (me->has_intro)
|
|
me->router.remove_neighbour(me->my_id);
|
|
}
|
|
}
|
|
|
|
bool QBConnection::is_send_in_flight() const
|
|
{return me->write_buffer.size() != 0;}
|
|
|
|
bool QBConnection::can_send_now() const
|
|
{return !is_send_in_flight() && me->pending_send.op_size();}
|
|
|
|
void QBConnection::try_send_pending()
|
|
{
|
|
if (can_send_now()) {
|
|
send(me->pending_send);
|
|
me->pending_send.Clear();
|
|
}
|
|
}
|
|
|
|
void QBConnection::send(msgs::QBMsg const&msg)
|
|
{
|
|
auto self = shared_from_this();
|
|
me->last_sent_ts = datetime::now();
|
|
|
|
msg.SerializeToString(&me->write_buffer);
|
|
bts_assert(me->write_buffer.size());
|
|
me->write_buffer_size = me->write_buffer.size();
|
|
|
|
std::array<asio::mutable_buffer,2> to_send;
|
|
to_send[0] = asio::buffer (&(me->write_buffer_size), sizeof(me->write_buffer_size));
|
|
to_send[1] = asio::buffer (&(me->write_buffer[0]), me->write_buffer.size());
|
|
|
|
LOG(QBus,Diag,
|
|
"Sending message from {} to {} size: {}",
|
|
me->my_desc , me->remote_desc , me->write_buffer_size);
|
|
|
|
asio::async_write (me->socket, to_send,
|
|
[this, self](system::error_code error, std::size_t) {
|
|
if (!error) {
|
|
me->write_buffer.clear();
|
|
try_send_pending();
|
|
}
|
|
else
|
|
close_connection(error.message());
|
|
});
|
|
}
|
|
|
|
}}
|
|
|
|
|