Files
spark/src/lib/quickbus/network/QBConnection.cpp
T

456 lines
14 KiB
C++

#include "QBConnection.h"
#include "lib/os/os.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/datetime.h"
#include "lib/logging/logging.h"
#include "lib/utils/bts_assert.h"
#include "lib/quickbus/routing/Router.h"
#include "lib/quickbus/network/endpoint_description.h"
#include "lib/quickbus/core/Advertisement.h"
#include "lib/quickbus/network/quickbus_msgs.pb.h"
#include "lib/config/protos/quickbus_cfg.pb.h"
#include <boost/algorithm/string/join.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
namespace bts {
namespace qbus {
using boost::asio::ip::tcp;
using namespace boost;
struct QBConnection::PImpl
{
PImpl(tcp::socket &s,Router &router,config::QuickbusNodeCfg const&cfg)
: socket(std::move(s)),
router(router),
pending_send(),
write_buffer(),
write_buffer_size(),
read_buffer(),
read_buffer_size(),
has_intro(false),
intro(),
my_id(),
my_desc(local_endpoint_description(socket)),
remote_desc(remote_endpoint_description(socket)),
cfg(cfg),
last_received_ts(datetime::now()),
last_sent_ts(datetime::now()),
hb_timer(socket.get_io_service())
{
write_buffer.reserve(100*1024);
read_buffer.reserve(100*1024);
}
tcp::socket socket;
Router &router;
msgs::QBMsg pending_send;
std::string write_buffer;
int32_t write_buffer_size;
std::string read_buffer;
int32_t read_buffer_size;
bool has_intro;
msgs::Intro intro;
NodeID my_id;
const std::string my_desc;
const std::string remote_desc;
const config::QuickbusNodeCfg cfg;
datetime::PTime last_received_ts;
datetime::PTime last_sent_ts;
boost::asio::deadline_timer hb_timer;
};
QBConnection::QBConnection (tcp::socket s,Router &router, config::QuickbusNodeCfg const&cfg)
:
me(new PImpl(s,router,cfg))
{}
QBConnection::~QBConnection()
{}
void QBConnection::init()
{
send_intro();
recv_intro();
check_hbs();
}
bool QBConnection::connected() const
{return me->socket.is_open();}
void QBConnection::send_intro()
{
LOG(QBus,Info,"Sending intro from {} to {}", me->my_desc , me->remote_desc );
bts_assert(!is_send_in_flight());
auto &intro = *me->pending_send.add_op()->mutable_intro();
intro.set_host_name(os::host_name());
intro.set_app_cmd_line(boost::algorithm::join(os::get_command_line()," "));
intro.set_node_id(me->router.id().value());
intro.set_pid(os::process_id());
try_send_pending();
}
void QBConnection::recv_intro()
{
recv_msg_header();
}
void QBConnection::check_hbs()
{
auto self = shared_from_this();
me->hb_timer.expires_from_now(boost::posix_time::milliseconds(me->cfg.hb_period_msecs()));
me->hb_timer.async_wait([self,this](const boost::system::error_code& ec) {
if (!ec && connected()) {
if (datetime::is_at_least_msecs_old (me->last_received_ts,
me->cfg.hb_period_msecs() *
me->cfg.max_missed_hbs()))
close_connection("No heartbeats received");
else {
if (!is_send_in_flight() &&
datetime::is_at_least_msecs_old (me->last_sent_ts, me->cfg.hb_period_msecs())) {
LOG(QBus,Diag,"{} sending HB to {}", me->my_desc ,me->remote_desc );
me->pending_send.add_op()->mutable_heartbeat();
try_send_pending();
}
check_hbs();
}
}
});
}
void QBConnection::recv_msg_header()
{
auto self = shared_from_this();
asio::async_read (me->socket,
asio::buffer (&me->read_buffer_size, sizeof(me->read_buffer_size)),
[this, self](system::error_code error, std::size_t ) {
if (!error)
recv_msg_body();
else
close_connection(error.message());
});
}
void QBConnection::recv_msg_body()
{
auto self = shared_from_this();
me->read_buffer.resize(me->read_buffer_size);
asio::async_read (me->socket,
asio::buffer (&me->read_buffer[0],me->read_buffer_size),
[this, self](system::error_code error, std::size_t ) {
if (!error) {
msgs::QBMsg m;
auto parse_ok = m.ParseFromArray (&(me->read_buffer[0]),
me->read_buffer.size ());
if (!parse_ok)
close_connection("Corrupt message");
else
handle_msg(&m);
}
else
recv_msg_body();
});
}
void QBConnection::handle_msg(msgs::QBMsg *msgs)
{
me->last_received_ts = datetime::now();
for (auto const&op : msgs->op()) {
if (op.op_case() == msgs::QBOp::kIntro) {
if (me->has_intro) {
close_connection("Received two intro messages");
return;
} else {
LOG(QBus,Info,
"Received intro from {} for {} intro: {} connection is owned by node: {}"
" tcp connection is local {} to remote {}",
me->remote_desc ,me->my_desc ,
utils::pb::to_string(op.intro()), me->router.id().value(),
me->my_desc, me->remote_desc );
me->has_intro = true;
me->intro=op.intro();
me->my_id = NodeID(me->intro.node_id());
if (me->router.is_neighbour(me->my_id)) {
LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" ,
me->my_desc , me->remote_desc , me->my_id.value());
if (me->socket.is_open())
me->socket.close();
return;
}
else {
LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." ,
me->my_desc , me->remote_desc , me->my_id.value());
me->router.add_neighbour(me->my_id,shared_from_this());
}
}
break;
}
if (!me->has_intro) {
close_connection("Expected intro");
return;
}
switch (op.op_case()) {
case msgs::QBOp::kIntro:
bts_assert(false);
break;
case msgs::QBOp::kNotification: {
LOG(QBus,Diag,
"{} received notification from {} for table {} sid: {} size: {}",
me->my_desc , me->remote_desc , op.notification().table_id(),
op.notification().subscription_id(),
op.notification().update().size());
auto &source_update = op.notification().update();
Update target_update;
target_update.resize(source_update.size());
for (int i = 0; i < source_update.size(); i++) {
auto &target_change = target_update[i];
auto &source_change = op.notification().update(i);
target_change.key = RecordKey(source_change.key());
target_change.value = std::make_shared<Record>(std::move(source_change.value()));
switch (source_change.type()) {
case msgs::Change::REMOVE:
target_change.type = Change::REMOVE;
break;
case msgs::Change::REPLACE:
target_change.type = Change::REPLACE;
break;
case msgs::Change::CREATE:
target_change.type = Change::CREATE;
break;
default:
bts_assert(false);
break;
}
}
me->router.notify(me->my_id,
TableID(op.notification().table_id()),
SubscriptionID(op.notification().subscription_id()),target_update);
break;
}
case msgs::QBOp::kAdvertisement: {
LOG(QBus,Info, "{} received advertisement from {} adv: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.advertisement()));
Advertisement ad(Topic(op.advertisement().topic()),op.advertisement().maintain_synopsis());
me->router.advertise(me->my_id, TableID(op.advertisement().table_id()),ad);
break;
}
case msgs::QBOp::kUnadvertisement:
LOG(QBus,Info, "{} received unadvertisement from {} unadv: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.unadvertisement()));
me->router.unadvertise(me->my_id, TableID(op.unadvertisement().table_id()));
break;
case msgs::QBOp::kSubscription:
LOG(QBus,Info, "{} received subscription from {} sub: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.subscription()));
me->router.subscribe(me->my_id, TableID(op.subscription().table_id()),
SubscriptionID(op.subscription().subscription_id()));
break;
case msgs::QBOp::kUnsubscription:
LOG(QBus,Info, "{} received unsubscription from {} unsub: {}" ,
me->my_desc , me->remote_desc , utils::pb::to_string(op.unsubscription()));
me->router.unsubscribe(me->my_id, TableID(op.unsubscription().table_id()));
break;
case msgs::QBOp::kHeartbeat:
LOG(QBus,Diag,"{} received heartbeat from {}", me->my_desc ,me->remote_desc) ;
break;
case msgs::QBOp::OP_NOT_SET:
LOG(QBus,Error,"{} received corrupted message from {}", me->my_desc ,me->remote_desc) ;
close_connection("Corrupted message");
return;
} // switch
} // for
recv_msg_header();
}
void QBConnection::advertise(NodeID const& sender, TableID const&id,Advertisement const&ad)
{
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_ad = *me->pending_send.add_op()->mutable_advertisement();
target_ad.set_topic(ad.topic.value());
target_ad.set_table_id(id.value());
target_ad.set_maintain_synopsis(ad.maintain_synopsis);
LOG(QBus,Info, "{} is sending an advertisement to {} ad: {}" ,
me->my_desc , me->remote_desc,utils::pb::to_string(target_ad));
try_send_pending();
}
void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID const&sid, Update const&u)
{
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_notification = *me->pending_send.add_op()->mutable_notification();
target_notification.set_table_id(id.value());
target_notification.set_subscription_id(sid.value());
auto &target_update = *target_notification.mutable_update();
target_update.Reserve(u.size());
for (auto &ch : u) {
auto *target_change = target_update.Add();
target_change->set_key(ch.key.value());
switch(ch.type) {
case Change::REMOVE:
target_change->set_type(msgs::Change::REMOVE);
break;
case Change::REPLACE:
*target_change->mutable_value() = ch.value->value();
target_change->set_type(msgs::Change::REPLACE);
break;
case Change::CREATE:
*target_change->mutable_value() = ch.value->value();
target_change->set_type(msgs::Change::CREATE);
break;
}
}
LOG(QBus,Diag,"{} is sending a notification to {} table id: {}",
me->my_desc , me->remote_desc , id.value() ) ;
try_send_pending();
}
void QBConnection::unadvertise(NodeID const&sender,TableID const&id)
{
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_unadvertise = *me->pending_send.add_op()->mutable_unadvertisement();
target_unadvertise.set_table_id(id.value());
LOG(QBus,Info,"{} is sending a unadvertisement to {} table id: {}",
me->my_desc , me->remote_desc , id.value()) ;
try_send_pending();
}
void QBConnection::subscribe(NodeID const&sender, TableID const&id,SubscriptionID const&sid)
{
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_subscribe = *me->pending_send.add_op()->mutable_subscription();
target_subscribe.set_table_id(id.value());
target_subscribe.set_subscription_id(sid.value());
LOG(QBus,Info,"{} is sending a subscription to {} sub: {}",
me->my_desc , me->remote_desc , utils::pb::to_string(target_subscribe));
try_send_pending();
}
void QBConnection::unsubscribe (NodeID const&sender, TableID const&id)
{
bts_assert(me->router.id() == sender);
bts_assert(me->has_intro);
auto &target_unsubscribe = *me->pending_send.add_op()->mutable_unsubscription();
target_unsubscribe.set_table_id(id.value());
LOG(QBus,Info,"{} is sending a unsubscription to {} table id: {}",
me->my_desc , me->remote_desc , id.value());
try_send_pending();
}
void QBConnection::close_connection(std::string const&reason)
{
if (me->socket.is_open()) {
LOG(QBus,Warn, "{} is closing its connection to {} reason: {}" ,
me->my_desc , me->remote_desc , reason);
me->socket.close();
if (me->has_intro)
me->router.remove_neighbour(me->my_id);
}
}
bool QBConnection::is_send_in_flight() const
{return me->write_buffer.size() != 0;}
bool QBConnection::can_send_now() const
{return !is_send_in_flight() && me->pending_send.op_size();}
void QBConnection::try_send_pending()
{
if (can_send_now()) {
send(me->pending_send);
me->pending_send.Clear();
}
}
void QBConnection::send(msgs::QBMsg const&msg)
{
auto self = shared_from_this();
me->last_sent_ts = datetime::now();
msg.SerializeToString(&me->write_buffer);
bts_assert(me->write_buffer.size());
me->write_buffer_size = me->write_buffer.size();
std::array<asio::mutable_buffer,2> to_send;
to_send[0] = asio::buffer (&(me->write_buffer_size), sizeof(me->write_buffer_size));
to_send[1] = asio::buffer (&(me->write_buffer[0]), me->write_buffer.size());
LOG(QBus,Diag,
"Sending message from {} to {} size: {}",
me->my_desc , me->remote_desc , me->write_buffer_size);
asio::async_write (me->socket, to_send,
[this, self](system::error_code error, std::size_t) {
if (!error) {
me->write_buffer.clear();
try_send_pending();
}
else
close_connection(error.message());
});
}
}}