#include "QBConnection.h" #include "lib/os/os.h" #include "lib/utils/PBUtils.h" #include "lib/utils/datetime.h" #include "lib/logging/logging.h" #include "lib/utils/bts_assert.h" #include "lib/quickbus/routing/Router.h" #include "lib/quickbus/network/endpoint_description.h" #include "lib/quickbus/core/Advertisement.h" #include "lib/quickbus/network/quickbus_msgs.pb.h" #include "lib/config/protos/quickbus_cfg.pb.h" #include #include namespace bts { namespace qbus { using boost::asio::ip::tcp; using namespace boost; struct QBConnection::PImpl { PImpl(tcp::socket &s,Router &router,config::QuickbusNodeCfg const&cfg) : socket(std::move(s)), router(router), pending_send(), write_buffer(), write_buffer_size(), read_buffer(), read_buffer_size(), has_intro(false), intro(), my_id(), my_desc(local_endpoint_description(socket)), remote_desc(remote_endpoint_description(socket)), cfg(cfg), last_received_ts(datetime::now()), last_sent_ts(datetime::now()), hb_timer(socket.get_io_service()) { write_buffer.reserve(100*1024); read_buffer.reserve(100*1024); } tcp::socket socket; Router &router; msgs::QBMsg pending_send; std::string write_buffer; int32_t write_buffer_size; std::string read_buffer; int32_t read_buffer_size; bool has_intro; msgs::Intro intro; NodeID my_id; const std::string my_desc; const std::string remote_desc; const config::QuickbusNodeCfg cfg; datetime::PTime last_received_ts; datetime::PTime last_sent_ts; boost::asio::deadline_timer hb_timer; }; QBConnection::QBConnection (tcp::socket s,Router &router, config::QuickbusNodeCfg const&cfg) : me(new PImpl(s,router,cfg)) {} QBConnection::~QBConnection() {} void QBConnection::init() { send_intro(); recv_intro(); check_hbs(); } bool QBConnection::connected() const {return me->socket.is_open();} void QBConnection::send_intro() { LOG(QBus,Info,"Sending intro from {} to {}", me->my_desc , me->remote_desc ); bts_assert(!is_send_in_flight()); auto &intro = *me->pending_send.add_op()->mutable_intro(); intro.set_host_name(os::host_name()); intro.set_app_cmd_line(boost::algorithm::join(os::get_command_line()," ")); intro.set_node_id(me->router.id().value()); intro.set_pid(os::process_id()); try_send_pending(); } void QBConnection::recv_intro() { recv_msg_header(); } void QBConnection::check_hbs() { auto self = shared_from_this(); me->hb_timer.expires_from_now(boost::posix_time::milliseconds(me->cfg.hb_period_msecs())); me->hb_timer.async_wait([self,this](const boost::system::error_code& ec) { if (!ec && connected()) { if (datetime::is_at_least_msecs_old (me->last_received_ts, me->cfg.hb_period_msecs() * me->cfg.max_missed_hbs())) close_connection("No heartbeats received"); else { if (!is_send_in_flight() && datetime::is_at_least_msecs_old (me->last_sent_ts, me->cfg.hb_period_msecs())) { LOG(QBus,Diag,"{} sending HB to {}", me->my_desc ,me->remote_desc ); me->pending_send.add_op()->mutable_heartbeat(); try_send_pending(); } check_hbs(); } } }); } void QBConnection::recv_msg_header() { auto self = shared_from_this(); asio::async_read (me->socket, asio::buffer (&me->read_buffer_size, sizeof(me->read_buffer_size)), [this, self](system::error_code error, std::size_t ) { if (!error) recv_msg_body(); else close_connection(error.message()); }); } void QBConnection::recv_msg_body() { auto self = shared_from_this(); me->read_buffer.resize(me->read_buffer_size); asio::async_read (me->socket, asio::buffer (&me->read_buffer[0],me->read_buffer_size), [this, self](system::error_code error, std::size_t ) { if (!error) { msgs::QBMsg m; auto parse_ok = m.ParseFromArray (&(me->read_buffer[0]), me->read_buffer.size ()); if (!parse_ok) close_connection("Corrupt message"); else handle_msg(&m); } else recv_msg_body(); }); } void QBConnection::handle_msg(msgs::QBMsg *msgs) { me->last_received_ts = datetime::now(); for (auto const&op : msgs->op()) { if (op.op_case() == msgs::QBOp::kIntro) { if (me->has_intro) { close_connection("Received two intro messages"); return; } else { LOG(QBus,Info, "Received intro from {} for {} intro: {} connection is owned by node: {}" " tcp connection is local {} to remote {}", me->remote_desc ,me->my_desc , utils::pb::to_string(op.intro()), me->router.id().value(), me->my_desc, me->remote_desc ); me->has_intro = true; me->intro=op.intro(); me->my_id = NodeID(me->intro.node_id()); if (me->router.is_neighbour(me->my_id)) { LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" , me->my_desc , me->remote_desc , me->my_id.value()); if (me->socket.is_open()) me->socket.close(); return; } else { LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." , me->my_desc , me->remote_desc , me->my_id.value()); me->router.add_neighbour(me->my_id,shared_from_this()); } } break; } if (!me->has_intro) { close_connection("Expected intro"); return; } switch (op.op_case()) { case msgs::QBOp::kIntro: bts_assert(false); break; case msgs::QBOp::kNotification: { LOG(QBus,Diag, "{} received notification from {} for table {} sid: {} size: {}", me->my_desc , me->remote_desc , op.notification().table_id(), op.notification().subscription_id(), op.notification().update().size()); auto &source_update = op.notification().update(); Update target_update; target_update.resize(source_update.size()); for (int i = 0; i < source_update.size(); i++) { auto &target_change = target_update[i]; auto &source_change = op.notification().update(i); target_change.key = RecordKey(source_change.key()); target_change.value = std::make_shared(std::move(source_change.value())); switch (source_change.type()) { case msgs::Change::REMOVE: target_change.type = Change::REMOVE; break; case msgs::Change::REPLACE: target_change.type = Change::REPLACE; break; case msgs::Change::CREATE: target_change.type = Change::CREATE; break; default: bts_assert(false); break; } } me->router.notify(me->my_id, TableID(op.notification().table_id()), SubscriptionID(op.notification().subscription_id()),target_update); break; } case msgs::QBOp::kAdvertisement: { LOG(QBus,Info, "{} received advertisement from {} adv: {}" , me->my_desc , me->remote_desc , utils::pb::to_string(op.advertisement())); Advertisement ad(Topic(op.advertisement().topic()),op.advertisement().maintain_synopsis()); me->router.advertise(me->my_id, TableID(op.advertisement().table_id()),ad); break; } case msgs::QBOp::kUnadvertisement: LOG(QBus,Info, "{} received unadvertisement from {} unadv: {}" , me->my_desc , me->remote_desc , utils::pb::to_string(op.unadvertisement())); me->router.unadvertise(me->my_id, TableID(op.unadvertisement().table_id())); break; case msgs::QBOp::kSubscription: LOG(QBus,Info, "{} received subscription from {} sub: {}" , me->my_desc , me->remote_desc , utils::pb::to_string(op.subscription())); me->router.subscribe(me->my_id, TableID(op.subscription().table_id()), SubscriptionID(op.subscription().subscription_id())); break; case msgs::QBOp::kUnsubscription: LOG(QBus,Info, "{} received unsubscription from {} unsub: {}" , me->my_desc , me->remote_desc , utils::pb::to_string(op.unsubscription())); me->router.unsubscribe(me->my_id, TableID(op.unsubscription().table_id())); break; case msgs::QBOp::kHeartbeat: LOG(QBus,Diag,"{} received heartbeat from {}", me->my_desc ,me->remote_desc) ; break; case msgs::QBOp::OP_NOT_SET: LOG(QBus,Error,"{} received corrupted message from {}", me->my_desc ,me->remote_desc) ; close_connection("Corrupted message"); return; } // switch } // for recv_msg_header(); } void QBConnection::advertise(NodeID const& sender, TableID const&id,Advertisement const&ad) { bts_assert(me->router.id() == sender); bts_assert(me->has_intro); auto &target_ad = *me->pending_send.add_op()->mutable_advertisement(); target_ad.set_topic(ad.topic.value()); target_ad.set_table_id(id.value()); target_ad.set_maintain_synopsis(ad.maintain_synopsis); LOG(QBus,Info, "{} is sending an advertisement to {} ad: {}" , me->my_desc , me->remote_desc,utils::pb::to_string(target_ad)); try_send_pending(); } void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID const&sid, Update const&u) { bts_assert(me->router.id() == sender); bts_assert(me->has_intro); auto &target_notification = *me->pending_send.add_op()->mutable_notification(); target_notification.set_table_id(id.value()); target_notification.set_subscription_id(sid.value()); auto &target_update = *target_notification.mutable_update(); target_update.Reserve(u.size()); for (auto &ch : u) { auto *target_change = target_update.Add(); target_change->set_key(ch.key.value()); switch(ch.type) { case Change::REMOVE: target_change->set_type(msgs::Change::REMOVE); break; case Change::REPLACE: *target_change->mutable_value() = ch.value->value(); target_change->set_type(msgs::Change::REPLACE); break; case Change::CREATE: *target_change->mutable_value() = ch.value->value(); target_change->set_type(msgs::Change::CREATE); break; } } LOG(QBus,Diag,"{} is sending a notification to {} table id: {}", me->my_desc , me->remote_desc , id.value() ) ; try_send_pending(); } void QBConnection::unadvertise(NodeID const&sender,TableID const&id) { bts_assert(me->router.id() == sender); bts_assert(me->has_intro); auto &target_unadvertise = *me->pending_send.add_op()->mutable_unadvertisement(); target_unadvertise.set_table_id(id.value()); LOG(QBus,Info,"{} is sending a unadvertisement to {} table id: {}", me->my_desc , me->remote_desc , id.value()) ; try_send_pending(); } void QBConnection::subscribe(NodeID const&sender, TableID const&id,SubscriptionID const&sid) { bts_assert(me->router.id() == sender); bts_assert(me->has_intro); auto &target_subscribe = *me->pending_send.add_op()->mutable_subscription(); target_subscribe.set_table_id(id.value()); target_subscribe.set_subscription_id(sid.value()); LOG(QBus,Info,"{} is sending a subscription to {} sub: {}", me->my_desc , me->remote_desc , utils::pb::to_string(target_subscribe)); try_send_pending(); } void QBConnection::unsubscribe (NodeID const&sender, TableID const&id) { bts_assert(me->router.id() == sender); bts_assert(me->has_intro); auto &target_unsubscribe = *me->pending_send.add_op()->mutable_unsubscription(); target_unsubscribe.set_table_id(id.value()); LOG(QBus,Info,"{} is sending a unsubscription to {} table id: {}", me->my_desc , me->remote_desc , id.value()); try_send_pending(); } void QBConnection::close_connection(std::string const&reason) { if (me->socket.is_open()) { LOG(QBus,Warn, "{} is closing its connection to {} reason: {}" , me->my_desc , me->remote_desc , reason); me->socket.close(); if (me->has_intro) me->router.remove_neighbour(me->my_id); } } bool QBConnection::is_send_in_flight() const {return me->write_buffer.size() != 0;} bool QBConnection::can_send_now() const {return !is_send_in_flight() && me->pending_send.op_size();} void QBConnection::try_send_pending() { if (can_send_now()) { send(me->pending_send); me->pending_send.Clear(); } } void QBConnection::send(msgs::QBMsg const&msg) { auto self = shared_from_this(); me->last_sent_ts = datetime::now(); msg.SerializeToString(&me->write_buffer); bts_assert(me->write_buffer.size()); me->write_buffer_size = me->write_buffer.size(); std::array to_send; to_send[0] = asio::buffer (&(me->write_buffer_size), sizeof(me->write_buffer_size)); to_send[1] = asio::buffer (&(me->write_buffer[0]), me->write_buffer.size()); LOG(QBus,Diag, "Sending message from {} to {} size: {}", me->my_desc , me->remote_desc , me->write_buffer_size); asio::async_write (me->socket, to_send, [this, self](system::error_code error, std::size_t) { if (!error) { me->write_buffer.clear(); try_send_pending(); } else close_connection(error.message()); }); } }}