Files
spark/src/lib/mercury/broker/CoreEventBroker.cpp
T
Pedro Pinto 8e00da5a72 SPARK-1653, The initial crash that john reported turned out to be a monster.
Events were getting re-ordered in our event queue, possibly due to
this:

https://github.com/cameron314/concurrentqueue/issues/126

We abandoned the moody camel queue.
2018-08-30 21:49:01 +00:00

473 lines
16 KiB
C++

#include "CoreEventBroker.h"
#include "lib/utils/Puid.h"
#include "lib/utils/bts_assert.h"
#include "lib/mercury/synopsis/ISynopsisBuilder.h"
#include "lib/mercury/topics/ITopic.h"
#include <algorithm>
#include <boost/container/flat_map.hpp>
namespace bts {
namespace mercury {
template <class K, class V> using Map = boost::container::flat_map<K, V>;
struct CoreEventBroker::NoReenter {
NoReenter(bool &inside) : inside(inside) {
bts_assert(!inside);
inside = true;
}
~NoReenter() { inside = false; }
bool &inside;
};
struct CoreEventBroker::InternalSubFilter {
TopicFilter matches;
InternalEventHandler handler;
};
struct CoreEventBroker::StreamRec {
StreamRec(CoreEventBroker &owner, IEventBrokerPtr const &src, StreamID sid,
TopicPtr const &topic)
: owner(owner), local(false), src(src), sid(sid), topic(topic),
internal_subs(), external_subs(), sub_id(),
synopsis_builder(topic->make_synopsis_builder()),
topic_tag(topic->to_str()) {}
StreamRec(CoreEventBroker &owner, StreamID sid, TopicPtr const &topic)
: owner(owner), local(true), src(), sid(sid), topic(topic),
internal_subs(), external_subs(), sub_id(),
synopsis_builder(topic->make_synopsis_builder()),
topic_tag(topic->to_str()) {}
~StreamRec() {
for (auto &isub : internal_subs)
isub.on_event(sid, EventType::END_STREAM, nullptr);
}
struct InternalSubRec {
InternalSubID sub_id;
InternalEventHandler on_event;
};
struct SubRec {
SubID sub_id;
IEventBrokerPtr neighbour;
};
CoreEventBroker &owner;
const bool local;
const IEventBrokerPtr src;
const StreamID sid;
const TopicPtr topic;
std::vector<InternalSubRec> internal_subs;
std::vector<SubRec> external_subs;
SubID sub_id;
SynopsisBuilderPtr synopsis_builder;
const std::string topic_tag;
bool is_local() const { return local; }
bool is_active() const {
return is_local() || internal_subs.size() || external_subs.size();
}
EventPayload *get_synopsis() { return synopsis_builder->get_synopsis(); }
void add_external_sub(IEventBrokerPtr const &node, SubID subid) {
auto sender = node->id();
bts_assert(std::find_if(external_subs.begin(), external_subs.end(),
[sender](auto const &sub_rec) {
return sub_rec.neighbour->id() == sender;
}) == external_subs.end());
// record wheter we are already active for this stream
auto was_active = is_active();
// add subscriber to collection
external_subs.push_back(SubRec{subid, node});
// trigger a subscription if appropriate
if (!was_active)
forward_sub();
else {
// we were already active so if we have a synopsis forward it
auto synopsis = synopsis_builder->get_synopsis();
if (synopsis) {
LOG(Mercury, Info, "CoreEventBroker {} sending synopsis for sid: "
"{} to broker {} (sub_id:{} topic: {})",
owner.id().value(), sid.value(), sender.value(), subid.value(),
topic_tag);
node->notify(owner.id(), sid, subid, *synopsis);
} else
LOG(Mercury, Info,
"CoreEventBroker {} does not have synopsis for sid: "
"{} for broker {} (sub_id:{} topic:{})",
owner.id().value(), sid.value(), sender.value(), subid.value(),
topic_tag);
}
}
void remove_external_sub(EventBrokerID sender, SubID subid) {
auto it = std::find_if(external_subs.begin(), external_subs.end(),
[sender](auto const &sub_rec) {
return sub_rec.neighbour->id() == sender;
});
bts_assert(it != external_subs.end());
bts_assert(it->sub_id == subid);
external_subs.erase(it);
if (!is_active())
forward_unsub();
}
void add_internal_sub(InternalEventHandler const &handler,
InternalSubID sub_id) {
auto it =
std::find_if(internal_subs.begin(), internal_subs.end(),
[sub_id](auto &sub) { return sub.sub_id == sub_id; });
bts_assert(it == internal_subs.end());
auto was_active = is_active();
internal_subs.push_back(InternalSubRec{sub_id, handler});
handler(sid, EventType::BEGIN_STREAM, get_synopsis());
if (!was_active)
forward_sub();
}
void remove_internal_sub(InternalSubID sub_id) {
auto it =
std::find_if(internal_subs.begin(), internal_subs.end(),
[sub_id](auto &sub) { return sub.sub_id == sub_id; });
bts_assert(it != internal_subs.end());
it->on_event(sid, EventType::END_SUB, nullptr);
internal_subs.erase(it);
if (!is_active())
forward_unsub();
}
void notify(EventPayload const &payload) {
// notify local subs
for (auto &isub : internal_subs)
isub.on_event(sid, EventType::NEW_EVENT, &payload);
// fwd to remote subs
for (auto &esub : external_subs)
esub.neighbour->notify(owner.id(), sid, esub.sub_id, payload);
// update synopsis
synopsis_builder->update(payload);
}
void forward_sub() {
bts_assert(!is_local());
bts_assert(internal_subs.size() || external_subs.size());
sub_id = owner.gen_external_sub_id();
LOG(Mercury, Info,
"CoreEventBroker {} issuing subscription for topic: {} sid: {} "
"to broker {} (sub_id:{})",
owner.id().value(),topic_tag , sid.value(), src->id().value(),
sub_id.value());
src->subscribe(owner.id(), sub_id, sid);
synopsis_builder->clear_synopsis();
}
void forward_unsub() {
bts_assert(!is_local());
bts_assert(internal_subs.empty() && external_subs.empty());
LOG(Mercury, Info, "CoreEventBroker {} canceling subscription for {} "
"from {} (sub_id: {})",
owner.id().value(), sid.value(), src->id().value(), sub_id.value());
src->unsubscribe(owner.id(), sub_id, sid);
}
};
struct CoreEventBroker::PImpl {
PImpl()
: streams(), neighbours(), internal_subs(),
my_id(utils::gen_id("event_broker")), last_external_sub_id(),
inside(false) {}
Map<StreamID, std::unique_ptr<StreamRec>> streams;
Map<EventBrokerID, IEventBrokerPtr> neighbours;
Map<InternalSubID, InternalSubFilter> internal_subs;
EventBrokerID my_id;
SubID last_external_sub_id;
bool inside;
};
CoreEventBroker::CoreEventBroker() : me(new PImpl()) {}
CoreEventBroker::~CoreEventBroker() {}
EventBrokerID CoreEventBroker::id() const { return me->my_id; }
void CoreEventBroker::add_neighbour(IEventBrokerPtr const &neighbour) {
LOG(Mercury, Info, "CoreEventBroker {} add_neighbour: {}",
me->my_id.value(), neighbour->id().value());
NoReenter nr(me->inside);
auto &n = me->neighbours[neighbour->id()];
bts_assert(!n);
n = neighbour;
for (auto const &s : me->streams)
n->advertise(me->my_id, s.first, s.second->topic);
}
void CoreEventBroker::remove_neighbour(EventBrokerID removed_neighbour_id) {
LOG(Mercury, Info, "CoreEventBroker {} remove_neighbour broker {} ",
id().value(), removed_neighbour_id.value());
NoReenter nr(me->inside);
auto &n_rec = me->neighbours[removed_neighbour_id];
bts_assert(n_rec);
// collect removed neigbhours's streams and subscriptions
std::vector<StreamID> its_streams;
std::vector<std::pair<SubID, StreamID>> its_subscriptions;
for (auto &kvp : me->streams) {
auto &sid = kvp.first;
auto &sr = kvp.second;
// does the removed neighbour own this table?
if (sr->src->id() == removed_neighbour_id)
its_streams.push_back(sid);
else
// the removed node does not own this table, but it might be
// subscribed to it.
for (auto &s : sr->external_subs)
if (s.neighbour->id() == removed_neighbour_id) {
its_subscriptions.push_back(std::make_pair(sr->sub_id, sid));
break;
}
}
// unadvertise orphan streams
for (auto &sid : its_streams) {
LOG(Mercury, Info,
"CoreEventBroker {} sending unadvertisement for stream owned by dead "
"node: {} sid: {}",
id().value(), removed_neighbour_id.value(), sid.value());
_unadvertise(removed_neighbour_id, sid);
}
// cancel the orphan subscriptions
for (auto &p : its_subscriptions) {
auto sub_id = p.first;
auto sid = p.second;
LOG(Mercury, Info,
"CoreEventBroker {} canceling subscription for stream requested by "
"dead node: {} sid: {} (sub_id:{})",
id().value(), removed_neighbour_id.value(), sid.value(),
sub_id.value());
_unsubscribe(removed_neighbour_id, sub_id, sid);
}
// finally remove the node from our index
me->neighbours.erase(removed_neighbour_id);
}
void CoreEventBroker::advertise(EventBrokerID sender, StreamID sid,
TopicPtr const &topic) {
LOG(Mercury, Info,
"CoreEventBroker {} received advertisement from broker {} "
"sid: {} topic: {}",
id().value(), sender.value(), sid.value(), topic->to_str());
NoReenter nr(me->inside);
// create an entry for this stream
auto &srec = me->streams[sid];
bts_assert(!srec);
if (sender != id()) {
// external advertisement, must come from a known neighbour
auto const &node = me->neighbours[sender];
bts_assert(node);
srec = std::make_unique<StreamRec>(*this, node, sid, topic);
} else
srec = std::make_unique<StreamRec>(*this, sid, topic);
// forward to neighbours
for (auto &kvp : me->neighbours)
if (kvp.first != sender) {
LOG(Mercury, Info,
"CoreEventBroker {} forwarding advertisement for stream {} owned "
"by broker {} to: {} sid: {} ",
id().value(), topic->to_str(), sender.value(), kvp.first.value(),
sid.value());
kvp.second->advertise(me->my_id, sid, topic);
}
// match to existing internal subscriptions
bool found = false;
for (auto const &kvp : me->internal_subs) {
auto internal_sub_id = kvp.first;
auto const &f = kvp.second;
if (f.matches(*topic)) {
found = true;
LOG(Mercury, Info,
"CoreEventBroker {} matched internal sub {} to topic {} sid {}",
id().value(), internal_sub_id, topic->to_str(), sid.value());
srec->add_internal_sub(f.handler, internal_sub_id);
}
}
if (!found)
LOG(Mercury, Info,
"CoreEventBroker {} has no internal sub matches to topic {} sid {}",
id().value(), topic->to_str(), sid.value());
}
void CoreEventBroker::unadvertise(EventBrokerID sender, StreamID sid) {
LOG(Mercury, Info,
"CoreEventBroker {} received unadvertisement from broker {} sid: {}",
id().value(), sender.value(), sid.value());
NoReenter nr(me->inside);
_unadvertise(sender, sid);
}
void CoreEventBroker::notify(EventBrokerID sender, StreamID sid, SubID ex_subid,
EventPayload const &payload) {
LOG(Mercury, Diag,
"CoreEventBroker {} received notify from broker {} sid: {} sub_id:{}",
id().value(), sender.value(), sid.value(), ex_subid.value());
NoReenter nr(me->inside);
auto &srec = me->streams[sid];
if (!srec)
LOG(Mercury, Diag,
"Will crash CoreEventBroker {} received notify from broker {} sid: {} sub_id:{}",
id().value(), sender.value(), sid.value(), ex_subid.value());
bts_assert(srec);
bts_assert((srec->is_local() && sender == id()) ||
(!srec->is_local() && srec->src->id() == sender));
if (srec->is_local() || srec->sub_id == ex_subid)
srec->notify(payload);
}
void CoreEventBroker::subscribe(EventBrokerID sender, SubID ex_subid,
StreamID sid) {
LOG(Mercury, Info, "CoreEventBroker {} received subscribe from broker {} "
"sid: {} sub_id:{}",
id().value(), sender.value(), sid.value(), ex_subid.value());
NoReenter nr(me->inside);
bts_assert(sender != id());
auto &node = me->neighbours[sender];
bts_assert(node);
auto it = me->streams.find(sid);
if (it != me->streams.end())
it->second->add_external_sub(node, ex_subid);
else
LOG(Mercury, Info, "CoreEventBroker {} received external subscribe for "
"dead stream {}. ignoring",
id().value(), sid.value());
}
void CoreEventBroker::unsubscribe(EventBrokerID sender, SubID ex_subid,
StreamID sid) {
LOG(Mercury, Info, "CoreEventBroker {} received unsubscribe from broker {} "
"sid: {} sub_id:{}",
id().value(), sender.value(), sid.value(), ex_subid.value());
NoReenter nr(me->inside);
_unsubscribe(sender, ex_subid, sid);
}
void CoreEventBroker::internal_subscribe(InternalSubID in_subid,
TopicFilter const &f,
InternalEventHandler const &cb) {
LOG(Mercury, Info,
"CoreEventBroker {} received internal_subscribe (sub_id: {})",
id().value(), in_subid.value());
NoReenter nr(me->inside);
// no duplicates
bts_assert(me->internal_subs.find(in_subid) == me->internal_subs.end());
// add to internal sub list
auto &sf = me->internal_subs[in_subid];
sf.matches = f;
sf.handler = cb;
// cross check new sub with all known streams, and trigger sub/synopsis
// update if appropriate
bool found = false;
for (auto &kvp : me->streams) {
auto &srec = kvp.second;
if (sf.matches(*srec->topic)) {
found = true;
LOG(Mercury, Info,
"CoreEventBroker {} matched internal sub {} to topic {} sid {}",
id().value(), in_subid, srec->topic->to_str(), kvp.first.value());
srec->add_internal_sub(cb, in_subid);
}
}
if (!found)
LOG(Mercury, Info,
"CoreEventBroker {} found no topics to match internal sub {}",
id().value(), in_subid.value());
}
void CoreEventBroker::internal_unsubscribe(InternalSubID in_subid) {
LOG(Mercury, Info,
"CoreEventBroker {} received internal_unsubscribe (sub_id: {})",
id().value(), in_subid.value());
NoReenter nr(me->inside);
// must be a previously entered sub
auto it = me->internal_subs.find(in_subid);
bts_assert(it != me->internal_subs.end());
auto &matches = it->second.matches;
// for each stream that matched this filter remove the corresponding internal
// sub
for (auto &kvp : me->streams) {
auto &srec = kvp.second;
if (matches(*srec->topic))
srec->remove_internal_sub(in_subid);
}
}
SubID CoreEventBroker::gen_external_sub_id() {
me->last_external_sub_id = SubID(me->last_external_sub_id.value() + 1);
return me->last_external_sub_id;
}
void CoreEventBroker::_unadvertise(EventBrokerID sender, StreamID sid) {
auto it = me->streams.find(sid);
bts_assert(it != me->streams.end());
auto &srec = it->second;
bts_assert((sender == id() && srec->is_local()) ||
(!srec->is_local() && sender == srec->src->id()));
auto topic_str = srec->topic->to_str();
// remove from our index
me->streams.erase(it);
for (auto &kvp : me->neighbours)
if (kvp.first != sender) {
LOG(Mercury, Info, "CoreEventBroker {} forwarding unadvertisement for "
"stream {} owned "
"by broker {} to: {} sid: {}",
id().value(), topic_str, sender.value(), kvp.first.value(),
sid.value());
kvp.second->unadvertise(me->my_id, sid);
}
}
void CoreEventBroker::_unsubscribe(EventBrokerID sender, SubID sub_id,
StreamID sid) {
auto it = me->streams.find(sid);
if (it != me->streams.end()) {
auto &srec = it->second;
srec->remove_external_sub(sender, sub_id);
}
}
}
}