Events were getting re-ordered in our event queue, possibly due to this: https://github.com/cameron314/concurrentqueue/issues/126 We abandoned the moody camel queue.
473 lines
16 KiB
C++
473 lines
16 KiB
C++
#include "CoreEventBroker.h"
|
|
|
|
#include "lib/utils/Puid.h"
|
|
#include "lib/utils/bts_assert.h"
|
|
#include "lib/mercury/synopsis/ISynopsisBuilder.h"
|
|
#include "lib/mercury/topics/ITopic.h"
|
|
|
|
#include <algorithm>
|
|
#include <boost/container/flat_map.hpp>
|
|
|
|
namespace bts {
|
|
namespace mercury {
|
|
|
|
template <class K, class V> using Map = boost::container::flat_map<K, V>;
|
|
|
|
struct CoreEventBroker::NoReenter {
|
|
NoReenter(bool &inside) : inside(inside) {
|
|
bts_assert(!inside);
|
|
inside = true;
|
|
}
|
|
~NoReenter() { inside = false; }
|
|
bool &inside;
|
|
};
|
|
|
|
struct CoreEventBroker::InternalSubFilter {
|
|
TopicFilter matches;
|
|
InternalEventHandler handler;
|
|
};
|
|
|
|
struct CoreEventBroker::StreamRec {
|
|
StreamRec(CoreEventBroker &owner, IEventBrokerPtr const &src, StreamID sid,
|
|
TopicPtr const &topic)
|
|
: owner(owner), local(false), src(src), sid(sid), topic(topic),
|
|
internal_subs(), external_subs(), sub_id(),
|
|
synopsis_builder(topic->make_synopsis_builder()),
|
|
topic_tag(topic->to_str()) {}
|
|
|
|
StreamRec(CoreEventBroker &owner, StreamID sid, TopicPtr const &topic)
|
|
: owner(owner), local(true), src(), sid(sid), topic(topic),
|
|
internal_subs(), external_subs(), sub_id(),
|
|
synopsis_builder(topic->make_synopsis_builder()),
|
|
topic_tag(topic->to_str()) {}
|
|
|
|
~StreamRec() {
|
|
for (auto &isub : internal_subs)
|
|
isub.on_event(sid, EventType::END_STREAM, nullptr);
|
|
}
|
|
|
|
struct InternalSubRec {
|
|
InternalSubID sub_id;
|
|
InternalEventHandler on_event;
|
|
};
|
|
|
|
struct SubRec {
|
|
SubID sub_id;
|
|
IEventBrokerPtr neighbour;
|
|
};
|
|
|
|
CoreEventBroker &owner;
|
|
const bool local;
|
|
const IEventBrokerPtr src;
|
|
const StreamID sid;
|
|
const TopicPtr topic;
|
|
|
|
std::vector<InternalSubRec> internal_subs;
|
|
std::vector<SubRec> external_subs;
|
|
|
|
SubID sub_id;
|
|
SynopsisBuilderPtr synopsis_builder;
|
|
const std::string topic_tag;
|
|
|
|
bool is_local() const { return local; }
|
|
bool is_active() const {
|
|
return is_local() || internal_subs.size() || external_subs.size();
|
|
}
|
|
EventPayload *get_synopsis() { return synopsis_builder->get_synopsis(); }
|
|
|
|
void add_external_sub(IEventBrokerPtr const &node, SubID subid) {
|
|
auto sender = node->id();
|
|
bts_assert(std::find_if(external_subs.begin(), external_subs.end(),
|
|
[sender](auto const &sub_rec) {
|
|
return sub_rec.neighbour->id() == sender;
|
|
}) == external_subs.end());
|
|
|
|
// record wheter we are already active for this stream
|
|
auto was_active = is_active();
|
|
|
|
// add subscriber to collection
|
|
external_subs.push_back(SubRec{subid, node});
|
|
|
|
// trigger a subscription if appropriate
|
|
if (!was_active)
|
|
forward_sub();
|
|
else {
|
|
// we were already active so if we have a synopsis forward it
|
|
auto synopsis = synopsis_builder->get_synopsis();
|
|
if (synopsis) {
|
|
LOG(Mercury, Info, "CoreEventBroker {} sending synopsis for sid: "
|
|
"{} to broker {} (sub_id:{} topic: {})",
|
|
owner.id().value(), sid.value(), sender.value(), subid.value(),
|
|
topic_tag);
|
|
node->notify(owner.id(), sid, subid, *synopsis);
|
|
} else
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} does not have synopsis for sid: "
|
|
"{} for broker {} (sub_id:{} topic:{})",
|
|
owner.id().value(), sid.value(), sender.value(), subid.value(),
|
|
topic_tag);
|
|
}
|
|
}
|
|
|
|
void remove_external_sub(EventBrokerID sender, SubID subid) {
|
|
auto it = std::find_if(external_subs.begin(), external_subs.end(),
|
|
[sender](auto const &sub_rec) {
|
|
return sub_rec.neighbour->id() == sender;
|
|
});
|
|
bts_assert(it != external_subs.end());
|
|
bts_assert(it->sub_id == subid);
|
|
external_subs.erase(it);
|
|
if (!is_active())
|
|
forward_unsub();
|
|
}
|
|
|
|
void add_internal_sub(InternalEventHandler const &handler,
|
|
InternalSubID sub_id) {
|
|
auto it =
|
|
std::find_if(internal_subs.begin(), internal_subs.end(),
|
|
[sub_id](auto &sub) { return sub.sub_id == sub_id; });
|
|
bts_assert(it == internal_subs.end());
|
|
auto was_active = is_active();
|
|
internal_subs.push_back(InternalSubRec{sub_id, handler});
|
|
handler(sid, EventType::BEGIN_STREAM, get_synopsis());
|
|
if (!was_active)
|
|
forward_sub();
|
|
}
|
|
|
|
void remove_internal_sub(InternalSubID sub_id) {
|
|
auto it =
|
|
std::find_if(internal_subs.begin(), internal_subs.end(),
|
|
[sub_id](auto &sub) { return sub.sub_id == sub_id; });
|
|
bts_assert(it != internal_subs.end());
|
|
it->on_event(sid, EventType::END_SUB, nullptr);
|
|
internal_subs.erase(it);
|
|
if (!is_active())
|
|
forward_unsub();
|
|
}
|
|
|
|
void notify(EventPayload const &payload) {
|
|
// notify local subs
|
|
for (auto &isub : internal_subs)
|
|
isub.on_event(sid, EventType::NEW_EVENT, &payload);
|
|
|
|
// fwd to remote subs
|
|
for (auto &esub : external_subs)
|
|
esub.neighbour->notify(owner.id(), sid, esub.sub_id, payload);
|
|
|
|
// update synopsis
|
|
synopsis_builder->update(payload);
|
|
}
|
|
|
|
void forward_sub() {
|
|
bts_assert(!is_local());
|
|
bts_assert(internal_subs.size() || external_subs.size());
|
|
|
|
sub_id = owner.gen_external_sub_id();
|
|
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} issuing subscription for topic: {} sid: {} "
|
|
"to broker {} (sub_id:{})",
|
|
owner.id().value(),topic_tag , sid.value(), src->id().value(),
|
|
sub_id.value());
|
|
|
|
src->subscribe(owner.id(), sub_id, sid);
|
|
synopsis_builder->clear_synopsis();
|
|
}
|
|
|
|
void forward_unsub() {
|
|
bts_assert(!is_local());
|
|
bts_assert(internal_subs.empty() && external_subs.empty());
|
|
LOG(Mercury, Info, "CoreEventBroker {} canceling subscription for {} "
|
|
"from {} (sub_id: {})",
|
|
owner.id().value(), sid.value(), src->id().value(), sub_id.value());
|
|
src->unsubscribe(owner.id(), sub_id, sid);
|
|
}
|
|
};
|
|
|
|
struct CoreEventBroker::PImpl {
|
|
PImpl()
|
|
: streams(), neighbours(), internal_subs(),
|
|
my_id(utils::gen_id("event_broker")), last_external_sub_id(),
|
|
inside(false) {}
|
|
|
|
Map<StreamID, std::unique_ptr<StreamRec>> streams;
|
|
Map<EventBrokerID, IEventBrokerPtr> neighbours;
|
|
Map<InternalSubID, InternalSubFilter> internal_subs;
|
|
EventBrokerID my_id;
|
|
SubID last_external_sub_id;
|
|
bool inside;
|
|
};
|
|
|
|
CoreEventBroker::CoreEventBroker() : me(new PImpl()) {}
|
|
CoreEventBroker::~CoreEventBroker() {}
|
|
|
|
EventBrokerID CoreEventBroker::id() const { return me->my_id; }
|
|
|
|
void CoreEventBroker::add_neighbour(IEventBrokerPtr const &neighbour) {
|
|
LOG(Mercury, Info, "CoreEventBroker {} add_neighbour: {}",
|
|
me->my_id.value(), neighbour->id().value());
|
|
NoReenter nr(me->inside);
|
|
|
|
auto &n = me->neighbours[neighbour->id()];
|
|
bts_assert(!n);
|
|
n = neighbour;
|
|
|
|
for (auto const &s : me->streams)
|
|
n->advertise(me->my_id, s.first, s.second->topic);
|
|
}
|
|
|
|
void CoreEventBroker::remove_neighbour(EventBrokerID removed_neighbour_id) {
|
|
LOG(Mercury, Info, "CoreEventBroker {} remove_neighbour broker {} ",
|
|
id().value(), removed_neighbour_id.value());
|
|
NoReenter nr(me->inside);
|
|
|
|
auto &n_rec = me->neighbours[removed_neighbour_id];
|
|
bts_assert(n_rec);
|
|
|
|
// collect removed neigbhours's streams and subscriptions
|
|
std::vector<StreamID> its_streams;
|
|
std::vector<std::pair<SubID, StreamID>> its_subscriptions;
|
|
for (auto &kvp : me->streams) {
|
|
auto &sid = kvp.first;
|
|
auto &sr = kvp.second;
|
|
|
|
// does the removed neighbour own this table?
|
|
if (sr->src->id() == removed_neighbour_id)
|
|
its_streams.push_back(sid);
|
|
else
|
|
// the removed node does not own this table, but it might be
|
|
// subscribed to it.
|
|
for (auto &s : sr->external_subs)
|
|
if (s.neighbour->id() == removed_neighbour_id) {
|
|
its_subscriptions.push_back(std::make_pair(sr->sub_id, sid));
|
|
break;
|
|
}
|
|
}
|
|
|
|
// unadvertise orphan streams
|
|
for (auto &sid : its_streams) {
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} sending unadvertisement for stream owned by dead "
|
|
"node: {} sid: {}",
|
|
id().value(), removed_neighbour_id.value(), sid.value());
|
|
_unadvertise(removed_neighbour_id, sid);
|
|
}
|
|
|
|
// cancel the orphan subscriptions
|
|
for (auto &p : its_subscriptions) {
|
|
auto sub_id = p.first;
|
|
auto sid = p.second;
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} canceling subscription for stream requested by "
|
|
"dead node: {} sid: {} (sub_id:{})",
|
|
id().value(), removed_neighbour_id.value(), sid.value(),
|
|
sub_id.value());
|
|
_unsubscribe(removed_neighbour_id, sub_id, sid);
|
|
}
|
|
|
|
// finally remove the node from our index
|
|
me->neighbours.erase(removed_neighbour_id);
|
|
}
|
|
|
|
void CoreEventBroker::advertise(EventBrokerID sender, StreamID sid,
|
|
TopicPtr const &topic) {
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} received advertisement from broker {} "
|
|
"sid: {} topic: {}",
|
|
id().value(), sender.value(), sid.value(), topic->to_str());
|
|
NoReenter nr(me->inside);
|
|
|
|
// create an entry for this stream
|
|
auto &srec = me->streams[sid];
|
|
bts_assert(!srec);
|
|
|
|
if (sender != id()) {
|
|
// external advertisement, must come from a known neighbour
|
|
auto const &node = me->neighbours[sender];
|
|
bts_assert(node);
|
|
srec = std::make_unique<StreamRec>(*this, node, sid, topic);
|
|
} else
|
|
srec = std::make_unique<StreamRec>(*this, sid, topic);
|
|
|
|
// forward to neighbours
|
|
for (auto &kvp : me->neighbours)
|
|
if (kvp.first != sender) {
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} forwarding advertisement for stream {} owned "
|
|
"by broker {} to: {} sid: {} ",
|
|
id().value(), topic->to_str(), sender.value(), kvp.first.value(),
|
|
sid.value());
|
|
kvp.second->advertise(me->my_id, sid, topic);
|
|
}
|
|
|
|
// match to existing internal subscriptions
|
|
bool found = false;
|
|
for (auto const &kvp : me->internal_subs) {
|
|
auto internal_sub_id = kvp.first;
|
|
auto const &f = kvp.second;
|
|
if (f.matches(*topic)) {
|
|
found = true;
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} matched internal sub {} to topic {} sid {}",
|
|
id().value(), internal_sub_id, topic->to_str(), sid.value());
|
|
srec->add_internal_sub(f.handler, internal_sub_id);
|
|
}
|
|
}
|
|
if (!found)
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} has no internal sub matches to topic {} sid {}",
|
|
id().value(), topic->to_str(), sid.value());
|
|
}
|
|
|
|
void CoreEventBroker::unadvertise(EventBrokerID sender, StreamID sid) {
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} received unadvertisement from broker {} sid: {}",
|
|
id().value(), sender.value(), sid.value());
|
|
NoReenter nr(me->inside);
|
|
_unadvertise(sender, sid);
|
|
}
|
|
|
|
void CoreEventBroker::notify(EventBrokerID sender, StreamID sid, SubID ex_subid,
|
|
EventPayload const &payload) {
|
|
LOG(Mercury, Diag,
|
|
"CoreEventBroker {} received notify from broker {} sid: {} sub_id:{}",
|
|
id().value(), sender.value(), sid.value(), ex_subid.value());
|
|
NoReenter nr(me->inside);
|
|
|
|
auto &srec = me->streams[sid];
|
|
if (!srec)
|
|
LOG(Mercury, Diag,
|
|
"Will crash CoreEventBroker {} received notify from broker {} sid: {} sub_id:{}",
|
|
id().value(), sender.value(), sid.value(), ex_subid.value());
|
|
|
|
bts_assert(srec);
|
|
bts_assert((srec->is_local() && sender == id()) ||
|
|
(!srec->is_local() && srec->src->id() == sender));
|
|
if (srec->is_local() || srec->sub_id == ex_subid)
|
|
srec->notify(payload);
|
|
}
|
|
|
|
void CoreEventBroker::subscribe(EventBrokerID sender, SubID ex_subid,
|
|
StreamID sid) {
|
|
LOG(Mercury, Info, "CoreEventBroker {} received subscribe from broker {} "
|
|
"sid: {} sub_id:{}",
|
|
id().value(), sender.value(), sid.value(), ex_subid.value());
|
|
NoReenter nr(me->inside);
|
|
|
|
bts_assert(sender != id());
|
|
auto &node = me->neighbours[sender];
|
|
bts_assert(node);
|
|
|
|
auto it = me->streams.find(sid);
|
|
if (it != me->streams.end())
|
|
it->second->add_external_sub(node, ex_subid);
|
|
else
|
|
LOG(Mercury, Info, "CoreEventBroker {} received external subscribe for "
|
|
"dead stream {}. ignoring",
|
|
id().value(), sid.value());
|
|
}
|
|
|
|
void CoreEventBroker::unsubscribe(EventBrokerID sender, SubID ex_subid,
|
|
StreamID sid) {
|
|
LOG(Mercury, Info, "CoreEventBroker {} received unsubscribe from broker {} "
|
|
"sid: {} sub_id:{}",
|
|
id().value(), sender.value(), sid.value(), ex_subid.value());
|
|
NoReenter nr(me->inside);
|
|
_unsubscribe(sender, ex_subid, sid);
|
|
}
|
|
|
|
void CoreEventBroker::internal_subscribe(InternalSubID in_subid,
|
|
TopicFilter const &f,
|
|
InternalEventHandler const &cb) {
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} received internal_subscribe (sub_id: {})",
|
|
id().value(), in_subid.value());
|
|
NoReenter nr(me->inside);
|
|
|
|
// no duplicates
|
|
bts_assert(me->internal_subs.find(in_subid) == me->internal_subs.end());
|
|
|
|
// add to internal sub list
|
|
auto &sf = me->internal_subs[in_subid];
|
|
sf.matches = f;
|
|
sf.handler = cb;
|
|
|
|
// cross check new sub with all known streams, and trigger sub/synopsis
|
|
// update if appropriate
|
|
bool found = false;
|
|
for (auto &kvp : me->streams) {
|
|
auto &srec = kvp.second;
|
|
if (sf.matches(*srec->topic)) {
|
|
found = true;
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} matched internal sub {} to topic {} sid {}",
|
|
id().value(), in_subid, srec->topic->to_str(), kvp.first.value());
|
|
srec->add_internal_sub(cb, in_subid);
|
|
}
|
|
}
|
|
if (!found)
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} found no topics to match internal sub {}",
|
|
id().value(), in_subid.value());
|
|
}
|
|
|
|
void CoreEventBroker::internal_unsubscribe(InternalSubID in_subid) {
|
|
LOG(Mercury, Info,
|
|
"CoreEventBroker {} received internal_unsubscribe (sub_id: {})",
|
|
id().value(), in_subid.value());
|
|
NoReenter nr(me->inside);
|
|
|
|
// must be a previously entered sub
|
|
auto it = me->internal_subs.find(in_subid);
|
|
bts_assert(it != me->internal_subs.end());
|
|
auto &matches = it->second.matches;
|
|
|
|
// for each stream that matched this filter remove the corresponding internal
|
|
// sub
|
|
for (auto &kvp : me->streams) {
|
|
auto &srec = kvp.second;
|
|
if (matches(*srec->topic))
|
|
srec->remove_internal_sub(in_subid);
|
|
}
|
|
}
|
|
|
|
SubID CoreEventBroker::gen_external_sub_id() {
|
|
me->last_external_sub_id = SubID(me->last_external_sub_id.value() + 1);
|
|
return me->last_external_sub_id;
|
|
}
|
|
|
|
void CoreEventBroker::_unadvertise(EventBrokerID sender, StreamID sid) {
|
|
auto it = me->streams.find(sid);
|
|
bts_assert(it != me->streams.end());
|
|
|
|
auto &srec = it->second;
|
|
bts_assert((sender == id() && srec->is_local()) ||
|
|
(!srec->is_local() && sender == srec->src->id()));
|
|
|
|
auto topic_str = srec->topic->to_str();
|
|
|
|
// remove from our index
|
|
me->streams.erase(it);
|
|
|
|
for (auto &kvp : me->neighbours)
|
|
if (kvp.first != sender) {
|
|
LOG(Mercury, Info, "CoreEventBroker {} forwarding unadvertisement for "
|
|
"stream {} owned "
|
|
"by broker {} to: {} sid: {}",
|
|
id().value(), topic_str, sender.value(), kvp.first.value(),
|
|
sid.value());
|
|
kvp.second->unadvertise(me->my_id, sid);
|
|
}
|
|
}
|
|
|
|
void CoreEventBroker::_unsubscribe(EventBrokerID sender, SubID sub_id,
|
|
StreamID sid) {
|
|
auto it = me->streams.find(sid);
|
|
if (it != me->streams.end()) {
|
|
auto &srec = it->second;
|
|
srec->remove_external_sub(sender, sub_id);
|
|
}
|
|
}
|
|
}
|
|
}
|