Pubsub stats

This commit is contained in:
Taha Afzal committed 2024-02-28 19:13:43 -05:00
1 parent e8bfe89780
commit 4512e085f2
15 files changed
+560 -514

No files matched your search

@@ -72,7 +72,8 @@ namespace bts {
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A';
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, ViewStreamDatabase vsd) {
private readonly string _tag_name;
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -89,10 +90,11 @@ namespace bts {
_publish_period = TimeSpan.FromSeconds(publish_period_seconds);
_msg_size = payload_size;
_max_pubs = max_pubs;
_tag_name = tag_name;
}
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size)
: this(publish_period_seconds, max_pubs, msg_size, ViewStreamDatabase.instance) { }
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber() {
_vsd.addWorker(this);
@@ -117,7 +119,7 @@ namespace bts {
private void _subscribe_thread_func() {
// subscription is done in another thread
SubscriberCore sub_core = new SubscriberCore();
sub_core.subscribe("test_tag");
sub_core.subscribe(_tag_name);
while (_continue_sub_thread) {
// TODO recv_id
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
@@ -150,7 +152,7 @@ namespace bts {
}
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish(String.Empty, "test_tag", ser_bytes);
_pub_core.publish(String.Empty, _tag_name, ser_bytes);
}
private void _fire_dp() {
@@ -4,25 +4,23 @@ using System.Text;
using System.IO;
//using BCG.Utilities;
using mbts;
using System.Linq;
namespace BCG
{
namespace BCG {
/// <summary>
/// The list of publishers and subscribers for
/// the tag is contained herein. Its used for internal
/// purposes, most noticeably TagInfoStream
/// </summary>
public class TagPubSubInfo
{
public class TagPubSubInfo {
public string tag;
public IDictionary<string, string> subscribers,
publishers;
public ICollection<Pair<string, string>> deadPublishers;
public string server;
public ICollection<Pair<string, string>> dead_publishers;
public string pubsub_server;
public TagPubSubInfo(Stream s)
{
myReadFrom(s);
public TagPubSubInfo(Stream s) {
_read_from(s);
//if (tag == "System_AllSubscriptions" && publishers.Count == 0)
//{
@@ -31,82 +29,89 @@ namespace BCG
//}
}
public TagPubSubInfo(string theTag, IDictionary<string, string> theSubscribers,
IDictionary<string, string> theActivePublishers,
ICollection<Pair<string, string>> theDeadPublishers,
string theServer)
{
tag = theTag;
server = theServer;
subscribers = theSubscribers;
publishers = theActivePublishers;
deadPublishers = theDeadPublishers;
//if (tag == "System_AllSubscriptions" && publishers.Count == 0)
//{
// Console.WriteLine("No publisher, direct cons ?? wtf ??");
// System.Windows.Forms.MessageBox.Show("No publisher, direct cons ?? wtf ??");
//}
public TagPubSubInfo(string the_tag, IDictionary<string, string> the_subscribers,
IDictionary<string, string> the_active_publishers,
ICollection<Pair<string, string>> the_dead_publishers,
string the_pubsub_erver) {
tag = the_tag;
pubsub_server = the_pubsub_erver;
subscribers = the_subscribers;
publishers = the_active_publishers;
dead_publishers = the_dead_publishers;
}
/// <summary>
/// writes itself to the stream passed on as
/// the arg
/// {tag,subCount,subs,conns,pub,pubConn,server}
/// </summary>
/// <param name="s"></param>
public void writeTo(Stream s)
{
/// {tag,sub_count,<conn, app>*,pub_count<conn,app>*, pubsub_server}
public void write_to(Stream s) {
BinaryWriter bw = new BinaryWriter(s);
bw.Write(tag);
bw.Write(subscribers.Count);
foreach (KeyValuePair<string, string> p in subscribers)
{
foreach (KeyValuePair<string, string> p in subscribers) {
bw.Write(p.Key);
bw.Write(p.Value);
}
bw.Write(publishers.Count);
foreach (KeyValuePair<string, string> p in publishers)
{
foreach (KeyValuePair<string, string> p in publishers) {
bw.Write(p.Key);
bw.Write(p.Value);
}
bw.Write(deadPublishers.Count);
foreach (Pair<string, string> p in deadPublishers)
{
bw.Write(dead_publishers.Count);
foreach (Pair<string, string> p in dead_publishers) {
bw.Write(p.first);
bw.Write(p.second);
}
bw.Write(server);
bw.Write(pubsub_server);
}
public static byte[] serialize(IEnumerable<TagPubSubInfo> tpsis) {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(tpsis.Count());
foreach (var tpsi in tpsis) {
tpsi.write_to(str);
}
str.Position = 0;
return str.ToArray();
}
public static List<TagPubSubInfo> deserialize(byte[] data) {
List<TagPubSubInfo> rv = new List<TagPubSubInfo>();
MemoryStream str = new MemoryStream(data);
BinaryReader br = new BinaryReader(str);
var count = br.ReadInt32();
for (int i = 0; i < count; i++) {
TagPubSubInfo tsi = new TagPubSubInfo(str);
rv.Add(tsi);
}
return rv;
}
/// <summary>
/// constructs itself from the stream
/// </summary>
/// <param name="s"></param>
private void myReadFrom(Stream s)
{
private void _read_from(Stream s) {
BinaryReader br = new BinaryReader(s);
tag = br.ReadString();
int nSubs = br.ReadInt32();
int n_subs = br.ReadInt32();
subscribers = new Dictionary<string, string>(); ;
for (int i = 0; i < nSubs; i++)
{
for (int i = 0; i < n_subs; i++) {
subscribers.Add(br.ReadString(), br.ReadString());
}
publishers = new Dictionary<string, string>();
int nPubs = br.ReadInt32();
for (int i = 0; i < nPubs; i++)
{
int n_pubs = br.ReadInt32();
for (int i = 0; i < n_pubs; i++) {
publishers.Add(br.ReadString(), br.ReadString());
}
deadPublishers = new List<Pair<string, string>>();
int nDeadPubs = br.ReadInt32();
for (int i = 0; i < nDeadPubs; i++)
{
(deadPublishers as List<Pair<string, string>>).Add(new Pair<string, string>(br.ReadString(), br.ReadString()));
dead_publishers = new List<Pair<string, string>>();
int n_dead_pubs = br.ReadInt32();
List<Pair<string, string>> dead_pubs = new List<Pair<string, string>>();
for (int i = 0; i < n_dead_pubs; i++) {
dead_pubs.Add(new Pair<string, string>(br.ReadString(), br.ReadString()));
}
server = br.ReadString();
dead_publishers = dead_pubs;
pubsub_server = br.ReadString();
}
/// <summary>
@@ -116,54 +121,41 @@ namespace BCG
/// well (it had no publisher to begin with).
/// However if a tag is ever published it will never become an orphan
/// </summary>
public bool orphaned
{
get
{
return publishers.Count == 0 && subscribers.Count == 0;
}
}
// public bool orphaned {
// get {
// return publishers.Count == 0 && subscribers.Count == 0;
// }
// }
public string subscribersInString
{
get
{
string subString = "";
foreach (KeyValuePair<string, string> p in subscribers)
{
subString += (subString == "" ? "" : ",") + p.Value + ';' + p.Key;
public string subscribers_as_string {
get {
string sub_str = "";
foreach (KeyValuePair<string, string> p in subscribers) {
sub_str += (sub_str == "" ? "" : ",") + p.Value + ';' + p.Key;
}
return subString;
return sub_str;
}
}
public string publishersInString
{
get
{
string pubString = "";
foreach (KeyValuePair<string, string> p in publishers)
{
pubString += (pubString == "" ? "" : ",") + p.Value + ';' + p.Key;
public string publishers_as_string {
get {
string pub_string = "";
foreach (KeyValuePair<string, string> p in publishers) {
pub_string += (pub_string == "" ? "" : ",") + p.Value + ';' + p.Key;
}
return pubString;
return pub_string;
}
}
public string deadPublishersInString
{
get
{
string deadPubString = "";
foreach (Pair<string, string> p in deadPublishers)
{
public string dead_publishers_as_string {
get {
string dead_pub_string = "";
foreach (Pair<string, string> p in dead_publishers) {
if (p != null)
deadPubString += (deadPubString == "" ? "" : ",") + p.second + ';' + p.first;
else{}
//BlueEventLog.logEvent("DeadPublishers in deadPublishersInString is null", "Error");
dead_pub_string += (dead_pub_string == "" ? "" : ",") + p.second + ';' + p.first;
}
return deadPubString;
return dead_pub_string;
}
}
@@ -2,62 +2,81 @@ using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Runtime.CompilerServices;
using System.Linq;
namespace BCG
{
public class TagStatsInfo
{
namespace BCG {
public class TagStatsInfo {
public string server;
public string tag;
public int minSize = Int32.MaxValue, maxSize = Int32.MinValue, lastUpdateSize;
public double averageSize;
public int totalUpdates;
public long totalBytes;
public long lastUpdateTime;
public int min_size = Int32.MaxValue, max_size = Int32.MinValue, last_update_size;
public double average_size;
public int total_updates;
public long total_bytes;
public long last_update_time;
public TagStatsInfo(string theTag)
{
public TagStatsInfo(string theTag) {
server = ClientInfo.thisClient.machineName;
tag = theTag;
}
public TagStatsInfo(Stream str)
{
public TagStatsInfo(Stream str) {
BinaryReader br = new BinaryReader(str);
server = br.ReadString();
tag = br.ReadString();
minSize = br.ReadInt32();
maxSize = br.ReadInt32();
lastUpdateSize = br.ReadInt32();
averageSize = br.ReadDouble();
totalUpdates = br.ReadInt32();
totalBytes = br.ReadInt64();
lastUpdateTime = br.ReadInt64();
min_size = br.ReadInt32();
max_size = br.ReadInt32();
last_update_size = br.ReadInt32();
average_size = br.ReadDouble();
total_updates = br.ReadInt32();
total_bytes = br.ReadInt64();
last_update_time = br.ReadInt64();
}
public void update(int size)
{
totalUpdates++;
totalBytes += size;
averageSize = totalBytes / totalUpdates;
minSize = size < minSize ? size : minSize;
maxSize = size > maxSize ? size : maxSize;
lastUpdateSize = size;
lastUpdateTime = DateTime.Now.Ticks;
public void update(int size) {
total_updates++;
total_bytes += size;
average_size = total_bytes / total_updates;
min_size = size < min_size ? size : min_size;
max_size = size > max_size ? size : max_size;
last_update_size = size;
last_update_time = DateTime.Now.Ticks;
}
public void writeTo(Stream str)
{
public void write_to(Stream str) {
BinaryWriter bw = new BinaryWriter(str);
bw.Write(server);
bw.Write(tag);
bw.Write(minSize);
bw.Write(maxSize);
bw.Write(lastUpdateSize);
bw.Write(averageSize);
bw.Write(totalUpdates);
bw.Write(totalBytes);
bw.Write(lastUpdateTime);
bw.Write(min_size);
bw.Write(max_size);
bw.Write(last_update_size);
bw.Write(average_size);
bw.Write(total_updates);
bw.Write(total_bytes);
bw.Write(last_update_time);
}
public static byte[] serialize(IEnumerable<TagStatsInfo> items) {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(items.Count());
foreach (var i in items) {
i.write_to(str);
}
str.Position = 0;
return str.ToArray();
}
public static IEnumerable<TagStatsInfo> deserialize(byte[] data) {
List<TagStatsInfo> rv = new List<TagStatsInfo>();
MemoryStream str = new MemoryStream(data);
BinaryReader br = new BinaryReader(str);
int count = br.ReadInt32();
for (int i = 0; i < count; i++) {
TagStatsInfo ts_info = new TagStatsInfo(str);
rv.Add(ts_info);
}
return rv;
}
}
@@ -3,18 +3,40 @@ using System.Collections.Generic;
using System.Text;
using BCG.Utilities.General;
using System.IO;
using bts;
namespace BCG {
class Pubsubheartbeat {
public Int64 hb_time_epoch;
public byte[] serialize() {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(hb_time_epoch);
str.Position = 0;
return str.ToArray();
}
public Pubsubheartbeat() {
hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
public static Pubsubheartbeat deserialize(byte[] data) {
long hb_time = BitConverter.ToInt64(data);
return new Pubsubheartbeat() {
hb_time_epoch = hb_time,
};
}
}
/// <summary>
/// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
/// A subscriber can subscribe to it, and find out of a PubSub is alive or
/// not. Infact more so, if the subsriptions are working or not.
/// </summary>
internal class SystemHeartbeatPublisher : IWorker {
public static string tagType = "System_Heartbeat";
public static string tagName = "System_Heartbeat";
public static string tag_type = "System_Heartbeat";
public static string tag_name = "System_Heartbeat";
private static SystemHeartbeatPublisher mySelf;
private static SystemHeartbeatPublisher _self;
private SystemHeartbeatPublisher() {
@@ -23,24 +45,29 @@ namespace BCG {
///waits 2500 msecs before calling doWork. This 2500 msecs
///is only interceded by an external event (publication or
///subscription)
MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500));
MasterWorker.getInstance(true).addWorker(this,
TimeSpan.FromMilliseconds(500));
}
public static void init() {
if (mySelf == null)
mySelf = new SystemHeartbeatPublisher();
if (_self == null)
_self = new SystemHeartbeatPublisher();
}
#region IWorker Members
public void doWork() {
PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString);
MetaData md = new MetaData(pub);
MemoryStream ms = new MemoryStream();
md.save(ms);
BinaryWriter bw = new BinaryWriter(ms);
bw.Write(true);
PubSubMessageReactor.internalSend(ms.ToArray());
Pubsubheartbeat hb = new Pubsubheartbeat();
PublishMsg msg = new PublishMsg() {
connection = "Internal",
publishing_app = ClientInfo.thisClient.app,
tag = "pss_hb",
tag_type = "Internal",
is_compressed = false,
publish_time_epoch_ms = hb.hb_time_epoch,
data_to_publish = hb.serialize()
};
PubSubMessageReactor.internal_publish(msg);
}
#endregion
@@ -3,63 +3,76 @@ using System.Collections.Generic;
using System.Text;
using System.IO;
using BCG.Utilities.General;
using bts;
namespace BCG {
/// <summary>
/// Hooks on to the onPublish event. Mantains the statistics
/// for each tag. This also is an IPubSubServerTagHandler. The reason
/// being that since it already holds all the tag statistics,
/// theres no reason to have another class do the same. Hence,
/// Hooks on to the PubSUbState's onPublish event. Mantains the statistics
/// for each tag, and publishes the stats, on the System_AllStatistics tag,
/// using the PubSubMessageReactor internal_publish. Since this tag is being
/// published tot he Pubsub, , the state has to be kept somwhere, and thus this class
/// is also an IPubSubServerTagHandler, and returns the total state on getContent
/// it doesnt do anything on the update as it, already has the state.
///
/// On the doWork, it internaly publishes the delta.
/// </summary>
internal class SystemStatsPublisher : IPubSubServerTagHandler {
public static string tagType = "System_Stats";
public static string tagName = "System_AllStatistics";
private Dictionary<string, bool> myTagsReadyToPublish = new Dictionary<string, bool>();
private Dictionary<string, TagStatsInfo> myTagStatsMap = new Dictionary<string, TagStatsInfo>();
private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
private static SystemStatsPublisher mySelf = null;
public static string tag_type = "System_Stats";
public static string tag_name = "System_AllStatistics";
// these tags have been touched, need to publish in the next doWork
private HashSet<string> _dirty_tags = new HashSet<string>();
private Dictionary<string, TagStatsInfo> _tag_stats_map = new Dictionary<string, TagStatsInfo>();
private MasterWorker _master_worker = MasterWorker.getInstance(false);
private static SystemStatsPublisher _self = null;
private SystemStatsPublisher(PubSubState state) {
state.onPublishStats += new PubSubState.PubSubStatsDelegate(myHandlePublish);
myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StatisticsUpdatePeriod));
bts.Log.Gen.info("SystemStatsPublisher constructed");
state.onPublishStats += new PubSubState.PubSubStatsDelegate(_handle_publish);
_master_worker.addWorker(this, TimeSpan.FromMilliseconds(250));
}
public static void init(PubSubState state) {
mySelf = new SystemStatsPublisher(state);
bts.Log.Gen.info("SystemStatsPublisher.init");
_self = new SystemStatsPublisher(state);
}
public static IPubSubServerTagHandler getInstance() {
return mySelf;
public static IPubSubServerTagHandler get_instance() {
return _self;
}
void myHandlePublish(string tag, int size) {
if (tag != tagName) {
myTagsReadyToPublish[tag] = true;
if (!myTagStatsMap.ContainsKey(tag))
myTagStatsMap.Add(tag, new TagStatsInfo(tag));
myTagStatsMap[tag].update(size);
void _handle_publish(string tag, int size) {
bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}");
if (tag == tag_name)
return;
TagStatsInfo tag_stats = null;
if (!_tag_stats_map.TryGetValue(tag, out tag_stats)) {
tag_stats = new TagStatsInfo(tag);
_tag_stats_map.Add(tag, tag_stats);
}
tag_stats.update(size);
_dirty_tags.Add(tag);
}
#region IWorker Members
public void doWork() {
if (myTagsReadyToPublish.Count > 0) {
// TODO: PubSubServerMsg
PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString);
if (_dirty_tags.Count > 0) {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
MetaData md = new MetaData(pub);
md.save(str);
new BinaryWriter(str).Write(myTagsReadyToPublish.Count);
foreach (string tag in myTagsReadyToPublish.Keys) {
myTagStatsMap[tag].writeTo(str);
List<TagStatsInfo> dirty_ts_info = new List<TagStatsInfo>();
foreach (string tag in _dirty_tags) {
//_tag_stats_map[tag].write_to(str);
dirty_ts_info.Add(_tag_stats_map[tag]);
}
myTagsReadyToPublish.Clear();
PubSubMessageReactor.internalSend(str.ToArray());
_dirty_tags.Clear();
var bytes = TagStatsInfo.serialize(dirty_ts_info);
PublishMsg pub_msg = new PublishMsg() {
connection = "InternalPublish",
tag = SystemStatsPublisher.tag_name,
tag_type = SystemStatsPublisher.tag_type,
data_to_publish = bytes
};
PubSubMessageReactor.internal_publish(pub_msg);
}
}
@@ -71,15 +84,18 @@ namespace BCG {
}
public byte[] getContent(string tag) {
BinaryWriter bw = new BinaryWriter(new MemoryStream());
bw.Write(myTagStatsMap.Count);
foreach (TagStatsInfo t in myTagStatsMap.Values)
t.writeTo(bw.BaseStream);
return (bw.BaseStream as MemoryStream).ToArray();
var ser = TagStatsInfo.serialize(_tag_stats_map.Values);
return ser;
// MemoryStream str = new MemoryStream();
// BinaryWriter bw = new BinaryWriter(str);
// bw.Write(_tag_stats_map.Count);
// foreach (TagStatsInfo t in _tag_stats_map.Values)
// t.write_to(bw.BaseStream);
// return str.ToArray();
}
public string[] supportedTagTypes {
get { return new string[] { SystemStatsPublisher.tagType }; }
get { return new string[] { SystemStatsPublisher.tag_type }; }
}
public void register(InternalPublishDelegate callback) { }
@@ -5,6 +5,8 @@ using BCG.Utilities.General;
using System.IO;
using System.Threading;
using mbts;
using System.Threading.Tasks;
using bts;
namespace BCG {
/// <summary>
@@ -23,60 +25,61 @@ namespace BCG {
/// It uses the PubSubState to publish the entire map on getUpdates
/// </summary>
class SystemSubscriptionMapPublisher : IPubSubServerTagHandler {
private PubSubState myPubsubState;
private Dictionary<string, bool> myTagsReadyToPublish = new Dictionary<string, bool>();
private readonly string myIp = SysInfoManager.getIP();
public static string tagType = "System_Sub";
public static string tagName = "System_AllSubscriptions";
private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
private static IPubSubServerTagHandler mySelf;
private PubSubState _state;
private HashSet<string> _dirty_tags = new HashSet<string>();
private readonly string _ip = SysInfoManager.getIP();
public static string tag_type = "System_Sub";
public static string tag_name = "System_AllSubscriptions";
private MasterWorker _master_worker = MasterWorker.getInstance(false);
private static IPubSubServerTagHandler _self;
private SystemSubscriptionMapPublisher(PubSubState state) {
myPubsubState = state;
myPubsubState.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(myOnTagStateChanges);
myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StateUpdatePeriod));
bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor");
_state = state;
_state.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(_on_tag_state_changes);
_master_worker.addWorker(this, TimeSpan.FromMilliseconds(500));
}
public static void init(PubSubState state) {
mySelf = new SystemSubscriptionMapPublisher(state);
bts.Log.Gen.info("SystemSubscriptionMapPublisher.init");
_self = new SystemSubscriptionMapPublisher(state);
}
public static IPubSubServerTagHandler getInstance() {
return mySelf;
return _self;
}
private void myOnTagStateChanges(string[] tags) {
foreach (string t in tags)
myTagsReadyToPublish[t] = true;
private void _on_tag_state_changes(string[] tags) {
bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes len: {tags.Length}");
_dirty_tags.AddRange(tags);
}
private void myInternalPublish() {
if (myTagsReadyToPublish.Count > 0) {
///serialize and write a PubMetaData
PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString);
MemoryStream str = new MemoryStream();
MetaData md = new MetaData(pub);
md.save(str);
BinaryWriter bw = new BinaryWriter(str);
///count of the following items
bw.Write(myTagsReadyToPublish.Count);
///TagPubSubInfo serialized one by one
foreach (string tag in myTagsReadyToPublish.Keys) {
Dictionary<string, string> subscribers, activePubslishers;
ICollection<Pair<string, string>> deadPublishers;
myPubsubState.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp);
ti.writeTo(str);
private void _internal_publish() {
if (_dirty_tags.Count > 0) {
List<TagPubSubInfo> tpsis = new List<TagPubSubInfo>();
foreach (string tag in _dirty_tags) {
Dictionary<string, string> subscribers, active_pubslishers;
ICollection<Pair<string, string>> dead_publishers;
_state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers);
TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers,
dead_publishers, _ip);
tpsis.Add(ti);
}
myTagsReadyToPublish.Clear();
PubSubMessageReactor.internalSend(str.ToArray());
_dirty_tags.Clear();
var ser_bytes = TagPubSubInfo.serialize(tpsis);
PublishMsg pub_msg = new PublishMsg() {
publish_time_epoch_ms = 0,
is_compressed = false,
data_to_publish = ser_bytes
};
PubSubMessageReactor.internal_publish(pub_msg);
}
}
#region IWorker Members
public void doWork() {
myInternalPublish();
_internal_publish();
}
#endregion
@@ -86,22 +89,21 @@ namespace BCG {
public void update(string tag, Stream[] messageBodies) { }
public byte[] getContent(string t) {
ICollection<string> allTags = myPubsubState.all_tags;
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(allTags.Count);
foreach (string tag in allTags) {
ICollection<string> all_tags = _state.all_tags;
List<TagPubSubInfo> ti_list = new List<TagPubSubInfo>();
foreach (string tag in all_tags) {
Dictionary<string, string> subscribers, activePubslishers;
ICollection<Pair<string, string>> deadPublishers;
myPubsubState.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp);
ti.writeTo(str);
_state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip);
ti_list.Add(ti);
}
return str.ToArray();
var rv = TagPubSubInfo.serialize(ti_list);
return rv;
}
public string[] supportedTagTypes {
get { return new string[] { SystemSubscriptionMapPublisher.tagType }; }
get { return new string[] { SystemSubscriptionMapPublisher.tag_type }; }
}
public void register(InternalPublishDelegate callback) { }
@@ -28,6 +28,7 @@ namespace BCG {
private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds);
private SingleTagTimeSliceDelegate onSingleTagPub;
private HighPerformanceTimer mySingleTagTimeSliceTimer = new HighPerformanceTimer();
private static SyncQueue<PublishMsg> _internal_pub_queue = new SyncQueue<PublishMsg>();
//this thing keeps track of whether a given tag is compressed or not. It is used for logging messages.
/* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/
@@ -81,6 +82,11 @@ namespace BCG {
private void _polling_thread_func() {
TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500);
_msg_hub.activate();
Action handle_internal_pubs = () => {
var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000);
foreach (var msg in internal_pubs)
_handle_pub("Internal", msg);
};
while (true) {
try {
@@ -92,6 +98,7 @@ namespace BCG {
foreach (var kv in received_data) {
_handle_client_msg(kv.first, kv.second);
}
handle_internal_pubs();
_mainloop_master_worker.doWork();
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}");
@@ -181,8 +188,10 @@ namespace BCG {
//myReceivingQueue.enqueue(new Pair<string, byte[]>(internalPublisherConnection, data));
}
public static void internal_publish(PublishMsg pub_msg) {
//_handle_pub("InternalPublisher", pub_msg);
//private static PubsSubM
public static void internal_publish(PublishMsg msg) {
bts.Log.Gen.info($"PubsubMessageReactor.internal_publish msg: {msg}");
_internal_pub_queue.enqueue(msg);
}
@@ -4,19 +4,16 @@ using System.Linq;
using System.Text;
/*
* This class contains the recent chagnes of state of a single pubsub tag, with respect to who has subscribed, who has published, etc.
* This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc.
* It has a list of new subscription events.
*
*/
namespace BCG
{
public class PubSubSingleTagState
{
namespace BCG {
public class PubSubSingleTagState {
private string myTag;
public string tag
{
public string tag {
get { return myTag; }
set { myTag = value; }
set { myTag = value; }
}
//we have this here as SortedList so that we can compare for equality
@@ -27,24 +24,22 @@ namespace BCG
private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
public PubSubSingleTagState()
{
public PubSubSingleTagState() {
//the total state of the tag
myPublishers = new SortedList<string,string>();
mySubscribers = new SortedList<string,string>();
myPublishers = new SortedList<string, string>();
mySubscribers = new SortedList<string, string>();
//how has the state changed recentlty
myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
myRemovedPublishers= new Dictionary<string, PubSubSingleTagStateEvent>();
myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
}
//returns a list of all the events that need to be published.
public PubSubSingleTagStateEvent[] getNewEventsForPublishing()
{
public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
myNewSubscribers.Values.CopyTo(newEvents, 0);
@@ -54,7 +49,7 @@ namespace BCG
myRemovedSubscribers.Values.CopyTo(newEvents, index);
index += myRemovedSubscribers.Count;
myRemovedPublishers.Values.CopyTo(newEvents, index);
return newEvents;
}
@@ -62,19 +57,16 @@ namespace BCG
//call this after you have published the new events
//this is made public for code readability reasons
//i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
public void resetNewEvents()
{
public void resetNewEvents() {
myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
}
public void newPublish(PubSubSingleTagStateEvent newPublisher)
{
public void newPublish(PubSubSingleTagStateEvent newPublisher) {
//don't do anything here if we already knew about the publishers because we were already dealing with this publishers
if(!myPublishers.ContainsKey(newPublisher.app))
{
if (!myPublishers.ContainsKey(newPublisher.app)) {
myPublishers.Add(newPublisher.app, newPublisher.app);
//if we were going to delete it, that means it was already being published by this app
@@ -88,10 +80,8 @@ namespace BCG
}
}
public void removePublisher(PubSubSingleTagStateEvent removedPublisher)
{
if(myPublishers.ContainsKey(removedPublisher.app))
{
public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
if (myPublishers.ContainsKey(removedPublisher.app)) {
myPublishers.Remove(removedPublisher.app);
//same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
@@ -103,10 +93,8 @@ namespace BCG
}
public void newSubscription(PubSubSingleTagStateEvent newSubscriber)
{
if (!mySubscribers.ContainsKey(newSubscriber.app))
{
public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
if (!mySubscribers.ContainsKey(newSubscriber.app)) {
mySubscribers.Add(newSubscriber.app, newSubscriber.app);
if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
@@ -118,10 +106,8 @@ namespace BCG
}
}
public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber)
{
if(mySubscribers.ContainsKey(removedSubscriber.app))
{
public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
if (mySubscribers.ContainsKey(removedSubscriber.app)) {
mySubscribers.Remove(removedSubscriber.app);
//if we were going to add this subscriber, we no logner want to
//this means no event needs to be published
@@ -48,20 +48,20 @@ namespace BCG {
/// </summary>
/// <param name="sub"></param>
/// <param name="md"></param>
public void sub(SubMetaData sub) {
///tag based key ds update
foreach (string tag in sub.tags) {
if (!_tag_to_connection_map.ContainsKey(tag)) {
_tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
}
_tag_to_connection_map[tag].onSub(sub);
// public void sub(SubMetaData sub) {
// ///tag based key ds update
// foreach (string tag in sub.tags) {
// if (!_tag_to_connection_map.ContainsKey(tag)) {
// _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
// }
// _tag_to_connection_map[tag].onSub(sub);
if (onSingleTagChange != null)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
}
if (onTagMapChanged != null)
onTagMapChanged(sub.tags);
}
// if (onSingleTagChange != null)
// onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
// }
// if (onTagMapChanged != null)
// onTagMapChanged(sub.tags);
// }
public void handle_sub(bts.SubscribeMsg sub_msg) {
foreach (string tag in sub_msg.tags) {
@@ -80,23 +80,23 @@ namespace BCG {
// TODO
}
public void unsub(UnSubMetaData unsub) {
foreach (string tag in unsub.tags) {
if (_tag_to_connection_map.ContainsKey(tag)) {
///maybe check here, that if there are no subscribers
///and publishers for the tag then remove it.
_tag_to_connection_map[tag].onUnsub(unsub);
if (onSingleTagChange != null)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
} else {
string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
" the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
}
}
if (onTagMapChanged != null)
onTagMapChanged(unsub.tags);
}
// public void unsub(UnSubMetaData unsub) {
// foreach (string tag in unsub.tags) {
// if (_tag_to_connection_map.ContainsKey(tag)) {
// ///maybe check here, that if there are no subscribers
// ///and publishers for the tag then remove it.
// _tag_to_connection_map[tag].onUnsub(unsub);
// if (onSingleTagChange != null)
// onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
// } else {
// string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
// " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
// //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
// }
// }
// if (onTagMapChanged != null)
// onTagMapChanged(unsub.tags);
// }
public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) {
foreach (string tag in ubnsub_msg.tags) {
@@ -150,35 +150,34 @@ namespace BCG {
/// </summary>
/// <param name="tag"></param>
/// <param name="msg"></param>
public void updatePubData(PubMetaData pub, Stream msg) {
int sz = (int)(msg.Length - msg.Position);
bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
///this is the only place where SingleTagData should be created
if (!_tag_to_data_map.ContainsKey(pub.tag)) {
///first time data is published for this tag
_tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
}
_tag_to_data_map[pub.tag].pubUpdate(msg);
// public void updatePubData(PubMetaData pub, Stream msg) {
// int sz = (int)(msg.Length - msg.Position);
// bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
// ///this is the only place where SingleTagData should be created
// if (!_tag_to_data_map.ContainsKey(pub.tag)) {
// ///first time data is published for this tag
// _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
// }
// _tag_to_data_map[pub.tag].pubUpdate(msg);
if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
_tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
}
bool pubChanged = false;
_tag_to_connection_map[pub.tag].onPub(pub, out pubChanged);
if (pubChanged) {
if (onTagMapChanged != null)
onTagMapChanged(new string[] { pub.tag });
// if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
// _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
// }
// bool pubChanged = false;
// _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged);
// if (pubChanged) {
// if (onTagMapChanged != null)
// onTagMapChanged(new string[] { pub.tag });
if (onSingleTagChange != null)
onSingleTagChange(
new PubSubSingleTagStateEvent(pub.tag,
PubSubSingleTagStateEvent.EventType.publish,
pub.connection, pub.publishingApp));
}
if (onPublishStats != null)
onPublishStats(pub.tag, sz);
}
// if (onSingleTagChange != null)
// onSingleTagChange(
// new PubSubSingleTagStateEvent(pub.tag,
// PubSubSingleTagStateEvent.EventType.publish,
// pub.connection, pub.publishingApp));
// }
// if (onPublishStats != null)
// onPublishStats(pub.tag, sz);
// }
public void update_pub_data(bts.PublishMsg pub_msg) {
// TODO
@@ -20,12 +20,13 @@ namespace BCG {
private IPubSubServerTagHandler _tag_handler;
public SingleTagData(string tag, string tag_type) {
bts.Log.Gen.info("SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
_tag = tag;
_tag_type = tag_type;
if (_tag_type == SystemSubscriptionMapPublisher.tagType)
_tag_handler = SystemSubscriptionMapPublisher.getInstance();
else if (_tag_type == SystemStatsPublisher.tagType)
_tag_handler = SystemStatsPublisher.getInstance();
// if (_tag_type == SystemSubscriptionMapPublisher.tagType)
// _tag_handler = SystemSubscriptionMapPublisher.getInstance();
if (_tag_type == SystemStatsPublisher.tag_type)
_tag_handler = SystemStatsPublisher.get_instance();
else
// this is where it gets the stream handler
_tag_handler = TagType2HandlerMap.getHandler(_tag_type, true);
@@ -37,29 +38,6 @@ namespace BCG {
public string tag_type { get { return _tag_type; } }
internal void pubUpdate(Stream stream) {
string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
bts.Log.Gen.info($"SingleTagData.pubUpdate tag: {_tag} handler: {handler_str} stream: {stream.Position} / {stream.Length}");
if (_tag_handler != null) {
try {
_tag_handler.update(_tag, new Stream[] { stream });
} catch (Exception e) {
// BlueEventLog.logEvent(e);
string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + _tag_type +
" tag = " + _tag + " length = " + stream.Length + "\n" +
"tag stated not updated. Stack Trace : " + e.StackTrace;
Console.WriteLine(errMsg);
if (e is OutOfMemoryException)
throw e;
else
throw new ApplicationException(errMsg, e);
}
} else {
_last_published_data = new byte[stream.Length - stream.Position];
stream.Read(_last_published_data, 0, _last_published_data.Length);
}
}
internal void handle_publish(PublishMsg pub_msg) {
string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
bts.Log.Gen.info($"SingleTagData.handle_publish pub_msg: {pub_msg} handler: {handler_str} pub_msg: {pub_msg}");
@@ -21,9 +21,6 @@ client and sent over to the pubsubserver, where uints received ad deserialized
SubscriberCore:ISubscriberCore (retired now)
|
V
StaticSubscriberCore
|
V
SingleServerSubscriberCore:ISingleServerSubscriberCore
PublisherCore:IPublisherCore (transactions not supported for now)
@@ -35,7 +32,21 @@ PubSubServer:
PubSubMessageReactor (top level, called from main)
ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection)
PubSubState keeps the total pubsub data in 2 classes
Dictionary<string, SingleTagData> and
Dictionary<string, SingleTagConnectionData>
SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state)
SingleTagConnectionData contains connection vs sub, and connection vs pub
PubSubState calls
SystemSubscriptionMapPublisher.init(this);
SystemStatsPublisher.init(this);
SystemHeartbeatPublisher.init();
---------------------------------------------------------------
How internal tags are published?
PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data
onPublishStats is subscribed by SystemStatsPublisher
SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send
SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state
Streams:
PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
@@ -1,122 +1,78 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using BCG;
using BCG.Utilities.General;
namespace BCG {
using MathNet.Numerics.RootFinding;
namespace bts {
[DataStreamUserRespresentation("tagStats")]
public class PubSubServerTagStatsStream : ViewStream, IWorker {
private static Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>> myStreamMap =
new Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>>();
private SubscriberCore _sub_core;
private ViewStreamDatabase _vsd;
private SubscriberCore mySubCore;
private ViewStreamDatabase myVsd;
private Dictionary<string, IBlueRow> myTagToStatsMap = new Dictionary<string, IBlueRow>();
private KeysBasedTable2 myTable = new KeysBasedTable2(new BlueTableSchema("", new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("server", 1, typeof(string)),
new ColumnInfo("tag", 2, typeof(string)),
new ColumnInfo("nUpdates", 3, typeof(int)),
new ColumnInfo("totalBytes", 4, typeof(int)),
new ColumnInfo("minSize", 5, typeof(int)),
new ColumnInfo("maxSize", 6, typeof(int)),
new ColumnInfo("lastUpdateSize", 7, typeof(int)),
new ColumnInfo("averageSize", 8, typeof(double)),
new ColumnInfo("lastUpdateTime", 9, typeof(DateTime))),
new string[] { "tag" });
public static ViewStream getPubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL) {
//you can also supply a service name and it should work, as long as substations are enabled.
// if (PubSub3Config.instance.PS3_UseSubstations)
// {
// return new ThreadSafeStream(SubstationTagStatsStream.getSubstationTagStatsStream(vsd, serverURL));
// }
if (!myStreamMap.ContainsKey(vsd))
myStreamMap.Add(vsd, new Dictionary<string, ViewStream>());
if (!myStreamMap[vsd].ContainsKey(serverURL))
myStreamMap[vsd].Add(serverURL, new PubSubServerTagStatsStream(vsd, serverURL));
return myStreamMap[vsd][serverURL];
}
#region constructors to be used by the static factory method
public PubSubServerTagStatsStream(string serverURL)
: this(ViewStreamDatabase.instance, serverURL) { }
private PubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL) {
myVsd = vsd;
mySubCore = new SubscriberCore(new PubSubRouter(new PubSubRoute[1] { new PubSubRoute(serverURL, ".*") }));
}
#endregion
#region exposed constructors that use the static factory methods
public PubSubServerTagStatsStream() {
// if (PubSub3Config.instance.PS3_UseSubstations)
// mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(ViewStreamDatabase.instance, false)));
// else
mySetRealSource(new PubSubServerTagStatsStream(ViewStreamDatabase.instance, PubSub3Config.instance.getSubRouting()));
}
private IBlueTableSchema _schema = new BlueTableSchema("tagStats",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("server", 1, typeof(string)),
new ColumnInfo("tag", 2, typeof(string)),
new ColumnInfo("n_updates", 3, typeof(int)),
new ColumnInfo("total_bytes", 4, typeof(int)),
new ColumnInfo("min_size", 5, typeof(int)),
new ColumnInfo("max_size", 6, typeof(int)),
new ColumnInfo("last_update_size", 7, typeof(int)),
new ColumnInfo("average_size", 8, typeof(double)),
new ColumnInfo("last_update_time", 9, typeof(DateTime)));
private IDictionary<string, TagStatsInfo> _tag_map = new Dictionary<string, TagStatsInfo>();
public PubSubServerTagStatsStream(ViewStreamDatabase vsd) {
// if (PubSub3Config.instance.PS3_UseSubstations)
// mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(vsd, false)));
// else
mySetRealSource(new PubSubServerTagStatsStream(vsd, PubSub3Config.instance.getSubRouting()));
_vsd = vsd;
_sub_core = new SubscriberCore();
}
public PubSubServerTagStatsStream(ViewStreamDatabase vsd, PubSubRoute[] routes) {
List<ViewStream> vss = new List<ViewStream>();
foreach (string url in PubSub3Config.instance.getAllPubSubs())
vss.Add(getPubSubServerTagStatsStream(vsd, url));
mySetRealSource(new AppendStream(vss.ToArray()));
}
#endregion
public PubSubServerTagStatsStream()
: this(ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber() {
mySubCore.subscribe("System_AllStatistics");
myVsd.addWorker(this);
_sub_core.subscribe("System_AllStatistics");
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
mySubCore.unsubscribe("System_AllStatistics");
myVsd.removeWorker(this);
_sub_core.unsubscribe("System_AllStatistics");
_vsd.removeWorker(this);
}
#region IWorker Members
public void doWork() {
foreach (var u in mySubCore.get_updates(TimeSpan.Zero)) {
List<TagStatsInfo> tag_stats = new List<TagStatsInfo>();
MemoryStream str = new MemoryStream(u.data);
BinaryReader br = new BinaryReader(str);
int n = br.ReadInt32();
for (int i = 0; i < n; i++) {
tag_stats.Add(new TagStatsInfo(str));
var updates = _sub_core.get_updates(TimeSpan.Zero);
foreach (var u in updates) {
if (u.tag != "System_AllStatistics")
continue;
var tag_infos = TagStatsInfo.deserialize(u.data);
foreach (var ti in tag_infos) {
_tag_map[ti.tag] = ti;
}
myHandleStats(tag_stats);
}
_fire_dp();
}
#endregion
private void myHandleStats(List<TagStatsInfo> tagStats) {
BlueTable tagStatsRows = new BlueTable(myTable.schema);
int id = 0;
foreach (TagStatsInfo t in tagStats) {
tagStatsRows.addRow(new BlueRow(myTable.schema, id++, t.server, t.tag, t.totalUpdates, t.totalBytes,
t.minSize, t.maxSize, t.lastUpdateSize,
t.averageSize, new DateTime(t.lastUpdateTime)));
private void _fire_dp() {
BlueTable bt = new BlueTable(_schema);
var sorted_tags = _tag_map.Keys.OrderBy(t => t);
// sorting it to keep the tag => id mapping the same (helps with incremental udpates)
foreach (var tag in sorted_tags) {
TagStatsInfo? ts_info = _tag_map[tag];
IBlueRow nr = new BlueRow(_schema, bt.Count, ts_info.server, ts_info.tag,
ts_info.total_updates, ts_info.total_bytes,
ts_info.min_size, ts_info.max_size,
ts_info.last_update_size, ts_info.average_size,
ts_info.last_update_time);
bt.addRow(nr);
}
IBlueRowCollection dRows, aRows;
myTable.updateMultipleRows(tagStatsRows, out dRows, out aRows);
myFireEvent(new DataPacket(myTable, aRows, dRows));
var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true);
if (dp.addedRows.Count + dp.deletedRows.Count > 0)
myFireEvent(dp);
}
}
@@ -4,6 +4,7 @@ using System.Text;
using BCG.Utilities.General;
using System.Collections;
using System.IO;
using MongoDB.Driver.Core.WireProtocol.Messages;
namespace BCG {
/// <summary>
@@ -11,65 +12,35 @@ namespace BCG {
/// server.
/// </summary>
public class PubSubServerStateStream : ViewStream, IWorker {
private static Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>> _stream_map =
new Dictionary<ViewStreamDatabase, Dictionary<string, ViewStream>>();
private KeysBasedTable2 _table =
new KeysBasedTable2(new BlueTableSchema("tagInfo",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("tag", 1, typeof(string)),
new ColumnInfo("server", 2, typeof(string)),
new ColumnInfo("subscribers", 3, typeof(string)),
new ColumnInfo("publishers", 4, typeof(string)),
new ColumnInfo("deadPublishers", 5, typeof(string))),
new string[] { "tag" });
private IDictionary<string, TagPubSubInfo> _tag_map = new Dictionary<string, TagPubSubInfo>();
private readonly string _tag = "System_AllSubscriptions";
private IBlueTableSchema _schema = null;
private SubscriberCore _sub_core;
private ViewStreamDatabase _vsd;
private static ViewStream _get_pub_sub_server_state_stream(ViewStreamDatabase vsd, string server_url) {
if (!_stream_map.ContainsKey(vsd))
_stream_map.Add(vsd, new Dictionary<string, ViewStream>());
if (!_stream_map[vsd].ContainsKey(server_url))
_stream_map[vsd].Add(server_url, new PubSubServerStateStream(server_url, vsd));
return _stream_map[vsd][server_url];
}
// public static ViewStream getPubSubServerStateStream(ViewStreamDatabase vsd, string server_uRL)
// {
// return getPubSubServerStateStream(vsd, server_uRL);
// }
public PubSubServerStateStream(string server_url)
: this(server_url, ViewStreamDatabase.instance) { }
public PubSubServerStateStream(string server_url, ViewStreamDatabase vsd) {
_sub_core = new SubscriberCore(new PubSubRouter(new PubSubRoute(server_url, ".*")));
_vsd = vsd;
}
public PubSubServerStateStream()
: this(ViewStreamDatabase.instance) { }
public PubSubServerStateStream(ViewStreamDatabase vsd) {
mySetRealSource(new PubSubServerStateStream(vsd, PubSub3Config.instance.getSubRouting()));
// TODO, construct schema
_schema = BlueDataUtility.form_schema("pubsub_server_state",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("ps_server", TypeCode.String),
Tuple.Create("subscibers", TypeCode.String),
Tuple.Create("publisher", TypeCode.String),
Tuple.Create("dead_publishers", TypeCode.String));
_vsd = vsd;
_sub_core = new SubscriberCore();
}
public PubSubServerStateStream(ViewStreamDatabase vsd, PubSubRoute[] routes) {
List<ViewStream> vss = new List<ViewStream>();
foreach (var r in routes)
vss.Add(_get_pub_sub_server_state_stream(vsd, r.serverAddress));
mySetRealSource(new AppendStream(vss.ToArray()));
}
protected override void myOnFirstSubscriber() {
_sub_core.subscribe("System_AllSubscriptions");
_sub_core.subscribe(_tag);
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_sub_core.unsubscribe("System_AllSubscriptions");
_sub_core.unsubscribe(_tag);
_vsd.removeWorker(this);
}
@@ -77,35 +48,37 @@ namespace BCG {
public void doWork() {
var updates = _sub_core.get_updates(TimeSpan.Zero);
// this is where the pubsub updates are converted to rows
//this is where the pubsub updates are converted to rows
if (updates.Count() > 0) {
int id = 0;
BlueTable addedRows = new BlueTable(_table.schema);
foreach (var u in updates) {
BinaryReader br = new BinaryReader(new MemoryStream(u.data));
// TODO: check this
int n_updates = br.ReadInt32();
for (int i = 0; i < n_updates; i++) {
TagPubSubInfo tsi = new TagPubSubInfo(new MemoryStream(u.data));
addedRows.addRow(new BlueRow(_table.schema, id++, tsi.tag, tsi.server, tsi.subscribersInString, tsi.publishersInString, tsi.deadPublishersInString));
if (u.tag != _tag) {
continue;
}
var tpsi_list = TagPubSubInfo.deserialize(u.data);
foreach (var tsi in tpsi_list) {
_tag_map[tsi.tag] = tsi;
}
}
IBlueRowCollection dRows, aRows;
_table.updateMultipleRows(addedRows, out dRows, out aRows);
myFireEvent(new DataPacket(_table, aRows, dRows));
}
_fire_dp();
}
#endregion
// public static string convertURLToTag(string URL) {
// string[] urlWords = URL.Split('.');
// string[] lastPart = urlWords[3].Split(':');
// //the first one has "bluto://" in front of it
// //we want to chop that off
// string result = "bluto_" + urlWords[0].Substring(8) + "_" + urlWords[1] + "_" + urlWords[2] + "_" + lastPart[0] + "_" + lastPart[1];
private void _fire_dp() {
BlueTable bt = new BlueTable(_schema);
var sorted_tags = _tag_map.Keys.OrderBy(t => t);
foreach (var kv in _tag_map) {
var tsi = kv.Value;
IBlueRow r = new BlueRow(_schema, bt.Count, tsi.pubsub_server, tsi.subscribers_as_string,
tsi.publishers_as_string, tsi.dead_publishers_as_string);
bt.addRow(r);
}
var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true);
if (dp.addedRows.Count + dp.deletedRows.Count > 0)
myFireEvent(dp);
// return result;
// }
}
}
}
@@ -0,0 +1,76 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using MathNet.Numerics.RootFinding;
using ZstdSharp.Unsafe;
[DataStreamUserRespresentation("pss_hb")]
public class PubsubHeartbeatStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private SubscriberCore _subcore = new SubscriberCore();
private MutableBlueRow _row = null;
private readonly IBlueTableSchema _schema = null;
public PubsubHeartbeatStream(ViewStreamDatabase vsd) {
_vsd = vsd;
_schema = BlueDataUtility.form_schema("PS_HBB",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("hb_sent_time", TypeCode.Int64),
Tuple.Create("hb_recv_time", TypeCode.Int64),
Tuple.Create("travel_period", TypeCode.Int32),
Tuple.Create("time_since_last_recv_msecs", TypeCode.Int32)
);
}
public PubsubHeartbeatStream() :
this(ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber() {
_subcore.subscribe("pss_hb");
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_vsd.removeWorker(this);
_subcore.unsubscribe("pss_hb");
}
public void doWork() {
var updates = _subcore.get_updates(TimeSpan.FromMilliseconds(1));
if (updates.Count() > 0) {
foreach (var m in updates) {
if (m.data != null && m.data.Length > 0) {
BinaryReader br = new BinaryReader(new MemoryStream(m.data));
Int64 hb_sent_time = br.ReadInt64();
Int64 hb_recv_time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
_row = _row == null ? new MutableBlueRow(_schema) : _row;
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var time_since_last_recv_msecs = now - hb_recv_time;
_row.setValue("id", 0);
_row.setValue("hb_sent_time", hb_sent_time);
_row.setValue("hb_recv_time", hb_recv_time);
_row.setValue("travel_period", hb_recv_time - hb_sent_time);
_row.setValue("time_since_last_recv_msecs", time_since_last_recv_msecs);
// TODO, add to table and fire
}
}
} else {
if (_row != null) {
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var ms_since_last_recv = now - _row.get<Int64>("hb_recv_time");
_row.setValue("time_since_last_recv_msecs", ms_since_last_recv);
// TODO, add to table and fire
}
// update the row
}
if (_row != null) {
BlueTable bt = new BlueTable(_row.schema);
bt.addRow(_row);
var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true);
myFireEvent(dp);
}
}
}
}
@@ -26,7 +26,7 @@ namespace BCG {
public TagSynopsisStream(ViewStreamDatabase vsd, string route) {
ViewStream vs = new PubSubServerStateStream(route, vsd);
ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd);
vs = new MapStream(vs, this);
myWorkers.Add(vs as IWorker);
@@ -41,7 +41,7 @@ namespace BCG {
public TagSynopsisStream(ViewStreamDatabase vsd, ICollection<string> routes) {
List<ViewStream> vss = new List<ViewStream>();
foreach (string r in routes) {
ViewStream vs = new PubSubServerStateStream(r, vsd);
ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd);
vs = new MapStream(vs, this);
vss.Add(vs);
myWorkers.Add(vs as IWorker);