diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs
index c0dee5d3c..29e87e7a2 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs
@@ -72,7 +72,8 @@ namespace bts {
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A';
- public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, ViewStreamDatabase vsd) {
+ private readonly string _tag_name;
+ public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -89,10 +90,11 @@ namespace bts {
_publish_period = TimeSpan.FromSeconds(publish_period_seconds);
_msg_size = payload_size;
_max_pubs = max_pubs;
+ _tag_name = tag_name;
}
- public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size)
- : this(publish_period_seconds, max_pubs, msg_size, ViewStreamDatabase.instance) { }
+ public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
+ : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber() {
_vsd.addWorker(this);
@@ -117,7 +119,7 @@ namespace bts {
private void _subscribe_thread_func() {
// subscription is done in another thread
SubscriberCore sub_core = new SubscriberCore();
- sub_core.subscribe("test_tag");
+ sub_core.subscribe(_tag_name);
while (_continue_sub_thread) {
// TODO recv_id
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
@@ -150,7 +152,7 @@ namespace bts {
}
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
- _pub_core.publish(String.Empty, "test_tag", ser_bytes);
+ _pub_core.publish(String.Empty, _tag_name, ser_bytes);
}
private void _fire_dp() {
diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs
index 640f63278..c4e4b3db1 100644
--- a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs
+++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs
@@ -4,25 +4,23 @@ using System.Text;
using System.IO;
//using BCG.Utilities;
using mbts;
+using System.Linq;
-namespace BCG
-{
+namespace BCG {
///
/// The list of publishers and subscribers for
/// the tag is contained herein. Its used for internal
/// purposes, most noticeably TagInfoStream
///
- public class TagPubSubInfo
- {
+ public class TagPubSubInfo {
public string tag;
public IDictionary subscribers,
publishers;
- public ICollection> deadPublishers;
- public string server;
+ public ICollection> dead_publishers;
+ public string pubsub_server;
- public TagPubSubInfo(Stream s)
- {
- myReadFrom(s);
+ public TagPubSubInfo(Stream s) {
+ _read_from(s);
//if (tag == "System_AllSubscriptions" && publishers.Count == 0)
//{
@@ -31,82 +29,89 @@ namespace BCG
//}
}
- public TagPubSubInfo(string theTag, IDictionary theSubscribers,
- IDictionary theActivePublishers,
- ICollection> theDeadPublishers,
- string theServer)
- {
- tag = theTag;
- server = theServer;
- subscribers = theSubscribers;
- publishers = theActivePublishers;
- deadPublishers = theDeadPublishers;
-
- //if (tag == "System_AllSubscriptions" && publishers.Count == 0)
- //{
- // Console.WriteLine("No publisher, direct cons ?? wtf ??");
- // System.Windows.Forms.MessageBox.Show("No publisher, direct cons ?? wtf ??");
- //}
+ public TagPubSubInfo(string the_tag, IDictionary the_subscribers,
+ IDictionary the_active_publishers,
+ ICollection> the_dead_publishers,
+ string the_pubsub_erver) {
+ tag = the_tag;
+ pubsub_server = the_pubsub_erver;
+ subscribers = the_subscribers;
+ publishers = the_active_publishers;
+ dead_publishers = the_dead_publishers;
}
- ///
/// writes itself to the stream passed on as
/// the arg
- /// {tag,subCount,subs,conns,pub,pubConn,server}
- ///
- ///
- public void writeTo(Stream s)
- {
+ /// {tag,sub_count,*,pub_count*, pubsub_server}
+ public void write_to(Stream s) {
BinaryWriter bw = new BinaryWriter(s);
bw.Write(tag);
bw.Write(subscribers.Count);
- foreach (KeyValuePair p in subscribers)
- {
+ foreach (KeyValuePair p in subscribers) {
bw.Write(p.Key);
bw.Write(p.Value);
}
bw.Write(publishers.Count);
- foreach (KeyValuePair p in publishers)
- {
+ foreach (KeyValuePair p in publishers) {
bw.Write(p.Key);
bw.Write(p.Value);
}
- bw.Write(deadPublishers.Count);
- foreach (Pair p in deadPublishers)
- {
+ bw.Write(dead_publishers.Count);
+ foreach (Pair p in dead_publishers) {
bw.Write(p.first);
bw.Write(p.second);
}
- bw.Write(server);
+ bw.Write(pubsub_server);
+ }
+
+ public static byte[] serialize(IEnumerable tpsis) {
+ MemoryStream str = new MemoryStream();
+ BinaryWriter bw = new BinaryWriter(str);
+ bw.Write(tpsis.Count());
+ foreach (var tpsi in tpsis) {
+ tpsi.write_to(str);
+ }
+ str.Position = 0;
+ return str.ToArray();
+ }
+
+ public static List deserialize(byte[] data) {
+ List rv = new List();
+ MemoryStream str = new MemoryStream(data);
+ BinaryReader br = new BinaryReader(str);
+ var count = br.ReadInt32();
+ for (int i = 0; i < count; i++) {
+ TagPubSubInfo tsi = new TagPubSubInfo(str);
+ rv.Add(tsi);
+ }
+ return rv;
}
///
/// constructs itself from the stream
///
///
- private void myReadFrom(Stream s)
- {
+ private void _read_from(Stream s) {
BinaryReader br = new BinaryReader(s);
tag = br.ReadString();
- int nSubs = br.ReadInt32();
+ int n_subs = br.ReadInt32();
subscribers = new Dictionary(); ;
- for (int i = 0; i < nSubs; i++)
- {
+ for (int i = 0; i < n_subs; i++) {
subscribers.Add(br.ReadString(), br.ReadString());
}
publishers = new Dictionary();
- int nPubs = br.ReadInt32();
- for (int i = 0; i < nPubs; i++)
- {
+ int n_pubs = br.ReadInt32();
+ for (int i = 0; i < n_pubs; i++) {
publishers.Add(br.ReadString(), br.ReadString());
}
- deadPublishers = new List>();
- int nDeadPubs = br.ReadInt32();
- for (int i = 0; i < nDeadPubs; i++)
- {
- (deadPublishers as List>).Add(new Pair(br.ReadString(), br.ReadString()));
+ dead_publishers = new List>();
+ int n_dead_pubs = br.ReadInt32();
+ List> dead_pubs = new List>();
+ for (int i = 0; i < n_dead_pubs; i++) {
+ dead_pubs.Add(new Pair(br.ReadString(), br.ReadString()));
}
- server = br.ReadString();
+ dead_publishers = dead_pubs;
+ pubsub_server = br.ReadString();
}
///
@@ -116,54 +121,41 @@ namespace BCG
/// well (it had no publisher to begin with).
/// However if a tag is ever published it will never become an orphan
///
- public bool orphaned
- {
- get
- {
- return publishers.Count == 0 && subscribers.Count == 0;
- }
- }
+ // public bool orphaned {
+ // get {
+ // return publishers.Count == 0 && subscribers.Count == 0;
+ // }
+ // }
- public string subscribersInString
- {
- get
- {
- string subString = "";
- foreach (KeyValuePair p in subscribers)
- {
- subString += (subString == "" ? "" : ",") + p.Value + ';' + p.Key;
+ public string subscribers_as_string {
+ get {
+ string sub_str = "";
+ foreach (KeyValuePair p in subscribers) {
+ sub_str += (sub_str == "" ? "" : ",") + p.Value + ';' + p.Key;
}
- return subString;
+ return sub_str;
}
}
- public string publishersInString
- {
- get
- {
- string pubString = "";
- foreach (KeyValuePair p in publishers)
- {
- pubString += (pubString == "" ? "" : ",") + p.Value + ';' + p.Key;
+ public string publishers_as_string {
+ get {
+ string pub_string = "";
+ foreach (KeyValuePair p in publishers) {
+ pub_string += (pub_string == "" ? "" : ",") + p.Value + ';' + p.Key;
}
- return pubString;
+ return pub_string;
}
}
- public string deadPublishersInString
- {
- get
- {
- string deadPubString = "";
- foreach (Pair p in deadPublishers)
- {
+ public string dead_publishers_as_string {
+ get {
+ string dead_pub_string = "";
+ foreach (Pair p in dead_publishers) {
if (p != null)
- deadPubString += (deadPubString == "" ? "" : ",") + p.second + ';' + p.first;
- else{}
- //BlueEventLog.logEvent("DeadPublishers in deadPublishersInString is null", "Error");
+ dead_pub_string += (dead_pub_string == "" ? "" : ",") + p.second + ';' + p.first;
}
- return deadPubString;
+ return dead_pub_string;
}
}
diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs
index 6b79123d5..7a792f263 100644
--- a/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs
+++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs
@@ -2,62 +2,81 @@ using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
+using System.Runtime.CompilerServices;
+using System.Linq;
-namespace BCG
-{
- public class TagStatsInfo
- {
+namespace BCG {
+ public class TagStatsInfo {
public string server;
public string tag;
- public int minSize = Int32.MaxValue, maxSize = Int32.MinValue, lastUpdateSize;
- public double averageSize;
- public int totalUpdates;
- public long totalBytes;
- public long lastUpdateTime;
+ public int min_size = Int32.MaxValue, max_size = Int32.MinValue, last_update_size;
+ public double average_size;
+ public int total_updates;
+ public long total_bytes;
+ public long last_update_time;
- public TagStatsInfo(string theTag)
- {
+ public TagStatsInfo(string theTag) {
server = ClientInfo.thisClient.machineName;
tag = theTag;
}
- public TagStatsInfo(Stream str)
- {
+ public TagStatsInfo(Stream str) {
BinaryReader br = new BinaryReader(str);
server = br.ReadString();
tag = br.ReadString();
- minSize = br.ReadInt32();
- maxSize = br.ReadInt32();
- lastUpdateSize = br.ReadInt32();
- averageSize = br.ReadDouble();
- totalUpdates = br.ReadInt32();
- totalBytes = br.ReadInt64();
- lastUpdateTime = br.ReadInt64();
+ min_size = br.ReadInt32();
+ max_size = br.ReadInt32();
+ last_update_size = br.ReadInt32();
+ average_size = br.ReadDouble();
+ total_updates = br.ReadInt32();
+ total_bytes = br.ReadInt64();
+ last_update_time = br.ReadInt64();
}
- public void update(int size)
- {
- totalUpdates++;
- totalBytes += size;
- averageSize = totalBytes / totalUpdates;
- minSize = size < minSize ? size : minSize;
- maxSize = size > maxSize ? size : maxSize;
- lastUpdateSize = size;
- lastUpdateTime = DateTime.Now.Ticks;
+ public void update(int size) {
+ total_updates++;
+ total_bytes += size;
+ average_size = total_bytes / total_updates;
+ min_size = size < min_size ? size : min_size;
+ max_size = size > max_size ? size : max_size;
+ last_update_size = size;
+ last_update_time = DateTime.Now.Ticks;
}
- public void writeTo(Stream str)
- {
+ public void write_to(Stream str) {
BinaryWriter bw = new BinaryWriter(str);
bw.Write(server);
bw.Write(tag);
- bw.Write(minSize);
- bw.Write(maxSize);
- bw.Write(lastUpdateSize);
- bw.Write(averageSize);
- bw.Write(totalUpdates);
- bw.Write(totalBytes);
- bw.Write(lastUpdateTime);
+ bw.Write(min_size);
+ bw.Write(max_size);
+ bw.Write(last_update_size);
+ bw.Write(average_size);
+ bw.Write(total_updates);
+ bw.Write(total_bytes);
+ bw.Write(last_update_time);
+ }
+
+ public static byte[] serialize(IEnumerable items) {
+ MemoryStream str = new MemoryStream();
+ BinaryWriter bw = new BinaryWriter(str);
+ bw.Write(items.Count());
+ foreach (var i in items) {
+ i.write_to(str);
+ }
+ str.Position = 0;
+ return str.ToArray();
+ }
+
+ public static IEnumerable deserialize(byte[] data) {
+ List rv = new List();
+ MemoryStream str = new MemoryStream(data);
+ BinaryReader br = new BinaryReader(str);
+ int count = br.ReadInt32();
+ for (int i = 0; i < count; i++) {
+ TagStatsInfo ts_info = new TagStatsInfo(str);
+ rv.Add(ts_info);
+ }
+ return rv;
}
}
diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs
index 47375d018..fb506104b 100644
--- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs
+++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs
@@ -3,18 +3,40 @@ using System.Collections.Generic;
using System.Text;
using BCG.Utilities.General;
using System.IO;
+using bts;
namespace BCG {
+ class Pubsubheartbeat {
+ public Int64 hb_time_epoch;
+ public byte[] serialize() {
+ MemoryStream str = new MemoryStream();
+ BinaryWriter bw = new BinaryWriter(str);
+ bw.Write(hb_time_epoch);
+ str.Position = 0;
+ return str.ToArray();
+ }
+
+ public Pubsubheartbeat() {
+ hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+ }
+
+ public static Pubsubheartbeat deserialize(byte[] data) {
+ long hb_time = BitConverter.ToInt64(data);
+ return new Pubsubheartbeat() {
+ hb_time_epoch = hb_time,
+ };
+ }
+ }
///
/// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
/// A subscriber can subscribe to it, and find out of a PubSub is alive or
/// not. Infact more so, if the subsriptions are working or not.
///
internal class SystemHeartbeatPublisher : IWorker {
- public static string tagType = "System_Heartbeat";
- public static string tagName = "System_Heartbeat";
+ public static string tag_type = "System_Heartbeat";
+ public static string tag_name = "System_Heartbeat";
- private static SystemHeartbeatPublisher mySelf;
+ private static SystemHeartbeatPublisher _self;
private SystemHeartbeatPublisher() {
@@ -23,24 +45,29 @@ namespace BCG {
///waits 2500 msecs before calling doWork. This 2500 msecs
///is only interceded by an external event (publication or
///subscription)
- MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500));
+ MasterWorker.getInstance(true).addWorker(this,
+ TimeSpan.FromMilliseconds(500));
}
public static void init() {
- if (mySelf == null)
- mySelf = new SystemHeartbeatPublisher();
+ if (_self == null)
+ _self = new SystemHeartbeatPublisher();
}
#region IWorker Members
public void doWork() {
- PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString);
- MetaData md = new MetaData(pub);
- MemoryStream ms = new MemoryStream();
- md.save(ms);
- BinaryWriter bw = new BinaryWriter(ms);
- bw.Write(true);
- PubSubMessageReactor.internalSend(ms.ToArray());
+ Pubsubheartbeat hb = new Pubsubheartbeat();
+ PublishMsg msg = new PublishMsg() {
+ connection = "Internal",
+ publishing_app = ClientInfo.thisClient.app,
+ tag = "pss_hb",
+ tag_type = "Internal",
+ is_compressed = false,
+ publish_time_epoch_ms = hb.hb_time_epoch,
+ data_to_publish = hb.serialize()
+ };
+ PubSubMessageReactor.internal_publish(msg);
}
#endregion
diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs
index 61d7106ef..c47d2ce25 100644
--- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs
+++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs
@@ -3,63 +3,76 @@ using System.Collections.Generic;
using System.Text;
using System.IO;
using BCG.Utilities.General;
+using bts;
namespace BCG {
///
- /// Hooks on to the onPublish event. Mantains the statistics
- /// for each tag. This also is an IPubSubServerTagHandler. The reason
- /// being that since it already holds all the tag statistics,
- /// theres no reason to have another class do the same. Hence,
+ /// Hooks on to the PubSUbState's onPublish event. Mantains the statistics
+ /// for each tag, and publishes the stats, on the System_AllStatistics tag,
+ /// using the PubSubMessageReactor internal_publish. Since this tag is being
+ /// published tot he Pubsub, , the state has to be kept somwhere, and thus this class
+ /// is also an IPubSubServerTagHandler, and returns the total state on getContent
/// it doesnt do anything on the update as it, already has the state.
///
/// On the doWork, it internaly publishes the delta.
///
internal class SystemStatsPublisher : IPubSubServerTagHandler {
- public static string tagType = "System_Stats";
- public static string tagName = "System_AllStatistics";
- private Dictionary myTagsReadyToPublish = new Dictionary();
- private Dictionary myTagStatsMap = new Dictionary();
- private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
- private static SystemStatsPublisher mySelf = null;
+ public static string tag_type = "System_Stats";
+ public static string tag_name = "System_AllStatistics";
+ // these tags have been touched, need to publish in the next doWork
+ private HashSet _dirty_tags = new HashSet();
+ private Dictionary _tag_stats_map = new Dictionary();
+ private MasterWorker _master_worker = MasterWorker.getInstance(false);
+ private static SystemStatsPublisher _self = null;
private SystemStatsPublisher(PubSubState state) {
- state.onPublishStats += new PubSubState.PubSubStatsDelegate(myHandlePublish);
- myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StatisticsUpdatePeriod));
+ bts.Log.Gen.info("SystemStatsPublisher constructed");
+ state.onPublishStats += new PubSubState.PubSubStatsDelegate(_handle_publish);
+ _master_worker.addWorker(this, TimeSpan.FromMilliseconds(250));
}
public static void init(PubSubState state) {
- mySelf = new SystemStatsPublisher(state);
+ bts.Log.Gen.info("SystemStatsPublisher.init");
+ _self = new SystemStatsPublisher(state);
}
- public static IPubSubServerTagHandler getInstance() {
- return mySelf;
+ public static IPubSubServerTagHandler get_instance() {
+ return _self;
}
- void myHandlePublish(string tag, int size) {
- if (tag != tagName) {
- myTagsReadyToPublish[tag] = true;
- if (!myTagStatsMap.ContainsKey(tag))
- myTagStatsMap.Add(tag, new TagStatsInfo(tag));
- myTagStatsMap[tag].update(size);
+ void _handle_publish(string tag, int size) {
+ bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}");
+ if (tag == tag_name)
+ return;
+ TagStatsInfo tag_stats = null;
+ if (!_tag_stats_map.TryGetValue(tag, out tag_stats)) {
+ tag_stats = new TagStatsInfo(tag);
+ _tag_stats_map.Add(tag, tag_stats);
}
+ tag_stats.update(size);
+ _dirty_tags.Add(tag);
}
+
#region IWorker Members
public void doWork() {
- if (myTagsReadyToPublish.Count > 0) {
- // TODO: PubSubServerMsg
- PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString);
+ if (_dirty_tags.Count > 0) {
MemoryStream str = new MemoryStream();
- BinaryWriter bw = new BinaryWriter(str);
- MetaData md = new MetaData(pub);
- md.save(str);
- new BinaryWriter(str).Write(myTagsReadyToPublish.Count);
- foreach (string tag in myTagsReadyToPublish.Keys) {
- myTagStatsMap[tag].writeTo(str);
+ List dirty_ts_info = new List();
+ foreach (string tag in _dirty_tags) {
+ //_tag_stats_map[tag].write_to(str);
+ dirty_ts_info.Add(_tag_stats_map[tag]);
}
- myTagsReadyToPublish.Clear();
- PubSubMessageReactor.internalSend(str.ToArray());
+ _dirty_tags.Clear();
+ var bytes = TagStatsInfo.serialize(dirty_ts_info);
+ PublishMsg pub_msg = new PublishMsg() {
+ connection = "InternalPublish",
+ tag = SystemStatsPublisher.tag_name,
+ tag_type = SystemStatsPublisher.tag_type,
+ data_to_publish = bytes
+ };
+ PubSubMessageReactor.internal_publish(pub_msg);
}
}
@@ -71,15 +84,18 @@ namespace BCG {
}
public byte[] getContent(string tag) {
- BinaryWriter bw = new BinaryWriter(new MemoryStream());
- bw.Write(myTagStatsMap.Count);
- foreach (TagStatsInfo t in myTagStatsMap.Values)
- t.writeTo(bw.BaseStream);
- return (bw.BaseStream as MemoryStream).ToArray();
+ var ser = TagStatsInfo.serialize(_tag_stats_map.Values);
+ return ser;
+ // MemoryStream str = new MemoryStream();
+ // BinaryWriter bw = new BinaryWriter(str);
+ // bw.Write(_tag_stats_map.Count);
+ // foreach (TagStatsInfo t in _tag_stats_map.Values)
+ // t.write_to(bw.BaseStream);
+ // return str.ToArray();
}
public string[] supportedTagTypes {
- get { return new string[] { SystemStatsPublisher.tagType }; }
+ get { return new string[] { SystemStatsPublisher.tag_type }; }
}
public void register(InternalPublishDelegate callback) { }
diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs
index d3512e40d..f7a01f311 100644
--- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs
+++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs
@@ -5,6 +5,8 @@ using BCG.Utilities.General;
using System.IO;
using System.Threading;
using mbts;
+using System.Threading.Tasks;
+using bts;
namespace BCG {
///
@@ -23,60 +25,61 @@ namespace BCG {
/// It uses the PubSubState to publish the entire map on getUpdates
///
class SystemSubscriptionMapPublisher : IPubSubServerTagHandler {
- private PubSubState myPubsubState;
- private Dictionary myTagsReadyToPublish = new Dictionary();
- private readonly string myIp = SysInfoManager.getIP();
- public static string tagType = "System_Sub";
- public static string tagName = "System_AllSubscriptions";
- private MasterWorker myMasterWorker = MasterWorker.getInstance(false);
- private static IPubSubServerTagHandler mySelf;
+ private PubSubState _state;
+ private HashSet _dirty_tags = new HashSet();
+ private readonly string _ip = SysInfoManager.getIP();
+ public static string tag_type = "System_Sub";
+ public static string tag_name = "System_AllSubscriptions";
+ private MasterWorker _master_worker = MasterWorker.getInstance(false);
+ private static IPubSubServerTagHandler _self;
private SystemSubscriptionMapPublisher(PubSubState state) {
- myPubsubState = state;
- myPubsubState.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(myOnTagStateChanges);
- myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StateUpdatePeriod));
+ bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor");
+ _state = state;
+ _state.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(_on_tag_state_changes);
+ _master_worker.addWorker(this, TimeSpan.FromMilliseconds(500));
}
public static void init(PubSubState state) {
- mySelf = new SystemSubscriptionMapPublisher(state);
+ bts.Log.Gen.info("SystemSubscriptionMapPublisher.init");
+ _self = new SystemSubscriptionMapPublisher(state);
}
public static IPubSubServerTagHandler getInstance() {
- return mySelf;
+ return _self;
}
- private void myOnTagStateChanges(string[] tags) {
- foreach (string t in tags)
- myTagsReadyToPublish[t] = true;
+ private void _on_tag_state_changes(string[] tags) {
+ bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes len: {tags.Length}");
+ _dirty_tags.AddRange(tags);
}
- private void myInternalPublish() {
- if (myTagsReadyToPublish.Count > 0) {
- ///serialize and write a PubMetaData
- PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString);
- MemoryStream str = new MemoryStream();
- MetaData md = new MetaData(pub);
- md.save(str);
- BinaryWriter bw = new BinaryWriter(str);
- ///count of the following items
- bw.Write(myTagsReadyToPublish.Count);
- ///TagPubSubInfo serialized one by one
- foreach (string tag in myTagsReadyToPublish.Keys) {
- Dictionary subscribers, activePubslishers;
- ICollection> deadPublishers;
- myPubsubState.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
- TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp);
- ti.writeTo(str);
+ private void _internal_publish() {
+ if (_dirty_tags.Count > 0) {
+ List tpsis = new List();
+ foreach (string tag in _dirty_tags) {
+ Dictionary subscribers, active_pubslishers;
+ ICollection> dead_publishers;
+ _state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers);
+ TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers,
+ dead_publishers, _ip);
+ tpsis.Add(ti);
}
- myTagsReadyToPublish.Clear();
- PubSubMessageReactor.internalSend(str.ToArray());
+ _dirty_tags.Clear();
+ var ser_bytes = TagPubSubInfo.serialize(tpsis);
+ PublishMsg pub_msg = new PublishMsg() {
+ publish_time_epoch_ms = 0,
+ is_compressed = false,
+ data_to_publish = ser_bytes
+ };
+ PubSubMessageReactor.internal_publish(pub_msg);
}
}
#region IWorker Members
public void doWork() {
- myInternalPublish();
+ _internal_publish();
}
#endregion
@@ -86,22 +89,21 @@ namespace BCG {
public void update(string tag, Stream[] messageBodies) { }
public byte[] getContent(string t) {
- ICollection allTags = myPubsubState.all_tags;
- MemoryStream str = new MemoryStream();
- BinaryWriter bw = new BinaryWriter(str);
- bw.Write(allTags.Count);
- foreach (string tag in allTags) {
+ ICollection all_tags = _state.all_tags;
+ List ti_list = new List();
+ foreach (string tag in all_tags) {
Dictionary subscribers, activePubslishers;
ICollection> deadPublishers;
- myPubsubState.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
- TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp);
- ti.writeTo(str);
+ _state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers);
+ TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip);
+ ti_list.Add(ti);
}
- return str.ToArray();
+ var rv = TagPubSubInfo.serialize(ti_list);
+ return rv;
}
public string[] supportedTagTypes {
- get { return new string[] { SystemSubscriptionMapPublisher.tagType }; }
+ get { return new string[] { SystemSubscriptionMapPublisher.tag_type }; }
}
public void register(InternalPublishDelegate callback) { }
diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs
index 57fd4730b..ca27c30d6 100644
--- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs
+++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs
@@ -28,6 +28,7 @@ namespace BCG {
private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds);
private SingleTagTimeSliceDelegate onSingleTagPub;
private HighPerformanceTimer mySingleTagTimeSliceTimer = new HighPerformanceTimer();
+ private static SyncQueue _internal_pub_queue = new SyncQueue();
//this thing keeps track of whether a given tag is compressed or not. It is used for logging messages.
/* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/
@@ -81,6 +82,11 @@ namespace BCG {
private void _polling_thread_func() {
TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500);
_msg_hub.activate();
+ Action handle_internal_pubs = () => {
+ var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000);
+ foreach (var msg in internal_pubs)
+ _handle_pub("Internal", msg);
+ };
while (true) {
try {
@@ -92,6 +98,7 @@ namespace BCG {
foreach (var kv in received_data) {
_handle_client_msg(kv.first, kv.second);
}
+ handle_internal_pubs();
_mainloop_master_worker.doWork();
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}");
@@ -181,8 +188,10 @@ namespace BCG {
//myReceivingQueue.enqueue(new Pair(internalPublisherConnection, data));
}
- public static void internal_publish(PublishMsg pub_msg) {
- //_handle_pub("InternalPublisher", pub_msg);
+ //private static PubsSubM
+ public static void internal_publish(PublishMsg msg) {
+ bts.Log.Gen.info($"PubsubMessageReactor.internal_publish msg: {msg}");
+ _internal_pub_queue.enqueue(msg);
}
diff --git a/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs
index 36e3fc67f..ae9520df3 100644
--- a/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs
+++ b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs
@@ -4,19 +4,16 @@ using System.Linq;
using System.Text;
/*
- * This class contains the recent chagnes of state of a single pubsub tag, with respect to who has subscribed, who has published, etc.
+ * This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc.
* It has a list of new subscription events.
*
*/
-namespace BCG
-{
- public class PubSubSingleTagState
- {
+namespace BCG {
+ public class PubSubSingleTagState {
private string myTag;
- public string tag
- {
+ public string tag {
get { return myTag; }
- set { myTag = value; }
+ set { myTag = value; }
}
//we have this here as SortedList so that we can compare for equality
@@ -27,24 +24,22 @@ namespace BCG
private Dictionary myNewPublishers;
private Dictionary myRemovedPublishers;
-
- public PubSubSingleTagState()
- {
+
+ public PubSubSingleTagState() {
//the total state of the tag
- myPublishers = new SortedList();
- mySubscribers = new SortedList();
+ myPublishers = new SortedList();
+ mySubscribers = new SortedList();
//how has the state changed recentlty
myNewSubscribers = new Dictionary();
myNewPublishers = new Dictionary();
- myRemovedPublishers= new Dictionary();
+ myRemovedPublishers = new Dictionary();
myRemovedSubscribers = new Dictionary();
}
//returns a list of all the events that need to be published.
- public PubSubSingleTagStateEvent[] getNewEventsForPublishing()
- {
+ public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
myNewSubscribers.Values.CopyTo(newEvents, 0);
@@ -54,7 +49,7 @@ namespace BCG
myRemovedSubscribers.Values.CopyTo(newEvents, index);
index += myRemovedSubscribers.Count;
myRemovedPublishers.Values.CopyTo(newEvents, index);
-
+
return newEvents;
}
@@ -62,19 +57,16 @@ namespace BCG
//call this after you have published the new events
//this is made public for code readability reasons
//i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
- public void resetNewEvents()
- {
+ public void resetNewEvents() {
myNewSubscribers = new Dictionary();
myNewPublishers = new Dictionary();
myRemovedSubscribers = new Dictionary();
myRemovedPublishers = new Dictionary();
}
- public void newPublish(PubSubSingleTagStateEvent newPublisher)
- {
+ public void newPublish(PubSubSingleTagStateEvent newPublisher) {
//don't do anything here if we already knew about the publishers because we were already dealing with this publishers
- if(!myPublishers.ContainsKey(newPublisher.app))
- {
+ if (!myPublishers.ContainsKey(newPublisher.app)) {
myPublishers.Add(newPublisher.app, newPublisher.app);
//if we were going to delete it, that means it was already being published by this app
@@ -88,10 +80,8 @@ namespace BCG
}
}
- public void removePublisher(PubSubSingleTagStateEvent removedPublisher)
- {
- if(myPublishers.ContainsKey(removedPublisher.app))
- {
+ public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
+ if (myPublishers.ContainsKey(removedPublisher.app)) {
myPublishers.Remove(removedPublisher.app);
//same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
@@ -103,10 +93,8 @@ namespace BCG
}
- public void newSubscription(PubSubSingleTagStateEvent newSubscriber)
- {
- if (!mySubscribers.ContainsKey(newSubscriber.app))
- {
+ public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
+ if (!mySubscribers.ContainsKey(newSubscriber.app)) {
mySubscribers.Add(newSubscriber.app, newSubscriber.app);
if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
@@ -118,10 +106,8 @@ namespace BCG
}
}
- public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber)
- {
- if(mySubscribers.ContainsKey(removedSubscriber.app))
- {
+ public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
+ if (mySubscribers.ContainsKey(removedSubscriber.app)) {
mySubscribers.Remove(removedSubscriber.app);
//if we were going to add this subscriber, we no logner want to
//this means no event needs to be published
diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs
index 3c1719265..527dfd1fb 100644
--- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs
+++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs
@@ -48,20 +48,20 @@ namespace BCG {
///
///
///
- public void sub(SubMetaData sub) {
- ///tag based key ds update
- foreach (string tag in sub.tags) {
- if (!_tag_to_connection_map.ContainsKey(tag)) {
- _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
- }
- _tag_to_connection_map[tag].onSub(sub);
+ // public void sub(SubMetaData sub) {
+ // ///tag based key ds update
+ // foreach (string tag in sub.tags) {
+ // if (!_tag_to_connection_map.ContainsKey(tag)) {
+ // _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
+ // }
+ // _tag_to_connection_map[tag].onSub(sub);
- if (onSingleTagChange != null)
- onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
- }
- if (onTagMapChanged != null)
- onTagMapChanged(sub.tags);
- }
+ // if (onSingleTagChange != null)
+ // onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
+ // }
+ // if (onTagMapChanged != null)
+ // onTagMapChanged(sub.tags);
+ // }
public void handle_sub(bts.SubscribeMsg sub_msg) {
foreach (string tag in sub_msg.tags) {
@@ -80,23 +80,23 @@ namespace BCG {
// TODO
}
- public void unsub(UnSubMetaData unsub) {
- foreach (string tag in unsub.tags) {
- if (_tag_to_connection_map.ContainsKey(tag)) {
- ///maybe check here, that if there are no subscribers
- ///and publishers for the tag then remove it.
- _tag_to_connection_map[tag].onUnsub(unsub);
- if (onSingleTagChange != null)
- onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
- } else {
- string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
- " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
- //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
- }
- }
- if (onTagMapChanged != null)
- onTagMapChanged(unsub.tags);
- }
+ // public void unsub(UnSubMetaData unsub) {
+ // foreach (string tag in unsub.tags) {
+ // if (_tag_to_connection_map.ContainsKey(tag)) {
+ // ///maybe check here, that if there are no subscribers
+ // ///and publishers for the tag then remove it.
+ // _tag_to_connection_map[tag].onUnsub(unsub);
+ // if (onSingleTagChange != null)
+ // onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
+ // } else {
+ // string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
+ // " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
+ // //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
+ // }
+ // }
+ // if (onTagMapChanged != null)
+ // onTagMapChanged(unsub.tags);
+ // }
public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) {
foreach (string tag in ubnsub_msg.tags) {
@@ -150,35 +150,34 @@ namespace BCG {
///
///
///
- public void updatePubData(PubMetaData pub, Stream msg) {
- int sz = (int)(msg.Length - msg.Position);
- bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
- ///this is the only place where SingleTagData should be created
- if (!_tag_to_data_map.ContainsKey(pub.tag)) {
- ///first time data is published for this tag
- _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
- }
- _tag_to_data_map[pub.tag].pubUpdate(msg);
+ // public void updatePubData(PubMetaData pub, Stream msg) {
+ // int sz = (int)(msg.Length - msg.Position);
+ // bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
+ // ///this is the only place where SingleTagData should be created
+ // if (!_tag_to_data_map.ContainsKey(pub.tag)) {
+ // ///first time data is published for this tag
+ // _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
+ // }
+ // _tag_to_data_map[pub.tag].pubUpdate(msg);
- if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
- _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
- }
- bool pubChanged = false;
- _tag_to_connection_map[pub.tag].onPub(pub, out pubChanged);
- if (pubChanged) {
- if (onTagMapChanged != null)
- onTagMapChanged(new string[] { pub.tag });
+ // if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
+ // _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
+ // }
+ // bool pubChanged = false;
+ // _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged);
+ // if (pubChanged) {
+ // if (onTagMapChanged != null)
+ // onTagMapChanged(new string[] { pub.tag });
- if (onSingleTagChange != null)
- onSingleTagChange(
- new PubSubSingleTagStateEvent(pub.tag,
- PubSubSingleTagStateEvent.EventType.publish,
- pub.connection, pub.publishingApp));
- }
- if (onPublishStats != null)
- onPublishStats(pub.tag, sz);
-
- }
+ // if (onSingleTagChange != null)
+ // onSingleTagChange(
+ // new PubSubSingleTagStateEvent(pub.tag,
+ // PubSubSingleTagStateEvent.EventType.publish,
+ // pub.connection, pub.publishingApp));
+ // }
+ // if (onPublishStats != null)
+ // onPublishStats(pub.tag, sz);
+ // }
public void update_pub_data(bts.PublishMsg pub_msg) {
// TODO
diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs
index 1afcdbf8a..3a73093a5 100644
--- a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs
+++ b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs
@@ -20,12 +20,13 @@ namespace BCG {
private IPubSubServerTagHandler _tag_handler;
public SingleTagData(string tag, string tag_type) {
+ bts.Log.Gen.info("SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
_tag = tag;
_tag_type = tag_type;
- if (_tag_type == SystemSubscriptionMapPublisher.tagType)
- _tag_handler = SystemSubscriptionMapPublisher.getInstance();
- else if (_tag_type == SystemStatsPublisher.tagType)
- _tag_handler = SystemStatsPublisher.getInstance();
+ // if (_tag_type == SystemSubscriptionMapPublisher.tagType)
+ // _tag_handler = SystemSubscriptionMapPublisher.getInstance();
+ if (_tag_type == SystemStatsPublisher.tag_type)
+ _tag_handler = SystemStatsPublisher.get_instance();
else
// this is where it gets the stream handler
_tag_handler = TagType2HandlerMap.getHandler(_tag_type, true);
@@ -37,29 +38,6 @@ namespace BCG {
public string tag_type { get { return _tag_type; } }
- internal void pubUpdate(Stream stream) {
- string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
- bts.Log.Gen.info($"SingleTagData.pubUpdate tag: {_tag} handler: {handler_str} stream: {stream.Position} / {stream.Length}");
- if (_tag_handler != null) {
- try {
- _tag_handler.update(_tag, new Stream[] { stream });
- } catch (Exception e) {
- // BlueEventLog.logEvent(e);
- string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + _tag_type +
- " tag = " + _tag + " length = " + stream.Length + "\n" +
- "tag stated not updated. Stack Trace : " + e.StackTrace;
- Console.WriteLine(errMsg);
- if (e is OutOfMemoryException)
- throw e;
- else
- throw new ApplicationException(errMsg, e);
- }
- } else {
- _last_published_data = new byte[stream.Length - stream.Position];
- stream.Read(_last_published_data, 0, _last_published_data.Length);
- }
- }
-
internal void handle_publish(PublishMsg pub_msg) {
string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL";
bts.Log.Gen.info($"SingleTagData.handle_publish pub_msg: {pub_msg} handler: {handler_str} pub_msg: {pub_msg}");
diff --git a/src/DataStreamSuite/PubSub/documentation/readme.txt b/src/DataStreamSuite/PubSub/documentation/readme.txt
index c23845ff1..af0703b49 100644
--- a/src/DataStreamSuite/PubSub/documentation/readme.txt
+++ b/src/DataStreamSuite/PubSub/documentation/readme.txt
@@ -21,9 +21,6 @@ client and sent over to the pubsubserver, where uints received ad deserialized
SubscriberCore:ISubscriberCore (retired now)
|
V
-StaticSubscriberCore
-|
-V
SingleServerSubscriberCore:ISingleServerSubscriberCore
PublisherCore:IPublisherCore (transactions not supported for now)
@@ -35,7 +32,21 @@ PubSubServer:
PubSubMessageReactor (top level, called from main)
ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection)
-
+PubSubState keeps the total pubsub data in 2 classes
+Dictionary and
+Dictionary
+SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state)
+SingleTagConnectionData contains connection vs sub, and connection vs pub
+PubSubState calls
+SystemSubscriptionMapPublisher.init(this);
+SystemStatsPublisher.init(this);
+SystemHeartbeatPublisher.init();
+---------------------------------------------------------------
+How internal tags are published?
+PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data
+onPublishStats is subscribed by SystemStatsPublisher
+SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send
+SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state
Streams:
PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges ->
diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs
index c243cf851..2f2acabcb 100644
--- a/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs
+++ b/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs
@@ -1,122 +1,78 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using System.IO;
+
+using BCG;
using BCG.Utilities.General;
-namespace BCG {
+using MathNet.Numerics.RootFinding;
+
+namespace bts {
[DataStreamUserRespresentation("tagStats")]
public class PubSubServerTagStatsStream : ViewStream, IWorker {
- private static Dictionary> myStreamMap =
- new Dictionary>();
+ private SubscriberCore _sub_core;
+ private ViewStreamDatabase _vsd;
- private SubscriberCore mySubCore;
- private ViewStreamDatabase myVsd;
- private Dictionary myTagToStatsMap = new Dictionary();
- private KeysBasedTable2 myTable = new KeysBasedTable2(new BlueTableSchema("", new ColumnInfo("id", 0, typeof(int)),
- new ColumnInfo("server", 1, typeof(string)),
- new ColumnInfo("tag", 2, typeof(string)),
- new ColumnInfo("nUpdates", 3, typeof(int)),
- new ColumnInfo("totalBytes", 4, typeof(int)),
- new ColumnInfo("minSize", 5, typeof(int)),
- new ColumnInfo("maxSize", 6, typeof(int)),
- new ColumnInfo("lastUpdateSize", 7, typeof(int)),
- new ColumnInfo("averageSize", 8, typeof(double)),
- new ColumnInfo("lastUpdateTime", 9, typeof(DateTime))),
- new string[] { "tag" });
-
- public static ViewStream getPubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL) {
- //you can also supply a service name and it should work, as long as substations are enabled.
- // if (PubSub3Config.instance.PS3_UseSubstations)
- // {
- // return new ThreadSafeStream(SubstationTagStatsStream.getSubstationTagStatsStream(vsd, serverURL));
- // }
- if (!myStreamMap.ContainsKey(vsd))
- myStreamMap.Add(vsd, new Dictionary());
- if (!myStreamMap[vsd].ContainsKey(serverURL))
- myStreamMap[vsd].Add(serverURL, new PubSubServerTagStatsStream(vsd, serverURL));
- return myStreamMap[vsd][serverURL];
- }
-
- #region constructors to be used by the static factory method
-
- public PubSubServerTagStatsStream(string serverURL)
- : this(ViewStreamDatabase.instance, serverURL) { }
-
- private PubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL) {
- myVsd = vsd;
- mySubCore = new SubscriberCore(new PubSubRouter(new PubSubRoute[1] { new PubSubRoute(serverURL, ".*") }));
- }
-
- #endregion
-
- #region exposed constructors that use the static factory methods
-
- public PubSubServerTagStatsStream() {
- // if (PubSub3Config.instance.PS3_UseSubstations)
- // mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(ViewStreamDatabase.instance, false)));
- // else
- mySetRealSource(new PubSubServerTagStatsStream(ViewStreamDatabase.instance, PubSub3Config.instance.getSubRouting()));
-
- }
+ private IBlueTableSchema _schema = new BlueTableSchema("tagStats",
+ new ColumnInfo("id", 0, typeof(int)),
+ new ColumnInfo("server", 1, typeof(string)),
+ new ColumnInfo("tag", 2, typeof(string)),
+ new ColumnInfo("n_updates", 3, typeof(int)),
+ new ColumnInfo("total_bytes", 4, typeof(int)),
+ new ColumnInfo("min_size", 5, typeof(int)),
+ new ColumnInfo("max_size", 6, typeof(int)),
+ new ColumnInfo("last_update_size", 7, typeof(int)),
+ new ColumnInfo("average_size", 8, typeof(double)),
+ new ColumnInfo("last_update_time", 9, typeof(DateTime)));
+ private IDictionary _tag_map = new Dictionary();
public PubSubServerTagStatsStream(ViewStreamDatabase vsd) {
- // if (PubSub3Config.instance.PS3_UseSubstations)
- // mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(vsd, false)));
- // else
- mySetRealSource(new PubSubServerTagStatsStream(vsd, PubSub3Config.instance.getSubRouting()));
-
+ _vsd = vsd;
+ _sub_core = new SubscriberCore();
}
- public PubSubServerTagStatsStream(ViewStreamDatabase vsd, PubSubRoute[] routes) {
- List vss = new List();
- foreach (string url in PubSub3Config.instance.getAllPubSubs())
- vss.Add(getPubSubServerTagStatsStream(vsd, url));
- mySetRealSource(new AppendStream(vss.ToArray()));
- }
-
- #endregion
+ public PubSubServerTagStatsStream()
+ : this(ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber() {
- mySubCore.subscribe("System_AllStatistics");
- myVsd.addWorker(this);
+ _sub_core.subscribe("System_AllStatistics");
+ _vsd.addWorker(this);
+
}
protected override void myOnLastSubscriber() {
- mySubCore.unsubscribe("System_AllStatistics");
- myVsd.removeWorker(this);
+ _sub_core.unsubscribe("System_AllStatistics");
+ _vsd.removeWorker(this);
}
- #region IWorker Members
-
public void doWork() {
- foreach (var u in mySubCore.get_updates(TimeSpan.Zero)) {
- List tag_stats = new List();
- MemoryStream str = new MemoryStream(u.data);
- BinaryReader br = new BinaryReader(str);
- int n = br.ReadInt32();
- for (int i = 0; i < n; i++) {
- tag_stats.Add(new TagStatsInfo(str));
+ var updates = _sub_core.get_updates(TimeSpan.Zero);
+ foreach (var u in updates) {
+ if (u.tag != "System_AllStatistics")
+ continue;
+ var tag_infos = TagStatsInfo.deserialize(u.data);
+ foreach (var ti in tag_infos) {
+ _tag_map[ti.tag] = ti;
}
- myHandleStats(tag_stats);
}
+ _fire_dp();
}
- #endregion
-
- private void myHandleStats(List tagStats) {
- BlueTable tagStatsRows = new BlueTable(myTable.schema);
- int id = 0;
- foreach (TagStatsInfo t in tagStats) {
- tagStatsRows.addRow(new BlueRow(myTable.schema, id++, t.server, t.tag, t.totalUpdates, t.totalBytes,
- t.minSize, t.maxSize, t.lastUpdateSize,
- t.averageSize, new DateTime(t.lastUpdateTime)));
+ private void _fire_dp() {
+ BlueTable bt = new BlueTable(_schema);
+ var sorted_tags = _tag_map.Keys.OrderBy(t => t);
+ // sorting it to keep the tag => id mapping the same (helps with incremental udpates)
+ foreach (var tag in sorted_tags) {
+ TagStatsInfo? ts_info = _tag_map[tag];
+ IBlueRow nr = new BlueRow(_schema, bt.Count, ts_info.server, ts_info.tag,
+ ts_info.total_updates, ts_info.total_bytes,
+ ts_info.min_size, ts_info.max_size,
+ ts_info.last_update_size, ts_info.average_size,
+ ts_info.last_update_time);
+ bt.addRow(nr);
}
- IBlueRowCollection dRows, aRows;
- myTable.updateMultipleRows(tagStatsRows, out dRows, out aRows);
- myFireEvent(new DataPacket(myTable, aRows, dRows));
+ var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true);
+ if (dp.addedRows.Count + dp.deletedRows.Count > 0)
+ myFireEvent(dp);
}
}
diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs
index 436c379ac..b4c929b3e 100644
--- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs
+++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs
@@ -4,6 +4,7 @@ using System.Text;
using BCG.Utilities.General;
using System.Collections;
using System.IO;
+using MongoDB.Driver.Core.WireProtocol.Messages;
namespace BCG {
///
@@ -11,65 +12,35 @@ namespace BCG {
/// server.
///
public class PubSubServerStateStream : ViewStream, IWorker {
- private static Dictionary> _stream_map =
- new Dictionary>();
- private KeysBasedTable2 _table =
- new KeysBasedTable2(new BlueTableSchema("tagInfo",
- new ColumnInfo("id", 0, typeof(int)),
- new ColumnInfo("tag", 1, typeof(string)),
- new ColumnInfo("server", 2, typeof(string)),
- new ColumnInfo("subscribers", 3, typeof(string)),
- new ColumnInfo("publishers", 4, typeof(string)),
- new ColumnInfo("deadPublishers", 5, typeof(string))),
- new string[] { "tag" });
+ private IDictionary _tag_map = new Dictionary();
+ private readonly string _tag = "System_AllSubscriptions";
+ private IBlueTableSchema _schema = null;
+
private SubscriberCore _sub_core;
private ViewStreamDatabase _vsd;
- private static ViewStream _get_pub_sub_server_state_stream(ViewStreamDatabase vsd, string server_url) {
- if (!_stream_map.ContainsKey(vsd))
- _stream_map.Add(vsd, new Dictionary());
- if (!_stream_map[vsd].ContainsKey(server_url))
- _stream_map[vsd].Add(server_url, new PubSubServerStateStream(server_url, vsd));
- return _stream_map[vsd][server_url];
- }
-
-
- // public static ViewStream getPubSubServerStateStream(ViewStreamDatabase vsd, string server_uRL)
- // {
- // return getPubSubServerStateStream(vsd, server_uRL);
- // }
-
-
- public PubSubServerStateStream(string server_url)
- : this(server_url, ViewStreamDatabase.instance) { }
-
- public PubSubServerStateStream(string server_url, ViewStreamDatabase vsd) {
- _sub_core = new SubscriberCore(new PubSubRouter(new PubSubRoute(server_url, ".*")));
- _vsd = vsd;
- }
-
public PubSubServerStateStream()
: this(ViewStreamDatabase.instance) { }
public PubSubServerStateStream(ViewStreamDatabase vsd) {
- mySetRealSource(new PubSubServerStateStream(vsd, PubSub3Config.instance.getSubRouting()));
+ // TODO, construct schema
+ _schema = BlueDataUtility.form_schema("pubsub_server_state",
+ Tuple.Create("id", TypeCode.Int32),
+ Tuple.Create("ps_server", TypeCode.String),
+ Tuple.Create("subscibers", TypeCode.String),
+ Tuple.Create("publisher", TypeCode.String),
+ Tuple.Create("dead_publishers", TypeCode.String));
+ _vsd = vsd;
+ _sub_core = new SubscriberCore();
}
-
- public PubSubServerStateStream(ViewStreamDatabase vsd, PubSubRoute[] routes) {
- List vss = new List();
- foreach (var r in routes)
- vss.Add(_get_pub_sub_server_state_stream(vsd, r.serverAddress));
- mySetRealSource(new AppendStream(vss.ToArray()));
- }
-
protected override void myOnFirstSubscriber() {
- _sub_core.subscribe("System_AllSubscriptions");
+ _sub_core.subscribe(_tag);
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
- _sub_core.unsubscribe("System_AllSubscriptions");
+ _sub_core.unsubscribe(_tag);
_vsd.removeWorker(this);
}
@@ -77,35 +48,37 @@ namespace BCG {
public void doWork() {
var updates = _sub_core.get_updates(TimeSpan.Zero);
- // this is where the pubsub updates are converted to rows
+ //this is where the pubsub updates are converted to rows
if (updates.Count() > 0) {
int id = 0;
- BlueTable addedRows = new BlueTable(_table.schema);
foreach (var u in updates) {
- BinaryReader br = new BinaryReader(new MemoryStream(u.data));
- // TODO: check this
- int n_updates = br.ReadInt32();
- for (int i = 0; i < n_updates; i++) {
- TagPubSubInfo tsi = new TagPubSubInfo(new MemoryStream(u.data));
- addedRows.addRow(new BlueRow(_table.schema, id++, tsi.tag, tsi.server, tsi.subscribersInString, tsi.publishersInString, tsi.deadPublishersInString));
+ if (u.tag != _tag) {
+ continue;
+ }
+ var tpsi_list = TagPubSubInfo.deserialize(u.data);
+ foreach (var tsi in tpsi_list) {
+ _tag_map[tsi.tag] = tsi;
}
}
- IBlueRowCollection dRows, aRows;
- _table.updateMultipleRows(addedRows, out dRows, out aRows);
- myFireEvent(new DataPacket(_table, aRows, dRows));
}
+ _fire_dp();
}
#endregion
- // public static string convertURLToTag(string URL) {
- // string[] urlWords = URL.Split('.');
- // string[] lastPart = urlWords[3].Split(':');
- // //the first one has "bluto://" in front of it
- // //we want to chop that off
- // string result = "bluto_" + urlWords[0].Substring(8) + "_" + urlWords[1] + "_" + urlWords[2] + "_" + lastPart[0] + "_" + lastPart[1];
+ private void _fire_dp() {
+ BlueTable bt = new BlueTable(_schema);
+ var sorted_tags = _tag_map.Keys.OrderBy(t => t);
+ foreach (var kv in _tag_map) {
+ var tsi = kv.Value;
+ IBlueRow r = new BlueRow(_schema, bt.Count, tsi.pubsub_server, tsi.subscribers_as_string,
+ tsi.publishers_as_string, tsi.dead_publishers_as_string);
+ bt.addRow(r);
+ }
+ var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true);
+ if (dp.addedRows.Count + dp.deletedRows.Count > 0)
+ myFireEvent(dp);
- // return result;
- // }
+ }
}
}
diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs
new file mode 100644
index 000000000..62c2e75e4
--- /dev/null
+++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs
@@ -0,0 +1,76 @@
+namespace bts {
+ using BCG;
+ using BCG.Utilities.General;
+
+ using MathNet.Numerics.RootFinding;
+
+ using ZstdSharp.Unsafe;
+ [DataStreamUserRespresentation("pss_hb")]
+ public class PubsubHeartbeatStream : ViewStream, IWorker {
+ private ViewStreamDatabase _vsd;
+ private SubscriberCore _subcore = new SubscriberCore();
+ private MutableBlueRow _row = null;
+ private readonly IBlueTableSchema _schema = null;
+ public PubsubHeartbeatStream(ViewStreamDatabase vsd) {
+ _vsd = vsd;
+ _schema = BlueDataUtility.form_schema("PS_HBB",
+ Tuple.Create("id", TypeCode.Int32),
+ Tuple.Create("hb_sent_time", TypeCode.Int64),
+ Tuple.Create("hb_recv_time", TypeCode.Int64),
+ Tuple.Create("travel_period", TypeCode.Int32),
+ Tuple.Create("time_since_last_recv_msecs", TypeCode.Int32)
+ );
+ }
+ public PubsubHeartbeatStream() :
+ this(ViewStreamDatabase.instance) { }
+
+ protected override void myOnFirstSubscriber() {
+ _subcore.subscribe("pss_hb");
+ _vsd.addWorker(this);
+
+ }
+
+ protected override void myOnLastSubscriber() {
+ _vsd.removeWorker(this);
+ _subcore.unsubscribe("pss_hb");
+
+ }
+
+ public void doWork() {
+ var updates = _subcore.get_updates(TimeSpan.FromMilliseconds(1));
+ if (updates.Count() > 0) {
+ foreach (var m in updates) {
+ if (m.data != null && m.data.Length > 0) {
+ BinaryReader br = new BinaryReader(new MemoryStream(m.data));
+ Int64 hb_sent_time = br.ReadInt64();
+ Int64 hb_recv_time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+ _row = _row == null ? new MutableBlueRow(_schema) : _row;
+ var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+ var time_since_last_recv_msecs = now - hb_recv_time;
+ _row.setValue("id", 0);
+ _row.setValue("hb_sent_time", hb_sent_time);
+ _row.setValue("hb_recv_time", hb_recv_time);
+ _row.setValue("travel_period", hb_recv_time - hb_sent_time);
+ _row.setValue("time_since_last_recv_msecs", time_since_last_recv_msecs);
+ // TODO, add to table and fire
+ }
+ }
+ } else {
+ if (_row != null) {
+ var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+ var ms_since_last_recv = now - _row.get("hb_recv_time");
+ _row.setValue("time_since_last_recv_msecs", ms_since_last_recv);
+ // TODO, add to table and fire
+ }
+ // update the row
+ }
+ if (_row != null) {
+ BlueTable bt = new BlueTable(_row.schema);
+ bt.addRow(_row);
+ var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true);
+ myFireEvent(dp);
+ }
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs
index 63ee61405..9802493e0 100644
--- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs
+++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs
@@ -26,7 +26,7 @@ namespace BCG {
public TagSynopsisStream(ViewStreamDatabase vsd, string route) {
- ViewStream vs = new PubSubServerStateStream(route, vsd);
+ ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd);
vs = new MapStream(vs, this);
myWorkers.Add(vs as IWorker);
@@ -41,7 +41,7 @@ namespace BCG {
public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes) {
List vss = new List();
foreach (string r in routes) {
- ViewStream vs = new PubSubServerStateStream(r, vsd);
+ ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd);
vs = new MapStream(vs, this);
vss.Add(vs);
myWorkers.Add(vs as IWorker);