From 4512e085f2eaabb95a9b624e48fdb9632017aef5 Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Wed, 28 Feb 2024 19:13:43 -0500 Subject: [PATCH] Pubsub stats --- .../Primitives/Misc/PubSubTestStream.cs | 12 +- .../Client/InternalTags/TagPubSubInfo.cs | 170 +++++++++--------- .../Client/InternalTags/TagStatsInfo.cs | 95 ++++++---- .../SystemHeartbeatPublisher.cs | 53 ++++-- .../SystemStatsPublisher.cs | 92 ++++++---- .../SystemSubscriptionMapPublisher.cs | 92 +++++----- .../PubSub/Server/PubSubMessageReactor.cs | 13 +- .../PubSub/Server/PubSubSingleTagState.cs | 56 +++--- .../PubSub/Server/PubSubState.cs | 113 ++++++------ .../PubSub/Server/SingleTagData.cs | 32 +--- .../PubSub/documentation/readme.txt | 19 +- .../PS3/Stats/PubSubServerTagStatsStream.cs | 146 ++++++--------- .../Subscription/PubSubServerStateStream.cs | 101 ++++------- .../PS3/Subscription/PubsubHeartbeatStream.cs | 76 ++++++++ .../PS3/Subscription/TagSynopsisStream.cs | 4 +- 15 files changed, 560 insertions(+), 514 deletions(-) create mode 100644 src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs index c0dee5d3c..29e87e7a2 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs @@ -72,7 +72,8 @@ namespace bts { private PayloadMessage _last_sent_msg, _last_recvd_msg; private readonly int _payload_sz; private const byte _payload_byte = (byte)'A'; - public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, ViewStreamDatabase vsd) { + private readonly string _tag_name; + public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) { _vsd = vsd; _payload_sz = payload_size; var schema = BlueDataUtility.form_schema("PubSubTest", @@ -89,10 +90,11 @@ namespace bts { _publish_period = TimeSpan.FromSeconds(publish_period_seconds); _msg_size = payload_size; _max_pubs = max_pubs; + _tag_name = tag_name; } - public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size) - : this(publish_period_seconds, max_pubs, msg_size, ViewStreamDatabase.instance) { } + public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name) + : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { } protected override void myOnFirstSubscriber() { _vsd.addWorker(this); @@ -117,7 +119,7 @@ namespace bts { private void _subscribe_thread_func() { // subscription is done in another thread SubscriberCore sub_core = new SubscriberCore(); - sub_core.subscribe("test_tag"); + sub_core.subscribe(_tag_name); while (_continue_sub_thread) { // TODO recv_id var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250)); @@ -150,7 +152,7 @@ namespace bts { } var ser_bytes = _last_sent_msg.serialize(); _n_pubs++; - _pub_core.publish(String.Empty, "test_tag", ser_bytes); + _pub_core.publish(String.Empty, _tag_name, ser_bytes); } private void _fire_dp() { diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs index 640f63278..c4e4b3db1 100644 --- a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs +++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs @@ -4,25 +4,23 @@ using System.Text; using System.IO; //using BCG.Utilities; using mbts; +using System.Linq; -namespace BCG -{ +namespace BCG { /// /// The list of publishers and subscribers for /// the tag is contained herein. Its used for internal /// purposes, most noticeably TagInfoStream /// - public class TagPubSubInfo - { + public class TagPubSubInfo { public string tag; public IDictionary subscribers, publishers; - public ICollection> deadPublishers; - public string server; + public ICollection> dead_publishers; + public string pubsub_server; - public TagPubSubInfo(Stream s) - { - myReadFrom(s); + public TagPubSubInfo(Stream s) { + _read_from(s); //if (tag == "System_AllSubscriptions" && publishers.Count == 0) //{ @@ -31,82 +29,89 @@ namespace BCG //} } - public TagPubSubInfo(string theTag, IDictionary theSubscribers, - IDictionary theActivePublishers, - ICollection> theDeadPublishers, - string theServer) - { - tag = theTag; - server = theServer; - subscribers = theSubscribers; - publishers = theActivePublishers; - deadPublishers = theDeadPublishers; - - //if (tag == "System_AllSubscriptions" && publishers.Count == 0) - //{ - // Console.WriteLine("No publisher, direct cons ?? wtf ??"); - // System.Windows.Forms.MessageBox.Show("No publisher, direct cons ?? wtf ??"); - //} + public TagPubSubInfo(string the_tag, IDictionary the_subscribers, + IDictionary the_active_publishers, + ICollection> the_dead_publishers, + string the_pubsub_erver) { + tag = the_tag; + pubsub_server = the_pubsub_erver; + subscribers = the_subscribers; + publishers = the_active_publishers; + dead_publishers = the_dead_publishers; } - /// /// writes itself to the stream passed on as /// the arg - /// {tag,subCount,subs,conns,pub,pubConn,server} - /// - /// - public void writeTo(Stream s) - { + /// {tag,sub_count,*,pub_count*, pubsub_server} + public void write_to(Stream s) { BinaryWriter bw = new BinaryWriter(s); bw.Write(tag); bw.Write(subscribers.Count); - foreach (KeyValuePair p in subscribers) - { + foreach (KeyValuePair p in subscribers) { bw.Write(p.Key); bw.Write(p.Value); } bw.Write(publishers.Count); - foreach (KeyValuePair p in publishers) - { + foreach (KeyValuePair p in publishers) { bw.Write(p.Key); bw.Write(p.Value); } - bw.Write(deadPublishers.Count); - foreach (Pair p in deadPublishers) - { + bw.Write(dead_publishers.Count); + foreach (Pair p in dead_publishers) { bw.Write(p.first); bw.Write(p.second); } - bw.Write(server); + bw.Write(pubsub_server); + } + + public static byte[] serialize(IEnumerable tpsis) { + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + bw.Write(tpsis.Count()); + foreach (var tpsi in tpsis) { + tpsi.write_to(str); + } + str.Position = 0; + return str.ToArray(); + } + + public static List deserialize(byte[] data) { + List rv = new List(); + MemoryStream str = new MemoryStream(data); + BinaryReader br = new BinaryReader(str); + var count = br.ReadInt32(); + for (int i = 0; i < count; i++) { + TagPubSubInfo tsi = new TagPubSubInfo(str); + rv.Add(tsi); + } + return rv; } /// /// constructs itself from the stream /// /// - private void myReadFrom(Stream s) - { + private void _read_from(Stream s) { BinaryReader br = new BinaryReader(s); tag = br.ReadString(); - int nSubs = br.ReadInt32(); + int n_subs = br.ReadInt32(); subscribers = new Dictionary(); ; - for (int i = 0; i < nSubs; i++) - { + for (int i = 0; i < n_subs; i++) { subscribers.Add(br.ReadString(), br.ReadString()); } publishers = new Dictionary(); - int nPubs = br.ReadInt32(); - for (int i = 0; i < nPubs; i++) - { + int n_pubs = br.ReadInt32(); + for (int i = 0; i < n_pubs; i++) { publishers.Add(br.ReadString(), br.ReadString()); } - deadPublishers = new List>(); - int nDeadPubs = br.ReadInt32(); - for (int i = 0; i < nDeadPubs; i++) - { - (deadPublishers as List>).Add(new Pair(br.ReadString(), br.ReadString())); + dead_publishers = new List>(); + int n_dead_pubs = br.ReadInt32(); + List> dead_pubs = new List>(); + for (int i = 0; i < n_dead_pubs; i++) { + dead_pubs.Add(new Pair(br.ReadString(), br.ReadString())); } - server = br.ReadString(); + dead_publishers = dead_pubs; + pubsub_server = br.ReadString(); } /// @@ -116,54 +121,41 @@ namespace BCG /// well (it had no publisher to begin with). /// However if a tag is ever published it will never become an orphan /// - public bool orphaned - { - get - { - return publishers.Count == 0 && subscribers.Count == 0; - } - } + // public bool orphaned { + // get { + // return publishers.Count == 0 && subscribers.Count == 0; + // } + // } - public string subscribersInString - { - get - { - string subString = ""; - foreach (KeyValuePair p in subscribers) - { - subString += (subString == "" ? "" : ",") + p.Value + ';' + p.Key; + public string subscribers_as_string { + get { + string sub_str = ""; + foreach (KeyValuePair p in subscribers) { + sub_str += (sub_str == "" ? "" : ",") + p.Value + ';' + p.Key; } - return subString; + return sub_str; } } - public string publishersInString - { - get - { - string pubString = ""; - foreach (KeyValuePair p in publishers) - { - pubString += (pubString == "" ? "" : ",") + p.Value + ';' + p.Key; + public string publishers_as_string { + get { + string pub_string = ""; + foreach (KeyValuePair p in publishers) { + pub_string += (pub_string == "" ? "" : ",") + p.Value + ';' + p.Key; } - return pubString; + return pub_string; } } - public string deadPublishersInString - { - get - { - string deadPubString = ""; - foreach (Pair p in deadPublishers) - { + public string dead_publishers_as_string { + get { + string dead_pub_string = ""; + foreach (Pair p in dead_publishers) { if (p != null) - deadPubString += (deadPubString == "" ? "" : ",") + p.second + ';' + p.first; - else{} - //BlueEventLog.logEvent("DeadPublishers in deadPublishersInString is null", "Error"); + dead_pub_string += (dead_pub_string == "" ? "" : ",") + p.second + ';' + p.first; } - return deadPubString; + return dead_pub_string; } } diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs index 6b79123d5..7a792f263 100644 --- a/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs +++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagStatsInfo.cs @@ -2,62 +2,81 @@ using System; using System.Collections.Generic; using System.Text; using System.IO; +using System.Runtime.CompilerServices; +using System.Linq; -namespace BCG -{ - public class TagStatsInfo - { +namespace BCG { + public class TagStatsInfo { public string server; public string tag; - public int minSize = Int32.MaxValue, maxSize = Int32.MinValue, lastUpdateSize; - public double averageSize; - public int totalUpdates; - public long totalBytes; - public long lastUpdateTime; + public int min_size = Int32.MaxValue, max_size = Int32.MinValue, last_update_size; + public double average_size; + public int total_updates; + public long total_bytes; + public long last_update_time; - public TagStatsInfo(string theTag) - { + public TagStatsInfo(string theTag) { server = ClientInfo.thisClient.machineName; tag = theTag; } - public TagStatsInfo(Stream str) - { + public TagStatsInfo(Stream str) { BinaryReader br = new BinaryReader(str); server = br.ReadString(); tag = br.ReadString(); - minSize = br.ReadInt32(); - maxSize = br.ReadInt32(); - lastUpdateSize = br.ReadInt32(); - averageSize = br.ReadDouble(); - totalUpdates = br.ReadInt32(); - totalBytes = br.ReadInt64(); - lastUpdateTime = br.ReadInt64(); + min_size = br.ReadInt32(); + max_size = br.ReadInt32(); + last_update_size = br.ReadInt32(); + average_size = br.ReadDouble(); + total_updates = br.ReadInt32(); + total_bytes = br.ReadInt64(); + last_update_time = br.ReadInt64(); } - public void update(int size) - { - totalUpdates++; - totalBytes += size; - averageSize = totalBytes / totalUpdates; - minSize = size < minSize ? size : minSize; - maxSize = size > maxSize ? size : maxSize; - lastUpdateSize = size; - lastUpdateTime = DateTime.Now.Ticks; + public void update(int size) { + total_updates++; + total_bytes += size; + average_size = total_bytes / total_updates; + min_size = size < min_size ? size : min_size; + max_size = size > max_size ? size : max_size; + last_update_size = size; + last_update_time = DateTime.Now.Ticks; } - public void writeTo(Stream str) - { + public void write_to(Stream str) { BinaryWriter bw = new BinaryWriter(str); bw.Write(server); bw.Write(tag); - bw.Write(minSize); - bw.Write(maxSize); - bw.Write(lastUpdateSize); - bw.Write(averageSize); - bw.Write(totalUpdates); - bw.Write(totalBytes); - bw.Write(lastUpdateTime); + bw.Write(min_size); + bw.Write(max_size); + bw.Write(last_update_size); + bw.Write(average_size); + bw.Write(total_updates); + bw.Write(total_bytes); + bw.Write(last_update_time); + } + + public static byte[] serialize(IEnumerable items) { + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + bw.Write(items.Count()); + foreach (var i in items) { + i.write_to(str); + } + str.Position = 0; + return str.ToArray(); + } + + public static IEnumerable deserialize(byte[] data) { + List rv = new List(); + MemoryStream str = new MemoryStream(data); + BinaryReader br = new BinaryReader(str); + int count = br.ReadInt32(); + for (int i = 0; i < count; i++) { + TagStatsInfo ts_info = new TagStatsInfo(str); + rv.Add(ts_info); + } + return rv; } } diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs index 47375d018..fb506104b 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs @@ -3,18 +3,40 @@ using System.Collections.Generic; using System.Text; using BCG.Utilities.General; using System.IO; +using bts; namespace BCG { + class Pubsubheartbeat { + public Int64 hb_time_epoch; + public byte[] serialize() { + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + bw.Write(hb_time_epoch); + str.Position = 0; + return str.ToArray(); + } + + public Pubsubheartbeat() { + hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + } + + public static Pubsubheartbeat deserialize(byte[] data) { + long hb_time = BitConverter.ToInt64(data); + return new Pubsubheartbeat() { + hb_time_epoch = hb_time, + }; + } + } /// /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat. /// A subscriber can subscribe to it, and find out of a PubSub is alive or /// not. Infact more so, if the subsriptions are working or not. /// internal class SystemHeartbeatPublisher : IWorker { - public static string tagType = "System_Heartbeat"; - public static string tagName = "System_Heartbeat"; + public static string tag_type = "System_Heartbeat"; + public static string tag_name = "System_Heartbeat"; - private static SystemHeartbeatPublisher mySelf; + private static SystemHeartbeatPublisher _self; private SystemHeartbeatPublisher() { @@ -23,24 +45,29 @@ namespace BCG { ///waits 2500 msecs before calling doWork. This 2500 msecs ///is only interceded by an external event (publication or ///subscription) - MasterWorker.getInstance(true).addWorker(this, new TimeSpan(0, 0, 0, 0, 500)); + MasterWorker.getInstance(true).addWorker(this, + TimeSpan.FromMilliseconds(500)); } public static void init() { - if (mySelf == null) - mySelf = new SystemHeartbeatPublisher(); + if (_self == null) + _self = new SystemHeartbeatPublisher(); } #region IWorker Members public void doWork() { - PubMetaData pub = new PubMetaData(tagType, tagName, 1, "local", ClientInfo.thisClient.processString); - MetaData md = new MetaData(pub); - MemoryStream ms = new MemoryStream(); - md.save(ms); - BinaryWriter bw = new BinaryWriter(ms); - bw.Write(true); - PubSubMessageReactor.internalSend(ms.ToArray()); + Pubsubheartbeat hb = new Pubsubheartbeat(); + PublishMsg msg = new PublishMsg() { + connection = "Internal", + publishing_app = ClientInfo.thisClient.app, + tag = "pss_hb", + tag_type = "Internal", + is_compressed = false, + publish_time_epoch_ms = hb.hb_time_epoch, + data_to_publish = hb.serialize() + }; + PubSubMessageReactor.internal_publish(msg); } #endregion diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs index 61d7106ef..c47d2ce25 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs @@ -3,63 +3,76 @@ using System.Collections.Generic; using System.Text; using System.IO; using BCG.Utilities.General; +using bts; namespace BCG { /// - /// Hooks on to the onPublish event. Mantains the statistics - /// for each tag. This also is an IPubSubServerTagHandler. The reason - /// being that since it already holds all the tag statistics, - /// theres no reason to have another class do the same. Hence, + /// Hooks on to the PubSUbState's onPublish event. Mantains the statistics + /// for each tag, and publishes the stats, on the System_AllStatistics tag, + /// using the PubSubMessageReactor internal_publish. Since this tag is being + /// published tot he Pubsub, , the state has to be kept somwhere, and thus this class + /// is also an IPubSubServerTagHandler, and returns the total state on getContent /// it doesnt do anything on the update as it, already has the state. /// /// On the doWork, it internaly publishes the delta. /// internal class SystemStatsPublisher : IPubSubServerTagHandler { - public static string tagType = "System_Stats"; - public static string tagName = "System_AllStatistics"; - private Dictionary myTagsReadyToPublish = new Dictionary(); - private Dictionary myTagStatsMap = new Dictionary(); - private MasterWorker myMasterWorker = MasterWorker.getInstance(false); - private static SystemStatsPublisher mySelf = null; + public static string tag_type = "System_Stats"; + public static string tag_name = "System_AllStatistics"; + // these tags have been touched, need to publish in the next doWork + private HashSet _dirty_tags = new HashSet(); + private Dictionary _tag_stats_map = new Dictionary(); + private MasterWorker _master_worker = MasterWorker.getInstance(false); + private static SystemStatsPublisher _self = null; private SystemStatsPublisher(PubSubState state) { - state.onPublishStats += new PubSubState.PubSubStatsDelegate(myHandlePublish); - myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StatisticsUpdatePeriod)); + bts.Log.Gen.info("SystemStatsPublisher constructed"); + state.onPublishStats += new PubSubState.PubSubStatsDelegate(_handle_publish); + _master_worker.addWorker(this, TimeSpan.FromMilliseconds(250)); } public static void init(PubSubState state) { - mySelf = new SystemStatsPublisher(state); + bts.Log.Gen.info("SystemStatsPublisher.init"); + _self = new SystemStatsPublisher(state); } - public static IPubSubServerTagHandler getInstance() { - return mySelf; + public static IPubSubServerTagHandler get_instance() { + return _self; } - void myHandlePublish(string tag, int size) { - if (tag != tagName) { - myTagsReadyToPublish[tag] = true; - if (!myTagStatsMap.ContainsKey(tag)) - myTagStatsMap.Add(tag, new TagStatsInfo(tag)); - myTagStatsMap[tag].update(size); + void _handle_publish(string tag, int size) { + bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}"); + if (tag == tag_name) + return; + TagStatsInfo tag_stats = null; + if (!_tag_stats_map.TryGetValue(tag, out tag_stats)) { + tag_stats = new TagStatsInfo(tag); + _tag_stats_map.Add(tag, tag_stats); } + tag_stats.update(size); + _dirty_tags.Add(tag); } + #region IWorker Members public void doWork() { - if (myTagsReadyToPublish.Count > 0) { - // TODO: PubSubServerMsg - PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString); + if (_dirty_tags.Count > 0) { MemoryStream str = new MemoryStream(); - BinaryWriter bw = new BinaryWriter(str); - MetaData md = new MetaData(pub); - md.save(str); - new BinaryWriter(str).Write(myTagsReadyToPublish.Count); - foreach (string tag in myTagsReadyToPublish.Keys) { - myTagStatsMap[tag].writeTo(str); + List dirty_ts_info = new List(); + foreach (string tag in _dirty_tags) { + //_tag_stats_map[tag].write_to(str); + dirty_ts_info.Add(_tag_stats_map[tag]); } - myTagsReadyToPublish.Clear(); - PubSubMessageReactor.internalSend(str.ToArray()); + _dirty_tags.Clear(); + var bytes = TagStatsInfo.serialize(dirty_ts_info); + PublishMsg pub_msg = new PublishMsg() { + connection = "InternalPublish", + tag = SystemStatsPublisher.tag_name, + tag_type = SystemStatsPublisher.tag_type, + data_to_publish = bytes + }; + PubSubMessageReactor.internal_publish(pub_msg); } } @@ -71,15 +84,18 @@ namespace BCG { } public byte[] getContent(string tag) { - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - bw.Write(myTagStatsMap.Count); - foreach (TagStatsInfo t in myTagStatsMap.Values) - t.writeTo(bw.BaseStream); - return (bw.BaseStream as MemoryStream).ToArray(); + var ser = TagStatsInfo.serialize(_tag_stats_map.Values); + return ser; + // MemoryStream str = new MemoryStream(); + // BinaryWriter bw = new BinaryWriter(str); + // bw.Write(_tag_stats_map.Count); + // foreach (TagStatsInfo t in _tag_stats_map.Values) + // t.write_to(bw.BaseStream); + // return str.ToArray(); } public string[] supportedTagTypes { - get { return new string[] { SystemStatsPublisher.tagType }; } + get { return new string[] { SystemStatsPublisher.tag_type }; } } public void register(InternalPublishDelegate callback) { } diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs index d3512e40d..f7a01f311 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs @@ -5,6 +5,8 @@ using BCG.Utilities.General; using System.IO; using System.Threading; using mbts; +using System.Threading.Tasks; +using bts; namespace BCG { /// @@ -23,60 +25,61 @@ namespace BCG { /// It uses the PubSubState to publish the entire map on getUpdates /// class SystemSubscriptionMapPublisher : IPubSubServerTagHandler { - private PubSubState myPubsubState; - private Dictionary myTagsReadyToPublish = new Dictionary(); - private readonly string myIp = SysInfoManager.getIP(); - public static string tagType = "System_Sub"; - public static string tagName = "System_AllSubscriptions"; - private MasterWorker myMasterWorker = MasterWorker.getInstance(false); - private static IPubSubServerTagHandler mySelf; + private PubSubState _state; + private HashSet _dirty_tags = new HashSet(); + private readonly string _ip = SysInfoManager.getIP(); + public static string tag_type = "System_Sub"; + public static string tag_name = "System_AllSubscriptions"; + private MasterWorker _master_worker = MasterWorker.getInstance(false); + private static IPubSubServerTagHandler _self; private SystemSubscriptionMapPublisher(PubSubState state) { - myPubsubState = state; - myPubsubState.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(myOnTagStateChanges); - myMasterWorker.addWorker(this, new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_StateUpdatePeriod)); + bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor"); + _state = state; + _state.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(_on_tag_state_changes); + _master_worker.addWorker(this, TimeSpan.FromMilliseconds(500)); } public static void init(PubSubState state) { - mySelf = new SystemSubscriptionMapPublisher(state); + bts.Log.Gen.info("SystemSubscriptionMapPublisher.init"); + _self = new SystemSubscriptionMapPublisher(state); } public static IPubSubServerTagHandler getInstance() { - return mySelf; + return _self; } - private void myOnTagStateChanges(string[] tags) { - foreach (string t in tags) - myTagsReadyToPublish[t] = true; + private void _on_tag_state_changes(string[] tags) { + bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes len: {tags.Length}"); + _dirty_tags.AddRange(tags); } - private void myInternalPublish() { - if (myTagsReadyToPublish.Count > 0) { - ///serialize and write a PubMetaData - PubMetaData pub = new PubMetaData(tagType, tagName, 0, "local", ClientInfo.thisClient.processString); - MemoryStream str = new MemoryStream(); - MetaData md = new MetaData(pub); - md.save(str); - BinaryWriter bw = new BinaryWriter(str); - ///count of the following items - bw.Write(myTagsReadyToPublish.Count); - ///TagPubSubInfo serialized one by one - foreach (string tag in myTagsReadyToPublish.Keys) { - Dictionary subscribers, activePubslishers; - ICollection> deadPublishers; - myPubsubState.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers); - TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp); - ti.writeTo(str); + private void _internal_publish() { + if (_dirty_tags.Count > 0) { + List tpsis = new List(); + foreach (string tag in _dirty_tags) { + Dictionary subscribers, active_pubslishers; + ICollection> dead_publishers; + _state.get_all_connections(tag, out subscribers, out active_pubslishers, out dead_publishers); + TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, active_pubslishers, + dead_publishers, _ip); + tpsis.Add(ti); } - myTagsReadyToPublish.Clear(); - PubSubMessageReactor.internalSend(str.ToArray()); + _dirty_tags.Clear(); + var ser_bytes = TagPubSubInfo.serialize(tpsis); + PublishMsg pub_msg = new PublishMsg() { + publish_time_epoch_ms = 0, + is_compressed = false, + data_to_publish = ser_bytes + }; + PubSubMessageReactor.internal_publish(pub_msg); } } #region IWorker Members public void doWork() { - myInternalPublish(); + _internal_publish(); } #endregion @@ -86,22 +89,21 @@ namespace BCG { public void update(string tag, Stream[] messageBodies) { } public byte[] getContent(string t) { - ICollection allTags = myPubsubState.all_tags; - MemoryStream str = new MemoryStream(); - BinaryWriter bw = new BinaryWriter(str); - bw.Write(allTags.Count); - foreach (string tag in allTags) { + ICollection all_tags = _state.all_tags; + List ti_list = new List(); + foreach (string tag in all_tags) { Dictionary subscribers, activePubslishers; ICollection> deadPublishers; - myPubsubState.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers); - TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, myIp); - ti.writeTo(str); + _state.get_all_connections(tag, out subscribers, out activePubslishers, out deadPublishers); + TagPubSubInfo ti = new TagPubSubInfo(tag, subscribers, activePubslishers, deadPublishers, _ip); + ti_list.Add(ti); } - return str.ToArray(); + var rv = TagPubSubInfo.serialize(ti_list); + return rv; } public string[] supportedTagTypes { - get { return new string[] { SystemSubscriptionMapPublisher.tagType }; } + get { return new string[] { SystemSubscriptionMapPublisher.tag_type }; } } public void register(InternalPublishDelegate callback) { } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index 57fd4730b..ca27c30d6 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -28,6 +28,7 @@ namespace BCG { private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); private SingleTagTimeSliceDelegate onSingleTagPub; private HighPerformanceTimer mySingleTagTimeSliceTimer = new HighPerformanceTimer(); + private static SyncQueue _internal_pub_queue = new SyncQueue(); //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. /* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/ @@ -81,6 +82,11 @@ namespace BCG { private void _polling_thread_func() { TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500); _msg_hub.activate(); + Action handle_internal_pubs = () => { + var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000); + foreach (var msg in internal_pubs) + _handle_pub("Internal", msg); + }; while (true) { try { @@ -92,6 +98,7 @@ namespace BCG { foreach (var kv in received_data) { _handle_client_msg(kv.first, kv.second); } + handle_internal_pubs(); _mainloop_master_worker.doWork(); } catch (Exception ex) { bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}"); @@ -181,8 +188,10 @@ namespace BCG { //myReceivingQueue.enqueue(new Pair(internalPublisherConnection, data)); } - public static void internal_publish(PublishMsg pub_msg) { - //_handle_pub("InternalPublisher", pub_msg); + //private static PubsSubM + public static void internal_publish(PublishMsg msg) { + bts.Log.Gen.info($"PubsubMessageReactor.internal_publish msg: {msg}"); + _internal_pub_queue.enqueue(msg); } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs index 36e3fc67f..ae9520df3 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs @@ -4,19 +4,16 @@ using System.Linq; using System.Text; /* - * This class contains the recent chagnes of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. + * This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. * It has a list of new subscription events. * */ -namespace BCG -{ - public class PubSubSingleTagState - { +namespace BCG { + public class PubSubSingleTagState { private string myTag; - public string tag - { + public string tag { get { return myTag; } - set { myTag = value; } + set { myTag = value; } } //we have this here as SortedList so that we can compare for equality @@ -27,24 +24,22 @@ namespace BCG private Dictionary myNewPublishers; private Dictionary myRemovedPublishers; - - public PubSubSingleTagState() - { + + public PubSubSingleTagState() { //the total state of the tag - myPublishers = new SortedList(); - mySubscribers = new SortedList(); + myPublishers = new SortedList(); + mySubscribers = new SortedList(); //how has the state changed recentlty myNewSubscribers = new Dictionary(); myNewPublishers = new Dictionary(); - myRemovedPublishers= new Dictionary(); + myRemovedPublishers = new Dictionary(); myRemovedSubscribers = new Dictionary(); } //returns a list of all the events that need to be published. - public PubSubSingleTagStateEvent[] getNewEventsForPublishing() - { + public PubSubSingleTagStateEvent[] getNewEventsForPublishing() { PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count]; myNewSubscribers.Values.CopyTo(newEvents, 0); @@ -54,7 +49,7 @@ namespace BCG myRemovedSubscribers.Values.CopyTo(newEvents, index); index += myRemovedSubscribers.Count; myRemovedPublishers.Values.CopyTo(newEvents, index); - + return newEvents; } @@ -62,19 +57,16 @@ namespace BCG //call this after you have published the new events //this is made public for code readability reasons //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily - public void resetNewEvents() - { + public void resetNewEvents() { myNewSubscribers = new Dictionary(); myNewPublishers = new Dictionary(); myRemovedSubscribers = new Dictionary(); myRemovedPublishers = new Dictionary(); } - public void newPublish(PubSubSingleTagStateEvent newPublisher) - { + public void newPublish(PubSubSingleTagStateEvent newPublisher) { //don't do anything here if we already knew about the publishers because we were already dealing with this publishers - if(!myPublishers.ContainsKey(newPublisher.app)) - { + if (!myPublishers.ContainsKey(newPublisher.app)) { myPublishers.Add(newPublisher.app, newPublisher.app); //if we were going to delete it, that means it was already being published by this app @@ -88,10 +80,8 @@ namespace BCG } } - public void removePublisher(PubSubSingleTagStateEvent removedPublisher) - { - if(myPublishers.ContainsKey(removedPublisher.app)) - { + public void removePublisher(PubSubSingleTagStateEvent removedPublisher) { + if (myPublishers.ContainsKey(removedPublisher.app)) { myPublishers.Remove(removedPublisher.app); //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it @@ -103,10 +93,8 @@ namespace BCG } - public void newSubscription(PubSubSingleTagStateEvent newSubscriber) - { - if (!mySubscribers.ContainsKey(newSubscriber.app)) - { + public void newSubscription(PubSubSingleTagStateEvent newSubscriber) { + if (!mySubscribers.ContainsKey(newSubscriber.app)) { mySubscribers.Add(newSubscriber.app, newSubscriber.app); if (myRemovedSubscribers.ContainsKey(newSubscriber.app)) @@ -118,10 +106,8 @@ namespace BCG } } - public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) - { - if(mySubscribers.ContainsKey(removedSubscriber.app)) - { + public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) { + if (mySubscribers.ContainsKey(removedSubscriber.app)) { mySubscribers.Remove(removedSubscriber.app); //if we were going to add this subscriber, we no logner want to //this means no event needs to be published diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index 3c1719265..527dfd1fb 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -48,20 +48,20 @@ namespace BCG { /// /// /// - public void sub(SubMetaData sub) { - ///tag based key ds update - foreach (string tag in sub.tags) { - if (!_tag_to_connection_map.ContainsKey(tag)) { - _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag)); - } - _tag_to_connection_map[tag].onSub(sub); + // public void sub(SubMetaData sub) { + // ///tag based key ds update + // foreach (string tag in sub.tags) { + // if (!_tag_to_connection_map.ContainsKey(tag)) { + // _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag)); + // } + // _tag_to_connection_map[tag].onSub(sub); - if (onSingleTagChange != null) - onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app)); - } - if (onTagMapChanged != null) - onTagMapChanged(sub.tags); - } + // if (onSingleTagChange != null) + // onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app)); + // } + // if (onTagMapChanged != null) + // onTagMapChanged(sub.tags); + // } public void handle_sub(bts.SubscribeMsg sub_msg) { foreach (string tag in sub_msg.tags) { @@ -80,23 +80,23 @@ namespace BCG { // TODO } - public void unsub(UnSubMetaData unsub) { - foreach (string tag in unsub.tags) { - if (_tag_to_connection_map.ContainsKey(tag)) { - ///maybe check here, that if there are no subscribers - ///and publishers for the tag then remove it. - _tag_to_connection_map[tag].onUnsub(unsub); - if (onSingleTagChange != null) - onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app)); - } else { - string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + - " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app); - //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); - } - } - if (onTagMapChanged != null) - onTagMapChanged(unsub.tags); - } + // public void unsub(UnSubMetaData unsub) { + // foreach (string tag in unsub.tags) { + // if (_tag_to_connection_map.ContainsKey(tag)) { + // ///maybe check here, that if there are no subscribers + // ///and publishers for the tag then remove it. + // _tag_to_connection_map[tag].onUnsub(unsub); + // if (onSingleTagChange != null) + // onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app)); + // } else { + // string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + + // " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app); + // //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); + // } + // } + // if (onTagMapChanged != null) + // onTagMapChanged(unsub.tags); + // } public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) { foreach (string tag in ubnsub_msg.tags) { @@ -150,35 +150,34 @@ namespace BCG { /// /// /// - public void updatePubData(PubMetaData pub, Stream msg) { - int sz = (int)(msg.Length - msg.Position); - bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}"); - ///this is the only place where SingleTagData should be created - if (!_tag_to_data_map.ContainsKey(pub.tag)) { - ///first time data is published for this tag - _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type)); - } - _tag_to_data_map[pub.tag].pubUpdate(msg); + // public void updatePubData(PubMetaData pub, Stream msg) { + // int sz = (int)(msg.Length - msg.Position); + // bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}"); + // ///this is the only place where SingleTagData should be created + // if (!_tag_to_data_map.ContainsKey(pub.tag)) { + // ///first time data is published for this tag + // _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type)); + // } + // _tag_to_data_map[pub.tag].pubUpdate(msg); - if (!_tag_to_connection_map.ContainsKey(pub.tag)) { - _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag)); - } - bool pubChanged = false; - _tag_to_connection_map[pub.tag].onPub(pub, out pubChanged); - if (pubChanged) { - if (onTagMapChanged != null) - onTagMapChanged(new string[] { pub.tag }); + // if (!_tag_to_connection_map.ContainsKey(pub.tag)) { + // _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag)); + // } + // bool pubChanged = false; + // _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged); + // if (pubChanged) { + // if (onTagMapChanged != null) + // onTagMapChanged(new string[] { pub.tag }); - if (onSingleTagChange != null) - onSingleTagChange( - new PubSubSingleTagStateEvent(pub.tag, - PubSubSingleTagStateEvent.EventType.publish, - pub.connection, pub.publishingApp)); - } - if (onPublishStats != null) - onPublishStats(pub.tag, sz); - - } + // if (onSingleTagChange != null) + // onSingleTagChange( + // new PubSubSingleTagStateEvent(pub.tag, + // PubSubSingleTagStateEvent.EventType.publish, + // pub.connection, pub.publishingApp)); + // } + // if (onPublishStats != null) + // onPublishStats(pub.tag, sz); + // } public void update_pub_data(bts.PublishMsg pub_msg) { // TODO diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs index 1afcdbf8a..3a73093a5 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs @@ -20,12 +20,13 @@ namespace BCG { private IPubSubServerTagHandler _tag_handler; public SingleTagData(string tag, string tag_type) { + bts.Log.Gen.info("SingleTagData constructed for tag: {tag} tag_type: {tag_type}"); _tag = tag; _tag_type = tag_type; - if (_tag_type == SystemSubscriptionMapPublisher.tagType) - _tag_handler = SystemSubscriptionMapPublisher.getInstance(); - else if (_tag_type == SystemStatsPublisher.tagType) - _tag_handler = SystemStatsPublisher.getInstance(); + // if (_tag_type == SystemSubscriptionMapPublisher.tagType) + // _tag_handler = SystemSubscriptionMapPublisher.getInstance(); + if (_tag_type == SystemStatsPublisher.tag_type) + _tag_handler = SystemStatsPublisher.get_instance(); else // this is where it gets the stream handler _tag_handler = TagType2HandlerMap.getHandler(_tag_type, true); @@ -37,29 +38,6 @@ namespace BCG { public string tag_type { get { return _tag_type; } } - internal void pubUpdate(Stream stream) { - string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL"; - bts.Log.Gen.info($"SingleTagData.pubUpdate tag: {_tag} handler: {handler_str} stream: {stream.Position} / {stream.Length}"); - if (_tag_handler != null) { - try { - _tag_handler.update(_tag, new Stream[] { stream }); - } catch (Exception e) { - // BlueEventLog.logEvent(e); - string errMsg = "tag handler threw an exception " + e.Message + " for tag type = " + _tag_type + - " tag = " + _tag + " length = " + stream.Length + "\n" + - "tag stated not updated. Stack Trace : " + e.StackTrace; - Console.WriteLine(errMsg); - if (e is OutOfMemoryException) - throw e; - else - throw new ApplicationException(errMsg, e); - } - } else { - _last_published_data = new byte[stream.Length - stream.Position]; - stream.Read(_last_published_data, 0, _last_published_data.Length); - } - } - internal void handle_publish(PublishMsg pub_msg) { string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL"; bts.Log.Gen.info($"SingleTagData.handle_publish pub_msg: {pub_msg} handler: {handler_str} pub_msg: {pub_msg}"); diff --git a/src/DataStreamSuite/PubSub/documentation/readme.txt b/src/DataStreamSuite/PubSub/documentation/readme.txt index c23845ff1..af0703b49 100644 --- a/src/DataStreamSuite/PubSub/documentation/readme.txt +++ b/src/DataStreamSuite/PubSub/documentation/readme.txt @@ -21,9 +21,6 @@ client and sent over to the pubsubserver, where uints received ad deserialized SubscriberCore:ISubscriberCore (retired now) | V -StaticSubscriberCore -| -V SingleServerSubscriberCore:ISingleServerSubscriberCore PublisherCore:IPublisherCore (transactions not supported for now) @@ -35,7 +32,21 @@ PubSubServer: PubSubMessageReactor (top level, called from main) ManagedTCPMessageHub : IPubsubMessageHub (this is what handles each ITCPConnection for each connection) - +PubSubState keeps the total pubsub data in 2 classes +Dictionary and +Dictionary +SingleTagData has the tag and IPubSubServerTagHandler (or byte[]) (IPubSubServerTagHandler holds the data state) +SingleTagConnectionData contains connection vs sub, and connection vs pub +PubSubState calls +SystemSubscriptionMapPublisher.init(this); +SystemStatsPublisher.init(this); +SystemHeartbeatPublisher.init(); +--------------------------------------------------------------- +How internal tags are published? +PubSubState has onPublishStats (tag, sz) that is fired on update_pub_data +onPublishStats is subscribed by SystemStatsPublisher +SystemStatsPublisher.onPublishStats updates the state and on the next doWork calls PubSubMessageReactor.internal_send +SystemStatsPublisher is also an IPubSubServerTagHandler so SingleTagData calls it getContent to get full state Streams: PubTableBroker allows publishing of streams through createTable, addRow, deleteRow, changeRow, makeBatchChanges -> diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs index c243cf851..2f2acabcb 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Stats/PubSubServerTagStatsStream.cs @@ -1,122 +1,78 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.IO; + +using BCG; using BCG.Utilities.General; -namespace BCG { +using MathNet.Numerics.RootFinding; + +namespace bts { [DataStreamUserRespresentation("tagStats")] public class PubSubServerTagStatsStream : ViewStream, IWorker { - private static Dictionary> myStreamMap = - new Dictionary>(); + private SubscriberCore _sub_core; + private ViewStreamDatabase _vsd; - private SubscriberCore mySubCore; - private ViewStreamDatabase myVsd; - private Dictionary myTagToStatsMap = new Dictionary(); - private KeysBasedTable2 myTable = new KeysBasedTable2(new BlueTableSchema("", new ColumnInfo("id", 0, typeof(int)), - new ColumnInfo("server", 1, typeof(string)), - new ColumnInfo("tag", 2, typeof(string)), - new ColumnInfo("nUpdates", 3, typeof(int)), - new ColumnInfo("totalBytes", 4, typeof(int)), - new ColumnInfo("minSize", 5, typeof(int)), - new ColumnInfo("maxSize", 6, typeof(int)), - new ColumnInfo("lastUpdateSize", 7, typeof(int)), - new ColumnInfo("averageSize", 8, typeof(double)), - new ColumnInfo("lastUpdateTime", 9, typeof(DateTime))), - new string[] { "tag" }); - - public static ViewStream getPubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL) { - //you can also supply a service name and it should work, as long as substations are enabled. - // if (PubSub3Config.instance.PS3_UseSubstations) - // { - // return new ThreadSafeStream(SubstationTagStatsStream.getSubstationTagStatsStream(vsd, serverURL)); - // } - if (!myStreamMap.ContainsKey(vsd)) - myStreamMap.Add(vsd, new Dictionary()); - if (!myStreamMap[vsd].ContainsKey(serverURL)) - myStreamMap[vsd].Add(serverURL, new PubSubServerTagStatsStream(vsd, serverURL)); - return myStreamMap[vsd][serverURL]; - } - - #region constructors to be used by the static factory method - - public PubSubServerTagStatsStream(string serverURL) - : this(ViewStreamDatabase.instance, serverURL) { } - - private PubSubServerTagStatsStream(ViewStreamDatabase vsd, string serverURL) { - myVsd = vsd; - mySubCore = new SubscriberCore(new PubSubRouter(new PubSubRoute[1] { new PubSubRoute(serverURL, ".*") })); - } - - #endregion - - #region exposed constructors that use the static factory methods - - public PubSubServerTagStatsStream() { - // if (PubSub3Config.instance.PS3_UseSubstations) - // mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(ViewStreamDatabase.instance, false))); - // else - mySetRealSource(new PubSubServerTagStatsStream(ViewStreamDatabase.instance, PubSub3Config.instance.getSubRouting())); - - } + private IBlueTableSchema _schema = new BlueTableSchema("tagStats", + new ColumnInfo("id", 0, typeof(int)), + new ColumnInfo("server", 1, typeof(string)), + new ColumnInfo("tag", 2, typeof(string)), + new ColumnInfo("n_updates", 3, typeof(int)), + new ColumnInfo("total_bytes", 4, typeof(int)), + new ColumnInfo("min_size", 5, typeof(int)), + new ColumnInfo("max_size", 6, typeof(int)), + new ColumnInfo("last_update_size", 7, typeof(int)), + new ColumnInfo("average_size", 8, typeof(double)), + new ColumnInfo("last_update_time", 9, typeof(DateTime))); + private IDictionary _tag_map = new Dictionary(); public PubSubServerTagStatsStream(ViewStreamDatabase vsd) { - // if (PubSub3Config.instance.PS3_UseSubstations) - // mySetRealSource(new ThreadSafeStream(new SubstationTagStatsStream(vsd, false))); - // else - mySetRealSource(new PubSubServerTagStatsStream(vsd, PubSub3Config.instance.getSubRouting())); - + _vsd = vsd; + _sub_core = new SubscriberCore(); } - public PubSubServerTagStatsStream(ViewStreamDatabase vsd, PubSubRoute[] routes) { - List vss = new List(); - foreach (string url in PubSub3Config.instance.getAllPubSubs()) - vss.Add(getPubSubServerTagStatsStream(vsd, url)); - mySetRealSource(new AppendStream(vss.ToArray())); - } - - #endregion + public PubSubServerTagStatsStream() + : this(ViewStreamDatabase.instance) { } protected override void myOnFirstSubscriber() { - mySubCore.subscribe("System_AllStatistics"); - myVsd.addWorker(this); + _sub_core.subscribe("System_AllStatistics"); + _vsd.addWorker(this); + } protected override void myOnLastSubscriber() { - mySubCore.unsubscribe("System_AllStatistics"); - myVsd.removeWorker(this); + _sub_core.unsubscribe("System_AllStatistics"); + _vsd.removeWorker(this); } - #region IWorker Members - public void doWork() { - foreach (var u in mySubCore.get_updates(TimeSpan.Zero)) { - List tag_stats = new List(); - MemoryStream str = new MemoryStream(u.data); - BinaryReader br = new BinaryReader(str); - int n = br.ReadInt32(); - for (int i = 0; i < n; i++) { - tag_stats.Add(new TagStatsInfo(str)); + var updates = _sub_core.get_updates(TimeSpan.Zero); + foreach (var u in updates) { + if (u.tag != "System_AllStatistics") + continue; + var tag_infos = TagStatsInfo.deserialize(u.data); + foreach (var ti in tag_infos) { + _tag_map[ti.tag] = ti; } - myHandleStats(tag_stats); } + _fire_dp(); } - #endregion - - private void myHandleStats(List tagStats) { - BlueTable tagStatsRows = new BlueTable(myTable.schema); - int id = 0; - foreach (TagStatsInfo t in tagStats) { - tagStatsRows.addRow(new BlueRow(myTable.schema, id++, t.server, t.tag, t.totalUpdates, t.totalBytes, - t.minSize, t.maxSize, t.lastUpdateSize, - t.averageSize, new DateTime(t.lastUpdateTime))); + private void _fire_dp() { + BlueTable bt = new BlueTable(_schema); + var sorted_tags = _tag_map.Keys.OrderBy(t => t); + // sorting it to keep the tag => id mapping the same (helps with incremental udpates) + foreach (var tag in sorted_tags) { + TagStatsInfo? ts_info = _tag_map[tag]; + IBlueRow nr = new BlueRow(_schema, bt.Count, ts_info.server, ts_info.tag, + ts_info.total_updates, ts_info.total_bytes, + ts_info.min_size, ts_info.max_size, + ts_info.last_update_size, ts_info.average_size, + ts_info.last_update_time); + bt.addRow(nr); } - IBlueRowCollection dRows, aRows; - myTable.updateMultipleRows(tagStatsRows, out dRows, out aRows); - myFireEvent(new DataPacket(myTable, aRows, dRows)); + var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true); + if (dp.addedRows.Count + dp.deletedRows.Count > 0) + myFireEvent(dp); } } diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs index 436c379ac..b4c929b3e 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs @@ -4,6 +4,7 @@ using System.Text; using BCG.Utilities.General; using System.Collections; using System.IO; +using MongoDB.Driver.Core.WireProtocol.Messages; namespace BCG { /// @@ -11,65 +12,35 @@ namespace BCG { /// server. /// public class PubSubServerStateStream : ViewStream, IWorker { - private static Dictionary> _stream_map = - new Dictionary>(); - private KeysBasedTable2 _table = - new KeysBasedTable2(new BlueTableSchema("tagInfo", - new ColumnInfo("id", 0, typeof(int)), - new ColumnInfo("tag", 1, typeof(string)), - new ColumnInfo("server", 2, typeof(string)), - new ColumnInfo("subscribers", 3, typeof(string)), - new ColumnInfo("publishers", 4, typeof(string)), - new ColumnInfo("deadPublishers", 5, typeof(string))), - new string[] { "tag" }); + private IDictionary _tag_map = new Dictionary(); + private readonly string _tag = "System_AllSubscriptions"; + private IBlueTableSchema _schema = null; + private SubscriberCore _sub_core; private ViewStreamDatabase _vsd; - private static ViewStream _get_pub_sub_server_state_stream(ViewStreamDatabase vsd, string server_url) { - if (!_stream_map.ContainsKey(vsd)) - _stream_map.Add(vsd, new Dictionary()); - if (!_stream_map[vsd].ContainsKey(server_url)) - _stream_map[vsd].Add(server_url, new PubSubServerStateStream(server_url, vsd)); - return _stream_map[vsd][server_url]; - } - - - // public static ViewStream getPubSubServerStateStream(ViewStreamDatabase vsd, string server_uRL) - // { - // return getPubSubServerStateStream(vsd, server_uRL); - // } - - - public PubSubServerStateStream(string server_url) - : this(server_url, ViewStreamDatabase.instance) { } - - public PubSubServerStateStream(string server_url, ViewStreamDatabase vsd) { - _sub_core = new SubscriberCore(new PubSubRouter(new PubSubRoute(server_url, ".*"))); - _vsd = vsd; - } - public PubSubServerStateStream() : this(ViewStreamDatabase.instance) { } public PubSubServerStateStream(ViewStreamDatabase vsd) { - mySetRealSource(new PubSubServerStateStream(vsd, PubSub3Config.instance.getSubRouting())); + // TODO, construct schema + _schema = BlueDataUtility.form_schema("pubsub_server_state", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("ps_server", TypeCode.String), + Tuple.Create("subscibers", TypeCode.String), + Tuple.Create("publisher", TypeCode.String), + Tuple.Create("dead_publishers", TypeCode.String)); + _vsd = vsd; + _sub_core = new SubscriberCore(); } - - public PubSubServerStateStream(ViewStreamDatabase vsd, PubSubRoute[] routes) { - List vss = new List(); - foreach (var r in routes) - vss.Add(_get_pub_sub_server_state_stream(vsd, r.serverAddress)); - mySetRealSource(new AppendStream(vss.ToArray())); - } - protected override void myOnFirstSubscriber() { - _sub_core.subscribe("System_AllSubscriptions"); + _sub_core.subscribe(_tag); _vsd.addWorker(this); } protected override void myOnLastSubscriber() { - _sub_core.unsubscribe("System_AllSubscriptions"); + _sub_core.unsubscribe(_tag); _vsd.removeWorker(this); } @@ -77,35 +48,37 @@ namespace BCG { public void doWork() { var updates = _sub_core.get_updates(TimeSpan.Zero); - // this is where the pubsub updates are converted to rows + //this is where the pubsub updates are converted to rows if (updates.Count() > 0) { int id = 0; - BlueTable addedRows = new BlueTable(_table.schema); foreach (var u in updates) { - BinaryReader br = new BinaryReader(new MemoryStream(u.data)); - // TODO: check this - int n_updates = br.ReadInt32(); - for (int i = 0; i < n_updates; i++) { - TagPubSubInfo tsi = new TagPubSubInfo(new MemoryStream(u.data)); - addedRows.addRow(new BlueRow(_table.schema, id++, tsi.tag, tsi.server, tsi.subscribersInString, tsi.publishersInString, tsi.deadPublishersInString)); + if (u.tag != _tag) { + continue; + } + var tpsi_list = TagPubSubInfo.deserialize(u.data); + foreach (var tsi in tpsi_list) { + _tag_map[tsi.tag] = tsi; } } - IBlueRowCollection dRows, aRows; - _table.updateMultipleRows(addedRows, out dRows, out aRows); - myFireEvent(new DataPacket(_table, aRows, dRows)); } + _fire_dp(); } #endregion - // public static string convertURLToTag(string URL) { - // string[] urlWords = URL.Split('.'); - // string[] lastPart = urlWords[3].Split(':'); - // //the first one has "bluto://" in front of it - // //we want to chop that off - // string result = "bluto_" + urlWords[0].Substring(8) + "_" + urlWords[1] + "_" + urlWords[2] + "_" + lastPart[0] + "_" + lastPart[1]; + private void _fire_dp() { + BlueTable bt = new BlueTable(_schema); + var sorted_tags = _tag_map.Keys.OrderBy(t => t); + foreach (var kv in _tag_map) { + var tsi = kv.Value; + IBlueRow r = new BlueRow(_schema, bt.Count, tsi.pubsub_server, tsi.subscribers_as_string, + tsi.publishers_as_string, tsi.dead_publishers_as_string); + bt.addRow(r); + } + var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true); + if (dp.addedRows.Count + dp.deletedRows.Count > 0) + myFireEvent(dp); - // return result; - // } + } } } diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs new file mode 100644 index 000000000..62c2e75e4 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs @@ -0,0 +1,76 @@ +namespace bts { + using BCG; + using BCG.Utilities.General; + + using MathNet.Numerics.RootFinding; + + using ZstdSharp.Unsafe; + [DataStreamUserRespresentation("pss_hb")] + public class PubsubHeartbeatStream : ViewStream, IWorker { + private ViewStreamDatabase _vsd; + private SubscriberCore _subcore = new SubscriberCore(); + private MutableBlueRow _row = null; + private readonly IBlueTableSchema _schema = null; + public PubsubHeartbeatStream(ViewStreamDatabase vsd) { + _vsd = vsd; + _schema = BlueDataUtility.form_schema("PS_HBB", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("hb_sent_time", TypeCode.Int64), + Tuple.Create("hb_recv_time", TypeCode.Int64), + Tuple.Create("travel_period", TypeCode.Int32), + Tuple.Create("time_since_last_recv_msecs", TypeCode.Int32) + ); + } + public PubsubHeartbeatStream() : + this(ViewStreamDatabase.instance) { } + + protected override void myOnFirstSubscriber() { + _subcore.subscribe("pss_hb"); + _vsd.addWorker(this); + + } + + protected override void myOnLastSubscriber() { + _vsd.removeWorker(this); + _subcore.unsubscribe("pss_hb"); + + } + + public void doWork() { + var updates = _subcore.get_updates(TimeSpan.FromMilliseconds(1)); + if (updates.Count() > 0) { + foreach (var m in updates) { + if (m.data != null && m.data.Length > 0) { + BinaryReader br = new BinaryReader(new MemoryStream(m.data)); + Int64 hb_sent_time = br.ReadInt64(); + Int64 hb_recv_time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + _row = _row == null ? new MutableBlueRow(_schema) : _row; + var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var time_since_last_recv_msecs = now - hb_recv_time; + _row.setValue("id", 0); + _row.setValue("hb_sent_time", hb_sent_time); + _row.setValue("hb_recv_time", hb_recv_time); + _row.setValue("travel_period", hb_recv_time - hb_sent_time); + _row.setValue("time_since_last_recv_msecs", time_since_last_recv_msecs); + // TODO, add to table and fire + } + } + } else { + if (_row != null) { + var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var ms_since_last_recv = now - _row.get("hb_recv_time"); + _row.setValue("time_since_last_recv_msecs", ms_since_last_recv); + // TODO, add to table and fire + } + // update the row + } + if (_row != null) { + BlueTable bt = new BlueTable(_row.schema); + bt.addRow(_row); + var dp = BlueDataUtility.build_dp_inc(bt, myLastValue as DataPacket, true); + myFireEvent(dp); + } + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs index 63ee61405..9802493e0 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs @@ -26,7 +26,7 @@ namespace BCG { public TagSynopsisStream(ViewStreamDatabase vsd, string route) { - ViewStream vs = new PubSubServerStateStream(route, vsd); + ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd); vs = new MapStream(vs, this); myWorkers.Add(vs as IWorker); @@ -41,7 +41,7 @@ namespace BCG { public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes) { List vss = new List(); foreach (string r in routes) { - ViewStream vs = new PubSubServerStateStream(r, vsd); + ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd); vs = new MapStream(vs, this); vss.Add(vs); myWorkers.Add(vs as IWorker);