Files
spark/src/DataStreamSuite/PubSub/Server/PubSubState.cs
T
John Colvin 60a52b3922 cleanup: Fix numerous build warnings in DataStreamSuite
Nearly all warnings were due to nullability issues, since the concept of
nullable reference types did not exist when the DataStream code was
originally written.

* PubSubClientMessage
  - Normalize indentation/whitespace
  - Fix some typos
  - Address nullability issues to fix build warnings
  - Modernize code

* Compression related files
  - ZipEntry
  - ZipFile
  - ZipInputStream
  - TarHeader

* RemoteMessage
  - make 'message' nullable to resolve warnings elsewhere (in calling code)

* PubSubSingleTagStateEvent.cs
* PubSubMessageReactor
* PubSubState
* SystemHeartbeatPublisher
* SystemStatsPublisher
* SingleTagData
* ManagedTCPMessageHub
* PubsubDeserializationController
* PublisherCore
* InflaterHuffmanTree
* AsyncTCPConnection_APM
* ThreadSafeQueue
* ManagedSubscriber
* SubscriberCore
* PubSubServerManagedTCPConnection
* AsyncTCPConnection_APM
* SuperMapStream
* MapStream
* TradeBarsStream
2024-10-30 19:30:29 +00:00

226 lines
9.5 KiB
C#

using System;
//using System.Messaging;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;
using System.IO;
using mbts;
using System.Linq.Expressions;
using bts;
using mbts.utils;
using bts.collections;
namespace BCG {
/// <summary>
/// Mantains all the state that there is in the server.
/// This information is keyed off the tags (rather than
/// keyed off subscribers or publishers)
/// State information consists of 2 distinct sets :
/// _tag_to_data_map consists of all the data published to
/// this pubsub, for every tag
/// _tag_to_connection_map consists of all the subscription and
/// publishing information (ie the connection and aplications that
/// are subsribing or publishing the tag)
/// </summary>
///
public class PubSubState {
public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent);
private event PubSubStateSingleTagChangeDelegate? onSingleTagChange;
public delegate void PubSubStateChangeDelegate(string[] tags);
public event PubSubStateChangeDelegate? onTagMapChanged;
public delegate void PubSubStatsDelegate(string tag, int size);
public event PubSubStatsDelegate? onPublishStats;
private Dictionary<string, SingleTagData> _tag_to_data_map = new Dictionary<string, SingleTagData>();
private Dictionary<string, SingleTagConnectionData> _tag_to_connection_map = new Dictionary<string, SingleTagConnectionData>();
public PubSubState() {
SystemSubscriptionMapPublisher.init(this);
SystemStatsPublisher.init(this);
SystemHeartbeatPublisher.init();
}
#region subscription based state changes
public void handle_sub(bts.SubscribeMsg sub_msg) {
Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}");
ExceptionAssert.Argument.NotNullOrEmpty(sub_msg.tags);
ArgumentException.ThrowIfNullOrEmpty(sub_msg.connection);
foreach (string tag in sub_msg.tags) {
_tag_to_connection_map.GetOrAdd(tag, _ => new SingleTagConnectionData(tag)).handle_sub(sub_msg);
onSingleTagChange?.Invoke(new(
tag, PubSubSingleTagStateEvent.EventType.subscription, sub_msg.connection, sub_msg.subscribing_app));
}
onTagMapChanged?.Invoke(sub_msg.tags);
// TODO
}
public void handle_unsub(bts.UnsubscribeMsg unsub_msg) {
ExceptionAssert.Argument.NotNullOrEmpty(unsub_msg.tags);
ArgumentException.ThrowIfNullOrEmpty(unsub_msg.connection);
foreach (string tag in unsub_msg.tags) {
if (_tag_to_connection_map.TryGetValue(tag, out var conn_data)) {
///maybe check here, that if there are no subscribers
///and publishers for the tag then remove it.
conn_data.handle_unsub(unsub_msg);
onSingleTagChange?.Invoke(new(
tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub_msg.connection, unsub_msg.subscribing_app));
} else {
bts.Log.Gen.error($"PubSubState.handle_unsub ubnsub_msg: {unsub_msg} tag: {tag} was never subscribed, so unsub call is voided");
}
}
if (onTagMapChanged != null)
onTagMapChanged(unsub_msg.tags);
}
public void remove_connection(string connection) {
List<string> tags_effected = new List<string>();
foreach (KeyValuePair<string, SingleTagConnectionData> p in _tag_to_connection_map) {
///maybe check here, that if there are no subscribers
///and publishers for the tag then remove it.
bool wasPublisher = false;
bool wasSubscriber = false;
string tag = p.Key;
if (p.Value.remove_connection(connection, out wasPublisher, out wasSubscriber))
tags_effected.Add(tag);
if (onSingleTagChange != null) {
//we don't know the app or key qualifier or anything for these connection. that is unfortunate.
if (wasPublisher)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, ""));
if (wasSubscriber)
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, connection, ""));
}
}
if (onTagMapChanged != null)
onTagMapChanged(tags_effected.ToArray());
}
#endregion
#region data based state changes
public void update_pub_data(bts.PublishMsg pub_msg) {
ArgumentException.ThrowIfNullOrEmpty(pub_msg.tag);
ArgumentException.ThrowIfNullOrEmpty(pub_msg.tag_type);
ArgumentException.ThrowIfNullOrEmpty(pub_msg.connection);
ArgumentNullException.ThrowIfNull(pub_msg.data_to_publish);
try {
bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");
///this is the only place where SingleTagData should be created
if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) {
///first time data is published for this tag
_tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type));
}
_tag_to_data_map[pub_msg.tag].handle_publish(pub_msg);
if (!_tag_to_connection_map.ContainsKey(pub_msg.tag)) {
_tag_to_connection_map.Add(pub_msg.tag, new SingleTagConnectionData(pub_msg.tag));
}
bool pub_changed = false;
_tag_to_connection_map[pub_msg.tag].handle_pub(pub_msg, out pub_changed);
if (pub_changed) {
if (onTagMapChanged != null)
onTagMapChanged(new string[] { pub_msg.tag });
if (onSingleTagChange != null)
onSingleTagChange(
new PubSubSingleTagStateEvent(pub_msg.tag,
PubSubSingleTagStateEvent.EventType.publish,
pub_msg.connection, pub_msg.publishing_app));
}
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubState.update_pub_data ex: {ex} pub_msg: {pub_msg}");
}
onPublishStats?.Invoke(pub_msg.tag, pub_msg.data_to_publish.Length);
}
#endregion
public ICollection<string> get_subscriber_connections_for_tag(string tag) {
if (_tag_to_connection_map.ContainsKey(tag))
return _tag_to_connection_map[tag].subscriber_connections;
else
return new string[] { };
}
public ICollection<string> getSubscribingAppsForTag(string tag) {
if (_tag_to_connection_map.ContainsKey(tag))
return _tag_to_connection_map[tag].subscriping_apps;
else
return new string[] { };
}
public byte[] get_content(string tag) {
if (!_tag_to_data_map.ContainsKey(tag))
return null;
var rv = _tag_to_data_map[tag].get_current_content();
var rv_txt = StreamUtilityMethods.first_n_bytes_to_string(rv, 4);
bts.Log.Gen.info($"PubSubState.get_content tag: {tag} content len: {rv.Length} rv: {rv_txt}");
return rv;
}
public string getTagType(string tag) {
if (!_tag_to_data_map.ContainsKey(tag))
return null;
else
return _tag_to_data_map[tag].tag_type;
}
/// <summary>
/// returns all the applications 'connected' to one tag, which
/// includes a Dictionary of connection vs subscribers,
/// Dictionary of connections vs publishers and a connection, app
/// pair of all dead publishers
/// </summary>
/// <param name="tag"></param>
/// <param name="subscribers"></param>
/// <param name="activePublishers"></param>
/// <param name="deadPublishers"></param>
public void get_all_connections(string tag, out Dictionary<string, string> subscribers,
out Dictionary<string, string> activePublishers,
out ICollection<Pair<string, string>> deadPublishers) {
if (_tag_to_connection_map.ContainsKey(tag)) {
SingleTagConnectionData map = _tag_to_connection_map[tag];
subscribers = map.subscribers;
activePublishers = map.active_publishers;
deadPublishers = map.dead_publishers;
} else {
subscribers = activePublishers = new Dictionary<string, string>();
deadPublishers = new List<Pair<string, string>>();
}
}
public ICollection<string> all_tags {
get { return _tag_to_connection_map.Keys; }
}
public ICollection<string> tags_with_data {
get { return _tag_to_data_map.Keys; }
}
/// <summary>
/// given a list of tags, constructs a Dictionary of
/// subscribers vs tags
/// </summary>
/// <param name="tags"></param>
/// <returns></returns>
public Dictionary<string, List<string>> get_subscriber_connections_for_tags(IEnumerable<string> tags) {
Dictionary<string, List<string>> retVal = new Dictionary<string, List<string>>();
foreach (string tag in tags) {
if (_tag_to_connection_map.ContainsKey(tag)) {
foreach (string sub in _tag_to_connection_map[tag].subscriber_connections) {
if (!retVal.ContainsKey(sub))
retVal.Add(sub, new List<string>());
retVal[sub].Add(tag);
}
}
}
return retVal;
}
}
}