Nearly all warnings were due to nullability issues, since the concept of nullable reference types did not exist when the DataStream code was originally written. * PubSubClientMessage - Normalize indentation/whitespace - Fix some typos - Address nullability issues to fix build warnings - Modernize code * Compression related files - ZipEntry - ZipFile - ZipInputStream - TarHeader * RemoteMessage - make 'message' nullable to resolve warnings elsewhere (in calling code) * PubSubSingleTagStateEvent.cs * PubSubMessageReactor * PubSubState * SystemHeartbeatPublisher * SystemStatsPublisher * SingleTagData * ManagedTCPMessageHub * PubsubDeserializationController * PublisherCore * InflaterHuffmanTree * AsyncTCPConnection_APM * ThreadSafeQueue * ManagedSubscriber * SubscriberCore * PubSubServerManagedTCPConnection * AsyncTCPConnection_APM * SuperMapStream * MapStream * TradeBarsStream
226 lines
9.5 KiB
C#
226 lines
9.5 KiB
C#
using System;
|
|
//using System.Messaging;
|
|
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using System.Reflection;
|
|
using System.IO;
|
|
|
|
using mbts;
|
|
using System.Linq.Expressions;
|
|
using bts;
|
|
using mbts.utils;
|
|
using bts.collections;
|
|
|
|
namespace BCG {
|
|
/// <summary>
|
|
/// Mantains all the state that there is in the server.
|
|
/// This information is keyed off the tags (rather than
|
|
/// keyed off subscribers or publishers)
|
|
/// State information consists of 2 distinct sets :
|
|
/// _tag_to_data_map consists of all the data published to
|
|
/// this pubsub, for every tag
|
|
/// _tag_to_connection_map consists of all the subscription and
|
|
/// publishing information (ie the connection and aplications that
|
|
/// are subsribing or publishing the tag)
|
|
/// </summary>
|
|
///
|
|
public class PubSubState {
|
|
|
|
public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent);
|
|
private event PubSubStateSingleTagChangeDelegate? onSingleTagChange;
|
|
public delegate void PubSubStateChangeDelegate(string[] tags);
|
|
public event PubSubStateChangeDelegate? onTagMapChanged;
|
|
public delegate void PubSubStatsDelegate(string tag, int size);
|
|
public event PubSubStatsDelegate? onPublishStats;
|
|
|
|
private Dictionary<string, SingleTagData> _tag_to_data_map = new Dictionary<string, SingleTagData>();
|
|
private Dictionary<string, SingleTagConnectionData> _tag_to_connection_map = new Dictionary<string, SingleTagConnectionData>();
|
|
|
|
public PubSubState() {
|
|
SystemSubscriptionMapPublisher.init(this);
|
|
SystemStatsPublisher.init(this);
|
|
SystemHeartbeatPublisher.init();
|
|
}
|
|
|
|
#region subscription based state changes
|
|
|
|
public void handle_sub(bts.SubscribeMsg sub_msg) {
|
|
Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}");
|
|
ExceptionAssert.Argument.NotNullOrEmpty(sub_msg.tags);
|
|
ArgumentException.ThrowIfNullOrEmpty(sub_msg.connection);
|
|
foreach (string tag in sub_msg.tags) {
|
|
_tag_to_connection_map.GetOrAdd(tag, _ => new SingleTagConnectionData(tag)).handle_sub(sub_msg);
|
|
onSingleTagChange?.Invoke(new(
|
|
tag, PubSubSingleTagStateEvent.EventType.subscription, sub_msg.connection, sub_msg.subscribing_app));
|
|
}
|
|
onTagMapChanged?.Invoke(sub_msg.tags);
|
|
// TODO
|
|
}
|
|
|
|
public void handle_unsub(bts.UnsubscribeMsg unsub_msg) {
|
|
ExceptionAssert.Argument.NotNullOrEmpty(unsub_msg.tags);
|
|
ArgumentException.ThrowIfNullOrEmpty(unsub_msg.connection);
|
|
foreach (string tag in unsub_msg.tags) {
|
|
if (_tag_to_connection_map.TryGetValue(tag, out var conn_data)) {
|
|
///maybe check here, that if there are no subscribers
|
|
///and publishers for the tag then remove it.
|
|
conn_data.handle_unsub(unsub_msg);
|
|
onSingleTagChange?.Invoke(new(
|
|
tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub_msg.connection, unsub_msg.subscribing_app));
|
|
} else {
|
|
bts.Log.Gen.error($"PubSubState.handle_unsub ubnsub_msg: {unsub_msg} tag: {tag} was never subscribed, so unsub call is voided");
|
|
}
|
|
}
|
|
if (onTagMapChanged != null)
|
|
onTagMapChanged(unsub_msg.tags);
|
|
}
|
|
|
|
public void remove_connection(string connection) {
|
|
List<string> tags_effected = new List<string>();
|
|
foreach (KeyValuePair<string, SingleTagConnectionData> p in _tag_to_connection_map) {
|
|
///maybe check here, that if there are no subscribers
|
|
///and publishers for the tag then remove it.
|
|
bool wasPublisher = false;
|
|
bool wasSubscriber = false;
|
|
string tag = p.Key;
|
|
if (p.Value.remove_connection(connection, out wasPublisher, out wasSubscriber))
|
|
tags_effected.Add(tag);
|
|
|
|
if (onSingleTagChange != null) {
|
|
//we don't know the app or key qualifier or anything for these connection. that is unfortunate.
|
|
if (wasPublisher)
|
|
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.publisherDie, connection, ""));
|
|
if (wasSubscriber)
|
|
onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, connection, ""));
|
|
|
|
}
|
|
}
|
|
if (onTagMapChanged != null)
|
|
onTagMapChanged(tags_effected.ToArray());
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region data based state changes
|
|
public void update_pub_data(bts.PublishMsg pub_msg) {
|
|
ArgumentException.ThrowIfNullOrEmpty(pub_msg.tag);
|
|
ArgumentException.ThrowIfNullOrEmpty(pub_msg.tag_type);
|
|
ArgumentException.ThrowIfNullOrEmpty(pub_msg.connection);
|
|
ArgumentNullException.ThrowIfNull(pub_msg.data_to_publish);
|
|
try {
|
|
bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");
|
|
///this is the only place where SingleTagData should be created
|
|
if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) {
|
|
///first time data is published for this tag
|
|
_tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type));
|
|
}
|
|
_tag_to_data_map[pub_msg.tag].handle_publish(pub_msg);
|
|
|
|
if (!_tag_to_connection_map.ContainsKey(pub_msg.tag)) {
|
|
_tag_to_connection_map.Add(pub_msg.tag, new SingleTagConnectionData(pub_msg.tag));
|
|
}
|
|
bool pub_changed = false;
|
|
_tag_to_connection_map[pub_msg.tag].handle_pub(pub_msg, out pub_changed);
|
|
if (pub_changed) {
|
|
if (onTagMapChanged != null)
|
|
onTagMapChanged(new string[] { pub_msg.tag });
|
|
|
|
if (onSingleTagChange != null)
|
|
onSingleTagChange(
|
|
new PubSubSingleTagStateEvent(pub_msg.tag,
|
|
PubSubSingleTagStateEvent.EventType.publish,
|
|
pub_msg.connection, pub_msg.publishing_app));
|
|
}
|
|
} catch (Exception ex) {
|
|
bts.Log.Gen.error($"PubSubState.update_pub_data ex: {ex} pub_msg: {pub_msg}");
|
|
}
|
|
onPublishStats?.Invoke(pub_msg.tag, pub_msg.data_to_publish.Length);
|
|
}
|
|
|
|
#endregion
|
|
|
|
public ICollection<string> get_subscriber_connections_for_tag(string tag) {
|
|
if (_tag_to_connection_map.ContainsKey(tag))
|
|
return _tag_to_connection_map[tag].subscriber_connections;
|
|
else
|
|
return new string[] { };
|
|
}
|
|
|
|
public ICollection<string> getSubscribingAppsForTag(string tag) {
|
|
if (_tag_to_connection_map.ContainsKey(tag))
|
|
return _tag_to_connection_map[tag].subscriping_apps;
|
|
else
|
|
return new string[] { };
|
|
}
|
|
|
|
public byte[] get_content(string tag) {
|
|
if (!_tag_to_data_map.ContainsKey(tag))
|
|
return null;
|
|
var rv = _tag_to_data_map[tag].get_current_content();
|
|
var rv_txt = StreamUtilityMethods.first_n_bytes_to_string(rv, 4);
|
|
bts.Log.Gen.info($"PubSubState.get_content tag: {tag} content len: {rv.Length} rv: {rv_txt}");
|
|
return rv;
|
|
}
|
|
|
|
public string getTagType(string tag) {
|
|
if (!_tag_to_data_map.ContainsKey(tag))
|
|
return null;
|
|
else
|
|
return _tag_to_data_map[tag].tag_type;
|
|
}
|
|
|
|
/// <summary>
|
|
/// returns all the applications 'connected' to one tag, which
|
|
/// includes a Dictionary of connection vs subscribers,
|
|
/// Dictionary of connections vs publishers and a connection, app
|
|
/// pair of all dead publishers
|
|
/// </summary>
|
|
/// <param name="tag"></param>
|
|
/// <param name="subscribers"></param>
|
|
/// <param name="activePublishers"></param>
|
|
/// <param name="deadPublishers"></param>
|
|
public void get_all_connections(string tag, out Dictionary<string, string> subscribers,
|
|
out Dictionary<string, string> activePublishers,
|
|
out ICollection<Pair<string, string>> deadPublishers) {
|
|
if (_tag_to_connection_map.ContainsKey(tag)) {
|
|
SingleTagConnectionData map = _tag_to_connection_map[tag];
|
|
subscribers = map.subscribers;
|
|
activePublishers = map.active_publishers;
|
|
deadPublishers = map.dead_publishers;
|
|
} else {
|
|
subscribers = activePublishers = new Dictionary<string, string>();
|
|
deadPublishers = new List<Pair<string, string>>();
|
|
}
|
|
}
|
|
|
|
public ICollection<string> all_tags {
|
|
get { return _tag_to_connection_map.Keys; }
|
|
}
|
|
|
|
public ICollection<string> tags_with_data {
|
|
get { return _tag_to_data_map.Keys; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// given a list of tags, constructs a Dictionary of
|
|
/// subscribers vs tags
|
|
/// </summary>
|
|
/// <param name="tags"></param>
|
|
/// <returns></returns>
|
|
public Dictionary<string, List<string>> get_subscriber_connections_for_tags(IEnumerable<string> tags) {
|
|
Dictionary<string, List<string>> retVal = new Dictionary<string, List<string>>();
|
|
foreach (string tag in tags) {
|
|
if (_tag_to_connection_map.ContainsKey(tag)) {
|
|
foreach (string sub in _tag_to_connection_map[tag].subscriber_connections) {
|
|
if (!retVal.ContainsKey(sub))
|
|
retVal.Add(sub, new List<string>());
|
|
retVal[sub].Add(tag);
|
|
}
|
|
}
|
|
}
|
|
return retVal;
|
|
}
|
|
|
|
}
|
|
}
|