Files
spark/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs
T

242 lines
12 KiB
C#

using System;
using System.Collections.Generic;
using System.Threading;
using bts;
using Bts.PubSub;
using Spark.Core;
using System.Text;
using System.Diagnostics;
using Bts.PubSub.Server;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Hosting;
using System.Threading.Tasks;
namespace BCG {
// This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it.
// The only outward facing members of this class is start
// It has an IPubsubMessageHub, that it gets messages from, and updates the PubSubState (which really is all the pubsub server state)
class PubSubMessageReactor : IHostedService {
private IPubsubMessageHub _msg_hub;
// private delegate void MyMsgHandler(string sender, Stream msg, MetaData m);
// private Dictionary<string, MyMsgHandler> myOperationToHandler = new Dictionary<string, MyMsgHandler>();
private Thread _msg_polling_thread;
// all the data,list of subscribers etc are in the state
private IPubSubState _state;
private IInternalPublisherQueue _internal_pub_queue;
private Stopwatch _stop_watch = new Stopwatch();
TaskCompletionSource? executionTask;
CancellationTokenSource? executionCts;
public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, IOptions<PubSubConfig> config)
: this(state, internalPublisherQueue, config.Value) { }
public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, PubSubConfig config)
: this(
state,
internalPublisherQueue,
config.ServerPort.assert_value(),
TimeSpan.FromMilliseconds(config.HeartbeatInterval.assert_value()),
config.MissingHeartbeatsBeforeGc.assert_value(),
TimeSpan.FromMilliseconds(config.DisposedConnectionCheckPeriod.assert_value())) { }
public PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, int port, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval) {
bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
_state = state;
_internal_pub_queue = internalPublisherQueue;
_msg_polling_thread = new Thread(new ThreadStart(_polling_thread_func));
_msg_polling_thread.Priority = ThreadPriority.AboveNormal;
_msg_hub = new ManagedTCPMessageHub(port, heartbeatInterval, maxMissedHeartbeats, collectDisposedConnectionsInterval);
// this is the only event this class gets from hub
_msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection));
}
// the connection callback is the only event we get back from the message_hub.
private void _handle_client_connection(string client, PubSubCallbacks.ConnectionStatus conn_status) {
bts.Log.Gen.info($"PubsubMessageReactor._handle_client_connection client: {client} conn_status: {conn_status}");
///interested in disconnections only, as connections of no
///significance until they subscribe, in which case an entry
///will be made
if (conn_status == PubSubCallbacks.ConnectionStatus.DISCONNECTED)
_state.remove_connection(client);
}
public Task StartAsync(CancellationToken cancellationToken) {
if (cancellationToken.IsCancellationRequested) {
return Task.FromCanceled(cancellationToken);
}
executionCts = new();
_msg_polling_thread.Start();
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken) {
if (cancellationToken.IsCancellationRequested) {
return Task.FromCanceled(cancellationToken);
}
executionCts?.Cancel();
return executionTask?.Task ?? Task.CompletedTask;
}
/// <summary>
/// This is the main PubSubServer thread. It loops around and read messages from
/// the IPubsubMessageHub. It then deserializes the messages and calls the approrpriate
/// hanlder
/// </summary>
private void _polling_thread_func() {
Log.Gen.info("PubSubMessageReactor._polling_thread_func starting thread function _polling_thread_func");
TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500);
_msg_hub.activate();
Action handle_internal_pubs = () => {
Log.Gen.info($"PubSubMessageReactor._polling_thread_func handle_internal_pubs");
var internal_pubs = _internal_pub_queue.DequeueMultiple(TimeSpan.FromSeconds(1), 64000);
//Log.Gen.info($"PubSubMessageReactor._polling_thread_func internal_pubs count: {internal_pubs.Count}");
foreach (var msg in internal_pubs) {
try {
_handle_pub("Internal", msg, true);
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}");
}
}
};
while (executionCts is null || !executionCts.IsCancellationRequested) {
try {
//bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
var received_data = _msg_hub.get_all_updates();
var n_msgs = received_data.Count;
if (n_msgs == 0) {
handle_internal_pubs();
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
continue;
}
bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
foreach (var kv in received_data) {
// multiple pubs for the same tags should be batched here
ArgumentException.ThrowIfNullOrEmpty(kv.first);
ArgumentNullException.ThrowIfNull(kv.second);
_handle_client_msg(kv.first, kv.second);
}
handle_internal_pubs();
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}");
if (ex is OutOfMemoryException) {
throw;
}
}
}
}
public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
var len = end_pos - start_pos;
StringBuilder hex_builder = new StringBuilder(len * 2);
int i = 0;
foreach (byte b in byte_arr) {
if (i >= start_pos && i < end_pos)
hex_builder.Append($"{b:X2} ");
i++;
}
return hex_builder.ToString().Trim();
}
// all messages (polled from the message_hub) sent to the client are handled here
private void _handle_client_msg(string sender, byte[] msg) {
//_stop_watch.Restart();
var stop_watch = Stopwatch.StartNew();
//string msg_txt = to_hex(msg, 0, msg.Length);
Log.Gen.info($"PubSubMessageReactor._handle_client_msg sender: {sender} msg: {msg.Length}");
// TODO: reduce object creation, C# has an ObjectPool
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
ps_client_msg.consruct_deserialize(msg);
bts.Log.Gen.info($"PubsubMessageReactor._handle_client_msg sender: {sender} ps_client_msg: {ps_client_msg}");
switch (ps_client_msg.msg_type) {
case PubSubMessageType.SUBSCRIBE:
ArgumentNullException.ThrowIfNull(ps_client_msg.sub_msg);
_handle_sub(sender, ps_client_msg.sub_msg);
break;
case PubSubMessageType.UNSUBSCRIIBE:
ArgumentNullException.ThrowIfNull(ps_client_msg.unsub_msg);
_handle_unsub(sender, ps_client_msg.unsub_msg);
break;
case PubSubMessageType.TRANSACTION:
ArgumentNullException.ThrowIfNull(ps_client_msg.transaction_msg);
_handle_transaction(sender, ps_client_msg.transaction_msg);
break;
case PubSubMessageType.PUBLISH:
ArgumentNullException.ThrowIfNull(ps_client_msg.pub_msg);
_handle_pub(sender, ps_client_msg.pub_msg, false);
break;
}
stop_watch.Stop();
Log.Gen.info($"PubSubMessageReactor._handle_client_msg processed ps_client_msg: {ps_client_msg} processing time: {stop_watch.ElapsedMilliseconds} msecs");
}
private void _handle_sub(string sender, SubscribeMsg sub_msg) {
//TODO: we make this correction here, since C++ clients are not sending this correctly
bts.Log.Gen.info($"PubSubMessageReactor._handle_sub sender: {sender} sub_msg: {sub_msg}");
if (String.IsNullOrEmpty(sub_msg.connection) || String.Compare(sender, sub_msg.connection) != 0) {
bts.Log.Gen.error($"PubSubMessageReactor._handle_sub correcting sub_msg.connection sender: {sender} sub_msg.connection: {sub_msg.connection}");
sub_msg.connection = sender;
}
bts.Log.Gen.info($"PubSubMessageReactor._handle_sub(corrected) sender: {sender} sub_msg: {sub_msg}");
///update the state
_state.handle_sub(sub_msg); // state keeps the list of all subscribers
///to all these new subscriber s, send the existing data
foreach (string tag in sub_msg.tags.empty_if_null()) {
byte[] data = _state.get_content(tag);
//ie data has been published once
if (data != null) {
bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
string tag_type_ = _state.getTagType(tag);
var msg_to_send = new PubSubServerMessage() {
tag = tag,
tag_type = tag_type_,
data = data
};
bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub sending msg_to_send: {msg_to_send} connection: {sub_msg.connection}");
_msg_hub.send(sub_msg.connection, ps_msg: msg_to_send);
return;
} else {
bts.Log.Gen.error($"ManagedTCPMessgeHub._handle_sub could not retreive data for sub_msg: {sub_msg}");
}
}
}
private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) {
bts.Log.assert(!pub_msg.is_compressed);
//bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}");
// this is where pubsub's data state is updated
_state.update_pub_data(pub_msg);
ArgumentNullException.ThrowIfNull(pub_msg.tag);
var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag);
bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}");
bts.PubSubServerMessage msg_to_send = new PubSubServerMessage() {
tag = pub_msg.tag,
tag_type = pub_msg.tag_type,
data = pub_msg.data_to_publish
};
///for all the subscribers, sends the messages over straightaway
foreach (string client_url in sub_connections_for_tag) {
//bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sending tag: {pub_msg.tag} to {client_url}");
_msg_hub.send(client_url, msg_to_send);
}
// if we had to persist the state, this is where we'll do it
}
private void _handle_transaction(string sender, TransactionMsg transaction_msg) {
bts.Log.assert_d(false, $"transactions are not supported yet msg: {transaction_msg}");
}
private void _handle_unsub(string sender, UnsubscribeMsg ubnsub_msg) {
// TODO: check ubnsub_msg.connection
_state.handle_unsub(ubnsub_msg);
}
}
}