Files
spark/src/DataStreamSuite/PubSub/PublisherCore.cs
T
John Colvin ce2c3d930a cleanup: Fix numerous build warnings in DataStreamSuite
Nearly all warnings were due to nullability issues, since the concept of
nullable reference types did not exist when the DataStream code was
originally written.

* PubSubClientMessage
  - Normalize indentation/whitespace
  - Fix some typos
  - Address nullability issues to fix build warnings
  - Modernize code

* Compression related files
  - ZipEntry
  - ZipFile
  - ZipInputStream
  - TarHeader

* RemoteMessage
  - make 'message' nullable to resolve warnings elsewhere (in calling code)

* PubSubSingleTagStateEvent.cs
* PubSubMessageReactor
* PubSubState
* SystemHeartbeatPublisher
* SystemStatsPublisher
* SingleTagData
* ManagedTCPMessageHub
* PubsubDeserializationController
* PublisherCore
* InflaterHuffmanTree
* AsyncTCPConnection_APM
* ThreadSafeQueue
* ManagedSubscriber
* SubscriberCore
* PubSubServerManagedTCPConnection
* AsyncTCPConnection_APM
* SuperMapStream
* MapStream
* TradeBarsStream
2024-11-29 14:57:13 +00:00

232 lines
9.4 KiB
C#

using System;
using System.IO;
using System.Collections.Generic;
using System.Threading;
using bts;
namespace BCG {
// this class uses an internal queue + thread to serialize and process calls
// made through its public interface (IPublisherCore)
public class PublisherCore : IPublisherCore {
// used to allow for different transports
// to be used when communicating to a pubsubserver
protected PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(false);
// internal queue used to serialize server calls. All calls to PublisherCore are written to this quwue,
// dequeued by the message pump thread
protected SyncQueue<PubSubClientMessage> _main_queue = new SyncQueue<PubSubClientMessage>();
// all the data published through this instance of PublisherCore
// is kept there, in a form that can be used repopulate a pubsub server
// in the event of a conection re-establishment to the PubSubServer
protected PublishedCache _state = new PublishedCache();
// every PubSubServer is communicated with a adifferent IPubSubServerConnection object.
// a dictionary for fast retrieval of publishers
// tag -> IPubSubServerConnection
protected IDictionary<string, IPubSubServerConnection> _tag_sender_map =
new Dictionary<string, IPubSubServerConnection>();
// pubsub server urls vs IPubSubServerConnection
private Dictionary<RemoteConnectionInfo, IPubSubServerConnection> _connection_map =
new Dictionary<RemoteConnectionInfo, IPubSubServerConnection>();
// router object, used to select which pubsub should be used for each tag
protected IPubSubRouter? _router;
// flag indicating wether we are in the middle of a transaction or not
protected bool _is_in_transaction = false;
#region constructors
public PublisherCore()
// TODOds
: this(new PubSubRouter()) { }
public PublisherCore(string server_url)
: this(new PubSubRouter(new PubSubRoute(server_url, ".*"))) { }
public PublisherCore(IPubSubRouter r) {
_router = r;
Thread pump_thread = new Thread(new ThreadStart(_message_pump));
pump_thread.IsBackground = true;
pump_thread.Start();
}
public void Dispose() { }
#endregion
public int get_number_of_waiting_messages() { return _main_queue.getCount(); }
#region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
public void begin_transaction() {
bts.Log.Gen.warn("PublisherCore.begin_transaction transactions not supported anymore");
//_main_queue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transaction_begin));
}
public void commit_transaction() {
bts.Log.Gen.warn("PublisherCore.commit_transaction transactions not supported anymore");
// _main_queue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transaction_commit));
}
public void abort_transaction() {
bts.Log.Gen.warn("PublisherCore.commit_transaction transactions not supported anymore");
//_main_queue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transaction_abort));
}
public bool is_in_transaction { get { return false; } }
public void clear_state(string tag) {
// TODO
bts.Log.assert(false);
//_main_queue.enqueue(new PublisherCoreMessage(tag));
}
public void publish(PubSubClientMessage msg) {
Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
_main_queue.enqueue(msg);
}
public void publish(string tag_type, string tag, byte[] content) {
// makes a PubSubClientMsg and calls the other publish
bts.PublishMsg publish_msg = new bts.PublishMsg() {
connection = String.Empty, // this will be filled when actually send it out in the _publish function
publishing_app = ClientInfo.thisClient.processString,
tag = tag,
tag_type = tag_type,
is_compressed = false,
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeSeconds(),
data_to_publish = content
};
PubSubClientMessage msg = new PubSubClientMessage {
pub_msg = publish_msg,
msg_type = PubSubMessageType.PUBLISH
};
publish(msg);
}
#endregion
#region Message Pump Thread - this is where messages are dequeued and the action performed
// all the state manahement happens here, the public interface methods merely enqueue the operation
private void _message_pump() {
while (true) {
try {
while (true) {
PubSubClientMessage? msg = _main_queue.dequeue(100);
// only msg tyes expected right now (6/21/24) are PUBLISh type
if (msg != null) {
bts.Log.Gen.info($"PublisherCore._message_pump msg: {msg}");
switch (msg.msg_type) {
case PubSubMessageType.TRANSACTION:
bts.Log.assert_d(false, "transaction not supported for now");
break;
case PubSubMessageType.SUBSCRIBE:
case PubSubMessageType.UNSUBSCRIIBE:
bts.Log.assert_d(false, $"Incorrect msg: {msg}");
break;
case PubSubMessageType.PUBLISH:
ArgumentNullException.ThrowIfNull(msg.pub_msg);
_publish(msg.pub_msg);
break;
}
}
// all interactions with conn are called from the same thread
foreach (var conn in _connection_map.Values) {
conn.doWork();
}
}
} catch (Exception ex) {
bts.Log.Gen.error($"PublisherCore.myMessagePump exception ex: {ex}");
if (ex is OutOfMemoryException)
throw;
}
}
}
private void _begin_transaction() {
_is_in_transaction = true;
}
private void _commit_transaction() {
//_end_transaction(true);
}
private void _abort_transaction() {
//_end_transaction(false);
}
protected virtual void _publish(PublishMsg msg) {
bts.Log.Gen.info($"PublisherCore._publish msg: {msg}");
// update the state
_state.publish(msg);
// send the data out
ArgumentNullException.ThrowIfNull(msg.tag);
var sender = _get_or_build_sender(msg.tag);
// actually send the data out
msg.connection = sender.client_url;
sender.send(new PubSubClientMessage() {
pub_msg = msg,
msg_type = PubSubMessageType.PUBLISH
});
}
// tag -> server_url, server_url -> IPubSubServerConnection
protected IPubSubServerConnection _get_or_build_sender(string tag) {
if (!_tag_sender_map.TryGetValue(tag, out var tag_conn)) {
ArgumentNullException.ThrowIfNull(_router);
RemoteConnectionInfo server_rci = _router.get_pub_route_for_tag(tag);
if (!_connection_map.TryGetValue(server_rci, out tag_conn)) {
tag_conn = _connection_factory.get_connection(server_rci);
tag_conn.register_connection_callback((server_url, connected) => {
_handle_pubsubserver_reconnection(server_url, connected);
});
_connection_map.Add(server_rci, tag_conn);
}
_tag_sender_map.Add(tag, tag_conn);
}
return tag_conn;
}
///in case of the server coming up, it publishes all the stored state
///for that server, back to it
private void _handle_pubsubserver_reconnection(string server_url, bool connection) {
bts.Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} connection: {connection}");
if (connection) {
// find all tags that are being published to this server
foreach (var kv in _tag_sender_map) {
var tag = kv.Key;
var conn = kv.Value;
if (conn.server_url == server_url) {
var content = _state.get_content(tag);
byte[] content_bytes = content.Item2;
string tag_type = content.Item1;
string content_bytes_log = content_bytes == null ? "NULL" : content_bytes.Length.ToString();
Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} content_bytes: {content_bytes_log}");
bts.PublishMsg publish_msg = new bts.PublishMsg() {
connection = kv.Key,
publishing_app = ClientInfo.thisClient.processString,
tag = tag,
tag_type = tag_type,
is_compressed = false,
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeSeconds(),
data_to_publish = content_bytes
};
_publish(publish_msg);
}
}
}
}
}
#endregion
}