Nearly all warnings were due to nullability issues, since the concept of nullable reference types did not exist when the DataStream code was originally written. * PubSubClientMessage - Normalize indentation/whitespace - Fix some typos - Address nullability issues to fix build warnings - Modernize code * Compression related files - ZipEntry - ZipFile - ZipInputStream - TarHeader * RemoteMessage - make 'message' nullable to resolve warnings elsewhere (in calling code) * PubSubSingleTagStateEvent.cs * PubSubMessageReactor * PubSubState * SystemHeartbeatPublisher * SystemStatsPublisher * SingleTagData * ManagedTCPMessageHub * PubsubDeserializationController * PublisherCore * InflaterHuffmanTree * AsyncTCPConnection_APM * ThreadSafeQueue * ManagedSubscriber * SubscriberCore * PubSubServerManagedTCPConnection * AsyncTCPConnection_APM * SuperMapStream * MapStream * TradeBarsStream
232 lines
9.4 KiB
C#
232 lines
9.4 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Collections.Generic;
|
|
using System.Threading;
|
|
using bts;
|
|
|
|
namespace BCG {
|
|
|
|
|
|
// this class uses an internal queue + thread to serialize and process calls
|
|
// made through its public interface (IPublisherCore)
|
|
public class PublisherCore : IPublisherCore {
|
|
// used to allow for different transports
|
|
// to be used when communicating to a pubsubserver
|
|
protected PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(false);
|
|
|
|
// internal queue used to serialize server calls. All calls to PublisherCore are written to this quwue,
|
|
// dequeued by the message pump thread
|
|
protected SyncQueue<PubSubClientMessage> _main_queue = new SyncQueue<PubSubClientMessage>();
|
|
|
|
// all the data published through this instance of PublisherCore
|
|
// is kept there, in a form that can be used repopulate a pubsub server
|
|
// in the event of a conection re-establishment to the PubSubServer
|
|
protected PublishedCache _state = new PublishedCache();
|
|
|
|
// every PubSubServer is communicated with a adifferent IPubSubServerConnection object.
|
|
|
|
// a dictionary for fast retrieval of publishers
|
|
// tag -> IPubSubServerConnection
|
|
protected IDictionary<string, IPubSubServerConnection> _tag_sender_map =
|
|
new Dictionary<string, IPubSubServerConnection>();
|
|
// pubsub server urls vs IPubSubServerConnection
|
|
private Dictionary<RemoteConnectionInfo, IPubSubServerConnection> _connection_map =
|
|
new Dictionary<RemoteConnectionInfo, IPubSubServerConnection>();
|
|
|
|
// router object, used to select which pubsub should be used for each tag
|
|
protected IPubSubRouter? _router;
|
|
|
|
// flag indicating wether we are in the middle of a transaction or not
|
|
protected bool _is_in_transaction = false;
|
|
|
|
#region constructors
|
|
|
|
public PublisherCore()
|
|
// TODOds
|
|
: this(new PubSubRouter()) { }
|
|
|
|
public PublisherCore(string server_url)
|
|
: this(new PubSubRouter(new PubSubRoute(server_url, ".*"))) { }
|
|
|
|
public PublisherCore(IPubSubRouter r) {
|
|
_router = r;
|
|
Thread pump_thread = new Thread(new ThreadStart(_message_pump));
|
|
pump_thread.IsBackground = true;
|
|
pump_thread.Start();
|
|
}
|
|
|
|
public void Dispose() { }
|
|
#endregion
|
|
|
|
public int get_number_of_waiting_messages() { return _main_queue.getCount(); }
|
|
|
|
#region Main Thread - all library functions are called on the main thread and queued to be picked by the pump
|
|
|
|
|
|
public void begin_transaction() {
|
|
bts.Log.Gen.warn("PublisherCore.begin_transaction transactions not supported anymore");
|
|
//_main_queue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transaction_begin));
|
|
}
|
|
|
|
public void commit_transaction() {
|
|
bts.Log.Gen.warn("PublisherCore.commit_transaction transactions not supported anymore");
|
|
// _main_queue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transaction_commit));
|
|
}
|
|
|
|
public void abort_transaction() {
|
|
bts.Log.Gen.warn("PublisherCore.commit_transaction transactions not supported anymore");
|
|
//_main_queue.enqueue(new PublisherCoreMessage(PublisherCoreMessageType.transaction_abort));
|
|
}
|
|
|
|
public bool is_in_transaction { get { return false; } }
|
|
|
|
public void clear_state(string tag) {
|
|
// TODO
|
|
bts.Log.assert(false);
|
|
//_main_queue.enqueue(new PublisherCoreMessage(tag));
|
|
}
|
|
|
|
public void publish(PubSubClientMessage msg) {
|
|
Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
|
|
Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
|
|
bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
|
|
_main_queue.enqueue(msg);
|
|
}
|
|
|
|
public void publish(string tag_type, string tag, byte[] content) {
|
|
// makes a PubSubClientMsg and calls the other publish
|
|
bts.PublishMsg publish_msg = new bts.PublishMsg() {
|
|
connection = String.Empty, // this will be filled when actually send it out in the _publish function
|
|
publishing_app = ClientInfo.thisClient.processString,
|
|
tag = tag,
|
|
tag_type = tag_type,
|
|
is_compressed = false,
|
|
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeSeconds(),
|
|
data_to_publish = content
|
|
};
|
|
PubSubClientMessage msg = new PubSubClientMessage {
|
|
pub_msg = publish_msg,
|
|
msg_type = PubSubMessageType.PUBLISH
|
|
};
|
|
publish(msg);
|
|
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Message Pump Thread - this is where messages are dequeued and the action performed
|
|
// all the state manahement happens here, the public interface methods merely enqueue the operation
|
|
private void _message_pump() {
|
|
while (true) {
|
|
try {
|
|
while (true) {
|
|
PubSubClientMessage? msg = _main_queue.dequeue(100);
|
|
// only msg tyes expected right now (6/21/24) are PUBLISh type
|
|
if (msg != null) {
|
|
bts.Log.Gen.info($"PublisherCore._message_pump msg: {msg}");
|
|
switch (msg.msg_type) {
|
|
case PubSubMessageType.TRANSACTION:
|
|
bts.Log.assert_d(false, "transaction not supported for now");
|
|
break;
|
|
case PubSubMessageType.SUBSCRIBE:
|
|
case PubSubMessageType.UNSUBSCRIIBE:
|
|
bts.Log.assert_d(false, $"Incorrect msg: {msg}");
|
|
break;
|
|
case PubSubMessageType.PUBLISH:
|
|
ArgumentNullException.ThrowIfNull(msg.pub_msg);
|
|
_publish(msg.pub_msg);
|
|
break;
|
|
}
|
|
}
|
|
// all interactions with conn are called from the same thread
|
|
foreach (var conn in _connection_map.Values) {
|
|
conn.doWork();
|
|
}
|
|
}
|
|
} catch (Exception ex) {
|
|
bts.Log.Gen.error($"PublisherCore.myMessagePump exception ex: {ex}");
|
|
if (ex is OutOfMemoryException)
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
private void _begin_transaction() {
|
|
_is_in_transaction = true;
|
|
}
|
|
|
|
private void _commit_transaction() {
|
|
//_end_transaction(true);
|
|
}
|
|
|
|
private void _abort_transaction() {
|
|
//_end_transaction(false);
|
|
}
|
|
|
|
|
|
protected virtual void _publish(PublishMsg msg) {
|
|
bts.Log.Gen.info($"PublisherCore._publish msg: {msg}");
|
|
// update the state
|
|
_state.publish(msg);
|
|
// send the data out
|
|
ArgumentNullException.ThrowIfNull(msg.tag);
|
|
var sender = _get_or_build_sender(msg.tag);
|
|
// actually send the data out
|
|
msg.connection = sender.client_url;
|
|
sender.send(new PubSubClientMessage() {
|
|
pub_msg = msg,
|
|
msg_type = PubSubMessageType.PUBLISH
|
|
});
|
|
}
|
|
|
|
// tag -> server_url, server_url -> IPubSubServerConnection
|
|
protected IPubSubServerConnection _get_or_build_sender(string tag) {
|
|
if (!_tag_sender_map.TryGetValue(tag, out var tag_conn)) {
|
|
ArgumentNullException.ThrowIfNull(_router);
|
|
RemoteConnectionInfo server_rci = _router.get_pub_route_for_tag(tag);
|
|
if (!_connection_map.TryGetValue(server_rci, out tag_conn)) {
|
|
tag_conn = _connection_factory.get_connection(server_rci);
|
|
tag_conn.register_connection_callback((server_url, connected) => {
|
|
_handle_pubsubserver_reconnection(server_url, connected);
|
|
});
|
|
_connection_map.Add(server_rci, tag_conn);
|
|
}
|
|
_tag_sender_map.Add(tag, tag_conn);
|
|
}
|
|
return tag_conn;
|
|
}
|
|
|
|
///in case of the server coming up, it publishes all the stored state
|
|
///for that server, back to it
|
|
private void _handle_pubsubserver_reconnection(string server_url, bool connection) {
|
|
bts.Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} connection: {connection}");
|
|
if (connection) {
|
|
// find all tags that are being published to this server
|
|
foreach (var kv in _tag_sender_map) {
|
|
var tag = kv.Key;
|
|
var conn = kv.Value;
|
|
if (conn.server_url == server_url) {
|
|
var content = _state.get_content(tag);
|
|
byte[] content_bytes = content.Item2;
|
|
string tag_type = content.Item1;
|
|
string content_bytes_log = content_bytes == null ? "NULL" : content_bytes.Length.ToString();
|
|
Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} content_bytes: {content_bytes_log}");
|
|
bts.PublishMsg publish_msg = new bts.PublishMsg() {
|
|
connection = kv.Key,
|
|
publishing_app = ClientInfo.thisClient.processString,
|
|
tag = tag,
|
|
tag_type = tag_type,
|
|
is_compressed = false,
|
|
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeSeconds(),
|
|
data_to_publish = content_bytes
|
|
};
|
|
_publish(publish_msg);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
}
|