Fixing the bug where messages were being sent by the pubsubserver out of order

This commit is contained in:
Taha authored and John Colvin committed 2024-09-18 14:02:34 +00:00
1 parent 6604eaaea7
commit b55c4103df
8 files changed
+62 -135

No files matched your search

@@ -158,7 +158,7 @@ namespace BCG {
/// </summary>
/// <param name="message"></param>
public void onData(RemoteMessage message) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData connection: {_connection} message: {message}");
// TODO check for disconenction
switch (message.messageType) {
case RemoteMessageType.Data:
@@ -41,12 +41,18 @@ namespace bts {
// this is a blocking call
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port} starting blocking call AcceptTcpClient");
TcpClient tcp_client = _server.AcceptTcpClient();
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
ITCPConnection conn = _create_connection(tcp_client);
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now");
_connection_handler.onConnection(conn);
}
//_cbs(conn);
}
private static ITCPConnection _create_connection(TcpClient tcp_client) {
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
// ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
return conn;
}
}
}
@@ -1,10 +1,6 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using bts.utils;
using System;
using System.IO;
using System.Net.Sockets;
@@ -12,19 +8,30 @@ namespace bts {
using System.Threading;
using System.Net;
using System.Text;
using System.ComponentModel;
// this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
public class ConnectionID {
public readonly RemoteConnectionInfo self;
public readonly RemoteConnectionInfo peer;
public readonly Type conn_type;
public override string ToString() {
return $"{conn_type} self: {self} peer: {peer}";
}
public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
self = self_;
peer = peer_;
conn_type = conn_type_;
}
}
private const int DATA_MSG_SIGNAL = 0;
private const int HB_MSG_SIGNAL = 1;
private TcpClient _tcp_client;
private NetworkStream _nw_stream;
private ITCPMessageHandler _msg_handler;
private bool _is_alive = true;
RemoteConnectionInfo _peer_conn_info = null;
RemoteConnectionInfo _self_conn_info = null;
ConnectionID _connection_id;
public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
var len = end_pos - start_pos;
@@ -42,10 +49,8 @@ namespace bts {
public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
_tcp_client = tcp_client;
_nw_stream = _tcp_client.GetStream();
_peer_conn_info = get_peer_address();
_self_conn_info = get_self_address();
Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons _peer_conn_info: {_peer_conn_info} _self_conn_info: {_self_conn_info}");
//_detect_connection_status_change();
_connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
}
private void _detect_connection_status_change() {
@@ -63,10 +68,7 @@ namespace bts {
ThreadPool.QueueUserWorkItem(_ => testing_func());
}
public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) {
}
#region Sending
public void _send_fixed_size_msg(byte[] buffer) {
string buff_txt = to_hex(buffer, 0, buffer.Length);
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
@@ -101,29 +103,38 @@ namespace bts {
}
}
private void _send_forever() {
while (_is_alive) {
byte[] data = _send_queue.dequeue();
if (data != null) {
_send_one_msg(data);
}
}
}
private SyncQueue<byte[]> _send_queue = new SyncQueue<byte[]>();
public void send(byte[] data) {
var data_txt = to_hex(data, 0, data.Length);
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.send data: {data_txt}");
//Task.Run(() => _send_one_msg(data));
ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data));
//ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data));
_send_queue.enqueue(data);
}
#endregion
public void register(ITCPMessageHandler message_handler) {
_msg_handler = message_handler;
// start reading in a Task
// starting thread pool threads to keep reading and sending
ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
ThreadPool.QueueUserWorkItem(_ => _send_forever());
}
public RemoteConnectionInfo get_peer_address() {
//if (_peer_conn_info == null)
return _peer_conn_info = TCPSocketUtils.get_peer_address(_tcp_client.Client);
//return _peer_conn_info;
return _connection_id.peer;
}
public RemoteConnectionInfo get_self_address() {
if (_self_conn_info == null)
_self_conn_info = TCPSocketUtils.get_self_address(_tcp_client.Client);
return _self_conn_info;
return _connection_id.self;
}
public bool isDisposed { get { return !_is_alive; } }
@@ -132,7 +143,7 @@ namespace bts {
// send hb
}
#region Reading
// This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
private byte[] _read_fixed_size_msg(int n_bytes) {
if (n_bytes <= 0) {
@@ -199,10 +210,10 @@ namespace bts {
try {
OneRecvMessage one_msg = _read_one_message();
if (one_msg.msg_type == DATA_MSG_SIGNAL) {
_msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _peer_conn_info));
_msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
} else if (one_msg.msg_type == HB_MSG_SIGNAL) {
// keep a last hb time an dudpate it here atomically
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
} else {
throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
}
@@ -212,6 +223,7 @@ namespace bts {
}
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_msgs_forever exiting loop");
}
#endregion
public bool is_disposed => !_is_alive;
@@ -253,7 +265,7 @@ namespace bts {
if (ex is OutOfMemoryException)
throw ex;
}
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info));
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
}
}
}
@@ -268,9 +280,7 @@ namespace bts {
}
public override string ToString() {
RemoteConnectionInfo self = get_self_address(),
peer = get_peer_address();
return $"type: {GetType().ToString()} self: {self} peer: {peer} is_disposed: {isDisposed}";
return _connection_id.ToString();
}
}
}
@@ -27,7 +27,7 @@ namespace bts {
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server: {server_rci}");
TcpClient tcp_client = new TcpClient(server_rci.ip, server_rci.port);
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server new TcpClient returned, connected: {tcp_client.Connected}");
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
ITCPConnection conn = _create_connection(tcp_client);
var conn_rci = conn.get_peer_address();
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, server_rci: {server_rci} connection.peer_address: {conn_rci}");
Log.assert(server_rci.Equals(conn_rci), $"{server_rci}!={conn_rci}");
@@ -35,6 +35,12 @@ namespace bts {
handler.onConnection(conn);
}
private static ITCPConnection _create_connection(TcpClient tcp_client) {
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
// ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
return conn;
}
public static void _keep_trying_blocking_function(Action action, TimeSpan timeout, TimeSpan wait_before_retrying) {
Func<bool> try_once = () => {
var cts = new CancellationTokenSource();
@@ -21,7 +21,7 @@ namespace BCG {
/// <param name="data"></param>
/// <param name="confId"></param>
/// <param name="del"></param>
void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
//void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
/// <summary>
/// sends data to the other side. theres no guaranatee that
/// data is delievered.
@@ -214,7 +214,7 @@ namespace BCG {
_connection_map.TryGetValue(p.first, out conn);
if (conn != null) {
conn.send(p.second);
Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}");
Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sending data client: {p.first} size: {p.second.Length}");
} else {
string error_msg = $"sending data of sz: {p.second.Count()} to {p.first} failed, this client might have " +
"disconnected, in which case it will be garbage collected within 60 secs";
@@ -20,27 +20,9 @@ namespace BCG {
private Thread _msg_polling_thread = null;
// all the data,list of subscribers etc are in the state
private PubSubState _state = new PubSubState();
private bool myStop = false;
private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false);
// private double myProcessingWarningThreshold =
// Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold);
// private HighPerformanceTimer myTimer = new HighPerformanceTimer();
private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds);
private SingleTagTimeSliceDelegate onSingleTagPub;
private static SyncQueue<PublishMsg> _internal_pub_queue = new SyncQueue<PublishMsg>();
private Stopwatch _stop_watch = new Stopwatch();
//this thing keeps track of whether a given tag is compressed or not. It is used for logging messages.
/* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/
// private delegate void IdentityFoundDelegate(string identity, bool isRoot);
// private event IdentityFoundDelegate onIdentityFound;
// private void fireIdentityFound(string identity, bool isRoot) {
// if (onIdentityFound != null)
// onIdentityFound(identity, isRoot);
// }
//private Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>();
public PubSubMessageReactor(int port_no) {
bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
@@ -54,7 +36,7 @@ namespace BCG {
TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()),
PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(),
TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value()));
// this is the only event this class gets from hub
_msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection));
}
@@ -69,10 +51,6 @@ namespace BCG {
_state.remove_connection(client);
}
private void stop() {
myStop = true;
}
public void start() {
_msg_polling_thread.Start();
}
@@ -104,6 +82,10 @@ namespace BCG {
//bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
var received_data = _msg_hub.get_all_updates();
var n_msgs = received_data.Count;
if (n_msgs == 0) {
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
continue;
}
bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
foreach (var kv in received_data) {
// multiple pubs for the same tags should be batched here
@@ -42,28 +42,6 @@ namespace BCG {
#region subscription based state changes
/// <summary>
/// everytime a subscription is made, this gets called
/// by PubsubServer2. Adds a
/// TODO : subscibers sending sub twice from the client
/// </summary>
/// <param name="sub"></param>
/// <param name="md"></param>
// public void sub(SubMetaData sub) {
// ///tag based key ds update
// foreach (string tag in sub.tags) {
// if (!_tag_to_connection_map.ContainsKey(tag)) {
// _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
// }
// _tag_to_connection_map[tag].onSub(sub);
// if (onSingleTagChange != null)
// onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
// }
// if (onTagMapChanged != null)
// onTagMapChanged(sub.tags);
// }
public void handle_sub(bts.SubscribeMsg sub_msg) {
Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}");
foreach (string tag in sub_msg.tags) {
@@ -82,24 +60,6 @@ namespace BCG {
// TODO
}
// public void unsub(UnSubMetaData unsub) {
// foreach (string tag in unsub.tags) {
// if (_tag_to_connection_map.ContainsKey(tag)) {
// ///maybe check here, that if there are no subscribers
// ///and publishers for the tag then remove it.
// _tag_to_connection_map[tag].onUnsub(unsub);
// if (onSingleTagChange != null)
// onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
// } else {
// string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
// " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
// //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
// }
// }
// if (onTagMapChanged != null)
// onTagMapChanged(unsub.tags);
// }
public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) {
foreach (string tag in ubnsub_msg.tags) {
SingleTagConnectionData conn_data = null;
@@ -139,48 +99,11 @@ namespace BCG {
}
if (onTagMapChanged != null)
onTagMapChanged(tags_effected.ToArray());
}
#endregion
#region data based state changes
/// <summary>
/// this is the only place in the state class where the
/// data is modified ; happens when data is published
/// </summary>
/// <param name="tag"></param>
/// <param name="msg"></param>
// public void updatePubData(PubMetaData pub, Stream msg) {
// int sz = (int)(msg.Length - msg.Position);
// bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
// ///this is the only place where SingleTagData should be created
// if (!_tag_to_data_map.ContainsKey(pub.tag)) {
// ///first time data is published for this tag
// _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
// }
// _tag_to_data_map[pub.tag].pubUpdate(msg);
// if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
// _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
// }
// bool pubChanged = false;
// _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged);
// if (pubChanged) {
// if (onTagMapChanged != null)
// onTagMapChanged(new string[] { pub.tag });
// if (onSingleTagChange != null)
// onSingleTagChange(
// new PubSubSingleTagStateEvent(pub.tag,
// PubSubSingleTagStateEvent.EventType.publish,
// pub.connection, pub.publishingApp));
// }
// if (onPublishStats != null)
// onPublishStats(pub.tag, sz);
// }
public void update_pub_data(bts.PublishMsg pub_msg) {
try {
bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");