Fixing the bug where messages were being sent by the pubsubserver out of order

This commit is contained in:
Taha authored and John Colvin committed 2024-09-18 14:02:34 +00:00
1 parent 6604eaaea7
commit b55c4103df
8 files changed
+62 -135

No files matched your search

@@ -1,10 +1,6 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using bts.utils;
using System;
using System.IO;
using System.Net.Sockets;
@@ -12,19 +8,30 @@ namespace bts {
using System.Threading;
using System.Net;
using System.Text;
using System.ComponentModel;
// this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
public class ConnectionID {
public readonly RemoteConnectionInfo self;
public readonly RemoteConnectionInfo peer;
public readonly Type conn_type;
public override string ToString() {
return $"{conn_type} self: {self} peer: {peer}";
}
public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
self = self_;
peer = peer_;
conn_type = conn_type_;
}
}
private const int DATA_MSG_SIGNAL = 0;
private const int HB_MSG_SIGNAL = 1;
private TcpClient _tcp_client;
private NetworkStream _nw_stream;
private ITCPMessageHandler _msg_handler;
private bool _is_alive = true;
RemoteConnectionInfo _peer_conn_info = null;
RemoteConnectionInfo _self_conn_info = null;
ConnectionID _connection_id;
public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
var len = end_pos - start_pos;
@@ -42,10 +49,8 @@ namespace bts {
public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
_tcp_client = tcp_client;
_nw_stream = _tcp_client.GetStream();
_peer_conn_info = get_peer_address();
_self_conn_info = get_self_address();
Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons _peer_conn_info: {_peer_conn_info} _self_conn_info: {_self_conn_info}");
//_detect_connection_status_change();
_connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
}
private void _detect_connection_status_change() {
@@ -63,10 +68,7 @@ namespace bts {
ThreadPool.QueueUserWorkItem(_ => testing_func());
}
public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) {
}
#region Sending
public void _send_fixed_size_msg(byte[] buffer) {
string buff_txt = to_hex(buffer, 0, buffer.Length);
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
@@ -101,29 +103,38 @@ namespace bts {
}
}
private void _send_forever() {
while (_is_alive) {
byte[] data = _send_queue.dequeue();
if (data != null) {
_send_one_msg(data);
}
}
}
private SyncQueue<byte[]> _send_queue = new SyncQueue<byte[]>();
public void send(byte[] data) {
var data_txt = to_hex(data, 0, data.Length);
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.send data: {data_txt}");
//Task.Run(() => _send_one_msg(data));
ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data));
//ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data));
_send_queue.enqueue(data);
}
#endregion
public void register(ITCPMessageHandler message_handler) {
_msg_handler = message_handler;
// start reading in a Task
// starting thread pool threads to keep reading and sending
ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
ThreadPool.QueueUserWorkItem(_ => _send_forever());
}
public RemoteConnectionInfo get_peer_address() {
//if (_peer_conn_info == null)
return _peer_conn_info = TCPSocketUtils.get_peer_address(_tcp_client.Client);
//return _peer_conn_info;
return _connection_id.peer;
}
public RemoteConnectionInfo get_self_address() {
if (_self_conn_info == null)
_self_conn_info = TCPSocketUtils.get_self_address(_tcp_client.Client);
return _self_conn_info;
return _connection_id.self;
}
public bool isDisposed { get { return !_is_alive; } }
@@ -132,7 +143,7 @@ namespace bts {
// send hb
}
#region Reading
// This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
private byte[] _read_fixed_size_msg(int n_bytes) {
if (n_bytes <= 0) {
@@ -199,10 +210,10 @@ namespace bts {
try {
OneRecvMessage one_msg = _read_one_message();
if (one_msg.msg_type == DATA_MSG_SIGNAL) {
_msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _peer_conn_info));
_msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
} else if (one_msg.msg_type == HB_MSG_SIGNAL) {
// keep a last hb time an dudpate it here atomically
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
} else {
throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
}
@@ -212,6 +223,7 @@ namespace bts {
}
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_msgs_forever exiting loop");
}
#endregion
public bool is_disposed => !_is_alive;
@@ -253,7 +265,7 @@ namespace bts {
if (ex is OutOfMemoryException)
throw ex;
}
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info));
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
}
}
}
@@ -268,9 +280,7 @@ namespace bts {
}
public override string ToString() {
RemoteConnectionInfo self = get_self_address(),
peer = get_peer_address();
return $"type: {GetType().ToString()} self: {self} peer: {peer} is_disposed: {isDisposed}";
return _connection_id.ToString();
}
}
}