From b55c4103df80dcb3ac82137e330c3d66e6366c7a Mon Sep 17 00:00:00 2001 From: Taha Date: Thu, 27 Jun 2024 00:17:22 -0400 Subject: [PATCH] Fixing the bug where messages were being sent by the pubsubserver out of order --- .../PubSubServerManagedTCPConnection.cs | 2 +- ...AsyncTCPConnectionAcceptor_AsyncWrapper.cs | 8 +- .../AsyncTCPConnection_AsyncWrapper.cs | 70 +++++++++-------- .../AsyncTCPConnector_AsyncWrapper.cs | 8 +- .../PubSub/ManagedTCP/ITCPConnection.cs | 2 +- .../ManagedTCP/ManagedTCPMessageHub.cs | 2 +- .../PubSub/Server/PubSubMessageReactor.cs | 28 ++----- .../PubSub/Server/PubSubState.cs | 77 ------------------- 8 files changed, 62 insertions(+), 135 deletions(-) diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs index 329e23d71..34c7d0d4f 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -158,7 +158,7 @@ namespace BCG { /// /// public void onData(RemoteMessage message) { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}"); + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData connection: {_connection} message: {message}"); // TODO check for disconenction switch (message.messageType) { case RemoteMessageType.Data: diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs index 571a7b15d..856839e5e 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs @@ -41,12 +41,18 @@ namespace bts { // this is a blocking call Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port} starting blocking call AcceptTcpClient"); TcpClient tcp_client = _server.AcceptTcpClient(); - ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client); + ITCPConnection conn = _create_connection(tcp_client); Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now"); _connection_handler.onConnection(conn); } //_cbs(conn); } + + private static ITCPConnection _create_connection(TcpClient tcp_client) { + ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client); + // ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client); + return conn; + } } } \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs index 24bbb0323..bf6214ece 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs @@ -1,10 +1,6 @@ namespace bts { using BCG; using BCG.Utilities.General; - - using bts.utils; - - using System; using System.IO; using System.Net.Sockets; @@ -12,19 +8,30 @@ namespace bts { using System.Threading; using System.Net; using System.Text; - using System.ComponentModel; // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly) public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable { + public class ConnectionID { + public readonly RemoteConnectionInfo self; + public readonly RemoteConnectionInfo peer; + public readonly Type conn_type; + public override string ToString() { + return $"{conn_type} self: {self} peer: {peer}"; + } + public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) { + self = self_; + peer = peer_; + conn_type = conn_type_; + } + } private const int DATA_MSG_SIGNAL = 0; private const int HB_MSG_SIGNAL = 1; private TcpClient _tcp_client; private NetworkStream _nw_stream; private ITCPMessageHandler _msg_handler; private bool _is_alive = true; - RemoteConnectionInfo _peer_conn_info = null; - RemoteConnectionInfo _self_conn_info = null; + ConnectionID _connection_id; public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) { var len = end_pos - start_pos; @@ -42,10 +49,8 @@ namespace bts { public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) { _tcp_client = tcp_client; _nw_stream = _tcp_client.GetStream(); - _peer_conn_info = get_peer_address(); - _self_conn_info = get_self_address(); - Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons _peer_conn_info: {_peer_conn_info} _self_conn_info: {_self_conn_info}"); - //_detect_connection_status_change(); + _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType()); + Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}"); } private void _detect_connection_status_change() { @@ -63,10 +68,7 @@ namespace bts { ThreadPool.QueueUserWorkItem(_ => testing_func()); } - public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) { - - } - + #region Sending public void _send_fixed_size_msg(byte[] buffer) { string buff_txt = to_hex(buffer, 0, buffer.Length); Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}"); @@ -101,29 +103,38 @@ namespace bts { } } + private void _send_forever() { + while (_is_alive) { + byte[] data = _send_queue.dequeue(); + if (data != null) { + _send_one_msg(data); + } + } + } + + private SyncQueue _send_queue = new SyncQueue(); public void send(byte[] data) { var data_txt = to_hex(data, 0, data.Length); Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.send data: {data_txt}"); //Task.Run(() => _send_one_msg(data)); - ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data)); + //ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data)); + _send_queue.enqueue(data); } + #endregion public void register(ITCPMessageHandler message_handler) { _msg_handler = message_handler; - // start reading in a Task + // starting thread pool threads to keep reading and sending ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever()); + ThreadPool.QueueUserWorkItem(_ => _send_forever()); } public RemoteConnectionInfo get_peer_address() { - //if (_peer_conn_info == null) - return _peer_conn_info = TCPSocketUtils.get_peer_address(_tcp_client.Client); - //return _peer_conn_info; + return _connection_id.peer; } public RemoteConnectionInfo get_self_address() { - if (_self_conn_info == null) - _self_conn_info = TCPSocketUtils.get_self_address(_tcp_client.Client); - return _self_conn_info; + return _connection_id.self; } public bool isDisposed { get { return !_is_alive; } } @@ -132,7 +143,7 @@ namespace bts { // send hb } - + #region Reading // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg private byte[] _read_fixed_size_msg(int n_bytes) { if (n_bytes <= 0) { @@ -199,10 +210,10 @@ namespace bts { try { OneRecvMessage one_msg = _read_one_message(); if (one_msg.msg_type == DATA_MSG_SIGNAL) { - _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _peer_conn_info)); + _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer)); } else if (one_msg.msg_type == HB_MSG_SIGNAL) { // keep a last hb time an dudpate it here atomically - _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info)); + _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer)); } else { throw new ApplicationException($"Unknown message type: {one_msg.msg_type}"); } @@ -212,6 +223,7 @@ namespace bts { } Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_msgs_forever exiting loop"); } + #endregion public bool is_disposed => !_is_alive; @@ -253,7 +265,7 @@ namespace bts { if (ex is OutOfMemoryException) throw ex; } - _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info)); + _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer)); } } } @@ -268,9 +280,7 @@ namespace bts { } public override string ToString() { - RemoteConnectionInfo self = get_self_address(), - peer = get_peer_address(); - return $"type: {GetType().ToString()} self: {self} peer: {peer} is_disposed: {isDisposed}"; + return _connection_id.ToString(); } } } \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs index 1e79b5a73..d8ee8aa31 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs @@ -27,7 +27,7 @@ namespace bts { bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server: {server_rci}"); TcpClient tcp_client = new TcpClient(server_rci.ip, server_rci.port); bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server new TcpClient returned, connected: {tcp_client.Connected}"); - ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client); + ITCPConnection conn = _create_connection(tcp_client); var conn_rci = conn.get_peer_address(); bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, server_rci: {server_rci} connection.peer_address: {conn_rci}"); Log.assert(server_rci.Equals(conn_rci), $"{server_rci}!={conn_rci}"); @@ -35,6 +35,12 @@ namespace bts { handler.onConnection(conn); } + private static ITCPConnection _create_connection(TcpClient tcp_client) { + ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client); + // ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client); + return conn; + } + public static void _keep_trying_blocking_function(Action action, TimeSpan timeout, TimeSpan wait_before_retrying) { Func try_once = () => { var cts = new CancellationTokenSource(); diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs index 9eff4ab4d..9afb9f920 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs @@ -21,7 +21,7 @@ namespace BCG { /// /// /// - void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del); + //void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del); /// /// sends data to the other side. theres no guaranatee that /// data is delievered. diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs index 19fab973d..887bb4aef 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -214,7 +214,7 @@ namespace BCG { _connection_map.TryGetValue(p.first, out conn); if (conn != null) { conn.send(p.second); - Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}"); + Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sending data client: {p.first} size: {p.second.Length}"); } else { string error_msg = $"sending data of sz: {p.second.Count()} to {p.first} failed, this client might have " + "disconnected, in which case it will be garbage collected within 60 secs"; diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index 805543fa0..fa2107120 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -20,27 +20,9 @@ namespace BCG { private Thread _msg_polling_thread = null; // all the data,list of subscribers etc are in the state private PubSubState _state = new PubSubState(); - private bool myStop = false; private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false); - // private double myProcessingWarningThreshold = - // Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold); - // private HighPerformanceTimer myTimer = new HighPerformanceTimer(); - - private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); - private SingleTagTimeSliceDelegate onSingleTagPub; private static SyncQueue _internal_pub_queue = new SyncQueue(); private Stopwatch _stop_watch = new Stopwatch(); - //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. - - /* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/ - // private delegate void IdentityFoundDelegate(string identity, bool isRoot); - // private event IdentityFoundDelegate onIdentityFound; - // private void fireIdentityFound(string identity, bool isRoot) { - // if (onIdentityFound != null) - // onIdentityFound(identity, isRoot); - // } - - //private Dictionary myTagCompressionState = new Dictionary(); public PubSubMessageReactor(int port_no) { bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized"); @@ -54,7 +36,7 @@ namespace BCG { TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()), PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(), TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value())); - + // this is the only event this class gets from hub _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection)); } @@ -69,10 +51,6 @@ namespace BCG { _state.remove_connection(client); } - private void stop() { - myStop = true; - } - public void start() { _msg_polling_thread.Start(); } @@ -104,6 +82,10 @@ namespace BCG { //bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates"); var received_data = _msg_hub.get_all_updates(); var n_msgs = received_data.Count; + if (n_msgs == 0) { + System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250)); + continue; + } bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}"); foreach (var kv in received_data) { // multiple pubs for the same tags should be batched here diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index ed524157e..a901a012b 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -42,28 +42,6 @@ namespace BCG { #region subscription based state changes - /// - /// everytime a subscription is made, this gets called - /// by PubsubServer2. Adds a - /// TODO : subscibers sending sub twice from the client - /// - /// - /// - // public void sub(SubMetaData sub) { - // ///tag based key ds update - // foreach (string tag in sub.tags) { - // if (!_tag_to_connection_map.ContainsKey(tag)) { - // _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag)); - // } - // _tag_to_connection_map[tag].onSub(sub); - - // if (onSingleTagChange != null) - // onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app)); - // } - // if (onTagMapChanged != null) - // onTagMapChanged(sub.tags); - // } - public void handle_sub(bts.SubscribeMsg sub_msg) { Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}"); foreach (string tag in sub_msg.tags) { @@ -82,24 +60,6 @@ namespace BCG { // TODO } - // public void unsub(UnSubMetaData unsub) { - // foreach (string tag in unsub.tags) { - // if (_tag_to_connection_map.ContainsKey(tag)) { - // ///maybe check here, that if there are no subscribers - // ///and publishers for the tag then remove it. - // _tag_to_connection_map[tag].onUnsub(unsub); - // if (onSingleTagChange != null) - // onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app)); - // } else { - // string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" + - // " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app); - // //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); - // } - // } - // if (onTagMapChanged != null) - // onTagMapChanged(unsub.tags); - // } - public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) { foreach (string tag in ubnsub_msg.tags) { SingleTagConnectionData conn_data = null; @@ -139,48 +99,11 @@ namespace BCG { } if (onTagMapChanged != null) onTagMapChanged(tags_effected.ToArray()); - - } #endregion #region data based state changes - /// - /// this is the only place in the state class where the - /// data is modified ; happens when data is published - /// - /// - /// - // public void updatePubData(PubMetaData pub, Stream msg) { - // int sz = (int)(msg.Length - msg.Position); - // bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}"); - // ///this is the only place where SingleTagData should be created - // if (!_tag_to_data_map.ContainsKey(pub.tag)) { - // ///first time data is published for this tag - // _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type)); - // } - // _tag_to_data_map[pub.tag].pubUpdate(msg); - - // if (!_tag_to_connection_map.ContainsKey(pub.tag)) { - // _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag)); - // } - // bool pubChanged = false; - // _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged); - // if (pubChanged) { - // if (onTagMapChanged != null) - // onTagMapChanged(new string[] { pub.tag }); - - // if (onSingleTagChange != null) - // onSingleTagChange( - // new PubSubSingleTagStateEvent(pub.tag, - // PubSubSingleTagStateEvent.EventType.publish, - // pub.connection, pub.publishingApp)); - // } - // if (onPublishStats != null) - // onPublishStats(pub.tag, sz); - // } - public void update_pub_data(bts.PublishMsg pub_msg) { try { bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");