//This implementation uses APM the beginFunc, endFunc pattern // https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/asynchronous-programming-model-apm#beginning-an-asynchronous-operation /// /// This class, mantains a connection to a peer socket and using the APM /// model to communicate. It sends heartbeats at regular /// intervals (config). It is in active state when messages /// are sent and received without an error state. It is assumed /// dead state when either of the following happens /// BeginReceive returns a SocketError /// BeginSend returns a SocketError /// EndReceive returns a SocketError /// EndSend returns a SocketError /// As soon as it assumes dead state it stops sending or /// receiving any messages (including heartbeats). /// The user of this class can either detect that by /// missing heartbeats or checking the isConnected property. /// If the peer dies in a 'silent' way, where none of the above /// methods indicate an error, missing heartbeats is guaranteed /// to detect it. /// using System; using System.Collections.Generic; using System.Text; using System.Net; using System.Net.Sockets; using System.IO; using BCG.Utilities.General; using System.Runtime.InteropServices; using BCG; using System.Threading; using System.Threading.Tasks; using System.Linq; using Bts.PubSub; using Spark.Core; namespace bts { public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable { // this class is used as the argument of the completion function for BeginSend/Receive // serves as the base class for both Send and ReeceiveMessage classes. // since both TCP sends and receives happen in multiple transfers, this // provides a wrapper around copying the partial transfers into the complete buffer public class AsyncMessageBase { readonly string msg_id = Guid.NewGuid().ToString(); // every BeginSend or BeginReceive has a unique key associated with it public int key; // total msg that needs to be transfered public byte[] buffer; // as data is transferred in chunks, transfer_ptr moves ahead by amount of data transferred. // everything < transfer_ptr has been successfully transferred, // ie this is pointing to where the next transfer will start from public int transfer_ptr; protected int _n_transfers = 0; public bool completed_sync = false; public AsyncMessageBase(byte[] the_buffer, int the_offset) { buffer = the_buffer; transfer_ptr = the_offset; } public int remaining_size { get { return buffer.Length - transfer_ptr; } } public int original_size => buffer.Length; public void advance_transfer_pointer(int advance_by) { transfer_ptr += advance_by; _n_transfers++; if (transfer_ptr > original_size) { bts.Log.Gen.error($"transfer went beyond initial size {ToString()}"); } // TODO: log if it goes past the buffer } public bool fully_transfered() { return transfer_ptr >= original_size; } public override string ToString() { return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " + $"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}"; } } public class AsyncSendMessage : AsyncMessageBase { public string message_id; public SendConfirmationDelegate on_send_del; //private int _n_transfers = 0; public AsyncSendMessage(byte[] the_buffer, int the_offset) : this(the_buffer, the_offset, String.Empty, null) { } public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del) : base(the_buffer, the_transfer_ptr) { message_id = the_message_id; on_send_del = the_on_send_del; } public override string ToString() { return base.ToString(); } } public class AsyncReceiveMessage : AsyncMessageBase { public enum ReadState { reading_size, reading_data }; readonly ReadState read_state; public AsyncReceiveMessage(int size, ReadState the_read_state) : this(new byte[size], 0, the_read_state) { } public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state) : base(the_buffer, the_transfer_ptr) { read_state = the_read_state; } public int read_int32(int start_idx = 0) { return BitConverter.ToInt32(buffer, start_idx); } public byte[] read_bytes(int start_idx) { byte[] rv = new byte[buffer.Length - start_idx]; Array.Copy(buffer, start_idx, rv, 0, rv.Length); return rv; } public bool is_reading_size() => read_state == ReadState.reading_size; } public class ErrorMessage { public readonly SocketError? socket_error = null; public readonly Exception exception = null; public readonly bool is_exception = false; public readonly string context = String.Empty; public ErrorMessage(SocketError socket_error_, string context_) { socket_error = socket_error_; context = context_; } public ErrorMessage(Exception exception_, string context_) { is_exception = true; exception = exception_; context = context_; } public bool oom => exception != null && exception is OutOfMemoryException; public override string ToString() { if (is_exception) return $"ErrorMessage exception: {exception} context: {context}"; return $"socket_error: {socket_error} context: {context}"; } } private const int DATA_MSG_SIGNAL = 0; private const int HB_MSG_SIGNAL = 1; private int _send_key = 0, _receive_key = 0; private HashSet _read_key_map = new HashSet(), _send_key_map = new HashSet(); private readonly int _max_msg_size = Int32.MaxValue;// PubSubConfig.Instance.MaxPacketSize.assert_value(); private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info; private readonly ITCPConnection.ConnectionSide _connection_side; private readonly Socket _sock; private ITCPMessageHandler _message_handler = null; private bool _is_alive = true; private byte[] _cached_hb_message = null; private readonly TimeSpan _hb_period = TimeSpan.FromSeconds(1); // TODO get it from config private MasterWorker _master_worker = MasterWorker.getInstance(true); private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false; private DateTime _last_hb_received_time = new DateTime(1900, 1, 1); public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) { _sock = sock; _hb_period = hb_period; _connection_side = connection_side; _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true); #region testing options switch (connection_side) { case ITCPConnection.ConnectionSide.Server: TCPSocketUtils.set_server_socket_options(ref _sock); break; case ITCPConnection.ConnectionSide.Client: TCPSocketUtils.set_client_socket_options(ref _sock); break; } #endregion _peer_conn_info = get_peer_address(); _self_conn_info = get_self_address(); bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}"); _detect_connection_status_change(); } private void _detect_connection_status_change() { Action testing_func = () => { bool connected = _sock.Connected; while (true) { if (connected != _sock.Connected) Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}"); connected = _sock.Connected; System.Threading.Thread.Sleep(50); } }; ThreadPool.QueueUserWorkItem(_ => testing_func()); } public override string ToString() { return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}"; } #region ITCPConnection Members public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) { conf_id = Guid.NewGuid().ToString(); byte[] msg = _build_data_message(data); _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del)); } public void send(byte[] data) { //Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}"); byte[] msg = _build_data_message(data); //Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}"); _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); } public void register(ITCPMessageHandler messageHandler) { bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned"); //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered"); _message_handler = messageHandler; _master_worker.addWorker(this, _hb_period); // we send only when send is called but read continoulsy, starting here _read_next_msg(); } public RemoteConnectionInfo get_peer_address() { return TCPSocketUtils.get_peer_address(_sock); } public RemoteConnectionInfo get_self_address() { return TCPSocketUtils.get_peer_address(_sock); } public bool isDisposed { get { return !_is_alive; } } #endregion #region IWorker Members bool _last_connected = false; public void doWork() { if (!_is_alive) return; return; // disable hbs for now if (!_hb_sent_once) { bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time"); _hb_sent_once = true; } var msg = _build_hb_message(); _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); } private static int _tid() { var tid = System.Threading.Thread.CurrentThread.ManagedThreadId; return tid; } private static string _cb_txt(IAsyncResult res) { return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}"; } #endregion #region sending data private void _begin_or_resume_sending(AsyncSendMessage message) { //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}"); if (_is_alive) { try { SocketError begin_send_err; // since we get message on on_send we dont need to keep the state around message.key = _send_key++; var res = _sock.BeginSend(message.buffer, message.transfer_ptr, message.remaining_size, SocketFlags.None, out begin_send_err, new AsyncCallback(BeginSend_callback_function), message); if (begin_send_err != SocketError.Success) _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending")); } catch (Exception ex) { _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending")); } } } private void BeginSend_callback_function(IAsyncResult res) { if (res.CompletedSynchronously) { ThreadPool.QueueUserWorkItem(_ => _on_send_async(res)); } else _on_send_async(res); } private void _on_send_async(IAsyncResult res) { AsyncSendMessage message = res.AsyncState as AsyncSendMessage; if (_send_key_map.Contains(message.key)) { bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled"); return; } _send_key_map.Add(message.key); // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " + // $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}"); if (_is_alive) { try { SocketError send_err; int bytes_sent = _sock.EndSend(res, out send_err); //message.offset += bytesSent; message.advance_transfer_pointer(bytes_sent); if (bytes_sent == 0) { bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception"); _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"), "_on_send")); return; } bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " + $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}"); if (send_err == SocketError.Success) { // successful send of some bytes //if (message.offset == message.buffer.Length) { if (message.fully_transfered()) { // fully sent, if confirmation needs to be sent out if (!String.IsNullOrEmpty(message.message_id)) { message.on_send_del(message.message_id, true); } } else { // there is more to send _begin_or_resume_sending(message); } } else { _handle_error(new ErrorMessage(send_err, "_on_send")); } } catch (Exception ex) { _handle_error(new ErrorMessage(ex, "_on_send")); } } } /// /// Assembles the heartbeat message, and uses it for /// future references /// /// private byte[] _build_hb_message() { if (_cached_hb_message == null) { BinaryWriter bw = new BinaryWriter(new MemoryStream()); // the total msg =[msg_sz, msg_type] int msg_sz = 4; bw.Write(msg_sz); bw.Write(HB_MSG_SIGNAL); _cached_hb_message = (bw.BaseStream as MemoryStream).ToArray(); } return _cached_hb_message; } // writes size prefixed msg private byte[] _build_data_message(byte[] data) { // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter) MemoryStream str = new MemoryStream(); BinaryWriter bw = new BinaryWriter(str); // the total msg =[msg_sz, msg_type, data] Int32 msg_sz = data.Length + 4; bw.Write(msg_sz); // +4 for the type of msg // to indicate data vs hb bw.Write(DATA_MSG_SIGNAL); // for data // if we want to id every message, this is the place to do it bw.Write(data); str.Position = 0; //not needed really return str.ToArray(); } #endregion #region receiving data private void _read_next_msg() { _read_size(); } private void _read_size() { _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size)); } private void _read_data(int data_size) { // TODO, this could be a problem is the size is too big - maybe read it in chunks (later) _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data)); } private static string IAsyncResult_txt(IAsyncResult res) { return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}"; } private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) { //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}"); if (!_is_alive) return; try { SocketError read_sock_err; rcv_msg.key = _receive_key++; var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr, rcv_msg.remaining_size, SocketFlags.None, out read_sock_err, new AsyncCallback(BeginReceive_callback_function), rcv_msg); if (read_sock_err != SocketError.Success) { bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}"); _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading")); } } catch (Exception ex) { _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading")); } } private void _log_incoming_msg(IAsyncResult res) { // AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage; // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length); // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}"); } // this is the function passed to BeginReceive, we private void BeginReceive_callback_function(IAsyncResult res) { _log_incoming_msg(res); if (res.CompletedSynchronously) { ThreadPool.QueueUserWorkItem(_ => _on_read_async(res)); } else _on_read_async(res); } // this will always be called on the non main-thread. the reason this is on its own thread is that this function ends up // calling _begin_or_resume_reading, which in tuns calls _on_read_async. So to not cause a stack overflow, this is broken // always called on a different thread than _begin_or_resume_reading private void _on_read_async(IAsyncResult res) { AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage; if (_read_key_map.Contains(rcv_message.key)) { bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled"); return; } _read_key_map.Add(rcv_message.key); if (!_is_alive) return; try { SocketError read_err; Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}"); int bytes_read = _sock.EndReceive(res, out read_err); rcv_message.advance_transfer_pointer(bytes_read); Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}"); if (bytes_read == 0) { bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception"); //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring"); //System.Threading.Thread.Sleep(50); _handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read")); return; } //return; if (read_err == SocketError.Success) { if (rcv_message.fully_transfered()) { if (rcv_message.is_reading_size()) { var msg_size = _on_read_size_completely(rcv_message); if (msg_size != null && msg_size.HasValue) { _read_data(msg_size.Value); } return; } else { // data is read int msg_type = rcv_message.read_int32(); bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL); if (msg_type == DATA_MSG_SIGNAL) { _on_read_data_completely(rcv_message); _read_next_msg(); return; } else if (msg_type == HB_MSG_SIGNAL) { _on_read_hb_completely(rcv_message); _read_next_msg(); return; } // go back to reading size for a new message //_read_size(); } } else { // msg has not been fully rd, we have already advanced the transfer ptr _begin_or_resume_reading(rcv_msg: rcv_message); } } else { _handle_error(new ErrorMessage(read_err, "_on_read_async")); } } catch (Exception ex) { _handle_error(new ErrorMessage(ex, "_on_read_async")); } } #endregion private int? _on_read_size_completely(AsyncReceiveMessage msg) { bts.Log.assert(msg.is_reading_size()); bts.Log.assert(msg.fully_transfered()); bts.Log.assert(msg.original_size == 4); var msg_sz = msg.read_int32(); //var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length); //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}"); bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}"); if (msg_sz < 0 || msg_sz > _max_msg_size) { string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}"; _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely")); return null; } else { // sz is determined, now we read that many bytes //_read_data(msg_sz); return msg_sz; } } private void _on_read_hb_completely(AsyncReceiveMessage msg) { bts.Log.assert(!msg.is_reading_size()); bts.Log.assert(msg.fully_transfered()); bts.Log.assert(msg.original_size == 4); bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL); if (!_hb_received_yet) { _hb_received_yet = true; //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}"); } _last_hb_received_time = DateTime.Now; _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info)); } private void _on_read_data_completely(AsyncReceiveMessage msg) { bts.Log.assert(!msg.is_reading_size()); bts.Log.assert(msg.fully_transfered()); var type = msg.read_int32(0); bts.Log.assert(type == DATA_MSG_SIGNAL); bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_data_completely msg: {msg} type: {type}"); byte[] data = msg.read_bytes(4); ///callbacks can be made through thread pool to avoid this ///thread from being blocked, by this callback if (!_data_received_yet) { _data_received_yet = true; //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}"); } // could be an index here _message_handler.onData(new RemoteMessage(data, RemoteMessageType.Data, _peer_conn_info)); } #region IDisposable Members public void Dispose() { // socket is closed and disposed here lock (this) { bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}"); if (_is_alive) { _master_worker.removeWorker(this); _is_alive = false; bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}"); try { bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now"); _sock.Shutdown(SocketShutdown.Both); } catch (Exception ex) { String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket"); if (ex is OutOfMemoryException) throw ex; } _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info)); } } } #endregion private string _dipose_impetus = String.Empty; private void _handle_error(ErrorMessage err_msg) { lock (this) { Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); if (err_msg.oom) throw err_msg.exception; _dipose_impetus = $"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"; Dispose(); } } } }