Files
spark/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs
T
2024-07-17 02:02:01 -04:00

703 lines
26 KiB
C#

//This implementation uses APM the beginFunc, endFunc pattern
// https://learn.microsoft.com/en-us/dotnet/standard/asynchronous-programming-patterns/asynchronous-programming-model-apm#beginning-an-asynchronous-operation
/// <summary>
/// This class, mantains a connection to a peer socket and using the APM
/// model to communicate. It sends heartbeats at regular
/// intervals (config). It is in active state when messages
/// are sent and received without an error state. It is assumed
/// dead state when either of the following happens
/// BeginReceive returns a SocketError
/// BeginSend returns a SocketError
/// EndReceive returns a SocketError
/// EndSend returns a SocketError
/// As soon as it assumes dead state it stops sending or
/// receiving any messages (including heartbeats).
/// The user of this class can either detect that by
/// missing heartbeats or checking the isConnected property.
/// If the peer dies in a 'silent' way, where none of the above
/// methods indicate an error, missing heartbeats is guaranteed
/// to detect it.
/// </summary>
using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.IO;
using BCG.Utilities.General;
using System.Runtime.InteropServices;
using BCG;
using System.Threading;
using System.Threading.Tasks;
using System.Linq;
using Bts.PubSub;
using Spark.Core;
namespace bts
{
public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable
{
// this class is used as the argument of the completion function for BeginSend/Receive
// serves as the base class for both Send and ReeceiveMessage classes.
// since both TCP sends and receives happen in multiple transfers, this
// provides a wrapper around copying the partial transfers into the complete buffer
public class AsyncMessageBase
{
readonly string msg_id = Guid.NewGuid().ToString();
// every BeginSend or BeginReceive has a unique key associated with it
public int key;
// total msg that needs to be transfered
public byte[] buffer;
// as data is transferred in chunks, transfer_ptr moves ahead by amount of data transferred.
// everything < transfer_ptr has been successfully transferred,
// ie this is pointing to where the next transfer will start from
public int transfer_ptr;
protected int _n_transfers = 0;
public bool completed_sync = false;
public AsyncMessageBase(byte[] the_buffer, int the_offset)
{
buffer = the_buffer;
transfer_ptr = the_offset;
}
public int remaining_size
{
get
{
return buffer.Length - transfer_ptr;
}
}
public int original_size => buffer.Length;
public void advance_transfer_pointer(int advance_by)
{
transfer_ptr += advance_by;
_n_transfers++;
if (transfer_ptr > original_size)
{
bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
}
// TODO: log if it goes past the buffer
}
public bool fully_transfered()
{
return transfer_ptr >= original_size;
}
public override string ToString()
{
return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
$"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}";
}
}
public class AsyncSendMessage : AsyncMessageBase
{
public string message_id;
public SendConfirmationDelegate on_send_del;
//private int _n_transfers = 0;
public AsyncSendMessage(byte[] the_buffer, int the_offset)
: this(the_buffer, the_offset, String.Empty, null) { }
public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
: base(the_buffer, the_transfer_ptr)
{
message_id = the_message_id;
on_send_del = the_on_send_del;
}
public override string ToString()
{
return base.ToString();
}
}
public class AsyncReceiveMessage : AsyncMessageBase
{
public enum ReadState
{
reading_size,
reading_data
};
readonly ReadState read_state;
public AsyncReceiveMessage(int size, ReadState the_read_state) :
this(new byte[size], 0, the_read_state)
{ }
public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
: base(the_buffer, the_transfer_ptr)
{
read_state = the_read_state;
}
public int read_int32(int start_idx = 0)
{
return BitConverter.ToInt32(buffer, start_idx);
}
public byte[] read_bytes(int start_idx)
{
byte[] rv = new byte[buffer.Length - start_idx];
Array.Copy(buffer, start_idx, rv, 0, rv.Length);
return rv;
}
public bool is_reading_size() => read_state == ReadState.reading_size;
}
public class ErrorMessage
{
public readonly SocketError? socket_error = null;
public readonly Exception exception = null;
public readonly bool is_exception = false;
public readonly string context = String.Empty;
public ErrorMessage(SocketError socket_error_, string context_)
{
socket_error = socket_error_;
context = context_;
}
public ErrorMessage(Exception exception_, string context_)
{
is_exception = true;
exception = exception_;
context = context_;
}
public bool oom => exception != null && exception is OutOfMemoryException;
public override string ToString()
{
if (is_exception)
return $"ErrorMessage exception: {exception} context: {context}";
return $"socket_error: {socket_error} context: {context}";
}
}
private const int DATA_MSG_SIGNAL = 0;
private const int HB_MSG_SIGNAL = 1;
private int _send_key = 0, _receive_key = 0;
private HashSet<int> _read_key_map = new HashSet<int>(),
_send_key_map = new HashSet<int>();
private readonly int _max_msg_size = Int32.MaxValue;// PubSubConfig.Instance.MaxPacketSize.assert_value();
private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info;
private readonly ITCPConnection.ConnectionSide _connection_side;
private readonly Socket _sock;
private ITCPMessageHandler _message_handler = null;
private bool _is_alive = true;
private byte[] _cached_hb_message = null;
private readonly TimeSpan _hb_period = TimeSpan.FromSeconds(1); // TODO get it from config
private MasterWorker _master_worker = MasterWorker.getInstance(true);
private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);
public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side)
{
_sock = sock;
_hb_period = hb_period;
_connection_side = connection_side;
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
#region testing options
switch (connection_side)
{
case ITCPConnection.ConnectionSide.Server:
TCPSocketUtils.set_server_socket_options(ref _sock);
break;
case ITCPConnection.ConnectionSide.Client:
TCPSocketUtils.set_client_socket_options(ref _sock);
break;
}
#endregion
_peer_conn_info = get_peer_address();
_self_conn_info = get_self_address();
bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}");
_detect_connection_status_change();
}
private void _detect_connection_status_change()
{
Action testing_func = () =>
{
bool connected = _sock.Connected;
while (true)
{
if (connected != _sock.Connected)
Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}");
connected = _sock.Connected;
System.Threading.Thread.Sleep(50);
}
};
ThreadPool.QueueUserWorkItem(_ => testing_func());
}
public override string ToString()
{
return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
}
#region ITCPConnection Members
public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
{
conf_id = Guid.NewGuid().ToString();
byte[] msg = _build_data_message(data);
_begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
}
public void send(byte[] data)
{
//Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
byte[] msg = _build_data_message(data);
//Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
_begin_or_resume_sending(new AsyncSendMessage(msg, 0));
}
public void register(ITCPMessageHandler messageHandler)
{
bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
//bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
_message_handler = messageHandler;
_master_worker.addWorker(this, _hb_period);
// we send only when send is called but read continoulsy, starting here
_read_next_msg();
}
public RemoteConnectionInfo get_peer_address()
{
return TCPSocketUtils.get_peer_address(_sock);
}
public RemoteConnectionInfo get_self_address()
{
return TCPSocketUtils.get_peer_address(_sock);
}
public bool isDisposed
{
get { return !_is_alive; }
}
#endregion
#region IWorker Members
bool _last_connected = false;
public void doWork()
{
if (!_is_alive)
return;
return;
// disable hbs for now
if (!_hb_sent_once)
{
bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
_hb_sent_once = true;
}
var msg = _build_hb_message();
_begin_or_resume_sending(new AsyncSendMessage(msg, 0));
}
private static int _tid()
{
var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
return tid;
}
private static string _cb_txt(IAsyncResult res)
{
return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
}
#endregion
#region sending data
private void _begin_or_resume_sending(AsyncSendMessage message)
{
//bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
if (_is_alive)
{
try
{
SocketError begin_send_err;
// since we get message on on_send we dont need to keep the state around
message.key = _send_key++;
var res = _sock.BeginSend(message.buffer, message.transfer_ptr, message.remaining_size, SocketFlags.None,
out begin_send_err, new AsyncCallback(BeginSend_callback_function), message);
if (begin_send_err != SocketError.Success)
_handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
}
catch (Exception ex)
{
_handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
}
}
}
private void BeginSend_callback_function(IAsyncResult res)
{
if (res.CompletedSynchronously)
{
ThreadPool.QueueUserWorkItem(_ => _on_send_async(res));
}
else
_on_send_async(res);
}
private void _on_send_async(IAsyncResult res)
{
AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
if (_send_key_map.Contains(message.key))
{
bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
return;
}
_send_key_map.Add(message.key);
// bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
// $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
if (_is_alive)
{
try
{
SocketError send_err;
int bytes_sent = _sock.EndSend(res, out send_err);
//message.offset += bytesSent;
message.advance_transfer_pointer(bytes_sent);
if (bytes_sent == 0)
{
bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
_handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
"_on_send"));
return;
}
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
$"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}");
if (send_err == SocketError.Success)
{
// successful send of some bytes
//if (message.offset == message.buffer.Length) {
if (message.fully_transfered())
{
// fully sent, if confirmation needs to be sent out
if (!String.IsNullOrEmpty(message.message_id))
{
message.on_send_del(message.message_id, true);
}
}
else
{
// there is more to send
_begin_or_resume_sending(message);
}
}
else
{
_handle_error(new ErrorMessage(send_err, "_on_send"));
}
}
catch (Exception ex)
{
_handle_error(new ErrorMessage(ex, "_on_send"));
}
}
}
/// <summary>
/// Assembles the heartbeat message, and uses it for
/// future references
/// </summary>
/// <returns></returns>
private byte[] _build_hb_message()
{
if (_cached_hb_message == null)
{
BinaryWriter bw = new BinaryWriter(new MemoryStream());
// the total msg =[msg_sz, msg_type]
int msg_sz = 4;
bw.Write(msg_sz);
bw.Write(HB_MSG_SIGNAL);
_cached_hb_message = (bw.BaseStream as MemoryStream).ToArray();
}
return _cached_hb_message;
}
// writes size prefixed msg
private byte[] _build_data_message(byte[] data)
{
// TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
// the total msg =[msg_sz, msg_type, data]
Int32 msg_sz = data.Length + 4;
bw.Write(msg_sz); // +4 for the type of msg
// to indicate data vs hb
bw.Write(DATA_MSG_SIGNAL); // for data
// if we want to id every message, this is the place to do it
bw.Write(data);
str.Position = 0; //not needed really
return str.ToArray();
}
#endregion
#region receiving data
private void _read_next_msg()
{
_read_size();
}
private void _read_size()
{
_begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
}
private void _read_data(int data_size)
{
// TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
_begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
}
private static string IAsyncResult_txt(IAsyncResult res)
{
return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}";
}
private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg)
{
//bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
if (!_is_alive)
return;
try
{
SocketError read_sock_err;
rcv_msg.key = _receive_key++;
var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
rcv_msg.remaining_size,
SocketFlags.None, out read_sock_err,
new AsyncCallback(BeginReceive_callback_function),
rcv_msg);
if (read_sock_err != SocketError.Success)
{
bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
_handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
}
}
catch (Exception ex)
{
_handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
}
}
private void _log_incoming_msg(IAsyncResult res)
{
// AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
// var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
// Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
}
// this is the function passed to BeginReceive, we
private void BeginReceive_callback_function(IAsyncResult res)
{
_log_incoming_msg(res);
if (res.CompletedSynchronously)
{
ThreadPool.QueueUserWorkItem(_ => _on_read_async(res));
}
else
_on_read_async(res);
}
// this will always be called on the non main-thread. the reason this is on its own thread is that this function ends up
// calling _begin_or_resume_reading, which in tuns calls _on_read_async. So to not cause a stack overflow, this is broken
// always called on a different thread than _begin_or_resume_reading
private void _on_read_async(IAsyncResult res)
{
AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
if (_read_key_map.Contains(rcv_message.key))
{
bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
return;
}
_read_key_map.Add(rcv_message.key);
if (!_is_alive)
return;
try
{
SocketError read_err;
Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}");
int bytes_read = _sock.EndReceive(res, out read_err);
rcv_message.advance_transfer_pointer(bytes_read);
Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}");
if (bytes_read == 0)
{
bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
//bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring");
//System.Threading.Thread.Sleep(50);
_handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read"));
return;
}
//return;
if (read_err == SocketError.Success)
{
if (rcv_message.fully_transfered())
{
if (rcv_message.is_reading_size())
{
var msg_size = _on_read_size_completely(rcv_message);
if (msg_size != null && msg_size.HasValue)
{
_read_data(msg_size.Value);
}
return;
}
else
{
// data is read
int msg_type = rcv_message.read_int32();
bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
if (msg_type == DATA_MSG_SIGNAL)
{
_on_read_data_completely(rcv_message);
_read_next_msg();
return;
}
else if (msg_type == HB_MSG_SIGNAL)
{
_on_read_hb_completely(rcv_message);
_read_next_msg();
return;
}
// go back to reading size for a new message
//_read_size();
}
}
else
{
// msg has not been fully rd, we have already advanced the transfer ptr
_begin_or_resume_reading(rcv_msg: rcv_message);
}
}
else
{
_handle_error(new ErrorMessage(read_err, "_on_read_async"));
}
}
catch (Exception ex)
{
_handle_error(new ErrorMessage(ex, "_on_read_async"));
}
}
#endregion
private int? _on_read_size_completely(AsyncReceiveMessage msg)
{
bts.Log.assert(msg.is_reading_size());
bts.Log.assert(msg.fully_transfered());
bts.Log.assert(msg.original_size == 4);
var msg_sz = msg.read_int32();
//var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
//bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}");
if (msg_sz < 0 || msg_sz > _max_msg_size)
{
string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
_handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely"));
return null;
}
else
{
// sz is determined, now we read that many bytes
//_read_data(msg_sz);
return msg_sz;
}
}
private void _on_read_hb_completely(AsyncReceiveMessage msg)
{
bts.Log.assert(!msg.is_reading_size());
bts.Log.assert(msg.fully_transfered());
bts.Log.assert(msg.original_size == 4);
bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
if (!_hb_received_yet)
{
_hb_received_yet = true;
//bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
}
_last_hb_received_time = DateTime.Now;
_message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
}
private void _on_read_data_completely(AsyncReceiveMessage msg)
{
bts.Log.assert(!msg.is_reading_size());
bts.Log.assert(msg.fully_transfered());
var type = msg.read_int32(0);
bts.Log.assert(type == DATA_MSG_SIGNAL);
bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_data_completely msg: {msg} type: {type}");
byte[] data = msg.read_bytes(4);
///callbacks can be made through thread pool to avoid this
///thread from being blocked, by this callback
if (!_data_received_yet)
{
_data_received_yet = true;
//bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
}
// could be an index here
_message_handler.onData(new RemoteMessage(data, RemoteMessageType.Data, _peer_conn_info));
}
#region IDisposable Members
public void Dispose()
{
// socket is closed and disposed here
lock (this)
{
bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}");
if (_is_alive)
{
_master_worker.removeWorker(this);
_is_alive = false;
bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
try
{
bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now");
_sock.Shutdown(SocketShutdown.Both);
}
catch (Exception ex)
{
String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
if (ex is OutOfMemoryException)
throw ex;
}
_message_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info));
}
}
}
#endregion
private string _dipose_impetus = String.Empty;
private void _handle_error(ErrorMessage err_msg)
{
lock (this)
{
Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
if (err_msg.oom)
throw err_msg.exception;
_dipose_impetus = $"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}";
Dispose();
}
}
}
}