Files
spark/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs
T
2023-09-30 12:53:51 -04:00

542 lines
20 KiB
C#

using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.IO;
//using BCG.Utilities;
using BCG.Utilities.General;
using System.Runtime.InteropServices;
namespace BCG
{
/// <summary>
/// This class, mantains a connection to a peer socket, using
/// the .Net async TCP library. It sends heartbeats at regular
/// intervals (config). It is in active states when messages
/// are sent and received without an error state. It assumed
/// dead state when either of the following happens
/// BeginReceive returns a SocketError
/// BeginSend returns a SocketError
/// EndReceive returns a SocketError
/// EndSend returns a SocketError
/// As soon as it assumes dead state it stops sending or
/// receiving any messages (including heartbeats).
/// The user of this class can either detect that by
/// missing heartbeats or checking the isConnected property.
/// Missing should be the more general way, since if the
/// peer dies in a 'silent' way, where none of the above
/// methods indicate an error, missing heartbeats is guaranteed
/// to detect it.
/// </summary>
public class AsyncTCPConnection : ITCPConnection, IWorker, IDisposable
{
/// <summary>
/// This is used as a state for the Async send. Reason being
/// that if onSend indicates that it could not send the full
/// data (ie EndSend does not return the full size), the offset
/// is used to resend the data
/// </summary>
public class AsyncMessage
{
public byte[] buffer;
public int offset;
public string messageId;
public SendConfirmationDelegate onSendDel;
public AsyncMessage(byte[] theBuffer, int theOffset)
: this(theBuffer, theOffset, String.Empty, null) { }
public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel)
{
buffer = theBuffer;
offset = theOffset;
messageId = theMessageId;
onSendDel = theOnSendDel;
}
public int length
{
get
{
return buffer.Length - offset;
}
}
}
enum ReadState
{
readingSize,
readingData
};
#region work around for .Net socket's LocalEndPoint bug
[StructLayout(LayoutKind.Sequential)]
private struct in_addr
{
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
public byte[] sin_addr;
}
[StructLayout(LayoutKind.Sequential)]
private struct sockaddr
{
public short sin_family;
public ushort sin_port;
public in_addr sin_addr;
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
public byte[] sin_zero;
}
[DllImport("wsock32.dll")]
private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen);
[DllImport("wsock32.dll")]
private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen);
#endregion
private readonly int myMaxMessageSize = PubSub3Config.instance.PS3_MaxPacketSize;
private RemoteConnectionInfo myPeer, mySelf;
private Socket mySock;
private ITCPMessageHandler myMessageHandler;
private byte[] myCurrentReadBuffer;
private int myCurrentReadBufferOffset;
private ReadState myCurrentReadState;
private bool myAlive = true;
private byte[] myHBMessage;
private TimeSpan myHBTimeSpan;
private MasterWorker myMasterWorker = MasterWorker.getInstance(true);
private bool myHBReceived = false, myDataReceived = false, myHBSent = false;
private DateTime myLastHBReceivedTime = new DateTime(1900, 1, 1);
public AsyncTCPConnection(Socket sock, TimeSpan heartBeatTimeSpan)
{
myHBTimeSpan = heartBeatTimeSpan;
mySock = sock;
mySock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
string ip;
int port;
getPeerAddress(out ip, out port);
myPeer = new RemoteConnectionInfo(ip, port);
getSelfAddress(out ip, out port);
mySelf = new RemoteConnectionInfo(ip, port);
// BlueEventLog.logEvent(String.Format("AsyncTCPConnection created : self = {0} , peer = {1}",
// mySelf.ToString(), myPeer.ToString()), PS3LogLevels.connection);
}
public override string ToString()
{
return String.Format("peer = {0}, self = {1}", myPeer.ToString(), mySelf.ToString());
}
#region ITCPConnection Members
public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del)
{
confId = Guid.NewGuid().ToString();
byte[] msg = myBuildDataMessage(data);
myBeginOrResumeSending(new AsyncMessage(msg, 0, confId, del));
}
public void send(byte[] data)
{
byte[] msg = myBuildDataMessage(data);
myBeginOrResumeSending(new AsyncMessage(msg, 0));
}
public void register(ITCPMessageHandler messageHandler)
{
// BlueEventLog.logAssert(myMessageHandler == null, "Message Handler is already assigned");
// BlueEventLog.logEvent(String.Format("AsyncTCPConnection for peer = {0} being registered", myPeer.ToString()),
// PS3LogLevels.connection);
myMessageHandler = messageHandler;
myMasterWorker.addWorker(this, myHBTimeSpan);
myReadSize();
}
public void getPeerAddress(out string ip, out int port)
{
ip = "";
port = 0;
if (myAlive)
{
IPEndPoint ipEp = mySock.RemoteEndPoint as IPEndPoint;
ip = ipEp.Address.ToString();
port = ipEp.Port;
}
///.Net has some problems getting the port, so using
///native code for that
//sockaddr addr = new sockaddr();
//int namelen = 16;
//getsockname(mySock.Handle, ref addr, ref namelen);
//port = addr.sin_port;
}
public void getSelfAddress(out string ip, out int port)
{
ip = "";
port = 0;
if (myAlive)
{
IPEndPoint ipEp = mySock.LocalEndPoint as IPEndPoint;
ip = ipEp.Address.ToString();
port = ipEp.Port;
}
///due to this bug in .Net socket, we have to resort to
///native code
//sockaddr addr = new sockaddr();
//int namelen = 16;
//getsockname(mySock.Handle, ref addr, ref namelen);
//port = addr.sin_port;
}
public bool isDisposed
{
get { return !myAlive; }
}
#endregion
#region IWorker Members
bool _last_connected = false;
public void doWork()
{
if (mySock != null)
//if (mySock.Connected != _last_connected)
{
Console.WriteLine($"AsyncTCPConnection.doWork connected: {mySock.Connected}");
_last_connected = mySock.Connected;
}
if (!myHBSent)
{
// BlueEventLog.logEvent(String.Format("Sending heartbeat to peer {0} for the first time", myPeer.ToString()),
// PS3LogLevels.connection);
myHBSent = true;
}
myBeginOrResumeSending(new AsyncMessage(myBuildHeartbeatMessage(), 0));
}
#endregion
#region sending data
private void myBeginOrResumeSending(AsyncMessage message)
{
if (myAlive)
{
try
{
//Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} connected: {mySock.Connected}");
SocketError beginSendErr;
mySock.BeginSend(message.buffer, message.offset, message.length, SocketFlags.None,
out beginSendErr, new AsyncCallback(myOnSend), message);
Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} socker_error: {beginSendErr} connected: {mySock.Connected}");
if (beginSendErr != SocketError.Success)
myHandleError(beginSendErr);
}
catch (Exception ex)
{
myHandleError(ex);
}
}
}
private void myOnSend(IAsyncResult res)
{
if (myAlive)
{
try
{
SocketError writeErr;
int bytesSent = mySock.EndSend(res, out writeErr);
Console.WriteLine($"AsyncTCPConnection.myOnSend bytes: {bytesSent} writeErr: {writeErr} connected: {mySock.Connected}");
AsyncMessage message = res.AsyncState as AsyncMessage;
message.offset += bytesSent;
if (writeErr == SocketError.Success)
{
if (message.offset == message.buffer.Length)
{
// if confirmation needs to be sent
if (String.Empty != message.messageId)
{
message.onSendDel(message.messageId, true);
}
}
else
{
///resume writing rest of the message
// BlueEventLog.logEvent("could not send the whole message all at once , messageSize = " +
// message.buffer.Length + " offset = " + message.offset, PS3LogLevels.info);
myBeginOrResumeSending(message);
}
}
else
{
myHandleError(writeErr);
}
}
catch (Exception ex)
{
myHandleError(ex);
}
}
}
/// <summary>
/// Assembles the heartbeat message, and uses it for
/// future references
/// </summary>
/// <returns></returns>
private byte[] myBuildHeartbeatMessage()
{
if (myHBMessage == null)
{
BinaryWriter bw = new BinaryWriter(new MemoryStream());
int sz = 4;
int msgType = 1;
bw.Write(sz);
bw.Write(msgType);
myHBMessage = (bw.BaseStream as MemoryStream).ToArray();
}
return myHBMessage;
}
/// <summary>
/// Assembles the data message. This is necessary
/// for the other side to understand the message. This can also
/// be achieved by not forming the message and use the
/// BeginSend (List<>) function rather than BeginSend (byte[])
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
private byte[] myBuildDataMessage(byte[] data)
{
BinaryWriter bw = new BinaryWriter(new MemoryStream());
bw.Write(data.Length + 8);
bw.Write(0);
bw.Write(0); ///mid for backward compatibility
bw.Write(data);
return (bw.BaseStream as MemoryStream).ToArray();
}
#endregion
#region receiving data
private void myReadSize()
{
myCurrentReadBuffer = new byte[4];
myCurrentReadBufferOffset = 0;
myCurrentReadState = ReadState.readingSize;
myBeginOrResumeReading();
}
private void myReadData(int dataSize)
{
myCurrentReadBuffer = new byte[dataSize];
myCurrentReadBufferOffset = 0;
myCurrentReadState = ReadState.readingData;
myBeginOrResumeReading();
}
private void myBeginOrResumeReading()
{
if (myAlive)
{
try
{
int beginReceiveAttempts = 0;
while (true)
{
beginReceiveAttempts++;
SocketError readSockErr;
mySock.BeginReceive(myCurrentReadBuffer, myCurrentReadBufferOffset,
myCurrentReadBuffer.Length - myCurrentReadBufferOffset,
SocketFlags.None, out readSockErr,
new AsyncCallback(myOnRead),
myCurrentReadBuffer);
Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeReading socket_error: {readSockErr} connected: {mySock.Connected}");
if (readSockErr == SocketError.NoBufferSpaceAvailable)
{
string msg = String.Format("Not enough buffer space to receive {0} bytes. Attempt: {1} of {2}.",
myCurrentReadBuffer.Length - myCurrentReadBufferOffset,
beginReceiveAttempts, PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1);
//BlueEventLog.logEvent(msg, PS3LogLevels.disconnection);
if (beginReceiveAttempts < (PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1))
System.Threading.Thread.Sleep(PubSub3Config.instance.PS3_millisecondsBetweenNoBufferSpaceRetries);
else
{
//BlueEventLog.logEvent("Could not recover from NoBufferSpace condition. Breaking connection.", PS3LogLevels.disconnection);
myHandleError(readSockErr);
}
}
else if (readSockErr == SocketError.Success)
{
if (beginReceiveAttempts > 1)
// BlueEventLog.logEvent(String.Format("Recovered from NoBufferSpace after {0} tries",beginReceiveAttempts-1) ,
// PS3LogLevels.disconnection);
break;
}
else
myHandleError(readSockErr);
}
}
catch (Exception ex)
{
myHandleError(ex);
}
}
}
private void myOnRead(IAsyncResult res)
{
if (myAlive)
{
try
{
SocketError readErr;
Console.WriteLine($"AsyncTCPConnection.myOnRead pre connected: {mySock.Connected}");
int bytesRead = mySock.EndReceive(res, out readErr);
Console.WriteLine($"AsyncTCPConnection.myOnRead bytes: {bytesRead} readErr: {readErr} connected: {mySock.Connected}");
myCurrentReadBufferOffset += bytesRead;
if (readErr == SocketError.Success)
{
if (myCurrentReadBufferOffset == myCurrentReadBuffer.Length)
{
///read the whole of the buffer
if (myCurrentReadState == ReadState.readingSize)
{
///the size has already been read, now convert
///it into an int, and start reading the data
int dataSize = new BinaryReader(new MemoryStream(myCurrentReadBuffer)).ReadInt32();
if (dataSize > myMaxMessageSize)
{
string errMessage = "Encountered malformed message, size indicated = " + dataSize;
//BlueEventLog.logEvent(errMessage, "Error");
myHandleError(new ApplicationException(errMessage));
}
else
myReadData(dataSize);
}
else
{
///the whole data has been finally read, time to
///tell the subscriber about it
BinaryReader br = new BinaryReader(new MemoryStream(myCurrentReadBuffer));
int msgType = br.ReadInt32();
if (msgType == 0)
{
///skips the message ID
br.ReadInt32();
byte[] data = br.ReadBytes(myCurrentReadBuffer.Length - 8);
///callbacks can be made through thread pool to avoid this
///thread from being blocked, by this callback
if (!myDataReceived)
{
myDataReceived = true;
Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}");
}
myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer));
}
else if (msgType == 1)
{
if (!myHBReceived)
{
myHBReceived = true;
Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}");
}
myLastHBReceivedTime = DateTime.Now;
myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer));
}
myReadSize();
}
}
else
{
///still part of the buffer left to read
///continue reading
myBeginOrResumeReading();
}
}
else
{
myHandleError(readErr);
}
}
catch (Exception ex)
{
myHandleError(ex);
}
}
}
#endregion
#region IDisposable Members
public void Dispose()
{
Console.WriteLine("AsyncTCPConnection.Dispose()");
lock (this)
{
if (myAlive)
{
myMasterWorker.removeWorker(this);
myAlive = false;
// BlueEventLog.logEvent(String.Format("AsycnTCPConnection with peer {0} being disposed, last HB was received at {1}" ,
// myPeer.ToString(), myLastHBReceivedTime.ToLongTimeString()),
// PS3LogLevels.disconnection);
try
{
mySock.Close();
}
catch (Exception ex)
{
String.Format("Exception of type {0} thrown while trying to close socket connected to {1} , with" +
" the following exception msg {2}", ex.GetType().ToString(), myPeer.ToString(), ex.Message);
//BlueEventLog.logEvent(ex);
if (ex is OutOfMemoryException)
throw ex;
}
}
}
}
#endregion
private void myHandleError(object err)
{
lock (this)
{
string errMsg = "";
if (err is Exception)
{
errMsg = String.Format("AsyncTCPConnection.Dispose Encountered an exception of type '{0}' on the socket connected to ip = {1} , port = {2} ; " +
"Exception message = {3} , if the socket has not been disposed, it will be disposed now", err.GetType(), myPeer.ip, myPeer.port, (err as Exception).Message);
if (err is OutOfMemoryException)
throw (err as OutOfMemoryException);
}
else if (err is SocketError)
{
errMsg = String.Format("AsyncTCPConnection.Dispose Encountered a SocketError of type '{0}' on the socket connected to ip = {1} , port = {2} ; " +
"if the socket has not been disposed, it will be disposed now", err.ToString(), myPeer.ip, myPeer.port);
Console.WriteLine($"AsyncTCPConnection.myHandleError errMsg: {errMsg}");
}
//BlueEventLog.logEvent(errMsg, PS3LogLevels.disconnection);
if (myAlive)
{
Dispose();
}
}
}
}
}