From da41a23c0e446aa2fea7e662b9b59a6e24a3cdce Mon Sep 17 00:00:00 2001 From: arhamGIT <59977509+arhamGIT@users.noreply.github.com> Date: Wed, 11 Jan 2023 06:32:07 -0500 Subject: [PATCH] thread safe socket added --- .../Core/WebsocketConnectionGateway.cs | 290 ++++++++++++------ .../spark_websocket/Core/WebsocketDataSync.cs | 2 +- 2 files changed, 205 insertions(+), 87 deletions(-) diff --git a/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs b/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs index 9b4a0b9f1..fce4ce107 100644 --- a/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs +++ b/src/client/spark_websocket/Core/WebsocketConnectionGateway.cs @@ -5,16 +5,107 @@ using System.Collections.Concurrent; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; namespace bts.websocket_core { + enum MessageType { + Message, + Close + } + /// + /// struct that holds client websocket and que + /// + struct Connection { + public WebSocket Ws { get; } + public ConcurrentQueue> OutgoingQueue { get; } + public Connection(WebSocket websocket) { + Ws = websocket; + OutgoingQueue = new ConcurrentQueue>(); + } + } + + class ConnectionsDictionary { + private ConcurrentDictionary connections; + public void AddConnection(WebsocketClientID id, Connection conn) { + var check = connections.TryAdd(id, conn); + if (!check) { + var log_txt = $"WebsocketDataSync.createID client_id: {id.ToString()} Already exits"; + bts.Log.Gen.warn(log_txt); + connections[id] = conn; + } + } + public void RemoveConnection(WebsocketClientID id) { + if (connections.ContainsKey(id)) { + if (connections.TryRemove(id, out Connection conn)) { + conn.Ws.Dispose(); + } + } + } + public WebSocket? GetSocket(WebsocketClientID id) => connections.ContainsKey(id) ? connections[id].Ws : null; + + public ConcurrentQueue>? GetSocketQueue(WebsocketClientID id) => connections.ContainsKey(id) ? connections[id].OutgoingQueue : null; + + public ConnectionsDictionary() { + connections = new ConcurrentDictionary(); + } + } + // class SocketsHandler { + // private WebApplication? app = null; + // private WebsocketServerParams? _server_params = null; + // public void start_listening(WebsocketServerParams server_params) { + // // start listening, assert that init has been called already + // bts.Log.assert(_server_params == null); + // bts.Log.assert(app == null); + + + // _server_params = server_params; + // var builder = WebApplication.CreateBuilder(); + // builder.WebHost.UseUrls(_server_params.Uri); + // app = builder.Build(); + // app.UseWebSockets(_server_params.get_ws_options()); + + // app.Map(_server_params.Default_end_point, async context => { + // if (context.WebSockets.IsWebSocketRequest) { + // using (var ws = await context.WebSockets.AcceptWebSocketAsync()) { + // await ProcessWebSocketConnectionAsync(ws); + // } + // } else { + // context.Response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest; + // } + // }); + // app.RunAsync(); + + // } + + // } + /// /// Handles all websocket related processing, interfaces with datasync, gets initial paramters from WebsocketServerParams /// + public class WebsocketConnectionGateway : IWebsocketConnectionGateway { - private IWebsocketDataSync? _sync = null; + + private WebsocketServerParams? _server_params = null; private WebApplication? app = null; + private IWebsocketDataSync? _sync = null; + private IWebsocketDataSync Sync { + get { + return (IWebsocketDataSync)GetNonNull(_sync); + } + set { + bts.Log.assert(_sync == null); + _sync = value; + } - // IDictionary clients = new Dictionary(); - private ConcurrentDictionary? clients = null; + } + private ConnectionsDictionary? connections = null; + private ConnectionsDictionary Connections { + get { + return (ConnectionsDictionary)GetNonNull(connections); + } + set { + bts.Log.assert(connections == null); + connections = value; + } + } /// /// To Avoid null reference warning, only called when object is not null, should only be called inside getNonNull, is separate function so can be changed later on @@ -43,58 +134,46 @@ namespace bts.websocket_core { return new Object(); } } + public static bool IsSocketOpen(WebSocket? ws) => ws != null && ws.State != WebSocketState.Closed && ws.State != WebSocketState.Aborted && ws.State != WebSocketState.None; + + private WebSocket? GetSocket(WebsocketClientID id, string error_message) { + var log = GetLogger(error_message); + var socket = Connections.GetSocket(id); + if (socket == null) { + log(" failed as could not find socket in dictionary"); + HandleDisconnection(id, error_message); + return null; + } + if (!IsSocketOpen(socket)) { + log(" failed as Socket closed"); + HandleDisconnection(id, error_message); + return null; + } + return socket; + } + private Action GetLogger(string base_text) => ((string additional_message) => { + base_text += additional_message; + bts.Log.Gen.info(base_text); + Console.WriteLine(base_text); + }); + /// + /// creates ID and adds it to clients, in case of collision just updates the previous value + /// + WebsocketClientID createID(WebSocket ws) { + var id = new WebsocketClientID(); + var sck = new Connection(ws); + Connections.AddConnection(id, sck); + return id; + } + /// /// Should be called in the start, stores pointer for sync and creates dictionary for clients, does not start server /// /// public void init(IWebsocketDataSync sync) { // first call - bts.Log.assert(_sync == null); - bts.Log.assert(clients == null); - _sync = sync; - clients = new ConcurrentDictionary(); - } - /// - /// stores the current clients, one client corresponds to one open socket which corresponds to one request, will update getter to c# convention - /// - private ConcurrentDictionary GetClients() => (ConcurrentDictionary)GetNonNull(clients); - - /// - /// will update getter to c# convention - /// - private IWebsocketDataSync GetSync() => (IWebsocketDataSync)GetNonNull(_sync); - - /// - /// creates ID and adds it to clients, in case of collision just updates the previous value - /// - WebsocketClientID createID(WebSocket ws) { - var id = new WebsocketClientID(); - var check = GetClients().TryAdd(id, ws); - if (!check) { - var log_txt = $"WebsocketDataSync.createID client_id: {id.ToString()} Already exits"; - bts.Log.Gen.warn(log_txt); - GetClients()[id] = ws; - } - return id; - } - - /// - /// wrapper to gets socket from clients dictionary against given id, returns null if not found - /// - private WebSocket? GetSocket(WebsocketClientID id) { - return GetClients().ContainsKey(id) ? GetClients()[id] : null; - - } - - /// - /// wrapper to remove socket from clients dictionary against given id, cleans up removed socket as well - /// - private void RemoveSocket(WebsocketClientID id) { - if (GetClients().ContainsKey(id)) { - if (GetClients().TryRemove(id, out WebSocket? ws)) { - ws.Dispose(); - } - } + Sync = sync; + Connections = new ConnectionsDictionary(); } /// @@ -103,10 +182,8 @@ namespace bts.websocket_core { /// 2. removes client from dictionary of active clients /// 3. asks sync to handle disconnection /// - public async Task HandleDisconnection(WebsocketClientID id, string optional_message = "") { - await SendDisconnectRequestAsync(id, optional_message); - RemoveSocket(id); - GetSync().handle_disconnection(id); + public void HandleDisconnection(WebsocketClientID id, string optional_message = "") { + TryDisconnect(id, optional_message); } /// @@ -119,7 +196,7 @@ namespace bts.websocket_core { // start listening, assert that init has been called already bts.Log.assert(_sync != null); bts.Log.assert(_server_params == null); - bts.Log.assert(clients != null); + bts.Log.assert(connections != null); bts.Log.assert(app == null); @@ -149,7 +226,10 @@ namespace bts.websocket_core { IEnumerable messages) { var tasks = new List(); foreach (var om in messages) { - tasks.Add(SendWebSocketRequestAsync(om.receiver_client_id, om.message_body)); + var q = Connections.GetSocketQueue(om.receiver_client_id); + if (q != null) { + q.Enqueue(Tuple.Create(MessageType.Message, om.message_body)); + } } } /// @@ -159,11 +239,32 @@ namespace bts.websocket_core { /// /// Task that keeps waiting for requests until the socket is closed public async Task ProcessWebSocketConnectionAsync(WebSocket ws) { - var request = createID(ws); - GetSync().handle_connection(request, null);//for now thinking this is async/ no IO - while (ws != null && ws.State == WebSocketState.Open) { - await ProcessWebSocketRequestAsync(request); + var id = createID(ws); + Sync.handle_connection(id, null);//for now thinking this is async/ no IO + var pushTask = Task.Run(() => ProcessWebSocketQueueAsync(id)); + await ProcessWebSocketRequestAsync(id); + await pushTask; + } + public async Task ProcessWebSocketQueueAsync(WebsocketClientID id) { + var socket = GetSocket(id, "WebsocketGateway.ProcessWebSocketQueueAsync could not process"); + var que = Connections.GetSocketQueue(id); + if (socket == null || que == null) { + return; } + do { + if (!que.TryDequeue(out var message)) { + await Task.Delay(TimeSpan.FromMilliseconds(100)); + continue; + } + if (message.Item1 == MessageType.Close) { + await SendDisconnectRequestAsync(id, message.Item2); + Connections.RemoveConnection(id); + Sync.handle_disconnection(id); + return; + } else if (message.Item1 == MessageType.Message) { + await SendWebSocketRequestAsync(id, message.Item2); + } + } while (IsSocketOpen(socket)); } /// /// 1. makes sure state is consistent @@ -175,9 +276,8 @@ namespace bts.websocket_core { /// id of client from whom the message is to be recieved /// Task that runs until socket is open public async Task ProcessWebSocketRequestAsync(WebsocketClientID id) { - var socket = GetSocket(id); - if (socket == null || socket.State != WebSocketState.Open) { - await HandleDisconnection(id, "WebsocketGateway.ProcessWebSocketRequestAsync could not receive Request as client closed"); + var socket = GetSocket(id, "WebsocketGateway.ProcessWebSocketRequestAsync could not process"); + if (socket == null) { return; } try { @@ -187,8 +287,8 @@ namespace bts.websocket_core { do { result = await socket.ReceiveAsync(buffer, CancellationToken.None); if (result.MessageType == WebSocketMessageType.Close) { - var log_txt = $"WebsocketGateway.ProcessWebSocketRequestAsync connection closed client_id: {id}"; - await HandleDisconnection(id, log_txt); + var log_txt = $"WebsocketGateway.ProcessWebSocketRequestAsync close request client_id: {id}"; + HandleDisconnection(id, log_txt); return; } if (buffer.Array != null)// added because of warning @@ -198,15 +298,15 @@ namespace bts.websocket_core { bts.Log.assert(false);//this should never happen } if (result.EndOfMessage) { - GetSync().handle_client_message(new WebsocketIncomingMessage(id, request)); + Sync.handle_client_message(new WebsocketIncomingMessage(id, request)); request = ""; } - } while (socket.State == WebSocketState.Open); + } while (IsSocketOpen(socket)); } catch (WebSocketException ex) { var ex_str = ex.ToString(); var log_txt = $"WebsocketGateway.ProcessWebSocketRequestAsync connection closed client_id: {id} Error: {ex_str}"; - await HandleDisconnection(id, log_txt); + HandleDisconnection(id, log_txt); } catch (OperationCanceledException) {//not sure about this read that this should just be ignored } } @@ -216,23 +316,42 @@ namespace bts.websocket_core { /// id of client to whom the Disconnect message should be sent /// /// task that completes if client disconnects or if there is an error + public void TryDisconnect(WebsocketClientID id, string message) { + var log = GetLogger($"WebsocketGateway.TryDisconnect tried closing client_id: {id} with message: {message}"); + var socket = Connections.GetSocket(id); + var que = Connections.GetSocketQueue(id); + if (socket == null || que == null) { + log(" failed as could not find socket in dictionary"); + Sync.handle_disconnection(id); + return; + } + if (!IsSocketOpen(socket)) { + if (socket.CloseStatus == WebSocketCloseStatus.NormalClosure) { + log(" already closed properly by other thread"); + return; + } + log(" already closed "); + Sync.handle_disconnection(id); + Connections.RemoveConnection(id); + return; + } + que.Enqueue(Tuple.Create(MessageType.Close, "")); + log(" added to que"); + return; + } public async Task SendDisconnectRequestAsync(WebsocketClientID id, string message) { - var log_txt = $"WebsocketGateway.SendDisconnectRequestAsync tried closing client_id: {id} with message: {message}"; - var socket = GetSocket(id); - if (socket != null && socket.State == WebSocketState.Open) { - try { - await socket.CloseAsync((WebSocketCloseStatus)1000, "", CancellationToken.None); - log_txt += " done succesfully"; - } catch { - log_txt += " failed as client didnt respond"; - } - } else if (socket == null) { - log_txt += " failed as could not find socket in dictionary"; - } else if (socket.State != WebSocketState.Open) { - log_txt += " Socket already closed"; + var log = GetLogger($"WebsocketGateway.SendDisconnectRequestAsync tried closing client_id: {id} with message: {message}"); + var sckt = Connections.GetSocket(id); + if (sckt == null) { + log(" failed as could not find socket in dictionary"); + return; + } + try { + await sckt.CloseAsync((WebSocketCloseStatus)1000, "", CancellationToken.None); + log(" done succesfully"); + } catch { + log(" failed as client didnt respond"); } - bts.Log.Gen.info(log_txt); - Console.WriteLine(log_txt); } /// @@ -242,9 +361,8 @@ namespace bts.websocket_core { /// message to be sent /// public async Task SendWebSocketRequestAsync(WebsocketClientID id, string message) { - var socket = GetSocket(id); - if (socket == null || socket.State != WebSocketState.Open) { - await HandleDisconnection(id, "WebsocketGateway.SendWebSocketRequestAsync could not send Request as client closed"); + var socket = GetSocket(id, "WebsocketGateway.SendWebSocketRequestAsync could not send Request"); + if (socket == null) { return; } try { @@ -252,7 +370,7 @@ namespace bts.websocket_core { } catch (WebSocketException ex) { var ex_str = ex.ToString(); var log_txt = $"WebsocketGateway.SendWebSocketRequestAsync connection closed client_id: {id} Error: {ex_str}"; - await HandleDisconnection(id, log_txt); + HandleDisconnection(id, log_txt); return; } } diff --git a/src/client/spark_websocket/Core/WebsocketDataSync.cs b/src/client/spark_websocket/Core/WebsocketDataSync.cs index 531905b73..ce1d05173 100644 --- a/src/client/spark_websocket/Core/WebsocketDataSync.cs +++ b/src/client/spark_websocket/Core/WebsocketDataSync.cs @@ -123,7 +123,7 @@ namespace bts.websocket_core { GetGateway().send_to_clients(messages); foreach (var key in toRemove) { ///example on how to disconnect, this should ideally happen in its own thread - GetGateway().HandleDisconnection(key, "closing client").Wait(); + GetGateway().HandleDisconnection(key, "closing client"); } Thread.Sleep(duration); }