From ca028c58ede3f6aeeeaec813ef2cfd3a99e8104c Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 5 Oct 2023 12:48:11 -0400 Subject: [PATCH] feat: Add Bts.DataStream.Websocket/Server --- .../Bts.DataStream.Websocket.csproj | 15 + .../ClientSubscription.cs | 9 + .../Bts.DataStream.Websocket/DataSync.cs | 301 ++++++++++++++++++ .../JObjectExtensions.cs | 34 ++ .../JsonParseResult.cs | 24 ++ .../JsonRpcErrorCode.cs | 6 + .../MultilineStringJsonConverter.cs | 48 +++ .../Bts.DataStream.Websocket/RequestType.cs | 8 + .../Bts.DataStream.Websocket/Session.cs | 27 ++ .../SubscriptionParams.cs | 110 +++++++ .../Bts.DataStream.Websocket/Usings.cs | 5 + .../ViewStreamUpdate.cs | 41 +++ .../Bts.DataStream.WebsocketServer.csproj | 18 ++ .../DataStreamWebsocketServer.conf | 7 + .../DataStreamWebsocketServerConfig.cs | 0 .../Bts.DataStream.WebsocketServer/Program.cs | 76 +++++ .../Bts.DataStream.WebsocketServer/README.md | 0 .../Bts.DataStream.WebsocketServer/Usings.cs | 7 + src/DataStreamSuite/DataStreamSuite.sln | 22 +- .../Collections/Extensions/ListExtensions.cs | 14 + .../utils/Exceptions/ExceptionAssert.cs | 2 +- 21 files changed, 768 insertions(+), 6 deletions(-) create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/JsonParseResult.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/JsonRpcErrorCode.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj create mode 100644 src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf create mode 100644 src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServerConfig.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs create mode 100644 src/DataStreamSuite/Bts.DataStream.WebsocketServer/README.md create mode 100644 src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj b/src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj new file mode 100644 index 000000000..0b510dd4e --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/Bts.DataStream.Websocket.csproj @@ -0,0 +1,15 @@ + + + + net7.0 + enable + enable + + + + + + + + + diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs new file mode 100644 index 000000000..53213b07a --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/ClientSubscription.cs @@ -0,0 +1,9 @@ +namespace Bts.DataStream.Websocket; + +class ClientSubscription(WebsocketClientID id, SubscriptionParams @params, IDisposable dispose) : IDisposable { + public SubscriptionParams Params => @params; + public WebsocketClientID Id => id; + + public void Dispose() => + dispose.Dispose(); +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs new file mode 100644 index 000000000..8b712fe4a --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/DataSync.cs @@ -0,0 +1,301 @@ +namespace Bts.DataStream.Websocket; + +using BCG; + +using bts; +using bts.api.core; +using bts.collections; + +using Bts.DataStreamExpressions; +using Bts.Websocket; + +using mbts.utils; + +using Spark.Core.Disposables; +using Spark.Core.Json; + +using System.Buffers; +using System.Reactive.Subjects; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +/// +/// Represents an I/O model (), that accepts websocket +/// requests (from clients) as input, and emits data/notifications (to clients) as output. +/// This class's chief role is managing data subscriptions to sparkd on the clients' behalf. +/// +public class DataSync(JsonSerializerOptions? serializerOpts = null) : + ISubject> { + protected readonly object lockObj = new(); + readonly JsonSerializerOptions serializerOpts = serializerOpts ?? JsonSerializerOptions.Default; + readonly Dictionary> clients = []; + + event Action>? requestSend; + + public void OnCompleted() => + Log.Api.info("DataSync incoming request stream has completed."); + + public void OnError(Exception error) => + Log.Api.error($"DataSync error: {error}"); + + public void OnNext(WebsocketConnectionUpdate value) { + switch (value.Type) { + case WebsocketConnectionUpdateType.IncomingMessage: + onIncomingMessage(value.IncomingMessage); + break; + case WebsocketConnectionUpdateType.Connect: + onConnect(value.Connect); + break; + case WebsocketConnectionUpdateType.Disconnect: + onDisconnect(value.Disconnect); + break; + default: + Log.assert(false, $"Unexpected WebsocketConnectionUpdate type: {value.Type}"); + break; + } + } + + public IDisposable Subscribe(IObserver> observer) { + lock (lockObj) { + requestSend += observer.OnNext; + return Disposable.Create(Dispose); + } + void Dispose() { + lock (lockObj) { + requestSend -= observer.OnNext; + } + } + } + + static JsonRpcId? extractRpcId(string messageBody) { + try { + return JsonNode.Parse(messageBody) + ?.AsObject() + ?.GetValueOrDefault("id") + ?.Deserialize(); + } catch { + return null; + } + } + + static string getRpcNotification(DataPacket dp) { + using (var buffer = new MemoryStream()) + using (var writer = new Utf8JsonWriter(buffer)) + using (writer.WriteObject()) { + writer.WriteString("method", "subscription"); + using (writer.WriteObject("params")) { + writer.Write("tableName", dp.schema.tableName); + writer.Write("columns", dp.schema.columnInfo); + var cellJsonFactories = dp.schema.columnInfo.Select(getCellJsonFactory).ToList(); + using (writer.WriteArray("added")) { + foreach (IBlueRow row in (dp.addedRows?.Cast()).empty_if_null()) { + using (writer.WriteObject()) { + for (int i = 0; i < cellJsonFactories.Count; i++) { + writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row)); + } + } + } + } + using (writer.WriteArray("deleted")) { + foreach (IBlueRow row in (dp.deletedRows?.Cast()).empty_if_null()) { + using (writer.WriteObject()) { + for (int i = 0; i < cellJsonFactories.Count; i++) { + writer.Write(dp.schema.columnInfo[i].colName, cellJsonFactories[i](row)); + } + } + } + } + } + writer.WriteString("jsonrpc", "2.0"); + writer.Flush(); + return Encoding.UTF8.GetString(buffer.ToArray()); + } + } + + static Func getCellJsonFactory(ColumnInfo colInfo) { + var t = colInfo.colType; + if (t == typeof(int)) { + return NullCheckWrapper(v => (int)v); + } + if (t == typeof(uint)) { + return NullCheckWrapper(v => (uint)v); + } + if (t == typeof(long)) { + return NullCheckWrapper(v => (long)v); + } + if (t == typeof(ulong)) { + return NullCheckWrapper(v => (ulong)v); + } + if (t == typeof(short)) { + return NullCheckWrapper(v => (short)v); + } + if (t == typeof(ushort)) { + return NullCheckWrapper(v => (ushort)v); + } + if (t == typeof(double)) { + return NullCheckWrapper(v => (double)v); + } + if (t == typeof(float)) { + return NullCheckWrapper(v => (float)v); + } + if (t == typeof(bool)) { + return NullCheckWrapper(v => (bool)v); + } + return NullCheckWrapper(v => (string)v); + + Func NullCheckWrapper(Func toJson) => + r => { + var obj = r[colInfo.colNo]; + if (obj is null) { + return null; + } + return toJson(obj); + }; + } + + void onConnect(WebsocketConnectNotification notification) { + var id = notification.Id; + lock (lockObj) { + clients.Add(id, []); + } + Log.Api.info($"DataStreamWebsocketDataSync client connected. clientId: {notification.Id}"); + if (notification.Message is not null) { + onIncomingMessage(notification.Message); + } + } + + void onDisconnect(WebsocketDisconnectNotification notification) { + Log.Api.warn($"DataStreamWebsocketDataSync client disconnected. clientId: {notification.Id}"); + unsubscribeAll(notification.Id); + clients.Remove(notification.Id); + } + + void onIncomingMessage(WebsocketIncomingMessage message) { + var id = message.sender_client_id; + if (!JsonRpcRequest.TryDeserialize(message, out var req, out var err)) { + onBadRequest(message, err); + return; + } + switch (req.Method!.ToLower()) { + case "subscribe": + subscribe(id, req); + break; + case "publish": + Log.Api.error("DataStreamWebsocket api 'publish' not implemented."); + // TODO + break; + case "unsubscribe": + unsubscribe(id, req); + break; + case "unsubscribe_all": + unsubscribeAll(id); + break; + default: + Log.Api.error($"Received request of unknown type: {req}"); + break; + } + } + + void sendResponse(WebsocketClientID id, JsonRpcId rpcId, T rpcParams) { + var rpcRes = JsonRpcResponse.Create(rpcId, rpcParams); + var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts); + send([new(id, rpcResStr)]); + } + + void sendError(WebsocketClientID id, JsonRpcId rpcId, JsonRpcError error) { + var rpcRes = JsonRpcResponse.CreateError(rpcId, error); + var rpcResStr = JsonSerializer.Serialize(rpcRes, serializerOpts); + send([new(id, rpcResStr)]); + } + + bool send(IEnumerable messages) { + lock (lockObj) { + if (requestSend is null) { + Log.Api.warn( + $"Unable to send message(s) because '{nameof(requestSend)}' handler is null"); + return false; + } + requestSend(messages); + return true; + } + } + + void subscribe(WebsocketClientID id, JsonRpcRequest req) { + var subParams = req.Params!.Deserialize()!; + var exp = DataStreamExpressionsInterpreter.Evaluate(subParams.Query!); + var stream = exp.getView(new Dictionary(0)); + var sub = new ClientSubscription(id, subParams, Disposable.Create(Dispose)); + lock (lockObj) { + clients[id].Add(sub); + sendResponse(id, req.Id!.Value, req.Params); + stream.onChange += OnChange; + } + + void OnChange(object? newValue) { + var dp = ExceptionAssert.of_type(newValue); + var notif = getRpcNotification(dp); + send([new(id, notif)]); + } + + void Dispose() => + stream.onChange -= OnChange; + } + + void unsubscribe(WebsocketClientID id, JsonRpcRequest req) { + var subParams = req.Params!.Deserialize(); + var subParamsStr = JsonSerializer.Serialize(subParams, serializerOpts); + // Minimize amount of code inside lock (just need it to find/remove the subscription) + List? subs = null; + ClientSubscription? sub = null; + lock (lockObj) { + subs = clients.GetValueOrDefault(id); + subs?.TryRemoveFirst(s => s!.Params.Equals(subParams), out sub); + } + if (subs is null) { + // Didn't find client (log/throw). + var msg = $"Unable to unsubscribe: did not find client. Id: {id}.\nParams:\n{subParamsStr}"; + Log.Api.error(msg); + throw new Exception(msg); + } + if (sub is null) { + // Found client, but didn't find subscription (log/throw). + var msg = $"Unable to unsubscribe: did not find the specified subscription. " + + $"Id: {id}. Request:\n{subParams}"; + Log.Api.error(msg); + throw new Exception(msg); + } + // Found client and subscription (dispose/remove subscription). + Log.Api.info($"Unsubscribing client. Id: {id}.\nParams:\n{subParamsStr}"); + sub.Dispose(); + } + + void unsubscribeAll(WebsocketClientID id) { + lock (lockObj) { + if (!clients.TryGetValue(id, out var subs)) { + throw new Exception( + $"Unable to unsubscribe: did not find client with id {id}"); + } + subs.DisposeAll(); + subs.Clear(); + } + } + + void onBadRequest(WebsocketIncomingMessage req, string? errorMsg) { + var rpcId = extractRpcId(req.message_body); + var logMsg = $"Recieved invalid request:\n{req.message_body}"; + if (errorMsg is not null) { + logMsg += $"\n\n**Exception**\n{errorMsg}"; + } + Log.Api.error(logMsg); + if (rpcId is null) { + return; + } + // Send error back to client + var rpcErr = JsonRpcError.Create( + (long)JsonRpcErrorCode.InvalidRequest, + $"Received invalid request: {errorMsg ?? "unknown error"}", + req.message_body); + sendError(req.sender_client_id, rpcId.Value, rpcErr); + } +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs new file mode 100644 index 000000000..687f7b7da --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/JObjectExtensions.cs @@ -0,0 +1,34 @@ +namespace Bts.DataStream.Websocket; + +using Newtonsoft.Json.Linq; + +using System.Diagnostics.CodeAnalysis; + +public static class JObjectExtensions { + public static bool TryParse( + this JObject filter, string key, [MaybeNull] out T obj, [NotNullWhen(false)] out string? error) { + obj = default; + var value = filter.GetValueOrDefault(key)?.ToString(); + if (value == null) { + error = $"{key} not present"; + return true; + } + return WebsocketJsonParsers.TryParse(value, key, out obj, out error); + } + + public static bool TryParse( + this JObject filter, string key, out T result, [NotNullWhen(false)] out string? error, T defaultValue) { + var value = filter.GetValueOrDefault(key)?.ToString(); + if (value == null) { + result = defaultValue; + error = $"{key} not present"; + return true; + } + if (WebsocketJsonParsers.TryParse(value, key, out var maybeRes, out error)) { + result = maybeRes ?? defaultValue; + return true; + } + result = defaultValue; + return false; + } +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/JsonParseResult.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/JsonParseResult.cs new file mode 100644 index 000000000..084fa514d --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/JsonParseResult.cs @@ -0,0 +1,24 @@ +namespace Bts.DataStream.Websocket; + +using Newtonsoft.Json.Linq; + +class JsonParseResult { + public string? MessageId { get; init; } + public RequestType Type { get; init; } + public JObject? Parameters { get; init; } + public string? Error { get; init; } + public bool Successful => Error is not null; + + public static JsonParseResult CreateError(string error) => + new() { + Error = error, + MessageId = "missing" + }; + + public static JsonParseResult Create(string messageId, RequestType type, JObject parameters) => + new() { + MessageId = messageId, + Parameters = parameters, + Type = type + }; +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/JsonRpcErrorCode.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/JsonRpcErrorCode.cs new file mode 100644 index 000000000..63da42224 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/JsonRpcErrorCode.cs @@ -0,0 +1,6 @@ +namespace Bts.DataStream.Websocket; + +public enum JsonRpcErrorCode { + None = 0, + InvalidRequest = 1, +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs new file mode 100644 index 000000000..4edda5593 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/MultilineStringJsonConverter.cs @@ -0,0 +1,48 @@ +namespace Bts.DataStream.Websocket; + +using Spark.Core; + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +class MultilineStringJsonConverter : JsonConverter { + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (typeToConvert != typeof(string)) { + throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert)); + } + return reader.TokenType switch { + JsonTokenType.String => StringImpl(ref reader), + JsonTokenType.StartArray => ArrayImpl(ref reader), + _ => throw new ArgumentException( + $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"), + }; + + static string? StringImpl(ref Utf8JsonReader reader) => + new(reader.GetString()!); + + static string? ArrayImpl(ref Utf8JsonReader reader) { + var lines = new List(2); + while (reader.Read()) { + switch (reader.TokenType) { + case JsonTokenType.String: + lines.Add(reader.GetString()!); + break; + case JsonTokenType.EndArray: + return string.Join("\n", lines); + } + } + throw new JsonException( + $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)"); + } + } + + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) { + var lines = value.Split("\n"); + if (lines.Length < 2) { + writer.WriteRawValue(value); + return; + } + writer.WriteRawValue($"[{", ".join(lines)}]"); + } +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs new file mode 100644 index 000000000..cc0413d95 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/RequestType.cs @@ -0,0 +1,8 @@ +namespace Bts.DataStream.Websocket; + +public enum RequestType { + Unspecified, + Subscribe, + Unsubscribe, + UnsubscribeAll, +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs new file mode 100644 index 000000000..61fa68d77 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/Session.cs @@ -0,0 +1,27 @@ +namespace Bts.DataStream.Websocket; + +using bts; + +using Bts.Websocket; + +using Google.Protobuf; + +using System.Text.Json; + +public static class Session { + static int sessionIdCounter; + + public static async Task Start( + WebsocketServerConfig config, JsonSerializerOptions? opts) { + opts ??= JsonSerializerOptions.Default; + var sessionId = Interlocked.Increment(ref sessionIdCounter); + var gateway = new WebsocketGatewayObservable(config); + var sync = new DataSync(opts); + using (gateway.Subscribe(sync)) + using (sync.Subscribe(gateway)) { + Log.Api.info($"Starting DataStream Websocket Session {sessionId}"); + Log.Api.info($"DataStream Websocket Session {sessionId} config:\n{JsonFormatter.Default.Format(config)}"); + await gateway.ConnectAsync(); + } + } +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs new file mode 100644 index 000000000..22de2e4b3 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/SubscriptionParams.cs @@ -0,0 +1,110 @@ +namespace Bts.DataStream.Websocket; + +using Spark.Core; + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + +public class SubscriptionParams : IEquatable { + static class PropNames { + public const string QUERY = "query"; + } + + public SubscriptionParams(string? query) { + Query = query; + } + + // Serializer supports plain string, or an array of lines for convenience + [MaybeNull] + [JsonPropertyName(PropNames.QUERY)] + [JsonConverter(typeof(MultilineStringJsonConverter))] + public string? Query { get; set; } + + public bool Equals(SubscriptionParams? other) => + other is not null && + Query == other.Query; + + class SubscriptionParamsConverter : JsonConverter { + public override bool CanConvert(Type typeToConvert) => + typeToConvert == typeof(SubscriptionParams); + + public override SubscriptionParams Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (typeToConvert != typeof(SubscriptionParams)) { + throw new ArgumentException($"Unable to parse {typeToConvert}", nameof(typeToConvert)); + } + return read(ref reader); + } + + SubscriptionParams read(ref Utf8JsonReader reader) { + return reader.TokenType switch { + JsonTokenType.String => stringImpl(ref reader), + JsonTokenType.StartArray => arrayImpl(ref reader), + JsonTokenType.StartObject => objectImpl(ref reader), + _ => throw new ArgumentException( + $"Unable to parse {typeof(SubscriptionParams)} from {reader.TokenType}"), + }; + + static SubscriptionParams stringImpl(ref Utf8JsonReader reader) => + new(reader.GetString()!); + + static SubscriptionParams arrayImpl(ref Utf8JsonReader reader) { + var lines = new List(2); + while (reader.Read()) { + switch (reader.TokenType) { + case JsonTokenType.String: + lines.Add(reader.GetString()!); + break; + case JsonTokenType.EndArray: + return new("\n".join(lines)); + } + } + throw new JsonException( + $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)"); + } + + static SubscriptionParams objectImpl(ref Utf8JsonReader reader) { + var lines = new List(2); + while (reader.Read()) { + switch (reader.TokenType) { + case JsonTokenType.PropertyName: + var propName = reader.GetString()!; + if (propName == PropNames.QUERY) { + switch (reader.TokenType) { + case JsonTokenType.StartArray: + return arrayImpl(ref reader); + case JsonTokenType.String: + return stringImpl(ref reader); + } + } + break; + case JsonTokenType.String: + lines.Add(reader.GetString()!); + break; + case JsonTokenType.EndArray: + return new("\n".join(lines)); + } + } + throw new JsonException( + $"Failed to parse {typeof(SubscriptionParams)} (did not reach end of array)"); + } + } + + public override void Write( + Utf8JsonWriter writer, SubscriptionParams value, JsonSerializerOptions options) { + var q = value.Query; + var lines = q?.Split("\n"); + if (lines is null || lines.Length < 2) { + writer.WriteString(PropNames.QUERY, value.Query); + return; + } + writer.WriteStartArray(PropNames.QUERY); + foreach (var l in lines) { + writer.WriteStringValue(l); + } + writer.WriteEndArray(); + } + } +} diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs new file mode 100644 index 000000000..ba56642fe --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/Usings.cs @@ -0,0 +1,5 @@ +global using bts.websocket_core; + +global using System; + +global using System.Collections.Immutable; diff --git a/src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs b/src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs new file mode 100644 index 000000000..b7737f659 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.Websocket/ViewStreamUpdate.cs @@ -0,0 +1,41 @@ +#nullable enable +using BCG; + +using Spark.Core.Json; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +class ViewStreamUpdate { + [JsonPropertyName("tableName")] + public string? TableName { get; set; } + + [JsonPropertyName("columns")] + public List? Columns { get; set; } + + [JsonPropertyName("addedRows")] + public List>? AddedRows { get; set; } + + [JsonPropertyName("deletedRows")] + public List>? DeletedRows { get; set; } + + public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) { + var schema = new BlueTableSchema([.. Columns!], TableName); + var addedRowsTable = new BlueTable(schema); + var deletedRowsTable = new BlueTable(schema); + if (applyChanges) { + state.deleteRows(deletedRowsTable); + state.addRows(addedRowsTable); + } + return new(state, addedRowsTable, deletedRowsTable); + } + + public static string ToJson() { + return ""; + } +} diff --git a/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj new file mode 100644 index 000000000..ee159b4c5 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Bts.DataStream.WebsocketServer.csproj @@ -0,0 +1,18 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + + + + + diff --git a/src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf new file mode 100644 index 000000000..0952e0781 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServer.conf @@ -0,0 +1,7 @@ +{ + "host": "localhost", + "port": 8080, + "scheme": "ws", + "default_end_point": "/", + "keep_alive_interval_secs": 1 +} diff --git a/src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServerConfig.cs b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/DataStreamWebsocketServerConfig.cs new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs new file mode 100644 index 000000000..6dfff6e40 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Program.cs @@ -0,0 +1,76 @@ +using bts.api.core; + +using Bts.DataStream.Websocket; + +using Google.Protobuf; + +using System.CommandLine.Parsing; +using System.Text.Json; + +static WebsocketServerConfig readConfigFile(string path) { + using var fs = File.OpenRead(path); + using var sr = new StreamReader(fs); + return JsonParser.Default.Parse(sr); +} + +static WebsocketServerConfig? readConfigArg(ArgumentResult result) { + var path = result.Tokens.Select(t => t.Value).FirstOrDefault() ?? string.Empty; + return readConfigFile(path); +} + +static WebsocketServerConfig getDefaultConfig() { + const string defaultCfgPath = "DataStreamWebsocketServer.conf"; + if (File.Exists(defaultCfgPath)) { + return readConfigFile(defaultCfgPath); + } + return new() { + Scheme = "http", + Host = "localhost", + Port = 8080, + DefaultEndPoint = "/", + KeepAliveIntervalSecs = 1 + }; +} + +static async Task run( + WebsocketServerConfig? wssConfig, string user, string password, string? sparkdConfigSource) { + var opts = new JsonSerializerOptions(JsonSerializerDefaults.Web); + wssConfig ??= getDefaultConfig(); + Console.WriteLine("Starting DataStreamWebsocketServer..."); + var serviceCtx = ServiceContext.create(sparkdConfigSource); + await serviceCtx.login(user, password); + await Session.Start(wssConfig, opts); + // if (sparkdConfigSource is not null && sparkdConfigSource.Length > 9999) { + // } else { + // await ChatGptSession.Start(); + // } +} + +var userOpt = new Option("--user", () => "bts"); +var passOpt = new Option("--password", () => string.Empty); +var sparkdConfigSourceOpt = new Option( + "--sparkd", description: "The sparkd config server url"); +// var bling = new ParseArgument(parseWsConfig); + +// WebsocketServerConfig? parseWsConfig(ArgumentResult result) { +// Token t; +// TokenType tt; +// Symbol s; +// throw new NotImplementedException(); +// } + +var configOpt = new Option( + "--config", readConfigArg, description: "WebsocketServerConfig file path"); +configOpt.SetDefaultValue(new()); +var rootCommand = new RootCommand("Starts a DataStream Websocket Server"); +rootCommand.AddOption(configOpt); +rootCommand.AddOption(userOpt); +rootCommand.AddOption(passOpt); +rootCommand.SetHandler(run, configOpt, userOpt, passOpt, sparkdConfigSourceOpt); + +try { + return await rootCommand.InvokeAsync(args); +} catch (Exception ex) { + Console.WriteLine($"Error: {ex}"); +} +return 1; diff --git a/src/DataStreamSuite/Bts.DataStream.WebsocketServer/README.md b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs new file mode 100644 index 000000000..671d64440 --- /dev/null +++ b/src/DataStreamSuite/Bts.DataStream.WebsocketServer/Usings.cs @@ -0,0 +1,7 @@ +global using Bts.Websocket; + +global using bts.websocket_core; + +global using System.CommandLine; + +global using System.Collections.Immutable; diff --git a/src/DataStreamSuite/DataStreamSuite.sln b/src/DataStreamSuite/DataStreamSuite.sln index 52990cb7c..e410bc801 100644 --- a/src/DataStreamSuite/DataStreamSuite.sln +++ b/src/DataStreamSuite/DataStreamSuite.sln @@ -44,7 +44,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server", "Bts.DataStreamWebApp.Server\Bts.DataStreamWebApp.Server.csproj", "{6DE5E4CE-423E-4571-81F7-539D268D6059}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.Websocket", "Bts.DataStream.Websocket\Bts.DataStream.Websocket.csproj", "{088655DA-3126-4F8A-9AC7-9061FDA891FF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStream.WebsocketServer", "Bts.DataStream.WebsocketServer\Bts.DataStream.WebsocketServer.csproj", "{35823BE0-C883-476A-8B30-D6097CBA1A8D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{EF492D73-C40F-4EEF-B917-4ADA7B30D728}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -139,9 +143,17 @@ Global {6DE5E4CE-423E-4571-81F7-539D268D6059}.Debug|Any CPU.Build.0 = Debug|Any CPU {6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.ActiveCfg = Release|Any CPU {6DE5E4CE-423E-4571-81F7-539D268D6059}.Release|Any CPU.Build.0 = Release|Any CPU - {09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.ActiveCfg = Debug|x64 - {09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.Build.0 = Debug|x64 - {09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.ActiveCfg = Release|x64 - {09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.Build.0 = Release|x64 + {088655DA-3126-4F8A-9AC7-9061FDA891FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {088655DA-3126-4F8A-9AC7-9061FDA891FF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {088655DA-3126-4F8A-9AC7-9061FDA891FF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {088655DA-3126-4F8A-9AC7-9061FDA891FF}.Release|Any CPU.Build.0 = Release|Any CPU + {35823BE0-C883-476A-8B30-D6097CBA1A8D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {35823BE0-C883-476A-8B30-D6097CBA1A8D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {35823BE0-C883-476A-8B30-D6097CBA1A8D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {35823BE0-C883-476A-8B30-D6097CBA1A8D}.Release|Any CPU.Build.0 = Release|Any CPU + {EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Debug|Any CPU.ActiveCfg = Debug|x64 + {EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Debug|Any CPU.Build.0 = Debug|x64 + {EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Release|Any CPU.ActiveCfg = Release|x64 + {EF492D73-C40F-4EEF-B917-4ADA7B30D728}.Release|Any CPU.Build.0 = Release|x64 EndGlobalSection EndGlobal diff --git a/src/client/spark_core/Collections/Extensions/ListExtensions.cs b/src/client/spark_core/Collections/Extensions/ListExtensions.cs index 904d084d1..c9abc76ff 100644 --- a/src/client/spark_core/Collections/Extensions/ListExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/ListExtensions.cs @@ -8,6 +8,7 @@ namespace bts.collections { using System; using System.Collections; using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; using System.Linq; public static class ListExtensions { @@ -413,6 +414,19 @@ namespace bts.collections { return false; } + public static bool TryRemoveFirst( + this IList source, Func predicate, [MaybeNullWhen(false)] out T result) { + for (int i = 0; i < source.Count; i++) { + if (predicate(source[i])) { + result = source[i]; + source.RemoveAt(i); + return true; + } + } + result = default; + return false; + } + public static IList WrapValue(T value) => new SingleValueList(value); diff --git a/src/client/spark_core/utils/Exceptions/ExceptionAssert.cs b/src/client/spark_core/utils/Exceptions/ExceptionAssert.cs index 19c73dc6d..f57155c0d 100644 --- a/src/client/spark_core/utils/Exceptions/ExceptionAssert.cs +++ b/src/client/spark_core/utils/Exceptions/ExceptionAssert.cs @@ -7,7 +7,7 @@ using System.Linq; using System.Numerics; namespace mbts.utils { - static class ExceptionAssert { + public static class ExceptionAssert { public static T of_type(object obj, string name = null) => obj is T ? (T)obj : throw new ArgumentException($"Expected {typeof(T)}, actually {(obj?.GetType()?.ToString() ?? "null")}.", name);