From bf913f2b9951ee730ab287af3ec1f08ced8a776b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 25 Jan 2024 14:45:03 -0600 Subject: [PATCH] chore: Add json facilities for DataStream websocket api --- .../DataStreamCore/BlueRowJsonConverter.cs | 61 ++++++++ .../DataPacketIncrementalIdsJsonConverter.cs | 31 ++++ .../DataPacketIncrementalIdsJsonSerializer.cs | 46 ++++++ .../DataStreamCore/DataPacketJsonConverter.cs | 143 ++++++++++++++++++ .../DataPacketJsonSerializer.cs | 49 ++++++ src/client/spark_core/Json/JsonExtensions.cs | 138 +++++++++++++++++ .../Json/JsonValueOrArrayConverter.cs | 29 ++++ 7 files changed, 497 insertions(+) create mode 100644 src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs create mode 100644 src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonConverter.cs create mode 100644 src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonSerializer.cs create mode 100644 src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs create mode 100644 src/DataStreamSuite/DataStreamCore/DataPacketJsonSerializer.cs create mode 100644 src/client/spark_core/Json/JsonExtensions.cs create mode 100644 src/client/spark_core/Json/JsonValueOrArrayConverter.cs diff --git a/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs new file mode 100644 index 000000000..f2fe33ea7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs @@ -0,0 +1,61 @@ +#nullable enable +using BCG; + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +class BlueRowJsonConverter : JsonConverter { + IBlueTableSchema? schema = null; + + public override IBlueRow Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (schema is null) { + throw new InvalidOperationException("Cannot read row from json because schema is null"); + } + var mbr = new MutableBlueRow(schema); + int colNum = 0; + switch (reader.TokenType) { + case JsonTokenType.StartArray: + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { + mbr.setValue(colNum++, readCellValue(ref reader)); + colNum++; + } + break; + case JsonTokenType.StartObject: + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) { + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new InvalidOperationException( + "Expected 'PropertyName' immediately after 'StartObject' token in row json"); + } + var colName = reader.GetString(); + if (!reader.Read()) { + throw new InvalidOperationException( + "Failed to read token immediately after 'PropertyName' token in row json"); + } + mbr.setValue(colName, readCellValue(ref reader)); + } + break; + } + if (mbr.Count != schema.columnInfo.Length) { + throw new InvalidOperationException( + $"Row json array length '{mbr.Count}' does not match the schema's " + + $"column length '{schema.columnInfo.Length}'."); + } + return mbr; + + static object? readCellValue(ref Utf8JsonReader reader) => + reader.TokenType switch { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.GetDouble(), + JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(), + _ => throw new InvalidOperationException("Failed to read cell value"), + }; + } + + public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) { + throw new NotImplementedException(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonConverter.cs new file mode 100644 index 000000000..3f1680fbd --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonConverter.cs @@ -0,0 +1,31 @@ +#nullable enable +using BCG; + +using Spark.Core.Json; + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +public class DataPacketIncrementalIdsJsonConverter : JsonConverter { + public override IDataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + throw new NotImplementedException(); + + public override void Write(Utf8JsonWriter writer, IDataPacket value, JsonSerializerOptions options) { + using var _ = writer.WriteObject(); + writer.WriteString("table", value.Schema.tableName); + value.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); + using (writer.WriteArray("deleted")) { + foreach (IBlueRow r in deletedRows) { + writer.WriteNumberValue(r.Id()); + } + } + using (writer.WriteArray("added")) { + foreach (IBlueRow r in addedRows) { + writer.WriteNumberValue(r.Id()); + } + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonSerializer.cs b/src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonSerializer.cs new file mode 100644 index 000000000..305ed3743 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacketIncrementalIdsJsonSerializer.cs @@ -0,0 +1,46 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.IO; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +/// +/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory +/// allocations on repeated calls to . For that reason, this class is NOT +/// thread safe. +/// +public class DataPacketIncrementalIdsJsonSerializer { + readonly MemoryStream buffer = new(); + readonly DataPacketIncrementalIdsJsonConverter converter; + + public DataPacketIncrementalIdsJsonSerializer(DataPacketIncrementalIdsJsonConverter? converter = null) => + this.converter = converter ?? new(); + + public string? Serialize(IDataPacket dp) { + buffer.Position = 0; + buffer.SetLength(0); + using (var jsonWriter = new Utf8JsonWriter(buffer)) { + var opts = new JsonSerializerOptions() { + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals + }; + converter.Write(jsonWriter, dp, opts); + } + var bytes = buffer.GetBuffer(); + var span = new ArraySegment(bytes, 0, (int)buffer.Length); + return Encoding.UTF8.GetString(span); + } + + public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) { + buffer.Position = 0; + buffer.SetLength(0); + var opts = new JsonSerializerOptions() { + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals + }; + converter.Write(jsonWriter, dp, opts); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs new file mode 100644 index 000000000..d7fbbe66d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs @@ -0,0 +1,143 @@ +#nullable enable +using BCG; + +using Spark.Core.Json; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +public class DataPacketJsonConverter : JsonConverter { + /// + /// Whether or not to include a field for current rows (i.e., snapshot of current state). By + /// default, only incremental changes are included ('added' and 'deleted'). + /// + public bool IncludeCurrent { get; set; } + + /// + /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays + /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans). + /// + public bool UseObjectRows { get; set; } + + public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (!reader.ReadObject()) { + return null; + } + if (!reader.ReadPropertyName(out var prop)) { + return null; + } + if (!reader.Read()) { + return null; + } + if (prop == "added") { + if (reader.TokenType != JsonTokenType.StartArray) { + return null; + } + while (true) { + if (!reader.Read()) { + break; + } + if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) { + return null; + } + switch (reader.TokenType) { + case JsonTokenType.String: + // DAMNIT! Need the schema. + break; + } + } + } + return null; + } + + public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) { + using var _ = writer.WriteObject(); + using (writer.WriteArray("columns")) { + foreach (var col in value.schema.columnInfo) { + using (writer.WriteObject()) { + writer.WriteString("name", col.colName); + writer.WriteNumber("index", col.colNo); + // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.) + writer.WriteString("type", col.type_code().ToString()); + } + } + } + + var columns = value.schema.columnInfo.Select(c => (Column: c, Write: GetCellValueWriter(c))).ToList(); + WriteRows("added", value.addedRows); + WriteRows("deleted", value.deletedRows); + if (IncludeCurrent) { + WriteRows("all", value.newRows); + } + + void WriteRows(string name, IBlueRowCollection rows) { + using (writer.WriteArray(name)) { + foreach (IBlueRow row in rows) { + WriteRow(row); + } + } + } + + void WriteRow(IBlueRow row) { + using var rowCase = UseObjectRows ? writer.WriteObject() : writer.WriteArray(); + for (int i = 0; i < columns.Count; i++) { + if (UseObjectRows) { + writer.WritePropertyName(columns[i].Column.colName); + } + columns[i].Write(row[i]); + } + } + + Action GetCellValueWriter(ColumnInfo colInfo) { + var t = colInfo.colType; + if (t == typeof(int)) { + return NullCheckWrapper(v => writer.WriteNumberValue((int)v)); + } + if (t == typeof(uint)) { + return NullCheckWrapper(v => writer.WriteNumberValue((uint)v)); + } + if (t == typeof(long)) { + return NullCheckWrapper(v => writer.WriteNumberValue((long)v)); + } + if (t == typeof(ulong)) { + return NullCheckWrapper(v => writer.WriteNumberValue((ulong)v)); + } + if (t == typeof(short)) { + return NullCheckWrapper(v => writer.WriteNumberValue((short)v)); + } + if (t == typeof(ushort)) { + return NullCheckWrapper(v => writer.WriteNumberValue((ushort)v)); + } + if (t == typeof(double)) { + return NullCheckWrapper(v => { + var dbl = Equals(v, DBNull.Value) ? double.NaN : (double)v; + JsonSerializer.Serialize(writer, dbl, options); + }); + } + if (t == typeof(float)) { + return NullCheckWrapper(v => { + var flt = Equals(v, DBNull.Value) ? float.NaN : (float)v; + writer.WriteNumberValue(flt); + }); + } + if (t == typeof(bool)) { + return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v)); + } + return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString())); + + Action NullCheckWrapper(Action writeNonNull) => + v => { + if (v is null) { + writer.WriteNullValue(); + } else { + writeNonNull(v); + } + }; + } + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketJsonSerializer.cs b/src/DataStreamSuite/DataStreamCore/DataPacketJsonSerializer.cs new file mode 100644 index 000000000..006fef179 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacketJsonSerializer.cs @@ -0,0 +1,49 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.IO; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +/// +/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory +/// allocations on repeated calls to . For that reason, this class is NOT +/// thread safe. +/// +public class DataPacketJsonSerializer { + readonly MemoryStream buffer = new(); + readonly DataPacketJsonConverter converter; + + public DataPacketJsonSerializer(DataPacketJsonConverter? converter = null) => + this.converter = converter ?? new() { + IncludeCurrent = true, + UseObjectRows = true, + }; + + public string? Serialize(DataPacket dp) { + buffer.Position = 0; + buffer.SetLength(0); + using (var jsonWriter = new Utf8JsonWriter(buffer)) { + var opts = new JsonSerializerOptions() { + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals + }; + converter.Write(jsonWriter, dp, opts); + } + var bytes = buffer.GetBuffer(); + var span = new ArraySegment(bytes, 0, (int)buffer.Length); + return Encoding.UTF8.GetString(span); + } + + public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) { + buffer.Position = 0; + buffer.SetLength(0); + var opts = new JsonSerializerOptions() { + NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals + }; + converter.Write(jsonWriter, dp, opts); + } +} diff --git a/src/client/spark_core/Json/JsonExtensions.cs b/src/client/spark_core/Json/JsonExtensions.cs new file mode 100644 index 000000000..9108aa55b --- /dev/null +++ b/src/client/spark_core/Json/JsonExtensions.cs @@ -0,0 +1,138 @@ +#nullable enable +namespace Spark.Core.Json; + +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Nodes; + +public static class JsonExtensions { + public static T As(this JsonNode node) => + node.GetValue(); + + public static JsonArray AsArrayOrEmpty(this JsonNode node) => + node as JsonArray ?? new(); + + public static TEnum AsEnum(this JsonNode node) where TEnum : struct, Enum { + var str = node.As(); + return Enum.Parse(str); + } + + public static string AsString(this JsonNode node) => + node.GetValue(); + + public static string AsStringOrEmpty(this JsonNode node) => + node is JsonValue value && value.TryGetValue(out string? str) ? str : string.Empty; + + public static JsonArray GetArrayOrEmpty(this JsonObject obj, string propertyName) => + obj.TryGetArray(propertyName, out var res) ? res : new(); + + public static T GetValue(this JsonObject obj, string propertyName) => + obj[propertyName]!.GetValue(); + + public static JsonObject GetObjectOrEmpty(this JsonObject obj, string propertyName) => + obj.TryGetObject(propertyName, out var res) ? res : new(); + + public static string GetStringOrEmpty(this JsonObject obj, string propertyName) => + obj.TryGetString(propertyName, out var res) ? res : string.Empty; + + public static bool ReadArray( + this Utf8JsonReader reader, [NotNullWhen(true)] out JsonArray? array) => + ReadArray(reader, JsonSerializerOptions.Default, out array); + + public static bool ReadArray( + this Utf8JsonReader reader, JsonSerializerOptions options, [NotNullWhen(true)] out JsonArray? array) { + array = JsonSerializer.Deserialize(ref reader, options); + return array is not null; + } + + public static bool ReadObject(this Utf8JsonReader reader) { + return reader.Read() && reader.TokenType == JsonTokenType.StartObject; + } + + public static bool ReadObject(this Utf8JsonReader reader, [NotNullWhen(true)] out JsonObject? obj) { + obj = JsonSerializer.Deserialize(ref reader); + return obj is not null; + } + + public static bool ReadPropertyName( + this Utf8JsonReader reader, [NotNullWhen(true)] out string? propertyName) { + if (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) { + propertyName = reader.GetString()!; + return true; + } + propertyName = null; + return false; + } + + public static bool ReadString( + this Utf8JsonReader reader, [NotNullWhen(true)] out string? str) => + ReadString(reader, JsonSerializerOptions.Default, out str); + + public static bool ReadString( + this Utf8JsonReader reader, JsonSerializerOptions options, [NotNullWhen(true)] out string? str) { + str = JsonSerializer.Deserialize(ref reader, options); + return str is not null; + } + + public static bool TryGetArray(this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonArray? result) { + result = obj.TryGetPropertyValue(propertyName, out var v) ? v as JsonArray : null; + return result is not null; + } + + public static bool TryGetObject( + this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonObject? result) { + result = obj.TryGetPropertyValue(propertyName, out var v) ? v as JsonObject : null; + return result is not null; + } + + public static bool TryGetString( + this JsonObject obj, string propertyName, [NotNullWhen(true)] out string? result) { + if (obj.TryGetPropertyValue(propertyName, out var v) && v is JsonValue value) { + return value.TryGetValue(out result); + } + result = null; + return false; + } + + public static IDisposable WriteArray(this Utf8JsonWriter writer) { + writer.WriteStartArray(); + return Disposable.Create(writer.WriteEndArray); + } + + public static IDisposable WriteArray(this Utf8JsonWriter writer, string propertyName) { + writer.WriteStartArray(propertyName); + return Disposable.Create(writer.WriteEndArray); + } + + public static void WriteArray(this Utf8JsonWriter writer, string propertyName, IEnumerable items) { + writer.WriteStartArray(propertyName); + foreach (var item in items) { + writer.WriteStringValue(item); + } + writer.WriteEndArray(); + } + + public static void Write(this Utf8JsonWriter writer, string propertyName, T node) { + writer.WritePropertyName(propertyName); + JsonSerializer.Serialize(writer, node); + } + + public static IDisposable WriteObject(this Utf8JsonWriter writer) { + writer.WriteStartObject(); + return Disposable.Create(writer.WriteEndObject); + } + + public static IDisposable WriteObject(this Utf8JsonWriter writer, string propertyName) { + writer.WritePropertyName(propertyName); + return writer.WriteObject(); + } + + public static void WriteObject(this Utf8JsonWriter writer, string propertyName, JsonObject obj) { + writer.WritePropertyName(propertyName); + JsonSerializer.Serialize(writer, obj); + } +} diff --git a/src/client/spark_core/Json/JsonValueOrArrayConverter.cs b/src/client/spark_core/Json/JsonValueOrArrayConverter.cs new file mode 100644 index 000000000..f2073c0d8 --- /dev/null +++ b/src/client/spark_core/Json/JsonValueOrArrayConverter.cs @@ -0,0 +1,29 @@ +#nullable enable +namespace Spark.Core.Json; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +/// +/// A JsonConverter that can parse input in the form of a (single) value, or an array. +/// +public class JsonValueOrArrayConverter : JsonConverter> { + public override List? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + reader.TokenType switch { + JsonTokenType.StartArray => + reader.ReadArray(options, out var array) + ? array.Select(n => n is null ? throw invalidJsonEx() : n.As()).ToList() + : throw invalidJsonEx(), + JsonTokenType.Null => null, + _ => new() { JsonSerializer.Deserialize(ref reader, options) ?? throw invalidJsonEx() } + }; + + public override void Write(Utf8JsonWriter writer, List value, JsonSerializerOptions options) => + JsonSerializer.Serialize(writer, value, options); + + static JsonException invalidJsonEx() => + new($"Failed to parse json (expected value or array of type {typeof(T)})"); +}