chore: Add json facilities for DataStream websocket api

This commit is contained in:
John Colvin committed 2025-11-03 16:13:17 -05:00
1 parent 003b0a3fa0
commit bf913f2b99
7 files changed
+497

No files matched your search

@@ -0,0 +1,61 @@
#nullable enable
using BCG;
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
class BlueRowJsonConverter : JsonConverter<IBlueRow> {
IBlueTableSchema? schema = null;
public override IBlueRow Read(
ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (schema is null) {
throw new InvalidOperationException("Cannot read row from json because schema is null");
}
var mbr = new MutableBlueRow(schema);
int colNum = 0;
switch (reader.TokenType) {
case JsonTokenType.StartArray:
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
mbr.setValue(colNum++, readCellValue(ref reader));
colNum++;
}
break;
case JsonTokenType.StartObject:
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
if (reader.TokenType != JsonTokenType.PropertyName) {
throw new InvalidOperationException(
"Expected 'PropertyName' immediately after 'StartObject' token in row json");
}
var colName = reader.GetString();
if (!reader.Read()) {
throw new InvalidOperationException(
"Failed to read token immediately after 'PropertyName' token in row json");
}
mbr.setValue(colName, readCellValue(ref reader));
}
break;
}
if (mbr.Count != schema.columnInfo.Length) {
throw new InvalidOperationException(
$"Row json array length '{mbr.Count}' does not match the schema's " +
$"column length '{schema.columnInfo.Length}'.");
}
return mbr;
static object? readCellValue(ref Utf8JsonReader reader) =>
reader.TokenType switch {
JsonTokenType.String => reader.GetString(),
JsonTokenType.Number => reader.GetDouble(),
JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
_ => throw new InvalidOperationException("Failed to read cell value"),
};
}
public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
throw new NotImplementedException();
}
}
@@ -0,0 +1,31 @@
#nullable enable
using BCG;
using Spark.Core.Json;
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
public class DataPacketIncrementalIdsJsonConverter : JsonConverter<IDataPacket> {
public override IDataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) =>
throw new NotImplementedException();
public override void Write(Utf8JsonWriter writer, IDataPacket value, JsonSerializerOptions options) {
using var _ = writer.WriteObject();
writer.WriteString("table", value.Schema.tableName);
value.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
using (writer.WriteArray("deleted")) {
foreach (IBlueRow r in deletedRows) {
writer.WriteNumberValue(r.Id());
}
}
using (writer.WriteArray("added")) {
foreach (IBlueRow r in addedRows) {
writer.WriteNumberValue(r.Id());
}
}
}
}
@@ -0,0 +1,46 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using System;
using System.IO;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
/// <summary>
/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory
/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
/// thread safe.
/// </summary>
public class DataPacketIncrementalIdsJsonSerializer {
readonly MemoryStream buffer = new();
readonly DataPacketIncrementalIdsJsonConverter converter;
public DataPacketIncrementalIdsJsonSerializer(DataPacketIncrementalIdsJsonConverter? converter = null) =>
this.converter = converter ?? new();
public string? Serialize(IDataPacket dp) {
buffer.Position = 0;
buffer.SetLength(0);
using (var jsonWriter = new Utf8JsonWriter(buffer)) {
var opts = new JsonSerializerOptions() {
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
};
converter.Write(jsonWriter, dp, opts);
}
var bytes = buffer.GetBuffer();
var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
return Encoding.UTF8.GetString(span);
}
public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) {
buffer.Position = 0;
buffer.SetLength(0);
var opts = new JsonSerializerOptions() {
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
};
converter.Write(jsonWriter, dp, opts);
}
}
@@ -0,0 +1,143 @@
#nullable enable
using BCG;
using Spark.Core.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
public class DataPacketJsonConverter : JsonConverter<DataPacket> {
/// <summary>
/// Whether or not to include a field for current rows (i.e., snapshot of current state). By
/// default, only incremental changes are included ('added' and 'deleted').
/// </summary>
public bool IncludeCurrent { get; set; }
/// <summary>
/// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
/// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
/// </summary>
public bool UseObjectRows { get; set; }
public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (!reader.ReadObject()) {
return null;
}
if (!reader.ReadPropertyName(out var prop)) {
return null;
}
if (!reader.Read()) {
return null;
}
if (prop == "added") {
if (reader.TokenType != JsonTokenType.StartArray) {
return null;
}
while (true) {
if (!reader.Read()) {
break;
}
if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
return null;
}
switch (reader.TokenType) {
case JsonTokenType.String:
// DAMNIT! Need the schema.
break;
}
}
}
return null;
}
public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
using var _ = writer.WriteObject();
using (writer.WriteArray("columns")) {
foreach (var col in value.schema.columnInfo) {
using (writer.WriteObject()) {
writer.WriteString("name", col.colName);
writer.WriteNumber("index", col.colNo);
// TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
writer.WriteString("type", col.type_code().ToString());
}
}
}
var columns = value.schema.columnInfo.Select(c => (Column: c, Write: GetCellValueWriter(c))).ToList();
WriteRows("added", value.addedRows);
WriteRows("deleted", value.deletedRows);
if (IncludeCurrent) {
WriteRows("all", value.newRows);
}
void WriteRows(string name, IBlueRowCollection rows) {
using (writer.WriteArray(name)) {
foreach (IBlueRow row in rows) {
WriteRow(row);
}
}
}
void WriteRow(IBlueRow row) {
using var rowCase = UseObjectRows ? writer.WriteObject() : writer.WriteArray();
for (int i = 0; i < columns.Count; i++) {
if (UseObjectRows) {
writer.WritePropertyName(columns[i].Column.colName);
}
columns[i].Write(row[i]);
}
}
Action<object?> GetCellValueWriter(ColumnInfo colInfo) {
var t = colInfo.colType;
if (t == typeof(int)) {
return NullCheckWrapper(v => writer.WriteNumberValue((int)v));
}
if (t == typeof(uint)) {
return NullCheckWrapper(v => writer.WriteNumberValue((uint)v));
}
if (t == typeof(long)) {
return NullCheckWrapper(v => writer.WriteNumberValue((long)v));
}
if (t == typeof(ulong)) {
return NullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
}
if (t == typeof(short)) {
return NullCheckWrapper(v => writer.WriteNumberValue((short)v));
}
if (t == typeof(ushort)) {
return NullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
}
if (t == typeof(double)) {
return NullCheckWrapper(v => {
var dbl = Equals(v, DBNull.Value) ? double.NaN : (double)v;
JsonSerializer.Serialize(writer, dbl, options);
});
}
if (t == typeof(float)) {
return NullCheckWrapper(v => {
var flt = Equals(v, DBNull.Value) ? float.NaN : (float)v;
writer.WriteNumberValue(flt);
});
}
if (t == typeof(bool)) {
return NullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
}
return NullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
Action<object?> NullCheckWrapper(Action<object> writeNonNull) =>
v => {
if (v is null) {
writer.WriteNullValue();
} else {
writeNonNull(v);
}
};
}
}
}
@@ -0,0 +1,49 @@
#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using System;
using System.IO;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
/// <summary>
/// This class seriailzes DataPackets to json strings. A common buffer is used to reduce memory
/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
/// thread safe.
/// </summary>
public class DataPacketJsonSerializer {
readonly MemoryStream buffer = new();
readonly DataPacketJsonConverter converter;
public DataPacketJsonSerializer(DataPacketJsonConverter? converter = null) =>
this.converter = converter ?? new() {
IncludeCurrent = true,
UseObjectRows = true,
};
public string? Serialize(DataPacket dp) {
buffer.Position = 0;
buffer.SetLength(0);
using (var jsonWriter = new Utf8JsonWriter(buffer)) {
var opts = new JsonSerializerOptions() {
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
};
converter.Write(jsonWriter, dp, opts);
}
var bytes = buffer.GetBuffer();
var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
return Encoding.UTF8.GetString(span);
}
public void Serialize(DataPacket dp, Utf8JsonWriter jsonWriter) {
buffer.Position = 0;
buffer.SetLength(0);
var opts = new JsonSerializerOptions() {
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
};
converter.Write(jsonWriter, dp, opts);
}
}
@@ -0,0 +1,138 @@
#nullable enable
namespace Spark.Core.Json;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Text.Json.Nodes;
public static class JsonExtensions {
public static T As<T>(this JsonNode node) =>
node.GetValue<T>();
public static JsonArray AsArrayOrEmpty(this JsonNode node) =>
node as JsonArray ?? new();
public static TEnum AsEnum<TEnum>(this JsonNode node) where TEnum : struct, Enum {
var str = node.As<string>();
return Enum.Parse<TEnum>(str);
}
public static string AsString(this JsonNode node) =>
node.GetValue<string>();
public static string AsStringOrEmpty(this JsonNode node) =>
node is JsonValue value && value.TryGetValue(out string? str) ? str : string.Empty;
public static JsonArray GetArrayOrEmpty(this JsonObject obj, string propertyName) =>
obj.TryGetArray(propertyName, out var res) ? res : new();
public static T GetValue<T>(this JsonObject obj, string propertyName) =>
obj[propertyName]!.GetValue<T>();
public static JsonObject GetObjectOrEmpty(this JsonObject obj, string propertyName) =>
obj.TryGetObject(propertyName, out var res) ? res : new();
public static string GetStringOrEmpty(this JsonObject obj, string propertyName) =>
obj.TryGetString(propertyName, out var res) ? res : string.Empty;
public static bool ReadArray(
this Utf8JsonReader reader, [NotNullWhen(true)] out JsonArray? array) =>
ReadArray(reader, JsonSerializerOptions.Default, out array);
public static bool ReadArray(
this Utf8JsonReader reader, JsonSerializerOptions options, [NotNullWhen(true)] out JsonArray? array) {
array = JsonSerializer.Deserialize<JsonArray>(ref reader, options);
return array is not null;
}
public static bool ReadObject(this Utf8JsonReader reader) {
return reader.Read() && reader.TokenType == JsonTokenType.StartObject;
}
public static bool ReadObject(this Utf8JsonReader reader, [NotNullWhen(true)] out JsonObject? obj) {
obj = JsonSerializer.Deserialize<JsonObject>(ref reader);
return obj is not null;
}
public static bool ReadPropertyName(
this Utf8JsonReader reader, [NotNullWhen(true)] out string? propertyName) {
if (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) {
propertyName = reader.GetString()!;
return true;
}
propertyName = null;
return false;
}
public static bool ReadString(
this Utf8JsonReader reader, [NotNullWhen(true)] out string? str) =>
ReadString(reader, JsonSerializerOptions.Default, out str);
public static bool ReadString(
this Utf8JsonReader reader, JsonSerializerOptions options, [NotNullWhen(true)] out string? str) {
str = JsonSerializer.Deserialize<string>(ref reader, options);
return str is not null;
}
public static bool TryGetArray(this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonArray? result) {
result = obj.TryGetPropertyValue(propertyName, out var v) ? v as JsonArray : null;
return result is not null;
}
public static bool TryGetObject(
this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonObject? result) {
result = obj.TryGetPropertyValue(propertyName, out var v) ? v as JsonObject : null;
return result is not null;
}
public static bool TryGetString(
this JsonObject obj, string propertyName, [NotNullWhen(true)] out string? result) {
if (obj.TryGetPropertyValue(propertyName, out var v) && v is JsonValue value) {
return value.TryGetValue(out result);
}
result = null;
return false;
}
public static IDisposable WriteArray(this Utf8JsonWriter writer) {
writer.WriteStartArray();
return Disposable.Create(writer.WriteEndArray);
}
public static IDisposable WriteArray(this Utf8JsonWriter writer, string propertyName) {
writer.WriteStartArray(propertyName);
return Disposable.Create(writer.WriteEndArray);
}
public static void WriteArray(this Utf8JsonWriter writer, string propertyName, IEnumerable<string> items) {
writer.WriteStartArray(propertyName);
foreach (var item in items) {
writer.WriteStringValue(item);
}
writer.WriteEndArray();
}
public static void Write<T>(this Utf8JsonWriter writer, string propertyName, T node) {
writer.WritePropertyName(propertyName);
JsonSerializer.Serialize(writer, node);
}
public static IDisposable WriteObject(this Utf8JsonWriter writer) {
writer.WriteStartObject();
return Disposable.Create(writer.WriteEndObject);
}
public static IDisposable WriteObject(this Utf8JsonWriter writer, string propertyName) {
writer.WritePropertyName(propertyName);
return writer.WriteObject();
}
public static void WriteObject(this Utf8JsonWriter writer, string propertyName, JsonObject obj) {
writer.WritePropertyName(propertyName);
JsonSerializer.Serialize(writer, obj);
}
}
@@ -0,0 +1,29 @@
#nullable enable
namespace Spark.Core.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
/// <summary>
/// A JsonConverter that can parse input in the form of a (single) value, or an array.
/// </summary>
public class JsonValueOrArrayConverter<T> : JsonConverter<List<T>> {
public override List<T>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) =>
reader.TokenType switch {
JsonTokenType.StartArray =>
reader.ReadArray(options, out var array)
? array.Select(n => n is null ? throw invalidJsonEx() : n.As<T>()).ToList()
: throw invalidJsonEx(),
JsonTokenType.Null => null,
_ => new() { JsonSerializer.Deserialize<T>(ref reader, options) ?? throw invalidJsonEx() }
};
public override void Write(Utf8JsonWriter writer, List<T> value, JsonSerializerOptions options) =>
JsonSerializer.Serialize(writer, value, options);
static JsonException invalidJsonEx() =>
new($"Failed to parse json (expected value or array of type {typeof(T)})");
}