diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs index 0557f1fe6..62e9b4f17 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -1,4 +1,5 @@ using System; +using System.Text.Json.Serialization; namespace BCG { /// @@ -56,9 +57,16 @@ namespace BCG { private Type myColType; private object myColValue; + [JsonPropertyName("colNo")] public int colNo { get { return myColNo; } } + + [JsonPropertyName("colName")] public string colName { get { return myColName; } } + + [JsonPropertyName("colType")] public Type colType { get { return myColType; } } + + [JsonIgnore] public object colValue {get{return myColValue;}} } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs new file mode 100644 index 000000000..f2fe33ea7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs @@ -0,0 +1,61 @@ +#nullable enable +using BCG; + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +class BlueRowJsonConverter : JsonConverter { + IBlueTableSchema? schema = null; + + public override IBlueRow Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (schema is null) { + throw new InvalidOperationException("Cannot read row from json because schema is null"); + } + var mbr = new MutableBlueRow(schema); + int colNum = 0; + switch (reader.TokenType) { + case JsonTokenType.StartArray: + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { + mbr.setValue(colNum++, readCellValue(ref reader)); + colNum++; + } + break; + case JsonTokenType.StartObject: + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) { + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new InvalidOperationException( + "Expected 'PropertyName' immediately after 'StartObject' token in row json"); + } + var colName = reader.GetString(); + if (!reader.Read()) { + throw new InvalidOperationException( + "Failed to read token immediately after 'PropertyName' token in row json"); + } + mbr.setValue(colName, readCellValue(ref reader)); + } + break; + } + if (mbr.Count != schema.columnInfo.Length) { + throw new InvalidOperationException( + $"Row json array length '{mbr.Count}' does not match the schema's " + + $"column length '{schema.columnInfo.Length}'."); + } + return mbr; + + static object? readCellValue(ref Utf8JsonReader reader) => + reader.TokenType switch { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.GetDouble(), + JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(), + _ => throw new InvalidOperationException("Failed to read cell value"), + }; + } + + public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) { + throw new NotImplementedException(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs new file mode 100644 index 000000000..6fda93d98 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs @@ -0,0 +1,155 @@ +#nullable enable +using BCG; + +using Spark.Core.Json; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +class DataPacketJsonConverter : JsonConverter { + /// + /// Whether or not to include a field for current rows (i.e., snapshot of current state). By + /// default, only incremental changes are included ('added' and 'deleted'). + /// + public bool IncludeCurrent { get; set; } + + /// + /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays + /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans). + /// + public bool UseObjectRows { get; set; } + + public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (!reader.ReadObject()) { + return null; + } + if (!reader.ReadPropertyName(out var prop)) { + return null; + } + if (!reader.Read()) { + return null; + } + if (prop == "added") { + var added = new List(); + if (reader.TokenType != JsonTokenType.StartArray) { + return null; + } + while (true) { + if (!reader.Read()) { + break; + } + if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) { + return null; + } + switch (reader.TokenType) { + case JsonTokenType.String: + // DAMNIT! Need the schema. + break; + } + } + } + return null; + } + + public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) { + using var _ = writer.WriteObject(); + using (writer.WriteArray("schema")) { + foreach (var col in value.schema.columnInfo) { + using (writer.WriteObject()) { + writer.WriteString("name", col.colName); + writer.WriteNumber("index", col.colNo); + // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.) + writer.WriteString("type", col.colType.ToString()); + } + } + } + + var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList(); + using (writer.WriteArray("added")) { + foreach (IBlueRow row in value.addedRows) { + using (writer.WriteArray()) { + for (int i = 0; i < cellWriters.Count; i++) { + cellWriters[i](row[i]); + } + } + } + } + using (writer.WriteArray("deleted")) { + foreach (IBlueRow row in value.deletedRows) { + using (writer.WriteArray()) { + for (int i = 0; i < cellWriters.Count; i++) { + cellWriters[i](row[i]); + } + } + } + } + if (IncludeCurrent) { + using (writer.WriteArray("current")) { + foreach (IBlueRow row in value.newRows) { + using (writer.WriteArray()) { + for (int i = 0; i < cellWriters.Count; i++) { + cellWriters[i](row[i]); + } + } + } + } + } + + Action getCellWriter(ColumnInfo colInfo) { + var writeVal = getCellValueWriter(colInfo); + if (!UseObjectRows) { + return writeVal; + } + return v => { + writer.WritePropertyName(colInfo.colName); + writeVal(v); + }; + } + + Action getCellValueWriter(ColumnInfo colInfo) { + var t = colInfo.colType; + if (t == typeof(int)) { + return nullCheckWrapper(v => writer.WriteNumberValue((int)v)); + } + if (t == typeof(uint)) { + return nullCheckWrapper(v => writer.WriteNumberValue((uint)v)); + } + if (t == typeof(long)) { + return nullCheckWrapper(v => writer.WriteNumberValue((long)v)); + } + if (t == typeof(ulong)) { + return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v)); + } + if (t == typeof(short)) { + return nullCheckWrapper(v => writer.WriteNumberValue((short)v)); + } + if (t == typeof(ushort)) { + return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v)); + } + if (t == typeof(double)) { + return nullCheckWrapper(v => writer.WriteNumberValue((double)v)); + } + if (t == typeof(float)) { + return nullCheckWrapper(v => writer.WriteNumberValue((float)v)); + } + if (t == typeof(bool)) { + return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v)); + } + return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString())); + + Action nullCheckWrapper(Action writeNonNull) => + v => { + if (v is null) { + writer.WriteNullValue(); + } else { + writeNonNull(v); + } + }; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs index f7d860617..8b589a275 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs @@ -1,104 +1,61 @@ +using mbts.utils; + +using Spark.Core.Json; + using System; -using BCG; -//using BCG.Utilities; -using System.Text.RegularExpressions; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Runtime.InteropServices; +using System.Text.RegularExpressions; -namespace BCG { - [JsonConverter(typeof(PubSubRouteJsonConverter))] - public class PubSubRoute : IComparable { - public string server_address; - public string tag_reg_exp; - //route names are used in intermediary code (i.e. while substations are being deployed) - //so that MC Hammer can advertise them. - public string name; - private Regex _regex = null; +namespace BCG; - public PubSubRoute() { } +public class PubSubRoute : IEquatable, IComparable { + [JsonConverter(typeof(JsonValueOrArrayConverter))] + [JsonPropertyName("server_address")] + public string? ServerAddress { get; set; } - public PubSubRoute(string the_server_address, string the_tag_reg_exp) { - server_address = the_server_address; - tag_reg_exp = the_tag_reg_exp; - } + [JsonPropertyName("tag")] + public string? TagRegExp { get; set; } + //route names are used in intermediary code (i.e. while substations are being deployed) + //so that MC Hammer can advertise them. + [JsonPropertyName("name")] + public string? Name { get; set; } - public string[] getAllPubSubs() { - return server_address.Split(';'); - } + static readonly IComparer comparer = new ComparerBuilder() { + p => p.ServerAddress, + p => p.TagRegExp, + }; - public Regex get_regex() { - return _regex ?? new Regex(tag_reg_exp); - } + Regex? myRegex; - #region IComparable Members + public PubSubRoute() { } - public int CompareTo(PubSubRoute other) { - return server_address == other.server_address ? tag_reg_exp.CompareTo(other.tag_reg_exp) - : server_address.CompareTo(other.server_address); - } - - #endregion - - public override bool Equals(object obj) { - if (!(obj is PubSubRoute) || obj == null) - return false; - else { - PubSubRoute other = obj as PubSubRoute; - return server_address.Equals(other.server_address) && - tag_reg_exp.Equals(other.tag_reg_exp); - } - } - - public override int GetHashCode() { - return base.GetHashCode(); - } - - public static PubSubRoute get_default_route() { - return new PubSubRoute("bluto://127.0.0.1:12345", ".*"); - } - - public override string ToString() { - return $"tag_reg_exp: {tag_reg_exp} server_address: {server_address}"; - } + public PubSubRoute(string theServerAddress, string theTagRegExp) { + ServerAddress = theServerAddress; + TagRegExp = theTagRegExp; } - public static class PubSubRouteExtensions { - public static bool IsMatch(this PubSubRoute route, string tag) => - route.get_regex().IsMatch(tag); + [MemberNotNull(nameof(TagRegExp), nameof(myRegex))] + public bool IsMatch(string tag) { + if (TagRegExp is null) { + throw new InvalidOperationException(); + } + myRegex ??= new Regex(TagRegExp); + return myRegex.IsMatch(tag); } - public class PubSubRouteJsonConverter : JsonConverter { - public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - var json_obj = JsonSerializer.Deserialize(ref reader); - if (json_obj is null) { - return null; - } - var rv = new PubSubRoute(); - ReadString("name", ref rv.name); - ReadString("server_address", ref rv.server_address); - ReadString("tag", ref rv.tag_reg_exp); - return rv; + public int CompareTo(PubSubRoute? other) => + other is null ? -1 : comparer.Compare(this, other); - void ReadString(string key, ref string valueOut) { - if (json_obj.TryGetPropertyValue(key, out var valueJson)) { - var valueStr = valueJson?.GetValue(); - // Only (over)write valueOut if non-null - if (valueStr is not null) { - valueOut = valueStr; - } - } - } - } + public bool Equals(PubSubRoute? other) => + other is not null + && ServerAddress == other.ServerAddress + && TagRegExp == other.TagRegExp; - public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) { - var jsonObj = new JsonObject { - ["name"] = value.name, - ["server_address"] = value.server_address, - ["tag"] = value.tag_reg_exp, - }; - JsonSerializer.Serialize(writer, jsonObj); - } - } + public override bool Equals(object? obj) => + Equals(obj as PubSubRoute); + + public override int GetHashCode() => + HashCode.Combine(ServerAddress, TagRegExp); } diff --git a/src/client/spark_core/Json/JsonExtensions.cs b/src/client/spark_core/Json/JsonExtensions.cs new file mode 100644 index 000000000..9108aa55b --- /dev/null +++ b/src/client/spark_core/Json/JsonExtensions.cs @@ -0,0 +1,138 @@ +#nullable enable +namespace Spark.Core.Json; + +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Nodes; + +public static class JsonExtensions { + public static T As(this JsonNode node) => + node.GetValue(); + + public static JsonArray AsArrayOrEmpty(this JsonNode node) => + node as JsonArray ?? new(); + + public static TEnum AsEnum(this JsonNode node) where TEnum : struct, Enum { + var str = node.As(); + return Enum.Parse(str); + } + + public static string AsString(this JsonNode node) => + node.GetValue(); + + public static string AsStringOrEmpty(this JsonNode node) => + node is JsonValue value && value.TryGetValue(out string? str) ? str : string.Empty; + + public static JsonArray GetArrayOrEmpty(this JsonObject obj, string propertyName) => + obj.TryGetArray(propertyName, out var res) ? res : new(); + + public static T GetValue(this JsonObject obj, string propertyName) => + obj[propertyName]!.GetValue(); + + public static JsonObject GetObjectOrEmpty(this JsonObject obj, string propertyName) => + obj.TryGetObject(propertyName, out var res) ? res : new(); + + public static string GetStringOrEmpty(this JsonObject obj, string propertyName) => + obj.TryGetString(propertyName, out var res) ? res : string.Empty; + + public static bool ReadArray( + this Utf8JsonReader reader, [NotNullWhen(true)] out JsonArray? array) => + ReadArray(reader, JsonSerializerOptions.Default, out array); + + public static bool ReadArray( + this Utf8JsonReader reader, JsonSerializerOptions options, [NotNullWhen(true)] out JsonArray? array) { + array = JsonSerializer.Deserialize(ref reader, options); + return array is not null; + } + + public static bool ReadObject(this Utf8JsonReader reader) { + return reader.Read() && reader.TokenType == JsonTokenType.StartObject; + } + + public static bool ReadObject(this Utf8JsonReader reader, [NotNullWhen(true)] out JsonObject? obj) { + obj = JsonSerializer.Deserialize(ref reader); + return obj is not null; + } + + public static bool ReadPropertyName( + this Utf8JsonReader reader, [NotNullWhen(true)] out string? propertyName) { + if (reader.Read() && reader.TokenType == JsonTokenType.PropertyName) { + propertyName = reader.GetString()!; + return true; + } + propertyName = null; + return false; + } + + public static bool ReadString( + this Utf8JsonReader reader, [NotNullWhen(true)] out string? str) => + ReadString(reader, JsonSerializerOptions.Default, out str); + + public static bool ReadString( + this Utf8JsonReader reader, JsonSerializerOptions options, [NotNullWhen(true)] out string? str) { + str = JsonSerializer.Deserialize(ref reader, options); + return str is not null; + } + + public static bool TryGetArray(this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonArray? result) { + result = obj.TryGetPropertyValue(propertyName, out var v) ? v as JsonArray : null; + return result is not null; + } + + public static bool TryGetObject( + this JsonObject obj, string propertyName, [NotNullWhen(true)] out JsonObject? result) { + result = obj.TryGetPropertyValue(propertyName, out var v) ? v as JsonObject : null; + return result is not null; + } + + public static bool TryGetString( + this JsonObject obj, string propertyName, [NotNullWhen(true)] out string? result) { + if (obj.TryGetPropertyValue(propertyName, out var v) && v is JsonValue value) { + return value.TryGetValue(out result); + } + result = null; + return false; + } + + public static IDisposable WriteArray(this Utf8JsonWriter writer) { + writer.WriteStartArray(); + return Disposable.Create(writer.WriteEndArray); + } + + public static IDisposable WriteArray(this Utf8JsonWriter writer, string propertyName) { + writer.WriteStartArray(propertyName); + return Disposable.Create(writer.WriteEndArray); + } + + public static void WriteArray(this Utf8JsonWriter writer, string propertyName, IEnumerable items) { + writer.WriteStartArray(propertyName); + foreach (var item in items) { + writer.WriteStringValue(item); + } + writer.WriteEndArray(); + } + + public static void Write(this Utf8JsonWriter writer, string propertyName, T node) { + writer.WritePropertyName(propertyName); + JsonSerializer.Serialize(writer, node); + } + + public static IDisposable WriteObject(this Utf8JsonWriter writer) { + writer.WriteStartObject(); + return Disposable.Create(writer.WriteEndObject); + } + + public static IDisposable WriteObject(this Utf8JsonWriter writer, string propertyName) { + writer.WritePropertyName(propertyName); + return writer.WriteObject(); + } + + public static void WriteObject(this Utf8JsonWriter writer, string propertyName, JsonObject obj) { + writer.WritePropertyName(propertyName); + JsonSerializer.Serialize(writer, obj); + } +} diff --git a/src/client/spark_core/Json/JsonValueOrArrayConverter.cs b/src/client/spark_core/Json/JsonValueOrArrayConverter.cs new file mode 100644 index 000000000..f2073c0d8 --- /dev/null +++ b/src/client/spark_core/Json/JsonValueOrArrayConverter.cs @@ -0,0 +1,29 @@ +#nullable enable +namespace Spark.Core.Json; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +/// +/// A JsonConverter that can parse input in the form of a (single) value, or an array. +/// +public class JsonValueOrArrayConverter : JsonConverter> { + public override List? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + reader.TokenType switch { + JsonTokenType.StartArray => + reader.ReadArray(options, out var array) + ? array.Select(n => n is null ? throw invalidJsonEx() : n.As()).ToList() + : throw invalidJsonEx(), + JsonTokenType.Null => null, + _ => new() { JsonSerializer.Deserialize(ref reader, options) ?? throw invalidJsonEx() } + }; + + public override void Write(Utf8JsonWriter writer, List value, JsonSerializerOptions options) => + JsonSerializer.Serialize(writer, value, options); + + static JsonException invalidJsonEx() => + new($"Failed to parse json (expected value or array of type {typeof(T)})"); +}