diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs index 9337d9420..82b9cb030 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -1,95 +1,103 @@ -using System; - -namespace BCG -{ - /// - /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable - /// - [Serializable] - public class ColumnInfo : IComparable - { - public static bool isAllowedType (Type t) - { - return - t.Equals (typeof (System.Int32)) || - t.Equals (typeof (System.Int64)) || - t.Equals (typeof (System.Double)) || - t.Equals (typeof (System.String)) || - t.Equals (typeof (System.DateTime)) ? true : - false; - } - - //the default column info does not cach values in a lookup table, therefore it is in group 0 - public ColumnInfo(string colName, int colNo, Type colType) - : this(colName, colNo, colType, 0) - { - - - } - - public ColumnInfo(string colName, int colNo, Type colType, short colGroup) - { - myColName = colName; - myColNo = colNo; - myColType = colType; - myLookupTableGroup = colGroup; - myColValue = null; - } - - - public ColumnInfo (string colName, int colNo, object colValue) - : this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;} - - public ColumnInfo (string colName, int colNo, Type colType, object colValue) - : this (colName, colNo, colType) - { - - myColValue = colValue == null ? DBNull.Value : colValue; - - } - - - public int CompareTo (object obj) - { - int otherColNo = ((ColumnInfo)(obj)).colNo; - if (myColNo < otherColNo) - return -1; - else if (myColNo > otherColNo) - return 1; - return 0; - } - - //public bool isCalculated{get{return myExpression != null;}} - - public bool isConstant {get{return myColValue != null;}} - - private int myColNo; - private string myColName; - private Type myColType; - //private string myExpression; - private object myColValue; - - //no table will ever have more than 65,536 columns - //if it does, we are in serious trouble - private short myLookupTableGroup; - - public int colNo{get{return myColNo;}} - public string colName {get{return myColName;}} - public Type colType {get{return myColType;}} - - - //this value corresponds to how the data in the column is serialized. - //if the group number is 0, the values are not stored in a table - //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together - //are stored together in a lookup table - - - public short lookupTableGroup - { - get { return myLookupTableGroup; } - set { myLookupTableGroup = value; } - } - //public string expression {get{return myExpression;}} - public object colValue {get{return myColValue;}} - } -} +using System; +using System.Text.Json.Serialization; + +namespace BCG +{ + /// + /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable + /// + [Serializable] + public class ColumnInfo : IComparable + { + public static bool isAllowedType (Type t) + { + return + t.Equals (typeof (System.Int32)) || + t.Equals (typeof (System.Int64)) || + t.Equals (typeof (System.Double)) || + t.Equals (typeof (System.String)) || + t.Equals (typeof (System.DateTime)) ? true : + false; + } + + //the default column info does not cach values in a lookup table, therefore it is in group 0 + public ColumnInfo(string colName, int colNo, Type colType) + : this(colName, colNo, colType, 0) + { + + + } + + public ColumnInfo(string colName, int colNo, Type colType, short colGroup) + { + myColName = colName; + myColNo = colNo; + myColType = colType; + myLookupTableGroup = colGroup; + myColValue = null; + } + + + public ColumnInfo (string colName, int colNo, object colValue) + : this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;} + + public ColumnInfo (string colName, int colNo, Type colType, object colValue) + : this (colName, colNo, colType) + { + + myColValue = colValue == null ? DBNull.Value : colValue; + + } + + + public int CompareTo (object obj) + { + int otherColNo = ((ColumnInfo)(obj)).colNo; + if (myColNo < otherColNo) + return -1; + else if (myColNo > otherColNo) + return 1; + return 0; + } + + //public bool isCalculated{get{return myExpression != null;}} + + public bool isConstant {get{return myColValue != null;}} + + private int myColNo; + private string myColName; + private Type myColType; + //private string myExpression; + private object myColValue; + + //no table will ever have more than 65,536 columns + //if it does, we are in serious trouble + private short myLookupTableGroup; + + [JsonPropertyName("colNo")] + public int colNo{get{return myColNo;}} + + [JsonPropertyName("colName")] + public string colName {get{return myColName;}} + + [JsonPropertyName("colType")] + public Type colType {get{return myColType;}} + + + //this value corresponds to how the data in the column is serialized. + //if the group number is 0, the values are not stored in a table + //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together + //are stored together in a lookup table + + + [JsonIgnore] + public short lookupTableGroup + { + get { return myLookupTableGroup; } + set { myLookupTableGroup = value; } + } + //public string expression {get{return myExpression;}} + [JsonIgnore] + public object colValue {get{return myColValue;}} + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs new file mode 100644 index 000000000..f2fe33ea7 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs @@ -0,0 +1,61 @@ +#nullable enable +using BCG; + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +class BlueRowJsonConverter : JsonConverter { + IBlueTableSchema? schema = null; + + public override IBlueRow Read( + ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (schema is null) { + throw new InvalidOperationException("Cannot read row from json because schema is null"); + } + var mbr = new MutableBlueRow(schema); + int colNum = 0; + switch (reader.TokenType) { + case JsonTokenType.StartArray: + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { + mbr.setValue(colNum++, readCellValue(ref reader)); + colNum++; + } + break; + case JsonTokenType.StartObject: + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) { + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new InvalidOperationException( + "Expected 'PropertyName' immediately after 'StartObject' token in row json"); + } + var colName = reader.GetString(); + if (!reader.Read()) { + throw new InvalidOperationException( + "Failed to read token immediately after 'PropertyName' token in row json"); + } + mbr.setValue(colName, readCellValue(ref reader)); + } + break; + } + if (mbr.Count != schema.columnInfo.Length) { + throw new InvalidOperationException( + $"Row json array length '{mbr.Count}' does not match the schema's " + + $"column length '{schema.columnInfo.Length}'."); + } + return mbr; + + static object? readCellValue(ref Utf8JsonReader reader) => + reader.TokenType switch { + JsonTokenType.String => reader.GetString(), + JsonTokenType.Number => reader.GetDouble(), + JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(), + _ => throw new InvalidOperationException("Failed to read cell value"), + }; + } + + public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) { + throw new NotImplementedException(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs new file mode 100644 index 000000000..8070b63a9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs @@ -0,0 +1,154 @@ +#nullable enable +using BCG; + +using Spark.Core.Json; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Bts.DataStreamCore; + +class DataPacketJsonConverter : JsonConverter { + /// + /// Whether or not to include a field for current rows (i.e., snapshot of current state). By + /// default, only incremental changes are included ('added' and 'deleted'). + /// + public bool IncludeCurrent { get; set; } + + /// + /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays + /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans). + /// + public bool UseObjectRows { get; set; } + + public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (!reader.ReadObject()) { + return null; + } + if (!reader.ReadPropertyName(out var prop)) { + return null; + } + if (!reader.Read()) { + return null; + } + if (prop == "added") { + var added = new List(); + if (reader.TokenType != JsonTokenType.StartArray) { + return null; + } + while (true) { + if (!reader.Read()) { + break; + } + if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) { + return null; + } + switch (reader.TokenType) { + case JsonTokenType.String: + // DAMNIT! Need the schema. + break; + } + } + } + } + + public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) { + using var _ = writer.WriteObject(); + using (writer.WriteArray("schema")) { + foreach (var col in value.schema.columnInfo) { + using (writer.WriteObject()) { + writer.WriteString("name", col.colName); + writer.WriteNumber("index", col.colNo); + // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.) + writer.WriteString("type", col.colType.ToString()); + } + } + } + + var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList(); + using (writer.WriteArray("added")) { + foreach (IBlueRow row in value.addedRows) { + using (writer.WriteArray()) { + for (int i = 0; i < cellWriters.Count; i++) { + cellWriters[i](row[i]); + } + } + } + } + using (writer.WriteArray("deleted")) { + foreach (IBlueRow row in value.deletedRows) { + using (writer.WriteArray()) { + for (int i = 0; i < cellWriters.Count; i++) { + cellWriters[i](row[i]); + } + } + } + } + if (IncludeCurrent) { + using (writer.WriteArray("current")) { + foreach (IBlueRow row in value.newRows) { + using (writer.WriteArray()) { + for (int i = 0; i < cellWriters.Count; i++) { + cellWriters[i](row[i]); + } + } + } + } + } + + Action getCellWriter(ColumnInfo colInfo) { + var writeVal = getCellValueWriter(colInfo); + if (!UseObjectRows) { + return writeVal; + } + return v => { + writer.WritePropertyName(colInfo.colName); + writeVal(v); + }; + } + + Action getCellValueWriter(ColumnInfo colInfo) { + var t = colInfo.colType; + if (t == typeof(int)) { + return nullCheckWrapper(v => writer.WriteNumberValue((int)v)); + } + if (t == typeof(uint)) { + return nullCheckWrapper(v => writer.WriteNumberValue((uint)v)); + } + if (t == typeof(long)) { + return nullCheckWrapper(v => writer.WriteNumberValue((long)v)); + } + if (t == typeof(ulong)) { + return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v)); + } + if (t == typeof(short)) { + return nullCheckWrapper(v => writer.WriteNumberValue((short)v)); + } + if (t == typeof(ushort)) { + return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v)); + } + if (t == typeof(double)) { + return nullCheckWrapper(v => writer.WriteNumberValue((double)v)); + } + if (t == typeof(float)) { + return nullCheckWrapper(v => writer.WriteNumberValue((float)v)); + } + if (t == typeof(bool)) { + return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v)); + } + return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString())); + + Action nullCheckWrapper(Action writeNonNull) => + v => { + if (v is null) { + writer.WriteNullValue(); + } else { + writeNonNull(v); + } + }; + } + } +}