diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs
index 9337d9420..82b9cb030 100644
--- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs
+++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs
@@ -1,95 +1,103 @@
-using System;
-
-namespace BCG
-{
- ///
- /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
- ///
- [Serializable]
- public class ColumnInfo : IComparable
- {
- public static bool isAllowedType (Type t)
- {
- return
- t.Equals (typeof (System.Int32)) ||
- t.Equals (typeof (System.Int64)) ||
- t.Equals (typeof (System.Double)) ||
- t.Equals (typeof (System.String)) ||
- t.Equals (typeof (System.DateTime)) ? true :
- false;
- }
-
- //the default column info does not cach values in a lookup table, therefore it is in group 0
- public ColumnInfo(string colName, int colNo, Type colType)
- : this(colName, colNo, colType, 0)
- {
-
-
- }
-
- public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
- {
- myColName = colName;
- myColNo = colNo;
- myColType = colType;
- myLookupTableGroup = colGroup;
- myColValue = null;
- }
-
-
- public ColumnInfo (string colName, int colNo, object colValue)
- : this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
-
- public ColumnInfo (string colName, int colNo, Type colType, object colValue)
- : this (colName, colNo, colType)
- {
-
- myColValue = colValue == null ? DBNull.Value : colValue;
-
- }
-
-
- public int CompareTo (object obj)
- {
- int otherColNo = ((ColumnInfo)(obj)).colNo;
- if (myColNo < otherColNo)
- return -1;
- else if (myColNo > otherColNo)
- return 1;
- return 0;
- }
-
- //public bool isCalculated{get{return myExpression != null;}}
-
- public bool isConstant {get{return myColValue != null;}}
-
- private int myColNo;
- private string myColName;
- private Type myColType;
- //private string myExpression;
- private object myColValue;
-
- //no table will ever have more than 65,536 columns
- //if it does, we are in serious trouble
- private short myLookupTableGroup;
-
- public int colNo{get{return myColNo;}}
- public string colName {get{return myColName;}}
- public Type colType {get{return myColType;}}
-
-
- //this value corresponds to how the data in the column is serialized.
- //if the group number is 0, the values are not stored in a table
- //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
- //are stored together in a lookup table
-
-
- public short lookupTableGroup
- {
- get { return myLookupTableGroup; }
- set { myLookupTableGroup = value; }
- }
- //public string expression {get{return myExpression;}}
- public object colValue {get{return myColValue;}}
- }
-}
+using System;
+using System.Text.Json.Serialization;
+
+namespace BCG
+{
+ ///
+ /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
+ ///
+ [Serializable]
+ public class ColumnInfo : IComparable
+ {
+ public static bool isAllowedType (Type t)
+ {
+ return
+ t.Equals (typeof (System.Int32)) ||
+ t.Equals (typeof (System.Int64)) ||
+ t.Equals (typeof (System.Double)) ||
+ t.Equals (typeof (System.String)) ||
+ t.Equals (typeof (System.DateTime)) ? true :
+ false;
+ }
+
+ //the default column info does not cach values in a lookup table, therefore it is in group 0
+ public ColumnInfo(string colName, int colNo, Type colType)
+ : this(colName, colNo, colType, 0)
+ {
+
+
+ }
+
+ public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
+ {
+ myColName = colName;
+ myColNo = colNo;
+ myColType = colType;
+ myLookupTableGroup = colGroup;
+ myColValue = null;
+ }
+
+
+ public ColumnInfo (string colName, int colNo, object colValue)
+ : this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
+
+ public ColumnInfo (string colName, int colNo, Type colType, object colValue)
+ : this (colName, colNo, colType)
+ {
+
+ myColValue = colValue == null ? DBNull.Value : colValue;
+
+ }
+
+
+ public int CompareTo (object obj)
+ {
+ int otherColNo = ((ColumnInfo)(obj)).colNo;
+ if (myColNo < otherColNo)
+ return -1;
+ else if (myColNo > otherColNo)
+ return 1;
+ return 0;
+ }
+
+ //public bool isCalculated{get{return myExpression != null;}}
+
+ public bool isConstant {get{return myColValue != null;}}
+
+ private int myColNo;
+ private string myColName;
+ private Type myColType;
+ //private string myExpression;
+ private object myColValue;
+
+ //no table will ever have more than 65,536 columns
+ //if it does, we are in serious trouble
+ private short myLookupTableGroup;
+
+ [JsonPropertyName("colNo")]
+ public int colNo{get{return myColNo;}}
+
+ [JsonPropertyName("colName")]
+ public string colName {get{return myColName;}}
+
+ [JsonPropertyName("colType")]
+ public Type colType {get{return myColType;}}
+
+
+ //this value corresponds to how the data in the column is serialized.
+ //if the group number is 0, the values are not stored in a table
+ //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
+ //are stored together in a lookup table
+
+
+ [JsonIgnore]
+ public short lookupTableGroup
+ {
+ get { return myLookupTableGroup; }
+ set { myLookupTableGroup = value; }
+ }
+ //public string expression {get{return myExpression;}}
+ [JsonIgnore]
+ public object colValue {get{return myColValue;}}
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs
new file mode 100644
index 000000000..f2fe33ea7
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamCore/BlueRowJsonConverter.cs
@@ -0,0 +1,61 @@
+#nullable enable
+using BCG;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class BlueRowJsonConverter : JsonConverter {
+ IBlueTableSchema? schema = null;
+
+ public override IBlueRow Read(
+ ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+ if (schema is null) {
+ throw new InvalidOperationException("Cannot read row from json because schema is null");
+ }
+ var mbr = new MutableBlueRow(schema);
+ int colNum = 0;
+ switch (reader.TokenType) {
+ case JsonTokenType.StartArray:
+ while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
+ mbr.setValue(colNum++, readCellValue(ref reader));
+ colNum++;
+ }
+ break;
+ case JsonTokenType.StartObject:
+ while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
+ if (reader.TokenType != JsonTokenType.PropertyName) {
+ throw new InvalidOperationException(
+ "Expected 'PropertyName' immediately after 'StartObject' token in row json");
+ }
+ var colName = reader.GetString();
+ if (!reader.Read()) {
+ throw new InvalidOperationException(
+ "Failed to read token immediately after 'PropertyName' token in row json");
+ }
+ mbr.setValue(colName, readCellValue(ref reader));
+ }
+ break;
+ }
+ if (mbr.Count != schema.columnInfo.Length) {
+ throw new InvalidOperationException(
+ $"Row json array length '{mbr.Count}' does not match the schema's " +
+ $"column length '{schema.columnInfo.Length}'.");
+ }
+ return mbr;
+
+ static object? readCellValue(ref Utf8JsonReader reader) =>
+ reader.TokenType switch {
+ JsonTokenType.String => reader.GetString(),
+ JsonTokenType.Number => reader.GetDouble(),
+ JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
+ _ => throw new InvalidOperationException("Failed to read cell value"),
+ };
+ }
+
+ public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
+ throw new NotImplementedException();
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs
new file mode 100644
index 000000000..8070b63a9
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamCore/DataPacketJsonConverter.cs
@@ -0,0 +1,154 @@
+#nullable enable
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Bts.DataStreamCore;
+
+class DataPacketJsonConverter : JsonConverter {
+ ///
+ /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
+ /// default, only incremental changes are included ('added' and 'deleted').
+ ///
+ public bool IncludeCurrent { get; set; }
+
+ ///
+ /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
+ /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
+ ///
+ public bool UseObjectRows { get; set; }
+
+ public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+ if (!reader.ReadObject()) {
+ return null;
+ }
+ if (!reader.ReadPropertyName(out var prop)) {
+ return null;
+ }
+ if (!reader.Read()) {
+ return null;
+ }
+ if (prop == "added") {
+ var added = new List();
+ if (reader.TokenType != JsonTokenType.StartArray) {
+ return null;
+ }
+ while (true) {
+ if (!reader.Read()) {
+ break;
+ }
+ if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
+ return null;
+ }
+ switch (reader.TokenType) {
+ case JsonTokenType.String:
+ // DAMNIT! Need the schema.
+ break;
+ }
+ }
+ }
+ }
+
+ public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
+ using var _ = writer.WriteObject();
+ using (writer.WriteArray("schema")) {
+ foreach (var col in value.schema.columnInfo) {
+ using (writer.WriteObject()) {
+ writer.WriteString("name", col.colName);
+ writer.WriteNumber("index", col.colNo);
+ // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
+ writer.WriteString("type", col.colType.ToString());
+ }
+ }
+ }
+
+ var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
+ using (writer.WriteArray("added")) {
+ foreach (IBlueRow row in value.addedRows) {
+ using (writer.WriteArray()) {
+ for (int i = 0; i < cellWriters.Count; i++) {
+ cellWriters[i](row[i]);
+ }
+ }
+ }
+ }
+ using (writer.WriteArray("deleted")) {
+ foreach (IBlueRow row in value.deletedRows) {
+ using (writer.WriteArray()) {
+ for (int i = 0; i < cellWriters.Count; i++) {
+ cellWriters[i](row[i]);
+ }
+ }
+ }
+ }
+ if (IncludeCurrent) {
+ using (writer.WriteArray("current")) {
+ foreach (IBlueRow row in value.newRows) {
+ using (writer.WriteArray()) {
+ for (int i = 0; i < cellWriters.Count; i++) {
+ cellWriters[i](row[i]);
+ }
+ }
+ }
+ }
+ }
+
+ Action