DataStreamCore Json stuff

This commit is contained in:
John Colvin committed 2024-02-08 19:04:11 -06:00
1 parent b0d90fc0e2
commit d4f02480df
3 files changed
+318 -95

No files matched your search

@@ -1,95 +1,103 @@
using System;
namespace BCG
{
/// <summary>
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
/// </summary>
[Serializable]
public class ColumnInfo : IComparable
{
public static bool isAllowedType (Type t)
{
return
t.Equals (typeof (System.Int32)) ||
t.Equals (typeof (System.Int64)) ||
t.Equals (typeof (System.Double)) ||
t.Equals (typeof (System.String)) ||
t.Equals (typeof (System.DateTime)) ? true :
false;
}
//the default column info does not cach values in a lookup table, therefore it is in group 0
public ColumnInfo(string colName, int colNo, Type colType)
: this(colName, colNo, colType, 0)
{
}
public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
{
myColName = colName;
myColNo = colNo;
myColType = colType;
myLookupTableGroup = colGroup;
myColValue = null;
}
public ColumnInfo (string colName, int colNo, object colValue)
: this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
public ColumnInfo (string colName, int colNo, Type colType, object colValue)
: this (colName, colNo, colType)
{
myColValue = colValue == null ? DBNull.Value : colValue;
}
public int CompareTo (object obj)
{
int otherColNo = ((ColumnInfo)(obj)).colNo;
if (myColNo < otherColNo)
return -1;
else if (myColNo > otherColNo)
return 1;
return 0;
}
//public bool isCalculated{get{return myExpression != null;}}
public bool isConstant {get{return myColValue != null;}}
private int myColNo;
private string myColName;
private Type myColType;
//private string myExpression;
private object myColValue;
//no table will ever have more than 65,536 columns
//if it does, we are in serious trouble
private short myLookupTableGroup;
public int colNo{get{return myColNo;}}
public string colName {get{return myColName;}}
public Type colType {get{return myColType;}}
//this value corresponds to how the data in the column is serialized.
//if the group number is 0, the values are not stored in a table
//if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
//are stored together in a lookup table
public short lookupTableGroup
{
get { return myLookupTableGroup; }
set { myLookupTableGroup = value; }
}
//public string expression {get{return myExpression;}}
public object colValue {get{return myColValue;}}
}
}
using System;
using System.Text.Json.Serialization;
namespace BCG
{
/// <summary>
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
/// </summary>
[Serializable]
public class ColumnInfo : IComparable
{
public static bool isAllowedType (Type t)
{
return
t.Equals (typeof (System.Int32)) ||
t.Equals (typeof (System.Int64)) ||
t.Equals (typeof (System.Double)) ||
t.Equals (typeof (System.String)) ||
t.Equals (typeof (System.DateTime)) ? true :
false;
}
//the default column info does not cach values in a lookup table, therefore it is in group 0
public ColumnInfo(string colName, int colNo, Type colType)
: this(colName, colNo, colType, 0)
{
}
public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
{
myColName = colName;
myColNo = colNo;
myColType = colType;
myLookupTableGroup = colGroup;
myColValue = null;
}
public ColumnInfo (string colName, int colNo, object colValue)
: this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
public ColumnInfo (string colName, int colNo, Type colType, object colValue)
: this (colName, colNo, colType)
{
myColValue = colValue == null ? DBNull.Value : colValue;
}
public int CompareTo (object obj)
{
int otherColNo = ((ColumnInfo)(obj)).colNo;
if (myColNo < otherColNo)
return -1;
else if (myColNo > otherColNo)
return 1;
return 0;
}
//public bool isCalculated{get{return myExpression != null;}}
public bool isConstant {get{return myColValue != null;}}
private int myColNo;
private string myColName;
private Type myColType;
//private string myExpression;
private object myColValue;
//no table will ever have more than 65,536 columns
//if it does, we are in serious trouble
private short myLookupTableGroup;
[JsonPropertyName("colNo")]
public int colNo{get{return myColNo;}}
[JsonPropertyName("colName")]
public string colName {get{return myColName;}}
[JsonPropertyName("colType")]
public Type colType {get{return myColType;}}
//this value corresponds to how the data in the column is serialized.
//if the group number is 0, the values are not stored in a table
//if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
//are stored together in a lookup table
[JsonIgnore]
public short lookupTableGroup
{
get { return myLookupTableGroup; }
set { myLookupTableGroup = value; }
}
//public string expression {get{return myExpression;}}
[JsonIgnore]
public object colValue {get{return myColValue;}}
}
}
@@ -0,0 +1,61 @@
#nullable enable
using BCG;
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
class BlueRowJsonConverter : JsonConverter<IBlueRow> {
IBlueTableSchema? schema = null;
public override IBlueRow Read(
ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (schema is null) {
throw new InvalidOperationException("Cannot read row from json because schema is null");
}
var mbr = new MutableBlueRow(schema);
int colNum = 0;
switch (reader.TokenType) {
case JsonTokenType.StartArray:
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
mbr.setValue(colNum++, readCellValue(ref reader));
colNum++;
}
break;
case JsonTokenType.StartObject:
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
if (reader.TokenType != JsonTokenType.PropertyName) {
throw new InvalidOperationException(
"Expected 'PropertyName' immediately after 'StartObject' token in row json");
}
var colName = reader.GetString();
if (!reader.Read()) {
throw new InvalidOperationException(
"Failed to read token immediately after 'PropertyName' token in row json");
}
mbr.setValue(colName, readCellValue(ref reader));
}
break;
}
if (mbr.Count != schema.columnInfo.Length) {
throw new InvalidOperationException(
$"Row json array length '{mbr.Count}' does not match the schema's " +
$"column length '{schema.columnInfo.Length}'.");
}
return mbr;
static object? readCellValue(ref Utf8JsonReader reader) =>
reader.TokenType switch {
JsonTokenType.String => reader.GetString(),
JsonTokenType.Number => reader.GetDouble(),
JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
_ => throw new InvalidOperationException("Failed to read cell value"),
};
}
public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
throw new NotImplementedException();
}
}
@@ -0,0 +1,154 @@
#nullable enable
using BCG;
using Spark.Core.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
class DataPacketJsonConverter : JsonConverter<DataPacket> {
/// <summary>
/// Whether or not to include a field for current rows (i.e., snapshot of current state). By
/// default, only incremental changes are included ('added' and 'deleted').
/// </summary>
public bool IncludeCurrent { get; set; }
/// <summary>
/// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
/// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
/// </summary>
public bool UseObjectRows { get; set; }
public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (!reader.ReadObject()) {
return null;
}
if (!reader.ReadPropertyName(out var prop)) {
return null;
}
if (!reader.Read()) {
return null;
}
if (prop == "added") {
var added = new List<MutableBlueRow>();
if (reader.TokenType != JsonTokenType.StartArray) {
return null;
}
while (true) {
if (!reader.Read()) {
break;
}
if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
return null;
}
switch (reader.TokenType) {
case JsonTokenType.String:
// DAMNIT! Need the schema.
break;
}
}
}
}
public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
using var _ = writer.WriteObject();
using (writer.WriteArray("schema")) {
foreach (var col in value.schema.columnInfo) {
using (writer.WriteObject()) {
writer.WriteString("name", col.colName);
writer.WriteNumber("index", col.colNo);
// TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
writer.WriteString("type", col.colType.ToString());
}
}
}
var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
using (writer.WriteArray("added")) {
foreach (IBlueRow row in value.addedRows) {
using (writer.WriteArray()) {
for (int i = 0; i < cellWriters.Count; i++) {
cellWriters[i](row[i]);
}
}
}
}
using (writer.WriteArray("deleted")) {
foreach (IBlueRow row in value.deletedRows) {
using (writer.WriteArray()) {
for (int i = 0; i < cellWriters.Count; i++) {
cellWriters[i](row[i]);
}
}
}
}
if (IncludeCurrent) {
using (writer.WriteArray("current")) {
foreach (IBlueRow row in value.newRows) {
using (writer.WriteArray()) {
for (int i = 0; i < cellWriters.Count; i++) {
cellWriters[i](row[i]);
}
}
}
}
}
Action<object?> getCellWriter(ColumnInfo colInfo) {
var writeVal = getCellValueWriter(colInfo);
if (!UseObjectRows) {
return writeVal;
}
return v => {
writer.WritePropertyName(colInfo.colName);
writeVal(v);
};
}
Action<object?> getCellValueWriter(ColumnInfo colInfo) {
var t = colInfo.colType;
if (t == typeof(int)) {
return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
}
if (t == typeof(uint)) {
return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
}
if (t == typeof(long)) {
return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
}
if (t == typeof(ulong)) {
return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
}
if (t == typeof(short)) {
return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
}
if (t == typeof(ushort)) {
return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
}
if (t == typeof(double)) {
return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
}
if (t == typeof(float)) {
return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
}
if (t == typeof(bool)) {
return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
}
return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
v => {
if (v is null) {
writer.WriteNullValue();
} else {
writeNonNull(v);
}
};
}
}
}