check point (bc i don't trust googledrive sync yet)

This commit is contained in:
John Colvin committed 2024-02-09 23:06:26 -06:00
1 parent 82278a4e94
commit b8bfc1058d
8 files changed
+331 -242

No files matched your search

@@ -74,13 +74,13 @@ namespace BCG
//if it does, we are in serious trouble
private short myLookupTableGroup;
[JsonPropertyName("colNo")]
[JsonPropertyName("index")]
public int colNo{get{return myColNo;}}
[JsonPropertyName("colName")]
[JsonPropertyName("name")]
public string colName {get{return myColName;}}
[JsonPropertyName("colType")]
[JsonPropertyName("type")]
public Type colType {get{return myColType;}}
@@ -1,61 +1,63 @@
#nullable enable
using BCG;
// #nullable enable
// using BCG;
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
// using Spark.Core.Json;
namespace Bts.DataStreamCore;
// using System;
// using System.Text.Json;
// using System.Text.Json.Nodes;
// using System.Text.Json.Serialization;
class BlueRowJsonConverter : JsonConverter<IBlueRow> {
IBlueTableSchema? schema = null;
// namespace Bts.DataStreamCore;
public override IBlueRow Read(
ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (schema is null) {
throw new InvalidOperationException("Cannot read row from json because schema is null");
}
var mbr = new MutableBlueRow(schema);
int colNum = 0;
switch (reader.TokenType) {
case JsonTokenType.StartArray:
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
mbr.setValue(colNum++, readCellValue(ref reader));
colNum++;
}
break;
case JsonTokenType.StartObject:
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
if (reader.TokenType != JsonTokenType.PropertyName) {
throw new InvalidOperationException(
"Expected 'PropertyName' immediately after 'StartObject' token in row json");
}
var colName = reader.GetString();
if (!reader.Read()) {
throw new InvalidOperationException(
"Failed to read token immediately after 'PropertyName' token in row json");
}
mbr.setValue(colName, readCellValue(ref reader));
}
break;
}
if (mbr.Count != schema.columnInfo.Length) {
throw new InvalidOperationException(
$"Row json array length '{mbr.Count}' does not match the schema's " +
$"column length '{schema.columnInfo.Length}'.");
}
return mbr;
// class BlueRowJsonConverter : JsonConverter<IBlueRow> {
// public override IBlueRow Read(
// ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
// var row = JsonSerializer.Deserialize<JsonObject>(ref reader, options)!;
// var schemaj = row["schema"]!.GetValue<JsonObject>() ??
// throw new InvalidOperationException("Failed to read schema");
// // var columns = schemaj["columns"]
// var mbr = new MutableBlueRow(schema);
// int colNum = 0;
// switch (reader.TokenType) {
// case JsonTokenType.StartArray:
// while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) {
// mbr.setValue(colNum++, readCellValue(ref reader));
// colNum++;
// }
// break;
// case JsonTokenType.StartObject:
// while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) {
// if (reader.TokenType != JsonTokenType.PropertyName) {
// throw new InvalidOperationException(
// "Expected 'PropertyName' immediately after 'StartObject' token in row json");
// }
// var colName = reader.GetString();
// if (!reader.Read()) {
// throw new InvalidOperationException(
// "Failed to read token immediately after 'PropertyName' token in row json");
// }
// mbr.setValue(colName, readCellValue(ref reader));
// }
// break;
// }
// if (mbr.Count != schema.columnInfo.Length) {
// throw new InvalidOperationException(
// $"Row json array length '{mbr.Count}' does not match the schema's " +
// $"column length '{schema.columnInfo.Length}'.");
// }
// return mbr;
static object? readCellValue(ref Utf8JsonReader reader) =>
reader.TokenType switch {
JsonTokenType.String => reader.GetString(),
JsonTokenType.Number => reader.GetDouble(),
JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
_ => throw new InvalidOperationException("Failed to read cell value"),
};
}
// static object? readCellValue(ref Utf8JsonReader reader) =>
// reader.TokenType switch {
// JsonTokenType.String => reader.GetString(),
// JsonTokenType.Number => reader.GetDouble(),
// JsonTokenType.True or JsonTokenType.False => reader.GetBoolean(),
// _ => throw new InvalidOperationException("Failed to read cell value"),
// };
// }
public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
throw new NotImplementedException();
}
}
// public override void Write(Utf8JsonWriter writer, IBlueRow value, JsonSerializerOptions options) {
// throw new NotImplementedException();
// }
// }
@@ -1,154 +1,186 @@
#nullable enable
using BCG;
// #nullable enable
// using BCG;
using Spark.Core.Json;
// using Spark.Core.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
// using System;
// using System.Collections.Generic;
// using System.Text.Json;
// using System.Text.Json.Nodes;
// using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
// namespace Bts.DataStreamCore;
class DataPacketJsonConverter : JsonConverter<DataPacket> {
/// <summary>
/// Whether or not to include a field for current rows (i.e., snapshot of current state). By
/// default, only incremental changes are included ('added' and 'deleted').
/// </summary>
public bool IncludeCurrent { get; set; }
// public class DataStreamUpdateJsonConverter : JsonConverter<DataStreamUpdate> {
// /// <summary>
// /// Whether or not to include a field for current rows (i.e., snapshot of current state). By
// /// default, only incremental changes are included ('added' and 'deleted').
// /// </summary>
// public bool IncludeCurrent { get; set; }
/// <summary>
/// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
/// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
/// </summary>
public bool UseObjectRows { get; set; }
// /// <summary>
// /// Whether to treat rows as objects (name/value pairs), or arrays (list of values). Arrays
// /// result in smaller payloads, but aren't as easy to read as name/value pairs (for mere humans).
// /// </summary>
// public bool UseObjectRows { get; set; }
public override DataPacket? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
if (!reader.ReadObject()) {
return null;
}
if (!reader.ReadPropertyName(out var prop)) {
return null;
}
if (!reader.Read()) {
return null;
}
if (prop == "added") {
var added = new List<MutableBlueRow>();
if (reader.TokenType != JsonTokenType.StartArray) {
return null;
}
while (true) {
if (!reader.Read()) {
break;
}
if (reader.TokenType != JsonTokenType.StartArray || !reader.Read()) {
return null;
}
switch (reader.TokenType) {
case JsonTokenType.String:
// DAMNIT! Need the schema.
break;
}
}
}
}
// public override DataStreamUpdate? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
// var dpj = JsonSerializer.Deserialize<JsonObject>(ref reader);
// if (dpj is null) {
// return null;
// }
// var tableName = dpj["tableName"]!.AsString();
// var columns = dpj["columns"]!.AsArray()
// .Select((c, i) => {
// var dataType = c!["type"]!.AsEnum<DataStreamDataType>();
// return (info: new ColumnInfo(c["name"]!.AsString(), i, dataType.ToType()), dataType);
// })
// .ToArray();
// var schema = new BlueTableSchema(columns.Cut1().ToArray(), tableName);
// return new() {
// TableName = tableName,
// AddedRows = ToRows(dpj.GetArrayOrEmpty("added")).ToList(),
// DeletedRows = ToRows(dpj.GetArrayOrEmpty("deleted")).ToList(),
// Columns = [.. schema.columnInfo]
// };
public override void Write(Utf8JsonWriter writer, DataPacket value, JsonSerializerOptions options) {
using var _ = writer.WriteObject();
using (writer.WriteArray("schema")) {
foreach (var col in value.schema.columnInfo) {
using (writer.WriteObject()) {
writer.WriteString("name", col.colName);
writer.WriteNumber("index", col.colNo);
// TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
writer.WriteString("type", col.colType.ToString());
}
}
}
// IEnumerable<IBlueRow> ToRows(JsonArray rows) {
// foreach (var row in rows) {
// var mbr = new MutableBlueRow(schema);
// switch (row!.GetValueKind()) {
// case JsonValueKind.Object:
// var rowObj = row.AsObject();
// for (int i = 0; i < columns.Length; i++) {
// SetCellValue(mbr, i, rowObj[columns[i].info.colName]!, columns[i].dataType);
// }
// break;
// case JsonValueKind.Array:
// var rowArr = row.AsArray();
// for (int i = 0; i < columns.Length; i++) {
// SetCellValue(mbr, i, rowArr[i]!, columns[i].dataType);
// }
// break;
// }
// yield return mbr;
// }
// }
var cellWriters = value.schema.columnInfo.Select(getCellWriter).ToList();
using (writer.WriteArray("added")) {
foreach (IBlueRow row in value.addedRows) {
using (writer.WriteArray()) {
for (int i = 0; i < cellWriters.Count; i++) {
cellWriters[i](row[i]);
}
}
}
}
using (writer.WriteArray("deleted")) {
foreach (IBlueRow row in value.deletedRows) {
using (writer.WriteArray()) {
for (int i = 0; i < cellWriters.Count; i++) {
cellWriters[i](row[i]);
}
}
}
}
if (IncludeCurrent) {
using (writer.WriteArray("current")) {
foreach (IBlueRow row in value.newRows) {
using (writer.WriteArray()) {
for (int i = 0; i < cellWriters.Count; i++) {
cellWriters[i](row[i]);
}
}
}
}
}
// static void SetCellValue(MutableBlueRow mbr, int i, JsonNode cell, DataStreamDataType dataType) {
// switch (dataType) {
// case DataStreamDataType.String:
// mbr.setValue(i, cell.GetValue<string>());
// break;
// case DataStreamDataType.Int32:
// mbr.setValue(i, cell.GetValue<int>());
// break;
// case DataStreamDataType.Int64:
// mbr.setValue(i, cell.GetValue<long>());
// break;
// case DataStreamDataType.Single:
// mbr.setValue(i, cell.GetValue<float>());
// break;
// case DataStreamDataType.Double:
// mbr.setValue(i, cell.GetValue<double>());
// break;
// }
// }
// }
Action<object?> getCellWriter(ColumnInfo colInfo) {
var writeVal = getCellValueWriter(colInfo);
if (!UseObjectRows) {
return writeVal;
}
return v => {
writer.WritePropertyName(colInfo.colName);
writeVal(v);
};
}
// public override void Write(Utf8JsonWriter writer, DataStreamUpdate value, JsonSerializerOptions options) {
// using var _ = writer.WriteObject();
// writer.WriteString("tableName", value.TableName);
// using (writer.WriteArray("columns")) {
// foreach (var col in value.Columns.empty_if_null()) {
// using (writer.WriteObject()) {
// writer.WriteString("name", col.colName);
// writer.WriteNumber("index", col.colNo);
// // TODO: use language agnostic type names (e.g., 'string', 'integer', 'float', 'date' etc.)
// writer.WriteString("type", col.colType.ToString());
// }
// }
// }
Action<object?> getCellValueWriter(ColumnInfo colInfo) {
var t = colInfo.colType;
if (t == typeof(int)) {
return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
}
if (t == typeof(uint)) {
return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
}
if (t == typeof(long)) {
return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
}
if (t == typeof(ulong)) {
return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
}
if (t == typeof(short)) {
return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
}
if (t == typeof(ushort)) {
return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
}
if (t == typeof(double)) {
return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
}
if (t == typeof(float)) {
return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
}
if (t == typeof(bool)) {
return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
}
return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
// var cellWriters = value.Columns!.Select(getCellWriter).ToList();
// using (writer.WriteArray("added")) {
// foreach (IBlueRow row in value.AddedRows!) {
// using (writer.WriteArray()) {
// for (int i = 0; i < cellWriters.Count; i++) {
// cellWriters[i](row[i]);
// }
// }
// }
// }
// using (writer.WriteArray("deleted")) {
// foreach (IBlueRow row in value.DeletedRows!) {
// using (writer.WriteArray()) {
// for (int i = 0; i < cellWriters.Count; i++) {
// cellWriters[i](row[i]);
// }
// }
// }
// }
// if (IncludeCurrent) {
// using (writer.WriteArray("current")) {
// foreach (IBlueRow row in value.newRows) {
// using (writer.WriteArray()) {
// for (int i = 0; i < cellWriters.Count; i++) {
// cellWriters[i](row[i]);
// }
// }
// }
// }
// }
Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
v => {
if (v is null) {
writer.WriteNullValue();
} else {
writeNonNull(v);
}
};
}
}
}
// Action<object?> getCellWriter(ColumnInfo colInfo) {
// var writeVal = getCellValueWriter(colInfo);
// if (!UseObjectRows) {
// return writeVal;
// }
// return v => {
// writer.WritePropertyName(colInfo.colName);
// writeVal(v);
// };
// }
// Action<object?> getCellValueWriter(ColumnInfo colInfo) {
// var t = colInfo.colType;
// if (t == typeof(int)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((int)v));
// }
// if (t == typeof(uint)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((uint)v));
// }
// if (t == typeof(long)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((long)v));
// }
// if (t == typeof(ulong)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((ulong)v));
// }
// if (t == typeof(short)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((short)v));
// }
// if (t == typeof(ushort)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((ushort)v));
// }
// if (t == typeof(double)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((double)v));
// }
// if (t == typeof(float)) {
// return nullCheckWrapper(v => writer.WriteNumberValue((float)v));
// }
// if (t == typeof(bool)) {
// return nullCheckWrapper(v => writer.WriteBooleanValue((bool)v));
// }
// return nullCheckWrapper(v => writer.WriteStringValue(v?.ToString()));
// Action<object?> nullCheckWrapper(Action<object> writeNonNull) =>
// v => {
// if (v is null) {
// writer.WriteNullValue();
// } else {
// writeNonNull(v);
// }
// };
// }
// }
// }
@@ -4,6 +4,7 @@
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<TargetFramework>net8.0</TargetFramework>
<Nullable>disable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
@@ -0,0 +1,11 @@
namespace Bts.DataStreamCore;
public enum DataStreamDataType {
Any,
Bytes,
String,
Int32,
Int64,
Single,
Double
}
@@ -0,0 +1,42 @@
#nullable enable
using Bts.DataStreamCore;
public static class DataStreamExtensions {
public static Type ToType(this DataStreamDataType dataType) =>
dataType switch {
DataStreamDataType.Any => typeof(object),
DataStreamDataType.String => typeof(string),
DataStreamDataType.Int32 => typeof(int),
DataStreamDataType.Int64 => typeof(long),
DataStreamDataType.Single => typeof(float),
DataStreamDataType.Double => typeof(double),
_ => throw new ArgumentException($"DataType '{dataType}' not supported.", nameof(dataType)),
};
public static DataStreamDataType FromType(Type type) {
if (type == typeof(object)) {
return DataStreamDataType.Any;
}
if (type == typeof(string)) {
return DataStreamDataType.String;
}
if (type == typeof(int)) {
return DataStreamDataType.Int32;
}
if (type == typeof(long)) {
return DataStreamDataType.Int64;
}
if (type == typeof(float)) {
return DataStreamDataType.Single;
}
if (type == typeof(double)) {
return DataStreamDataType.Double;
}
if (type == typeof(byte[])) {
return DataStreamDataType.Bytes;
}
throw new ArgumentException(
$"Unable to convert type '{type}' to {typeof(DataStreamDataType)}");
}
}
@@ -1,41 +1,41 @@
#nullable enable
using BCG;
using Spark.Core.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
class ViewStreamUpdate {
[JsonPropertyName("tableName")]
public string? TableName { get; set; }
[JsonPropertyName("columns")]
public List<ColumnInfo>? Columns { get; set; }
[JsonPropertyName("addedRows")]
public List<Dictionary<string, object?>>? AddedRows { get; set; }
[JsonPropertyName("deletedRows")]
public List<Dictionary<string, object?>>? DeletedRows { get; set; }
public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
var schema = new BlueTableSchema([.. Columns!], TableName);
var addedRowsTable = new BlueTable(schema);
var deletedRowsTable = new BlueTable(schema);
if (applyChanges) {
state.deleteRows(deletedRowsTable);
state.addRows(addedRowsTable);
}
return new(state, addedRowsTable, deletedRowsTable);
}
public static string ToJson() {
return "";
}
}
#nullable enable
using BCG;
using Spark.Core.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Bts.DataStreamCore;
public class DataStreamUpdate {
[JsonPropertyName("tableName")]
public string? TableName { get; set; }
[JsonPropertyName("columns")]
public List<ColumnInfo>? Columns { get; set; }
[JsonPropertyName("addedRows")]
public List<IBlueRow>? AddedRows { get; set; }
[JsonPropertyName("deletedRows")]
public List<IBlueRow>? DeletedRows { get; set; }
public DataPacket ToDataPacket(BlueTable state, bool applyChanges = true) {
var schema = new BlueTableSchema([.. Columns!], TableName);
var addedRowsTable = new BlueTable(schema);
var deletedRowsTable = new BlueTable(schema);
if (applyChanges) {
state.deleteRows(deletedRowsTable);
state.addRows(addedRowsTable);
}
return new(state, addedRowsTable, deletedRowsTable);
}
public static string ToJson() {
return "";
}
}
@@ -0,0 +1 @@
using System.Linq;