From 4db9cdd357e15015b4a2115d1f20800caefb8d00 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 22 Sep 2025 00:42:59 -0400 Subject: [PATCH] feat: Add DiffStream to compute row diffs (like pandas) --- .../DataStream/BlueOrb/DiffStream.cs | 163 ++++++++++++++++++ .../DataStream/BlueOrb/DiffTable.cs | 101 +++++++++++ .../DataStream/DataStreamExtensions.cs | 3 + .../DataStream/Primitives/Diff/DiffStream.cs | 2 +- .../DataStreamCore/BlueData/ColumnInfo.cs | 9 +- 5 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 src/DataStreamSuite/DataStream/BlueOrb/DiffStream.cs create mode 100644 src/DataStreamSuite/DataStream/BlueOrb/DiffTable.cs diff --git a/src/DataStreamSuite/DataStream/BlueOrb/DiffStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/DiffStream.cs new file mode 100644 index 000000000..bc48c2b28 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/DiffStream.cs @@ -0,0 +1,163 @@ +namespace bts.BlueOrb; + +using BCG; + +using Bts.DataStream; +using Bts.DataStreamCore; + +using Spark.Core.Collections; +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; + +public class DiffStream : ViewStream, IDisposable { + readonly ViewStream source; + readonly int period; + IBlueTableSchema? outputSchema; + IDisposable? sub; + BlueTableImmutable? lastFired; + + public DiffStream(ViewStream source, int period) { + this.source = source; + this.period = period; + } + + ~DiffStream() => + Dispose(); + + public void Dispose() => + Disposable.dispose(ref sub); + + protected override void myOnFirstSubscriber() { + Log.isnull(sub); + sub = source.Subscribe(OnSourceChange); + + void OnSourceChange(IDataPacket dp) { + if (outputSchema is null) { + var inputSchema = dp.Schema; + var outputCols = new ColumnInfo[inputSchema.columnInfo.Length]; + for (int i = 0; i < inputSchema.columnInfo.Length; i++) { + outputCols[i] = GetOutputColumn(inputSchema.columnInfo[i]); + } + outputSchema = new BlueTableSchema($"{inputSchema.tableName}_Diff{period}", outputCols); + } + lastFired ??= new(outputSchema); + var newTable = new DiffTable(dp.newRows, period); + var allRows = new List(); + foreach (IBlueRow r in newTable) { + var rvals = new object?[r.schema.columnInfo.Length]; + int x = 0; + foreach (var col in r.schema.columnInfo) { + var val = r[col.colNo]; + rvals[x++] = val; + } + allRows.Add(rvals); + } + var bd = lastFired.ToDataPacketBuilder(); + var removedIndices = new List(); + var addedIndices = new List(); + var ops = new List<(int Index, bool IsAdd)>(); + foreach (var r in dp.deletedRows) { + int id = r.Id(); + var rOut = lastFired[id]; + var idx = lastFired.IndexOf(rOut); + Log.assert(idx >= 0 && idx < lastFired.Count); + removedIndices.Add(idx); + ops.Add((idx, false)); + } + foreach (var r in dp.addedRows) { + var idx = dp.newRows.IndexOf(r); + Log.assert(idx >= 0 && idx < dp.newRows.Count); + addedIndices.Add(idx); + ops.Add((idx, true)); + } + ops.Sort(Comparer<(int Index, bool IsAdd)>.Create((a, b) => a.Index.CompareTo(b.Index))); + int rmCount = 0; + var removedRows = new Dictionary(); + for (int i = 0; i < ops.Count; i++) { + var op = ops[i]; + if (op.IsAdd) { + continue; + } + rmCount++; + int j = op.Index/* - rmCount*/; + var stop = Math.Min(j + period, lastFired.Count/* - rmCount*/); + // Remove the row that was removed in the incoming DataPacket. + bd.Remove(lastFired.GetRow(j).Id()); + for (j++; j < stop; j++) { + // Remove rows that changed as a result. We will add the updated versions + // of these rows in the next pass. + var r = lastFired.GetRow(j); + bd.Remove(r.Id()); + } + } + int addCount = ops.Count - rmCount; + for (int i = ops.Count - 1; i >= 0; i--) { + var op = ops[i]; + if (op.IsAdd) { + addCount--; + int j = op.Index; + var stop = Math.Min(j + period, newTable.Count); + for (; j < stop; j++) { + var r = newTable.GetRow(j); + bd.AddOrUpdate(r); + } + } else { + rmCount--; + int j = op.Index - rmCount + addCount; + var stop = Math.Min(j + period, newTable.Count); + for (; j < stop; j++) { + // Add rows that changed due to the removed row (the row with which they compute + // the diff has changed). + var r = newTable.GetRow(j); + bd.AddOrUpdate(r); + } + } + } + var dpOut = bd.ToDataPacket(out lastFired, false); + myFireEvent(dpOut); + } + + ColumnInfo GetOutputColumn(ColumnInfo inputColumn) { + return inputColumn.type_code() switch { + TypeCode.Boolean => inputColumn.Clone(), + TypeCode.Byte or TypeCode.SByte or TypeCode.Int16 => WithType(TypeCode.Int32), + TypeCode.UInt32 or TypeCode.UInt64 => WithType(TypeCode.Int64), + _ => inputColumn.Clone(), + }; + + ColumnInfo WithType(TypeCode typeCode) => + new(inputColumn.colName, inputColumn.colNo, typeCode); + } + } + + protected override void myOnLastSubscriber() => + Disposable.dispose(ref sub); + + static class ExpressionBuilder { + public static Func Int32DiffExpression(int colIndex) { + var colIndexParam = Expression.Constant(colIndex, typeof(int)); + var rowParam = Expression.Parameter(typeof(IBlueRow), "row"); + var getValMethod = typeof(ExpressionBuilder).GetMethod(nameof(getInt32Value)) + ?? throw new InvalidOperationException("Failed to find int32 value getter via reflection"); + var getVal = Expression.Lambda>(Expression.Call(null, getValMethod, rowParam, colIndexParam), rowParam).Compile(); + return ComputeDiff; + + int? ComputeDiff(IBlueRow leftRow, IBlueRow rightRow) { + var leftVal = getVal(leftRow); + var rightVal = getVal(rightRow); + if (leftVal.HasValue && rightVal.HasValue) { + return leftVal.Value - rightVal.Value; + } + return null; + } + } + + static int? getInt32Value(IBlueRow row, int colIndex) { + var obj = row[colIndex]; + return obj as int?; + } + } +} diff --git a/src/DataStreamSuite/DataStream/BlueOrb/DiffTable.cs b/src/DataStreamSuite/DataStream/BlueOrb/DiffTable.cs new file mode 100644 index 000000000..d1e86bc53 --- /dev/null +++ b/src/DataStreamSuite/DataStream/BlueOrb/DiffTable.cs @@ -0,0 +1,101 @@ +namespace bts.BlueOrb; + +using BCG; + +using bts.collections; + +using Bts.DataStreamCore; + +using Spark.Core.Collections; + +using System; +using System.Collections.Generic; +using System.Linq; + +public class DiffTable : IBlueRowCollection { + readonly IBlueRowCollection input; + readonly int period; + readonly IList[] columns; + + public DiffTable(IBlueRowCollection input, int period) { + this.input = input; + this.period = period; + var colInfos = input.schema.columnInfo.OrderBy(c => c.colNo).ToArray(); + if (colInfos.Length > 0) { + Log.assert(colInfos[0].colNo == 0 && colInfos[^1].colNo == colInfos.Length - 1); + } + columns = colInfos.WrapSelector(c => GetColumn(c.colNo), null).ToArray(); + } + + public IBlueRow this[int id] => + GetRow(input.IndexOf(input[id])); + + public IBlueTableSchema schema => input.schema; + + public int Count => input.Count; + + public bool containsID(int id) => + input.containsID(id); + + public IList GetColumn(int colIndex) { + var count = input.Count; + var colInfo = input.schema.columnInfo[colIndex]; + if (colInfo.colName == "id") { + return new StepList(0, 1, count).as_mutable().WrapSelector(GetId, null); + } + var typeCode = colInfo.type_code(); + return new StepList(0, 1, count).as_mutable().WrapSelector(GetValue, null); + + object? GetId(int rowIndex) { + var row = input.GetRow(rowIndex); + return row.Id(); + } + + object? GetValue(int rowIndex) { + var rightIndex = rowIndex - period; + if (rowIndex < 0 || rowIndex >= count || rightIndex < 0 || rightIndex >= count) { + return null; + } + var leftRow = input.GetRow(rowIndex); + var rightRow = input.GetRow(rightIndex); + return GetDiff(leftRow[colIndex], rightRow[colIndex], typeCode); + } + } + + public IEnumerator GetEnumerator() { + var count = input.Count; + for (int i = 0; i < count; i++) { + yield return GetRow(i); + } + } + + public IBlueRow GetRow(int index) => + new BlueRowColumnWrapper(input.schema, index, columns); + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + var inputRow = input[id]; + int idx = input.IndexOf(inputRow); + if (idx != -1) { + var row = GetRow(idx); + if (BlueRowEqualityComparer.Equals(item, row)) { + return idx; + } + } + return -1; + } + + static object? GetDiff(object? leftVal, object? rightVal, TypeCode colType) { + if (leftVal is null || rightVal is null) { + return null; + } + return colType switch { + TypeCode.Int32 => (int)leftVal - (int)rightVal, + TypeCode.Int64 => (long)leftVal - (long)rightVal, + TypeCode.Double => (double)leftVal - (double)rightVal, + TypeCode.String => ((string)leftVal).Replace((string)rightVal, string.Empty), + TypeCode.DateTime => new DateTime(((DateTime)leftVal).Ticks - ((DateTime)rightVal).Ticks), + _ => null, + }; + } +} diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index 3805c1f98..10a8ecbed 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -262,4 +262,7 @@ public static class DataStreamExtensions { }; } } + + public static ViewStream Diff(this ViewStream source, int period = 1) => + new bts.BlueOrb.DiffStream(source, period); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs index 85f95e9ee..83f36a843 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs @@ -6,7 +6,7 @@ namespace BCG { - [DataStreamUserRespresentation("diff")] + [DataStreamUserRespresentation("oldDiff")] public class DiffStream : ViewStream { ViewStream myLStream, myRStream; diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs index 089612243..c3c96b401 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -1,4 +1,6 @@ #nullable enable +using bts; + using System; using System.Text.Json.Serialization; @@ -7,7 +9,7 @@ namespace BCG { /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable /// [Serializable] - public class ColumnInfo : IEquatable, IComparable, IComparable { + public class ColumnInfo : IEquatable, IComparable, IComparable, ICloneable { //the default column info does not cach values in a lookup table, therefore it is in group 0 public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) { @@ -22,6 +24,8 @@ namespace BCG { } public static bool isAllowedType(Type type) => + type == typeof(bool) || + type == typeof(byte) || type == typeof(int) || type == typeof(long) || type == typeof(double) || @@ -34,6 +38,9 @@ namespace BCG { public int CompareTo(object? obj) => CompareTo(obj as ColumnInfo); + public ColumnInfo Clone() => + new(colName, colNo, colType); + public bool Equals(ColumnInfo? other) => other is not null && colNo == other.colNo