diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionEqualityComparer.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionEqualityComparer.cs new file mode 100644 index 000000000..62e77b85a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionEqualityComparer.cs @@ -0,0 +1,35 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public class BlueRowCollectionEqualityComparer : IEqualityComparer { + readonly IEqualityComparer rowComparer = BlueRowEqualityComparer.Instance; + + public static BlueRowCollectionEqualityComparer Instance { get; } = new(); + + public bool Equals(IBlueRowCollection? x, IBlueRowCollection? y) { + if (x is null) { + return y is null; + } + if (x.Count != y?.Count) { + return false; + } + return x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) + && x.Cast().OrderBy(r => r.Id()).SequenceEqual( + y.Cast().OrderBy(r => r.Id()), + rowComparer); + } + + public int GetHashCode([DisallowNull] IBlueRowCollection rows) { + var hash = new HashCode(); + hash.Add(rows.schema.columnInfo.Length); + hash.Add(rows.Count); + return hash.ToHashCode(); + } +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/DataPacketEqualityComparer.cs b/src/DataStreamSuite/DataStreamCore/BlueData/DataPacketEqualityComparer.cs new file mode 100644 index 000000000..6919d9caa --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/DataPacketEqualityComparer.cs @@ -0,0 +1,37 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public class DataPacketEqualityComparer : IEqualityComparer { + readonly IEqualityComparer rowCollectionComparer; + + public DataPacketEqualityComparer(IEqualityComparer? rowCollectionComparer = null) => + this.rowCollectionComparer = rowCollectionComparer ?? BlueRowCollectionEqualityComparer.Instance; + + public static DataPacketEqualityComparer Instance { get; } = new(); + + public bool Equals(DataPacket? x, DataPacket? y) { + if (x is null) { + return y is null; + } + return y is not null + && x.schema.columnInfo.SequenceEqual(y.schema.columnInfo) + && rowCollectionComparer.Equals(x.addedRows, y.addedRows) + && rowCollectionComparer.Equals(x.deletedRows, y.deletedRows) + && rowCollectionComparer.Equals(x.newRows, y.newRows); + } + + public int GetHashCode([DisallowNull] DataPacket obj) { + var hash = new HashCode(); + hash.Add(obj.schema.columnNames.Length); + hash.Add(obj.addedRows.Count); + hash.Add(obj.deletedRows.Count); + return hash.ToHashCode(); + } +} diff --git a/src/DataStreamSuite/DataStreamTests/FilterStreamTests.cs b/src/DataStreamSuite/DataStreamTests/FilterStreamTests.cs new file mode 100644 index 000000000..f49d63bab --- /dev/null +++ b/src/DataStreamSuite/DataStreamTests/FilterStreamTests.cs @@ -0,0 +1,160 @@ +namespace Bts.DataStreamTests; + +using BCG; + +using bts.collections; + +using Bts.DataStreamCore; + +using System.Collections; + +[TestFixture] +public class FilterStreamTests { + readonly BlueTableSchema schema = new( + new ColumnInfo[] { + new("id", 0, TypeCode.Int32), + new("value", 1, TypeCode.Double), + }, "test_table"); + + [Test] + public void TestAll() { + var driverStream = new TestDriverStream(schema, null); + var filterStream = new FilterStream(driverStream, "value<15"); + var expected = new Outcome(schema); + var actual = new List(); + filterStream.onChange += onFilterStreamChange; + + void onFilterStreamChange(object newValue) { + var dp = (DataPacket)newValue; + actual.Add(new(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows))); + } + + // Expected changes: + // -------------------------------------------------------------------------- + // | Step | Remove | Add | Included | Excluded | + // |------------------------------------------------------------------------| + // | 1 | | (1, 09) | | | + // | | | (2, 11) | (1, 9), (2, 11) | | + // | 2 | (2, 11) | *(2, 15) | (1, 9) | *(2, 15) | + // | 3 | (1, 09) | (1, 08) | (1, 8) | *(2, 15) | + // | 4 | (1, 08) | (1, 07) | | | + // | | *(2, 15) | (2, 09) | (1, 7), (2, 09) | | + // | 5 | | (3, 03) | (1, 7), (2, 09), (3, 03) | | + // | 6 | | *(4, 17) | (1, 7), (2, 09), (3, 03) | *(4, 17) | + // | 7 | (2, 09) | | (1, 7), (3, 03) | *(4, 17) | + // | 8 | *(4, 17) | | (1, 7), (3, 03) | | + // -------------------------------------------------------------------------- + // * This value should be excluded by the filter + + // 1) + expected.Add(addedRows: new[] { (1, 9), (2, 11) }); + // 2) + expected.Add(removedRows: new[] { (2, 11) }); + // 3) + expected.Add(removedRows: new[] { (1, 9) }, addedRows: new[] { (1, 8) }); + // 4) + expected.Add(removedRows: new[] { (1, 8) }, addedRows: new[] { (1, 7), (2, 9) }); + // 5) + expected.Add(addedRows: new[] { (3, 3) }); + // 6) * No change in items that pass filter + // 7) + expected.Add(removedRows: new[] { (2, 9) }); + // 8) * No change in items that pass filter + + // 'skipNum' tracks the number of ops that don't change the filtered rows + // (i.e., adding/removing items that are filtered out). After each step, + // we assert the output table has fired 'stepNum - skipNum' change events. + // After all steps have been performed, verify the output DataPackets have + // the correct changes (and were fired in the correct order). + int stepNum = 1; + int skipNum = 0; + // 1) + setInputRows((1, 9), (2, 11)); + // 2) + setInputRows((1, 9), (2, 15)); + // 3) + setInputRows((1, 8), (2, 15)); + // 4) + setInputRows((1, 7), (2, 9)); + // 5) + setInputRows((1, 7), (2, 9), (3, 3)); + // 6) * Add filtered item + skipNum++; + setInputRows((1, 7), (2, 9), (3, 3), (4, 17)); + // 7) + setInputRows((1, 7), (3, 3), (4, 17)); + // 8) * Remove filtered item + skipNum++; + setInputRows(new[] { (1, 7), (3, 3) }); + // All changes have been pushed through the input. Assert the output DataPackets are correct. + Assert.That(actual.SequenceEqual(expected, DataPacketEqualityComparer.Instance)); + + void setInputRows(params (int, int)[] rows) { + driverStream.Reset(rows.Select(getBlueRow)); + Assert.That(actual, Has.Count.EqualTo(stepNum - skipNum)); + stepNum++; + } + } + + static IBlueRow getBlueRow(IBlueTableSchema schema, (int Id, int Value) rowTup) { + var r = new MutableBlueRow(schema); + r.setValue("id", rowTup.Id); + r.setValue("value", rowTup.Value); + return r; + } + + static BlueTable getBlueTable(IBlueTableSchema schema, IEnumerable<(int Id, int Value)>? rows = null) { + var bt = new BlueTable(schema); + bt.addRows(rows.empty_if_null().Select(r => getBlueRow(schema, r))); + return bt; + } + + IBlueRow getBlueRow((int Id, int Value) rowTup) => + getBlueRow(schema, rowTup); + + /// + /// Test outcome, represented as a sequence of DataPackets. + /// + class Outcome : IReadOnlyList { + readonly IBlueTableSchema schema; + readonly List history = new(); + BlueTable? currBlueRows; + + public Outcome(IBlueTableSchema schema) => + this.schema = schema; + + public int Count => history.Count; + + public DataPacket? this[int index] => history[index]; + + public DataPacket? Add( + IEnumerable<(int Id, int Value)>? addedRows = null, + IEnumerable<(int Id, int Value)>? removedRows = null + ) { + if (addedRows is null && removedRows is null) { + history.Add(null); + return null; + } + addedRows ??= Array.Empty<(int, int)>(); + removedRows ??= Array.Empty<(int, int)>(); + var newBlueRows = new BlueTable(schema); + if (currBlueRows is not null) { + newBlueRows.addRows(currBlueRows); + } + var addedBlueRows = getBlueTable(schema, addedRows); + var removedBlueRows = getBlueTable(schema, removedRows); + newBlueRows.deleteRows(removedBlueRows); + newBlueRows.addRows(addedBlueRows); + currBlueRows = newBlueRows; + var rv = new DataPacket(newBlueRows, addedBlueRows, removedBlueRows); + history.Add(rv); + return rv; + } + + public IEnumerator GetEnumerator() => + history.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => + GetEnumerator(); + } +} diff --git a/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs b/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs new file mode 100644 index 000000000..c18e7ee02 --- /dev/null +++ b/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs @@ -0,0 +1,72 @@ +namespace Bts.DataStreamTests; + +using BCG; +using BCG.Utilities.General; + +using bts.collections; + +using Bts.DataStreamCore; + +public class TestDriverStream : ViewStream, IWorker { + readonly IBlueTableSchema schema; + readonly object lockObj = new(); + readonly ViewStreamDatabase? vsd; + BlueTable? lastPublishedTable; + BlueTable? currentTable; + + public TestDriverStream(IBlueTableSchema schema, ViewStreamDatabase? vsd = null) { + this.schema = schema; + this.vsd = vsd; + } + + bool publishOnDoWork => vsd is not null; + + public void Reset(params IBlueRow[] rows) => + Reset(rows.AsEnumerable()); + + public void Reset(IEnumerable rows) { + lock (lockObj) { + currentTable = new BlueTable(schema); + currentTable.AddRange(rows.Select(r => new KeyValuePair(r.Id(), r))); + if (!publishOnDoWork) { + syncAndFire(); + } + } + } + + protected override void myOnFirstSubscriber() => + vsd?.addWorker(this); + + protected override void myOnLastSubscriber() => + vsd?.removeWorker(this); + + void IWorker.doWork() { + lock (lockObj) { + if (!publishOnDoWork || currentTable is null || currentTable == lastPublishedTable) { + return; + } + syncAndFire(); + } + } + + void syncAndFire() { + if (currentTable is null) { + throw new ArgumentNullException(nameof(currentTable)); + } + var deletedRows = new BlueTable(schema); + var addedRows = new BlueTable(schema); + if (lastPublishedTable is null) { + addedRows.addRows(currentTable); + } else { + var comp = BlueRowEqualityComparer.Instance; + deletedRows.AsDictionary().SetItems(lastPublishedTable.Where(oldRow => + !(currentTable.AsDictionary().TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value)))); + addedRows.SetItems(currentTable.Where(newRow => + !(lastPublishedTable.AsDictionary().TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value)))); + } + lastPublishedTable = currentTable; + if (deletedRows.Count > 0 || addedRows.Count > 0) { + myFireEvent(new DataPacket(currentTable, addedRows, deletedRows)); + } + } +}