Files
spark/src/DataStreamSuite/DataStreamTests/DataPacketBuilder/DataPacketBuilderTests.cs
T
John Colvin 0d10c2ba5d new: Add GetColumn fn to IBlueRowCollection interface
This allows us to implement python/pandas functionality, much of which is based
on column operations.
2025-11-15 12:38:50 -05:00

294 lines
11 KiB
C#

namespace Bts.DataStreamTests;
using BCG;
using bts.api.core;
using bts.collections;
using Bts.DataStreamCore;
using mbts.utils;
using NUnit.Framework;
using System.Collections.Immutable;
using System.Linq;
[TestFixture]
public class DataPacketBuilderTests {
static readonly BlueTableSchema schema = new(
new ColumnInfo[] {
new("id", 0, TypeCode.Int32),
new("value", 1, TypeCode.Double),
}, "test_table");
[Test]
public static void TestAdd() {
var table = new BlueTableImmutable(schema);
var testRows = new (int, double)[] {
(0, 23),
(1, 25),
(2, 27)
};
var history = new List<DataPacket>();
// Add 1 row
var bd = table.ToDataPacketBuilder();
bd.AddOrUpdate(getBlueRow(testRows[0]));
history.Add(bd.ToDataPacket());
table = bd.ToImmutable();
assertDataPacket(
history[0],
newRows: new[] { testRows[0] },
addedRows: new[] { testRows[0] });
assertNewRows(history[0], testRows[0]);
assertAddedRows(history[0], testRows[0]);
assertRows(table, testRows[0]);
// Add 2 more rows
bd = table.ToDataPacketBuilder();
bd.AddOrUpdate(getBlueRow(testRows[1]));
bd.AddOrUpdate(getBlueRow(testRows[2]));
history.Add(bd.ToDataPacket());
assertDataPacketIncremental(
oldDp: history[0],
newDp: history[1],
newRows: new[] { testRows[0], testRows[1], testRows[2] },
addedRows: new[] { testRows[1], testRows[2] });
table = bd.ToImmutable();
assertRows(table, testRows[0], testRows[1], testRows[2]);
}
[Test]
public static void TestRemove() {
var testRows = new (int Id, double Value)[] {
(0, 23),
(1, 25),
(2, 27)
};
var table = getTable(testRows);
var history = new List<DataPacket> {
getDataPacket(testRows)
};
var bd = table.ToDataPacketBuilder();
bd.Remove(testRows[0].Id);
history.Add(bd.ToDataPacket());
assertDataPacketIncremental(
oldDp: history[0],
newDp: history[1],
newRows: new[] { testRows[1], testRows[2] },
removedRows: new[] { testRows[0] });
}
[Test]
public static void TestNetChange() {
// Init test data
var testRows = new (int Id, double Value)[] {
(0, 23),
(1, 25),
(2, 27)
};
var table = getTable(testRows);
var history = new List<DataPacket> {
getDataPacket(testRows)
};
// Create builder, remove a row, then add it back but with a different value (same id).
// Verify the old row is removed, and the new one added in the resultant DataPacket.
var bd = table.ToDataPacketBuilder();
var oldRow = testRows[1];
bd.Remove(oldRow.Id);
var newRow = (testRows[1].Id, testRows[1].Value + 1);
testRows[1] = newRow;
bd.AddOrUpdate(getBlueRow(newRow));
history.Add(bd.ToDataPacket());
assertChanges(addedRows: new[] { newRow }, removedRows: new[] { oldRow });
// Add a new row, remove it, then add it back. Verify the resultant DataPacket adds the new row.
newRow = (3, 29);
testRows = testRows.Append(newRow).ToArray();
bd.AddOrUpdate(getBlueRow(newRow));
bd.Remove(newRow.Id);
bd.AddOrUpdate(getBlueRow(newRow));
history.Add(bd.ToDataPacket());
assertChanges(addedRows: new[] { newRow });
void assertChanges(
(int Id, double Value)[]? addedRows = null, (int Id, double Value)[]? removedRows = null) {
assertDataPacketIncremental(
history[history.Count - 2],
history[history.Count - 1],
testRows,
addedRows: addedRows,
removedRows: removedRows);
}
}
[Test]
public static void TestNoNetChange() {
// Init test data
var testRows = new (int Id, double Value)[] {
(0, 23),
(1, 25),
(2, 27)
};
var table = getTable(testRows);
// Create builder, but don't mutate the table.
// Verify the resultant DataPacket has no changes.
var bd = table.ToDataPacketBuilder();
assertNoChanges();
var rowComp = BlueRowEqualityComparer.Instance;
// Update rows with their existing values (i.e., don't actually change anything).
// Verify the resultant DataPacket has no changes.
// Note: A row comparer arg is necessary for the builder to check equality w/ existing row.
bd.AddOrUpdate(getBlueRow(testRows[0]), rowComp);
bd.AddOrUpdate(getBlueRow(testRows[1]), rowComp);
assertNoChanges();
// Now do the same thing, but without the row comparer. Verify the resultant DataPacket
// has the phony changes.
bd.AddOrUpdate(getBlueRow(testRows[0]));
bd.AddOrUpdate(getBlueRow(testRows[1]));
assertPhonyChanges(testRows[0], testRows[1]);
// Add a new row, then remove it. Verify the resultant DataPacket has no changes
bd.AddOrUpdate(getBlueRow(3, 29));
bd.Remove(3);
assertNoChanges();
// Remove existing rows, then add them back. Verify the resultant DataPacket has no changes.
bd.Remove(testRows[0].Id);
bd.Remove(testRows[1].Id);
bd.AddOrUpdate(getBlueRow(testRows[0]), rowComp);
bd.AddOrUpdate(getBlueRow(testRows[1]), rowComp);
assertNoChanges();
// Now do the same thing, but without the row comparer. Verify the resultant DataPacket
// has the phony changes.
bd.Remove(testRows[0].Id);
bd.Remove(testRows[1].Id);
bd.AddOrUpdate(getBlueRow(testRows[0]));
bd.AddOrUpdate(getBlueRow(testRows[1]));
assertPhonyChanges(testRows[0], testRows[1]);
void assertNoChanges() {
var newDp = bd.ToDataPacket();
assertDataPacket(newDp, testRows);
var newTable = bd.ToImmutable();
assertRows(newTable, testRows);
}
void assertPhonyChanges(params (int Id, double Value)[] phonyChanges) {
var newDp = bd.ToDataPacket();
assertDataPacket(newDp, testRows, addedRows: phonyChanges, removedRows: phonyChanges);
var newTable = bd.ToImmutable();
assertRows(newTable, testRows);
}
}
static BlueTableImmutable getTable(params (int Id, double Value)[] rows) =>
new(schema, ImmutableSortedSet.CreateRange(
rows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup)))));
static MutableBlueRow getBlueRow((int id, double value) tup) =>
getBlueRow(tup.id, tup.value);
static MutableBlueRow getBlueRow(int id, double value) {
var r = new MutableBlueRow(schema);
r.setValue("id", id);
r.setValue("value", value);
return r;
}
static DataPacket getDataPacket(
(int Id, double Value)[] newRows,
(int Id, double Value)[]? addedRows = null,
(int Id, double Value)[]? removedRows = null) {
addedRows ??= Array.Empty<(int, double)>();
removedRows ??= Array.Empty<(int, double)>();
var kvpComp = new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key };
return new DataPacket(
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(newRows))),
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(addedRows))),
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(removedRows))));
static IEnumerable<KeyValuePair<int, IBlueRow>> GetRowKvps((int Id, double Value)[] rows) =>
rows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup)));
}
static (int, double) deconstructRow(IBlueRow row) =>
(row.get<int>("id"), row.get<double>("value"));
static void assertRows(BlueTableImmutable table, params (int, double)[] rows) =>
Assert.That(table.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
static void assertDataPacket(
DataPacket dp,
(int Id, double Value)[] newRows,
(int Id, double Value)[]? addedRows = null,
(int Id, double Value)[]? removedRows = null
) {
addedRows ??= Array.Empty<(int, double)>();
removedRows ??= Array.Empty<(int, double)>();
if (!newRows.Select(r => r.Id).IsDistinct()) {
throw new ArgumentException("Duplicate ids", nameof(newRows));
}
if (!addedRows.Select(r => r.Id).IsDistinct()) {
throw new ArgumentException("Duplicate ids", nameof(addedRows));
}
if (!removedRows.Select(r => r.Id).IsDistinct()) {
throw new ArgumentException("Duplicate ids", nameof(removedRows));
}
var changes = addedRows.Join(
removedRows, r => r.Id, r => r.Id, (ar, rr) => new Change<(int Id, double Value)>(rr, ar))
.ToDictionary(ch => ch.New.Id);
if (!changes.All(kv => kv.Key == kv.Value.New.Id && kv.Key == kv.Value.Old.Id)) {
throw new InvalidOperationException("Computed changes are invalid");
}
// strip out phony changes (i.e., where no row values actually changed).
var trueRemoves = removedRows.Where(r => !changes.ContainsKey(r.Id)).ToList();
if (removedRows.Length != trueRemoves.Count + changes.Count) {
throw new InvalidOperationException("Computed removes are invalid");
}
assertNewRows(dp, newRows);
assertAddedRows(dp, addedRows);
assertRemovedRows(dp, removedRows);
Assert.Multiple(() => {
Assert.That(addedRows.All(newRows.Contains));
Assert.That(!trueRemoves.Any(newRows.Contains));
});
}
static void assertDataPacketIncremental(
DataPacket oldDp,
DataPacket newDp,
(int Id, double Value)[] newRows,
(int Id, double Value)[]? addedRows = null,
(int Id, double Value)[]? removedRows = null) {
addedRows ??= Array.Empty<(int, double)>();
removedRows ??= Array.Empty<(int, double)>();
var removedIds = removedRows.Select(r => r.Id);
var addedIds = addedRows.Select(r => r.Id);
var trueAddedIds = addedIds.Except(removedIds);
var trueRemovedIds = removedIds.Except(addedIds);
assertDataPacket(newDp, newRows, addedRows, removedRows);
Assert.Multiple(() => {
Assert.That(addedIds.All(newDp.newRows.containsID));
Assert.That(!trueAddedIds.Any(oldDp.newRows.containsID));
Assert.That(removedIds.All(oldDp.newRows.containsID));
Assert.That(!trueRemovedIds.Any(newDp.newRows.containsID));
Assert.That(newDp.newRows.Count, Is.EqualTo(
oldDp.newRows.Count + addedRows.Length - removedRows.Length));
});
}
static void assertNewRows(DataPacket dp, params (int, double)[] rows) =>
Assert.That(dp.newRows.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
static void assertAddedRows(DataPacket dp, params (int, double)[] rows) =>
Assert.That(dp.addedRows.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
static void assertRemovedRows(DataPacket dp, params (int, double)[] rows) =>
Assert.That(dp.deletedRows.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
}