This allows us to implement python/pandas functionality, much of which is based on column operations.
294 lines
11 KiB
C#
294 lines
11 KiB
C#
namespace Bts.DataStreamTests;
|
|
|
|
using BCG;
|
|
|
|
using bts.api.core;
|
|
using bts.collections;
|
|
|
|
using Bts.DataStreamCore;
|
|
|
|
using mbts.utils;
|
|
|
|
using NUnit.Framework;
|
|
|
|
using System.Collections.Immutable;
|
|
using System.Linq;
|
|
|
|
[TestFixture]
|
|
public class DataPacketBuilderTests {
|
|
static readonly BlueTableSchema schema = new(
|
|
new ColumnInfo[] {
|
|
new("id", 0, TypeCode.Int32),
|
|
new("value", 1, TypeCode.Double),
|
|
}, "test_table");
|
|
|
|
[Test]
|
|
public static void TestAdd() {
|
|
var table = new BlueTableImmutable(schema);
|
|
var testRows = new (int, double)[] {
|
|
(0, 23),
|
|
(1, 25),
|
|
(2, 27)
|
|
};
|
|
var history = new List<DataPacket>();
|
|
// Add 1 row
|
|
var bd = table.ToDataPacketBuilder();
|
|
bd.AddOrUpdate(getBlueRow(testRows[0]));
|
|
history.Add(bd.ToDataPacket());
|
|
table = bd.ToImmutable();
|
|
assertDataPacket(
|
|
history[0],
|
|
newRows: new[] { testRows[0] },
|
|
addedRows: new[] { testRows[0] });
|
|
assertNewRows(history[0], testRows[0]);
|
|
assertAddedRows(history[0], testRows[0]);
|
|
assertRows(table, testRows[0]);
|
|
|
|
// Add 2 more rows
|
|
bd = table.ToDataPacketBuilder();
|
|
bd.AddOrUpdate(getBlueRow(testRows[1]));
|
|
bd.AddOrUpdate(getBlueRow(testRows[2]));
|
|
history.Add(bd.ToDataPacket());
|
|
assertDataPacketIncremental(
|
|
oldDp: history[0],
|
|
newDp: history[1],
|
|
newRows: new[] { testRows[0], testRows[1], testRows[2] },
|
|
addedRows: new[] { testRows[1], testRows[2] });
|
|
table = bd.ToImmutable();
|
|
assertRows(table, testRows[0], testRows[1], testRows[2]);
|
|
}
|
|
|
|
[Test]
|
|
public static void TestRemove() {
|
|
var testRows = new (int Id, double Value)[] {
|
|
(0, 23),
|
|
(1, 25),
|
|
(2, 27)
|
|
};
|
|
var table = getTable(testRows);
|
|
var history = new List<DataPacket> {
|
|
getDataPacket(testRows)
|
|
};
|
|
var bd = table.ToDataPacketBuilder();
|
|
bd.Remove(testRows[0].Id);
|
|
history.Add(bd.ToDataPacket());
|
|
assertDataPacketIncremental(
|
|
oldDp: history[0],
|
|
newDp: history[1],
|
|
newRows: new[] { testRows[1], testRows[2] },
|
|
removedRows: new[] { testRows[0] });
|
|
}
|
|
|
|
[Test]
|
|
public static void TestNetChange() {
|
|
// Init test data
|
|
var testRows = new (int Id, double Value)[] {
|
|
(0, 23),
|
|
(1, 25),
|
|
(2, 27)
|
|
};
|
|
var table = getTable(testRows);
|
|
var history = new List<DataPacket> {
|
|
getDataPacket(testRows)
|
|
};
|
|
|
|
// Create builder, remove a row, then add it back but with a different value (same id).
|
|
// Verify the old row is removed, and the new one added in the resultant DataPacket.
|
|
var bd = table.ToDataPacketBuilder();
|
|
var oldRow = testRows[1];
|
|
bd.Remove(oldRow.Id);
|
|
var newRow = (testRows[1].Id, testRows[1].Value + 1);
|
|
testRows[1] = newRow;
|
|
bd.AddOrUpdate(getBlueRow(newRow));
|
|
history.Add(bd.ToDataPacket());
|
|
assertChanges(addedRows: new[] { newRow }, removedRows: new[] { oldRow });
|
|
|
|
// Add a new row, remove it, then add it back. Verify the resultant DataPacket adds the new row.
|
|
newRow = (3, 29);
|
|
testRows = testRows.Append(newRow).ToArray();
|
|
bd.AddOrUpdate(getBlueRow(newRow));
|
|
bd.Remove(newRow.Id);
|
|
bd.AddOrUpdate(getBlueRow(newRow));
|
|
history.Add(bd.ToDataPacket());
|
|
assertChanges(addedRows: new[] { newRow });
|
|
|
|
void assertChanges(
|
|
(int Id, double Value)[]? addedRows = null, (int Id, double Value)[]? removedRows = null) {
|
|
assertDataPacketIncremental(
|
|
history[history.Count - 2],
|
|
history[history.Count - 1],
|
|
testRows,
|
|
addedRows: addedRows,
|
|
removedRows: removedRows);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public static void TestNoNetChange() {
|
|
// Init test data
|
|
var testRows = new (int Id, double Value)[] {
|
|
(0, 23),
|
|
(1, 25),
|
|
(2, 27)
|
|
};
|
|
var table = getTable(testRows);
|
|
|
|
// Create builder, but don't mutate the table.
|
|
// Verify the resultant DataPacket has no changes.
|
|
var bd = table.ToDataPacketBuilder();
|
|
assertNoChanges();
|
|
var rowComp = BlueRowEqualityComparer.Instance;
|
|
|
|
// Update rows with their existing values (i.e., don't actually change anything).
|
|
// Verify the resultant DataPacket has no changes.
|
|
// Note: A row comparer arg is necessary for the builder to check equality w/ existing row.
|
|
bd.AddOrUpdate(getBlueRow(testRows[0]), rowComp);
|
|
bd.AddOrUpdate(getBlueRow(testRows[1]), rowComp);
|
|
assertNoChanges();
|
|
// Now do the same thing, but without the row comparer. Verify the resultant DataPacket
|
|
// has the phony changes.
|
|
bd.AddOrUpdate(getBlueRow(testRows[0]));
|
|
bd.AddOrUpdate(getBlueRow(testRows[1]));
|
|
assertPhonyChanges(testRows[0], testRows[1]);
|
|
|
|
// Add a new row, then remove it. Verify the resultant DataPacket has no changes
|
|
bd.AddOrUpdate(getBlueRow(3, 29));
|
|
bd.Remove(3);
|
|
assertNoChanges();
|
|
|
|
// Remove existing rows, then add them back. Verify the resultant DataPacket has no changes.
|
|
bd.Remove(testRows[0].Id);
|
|
bd.Remove(testRows[1].Id);
|
|
bd.AddOrUpdate(getBlueRow(testRows[0]), rowComp);
|
|
bd.AddOrUpdate(getBlueRow(testRows[1]), rowComp);
|
|
assertNoChanges();
|
|
// Now do the same thing, but without the row comparer. Verify the resultant DataPacket
|
|
// has the phony changes.
|
|
bd.Remove(testRows[0].Id);
|
|
bd.Remove(testRows[1].Id);
|
|
bd.AddOrUpdate(getBlueRow(testRows[0]));
|
|
bd.AddOrUpdate(getBlueRow(testRows[1]));
|
|
assertPhonyChanges(testRows[0], testRows[1]);
|
|
|
|
void assertNoChanges() {
|
|
var newDp = bd.ToDataPacket();
|
|
assertDataPacket(newDp, testRows);
|
|
var newTable = bd.ToImmutable();
|
|
assertRows(newTable, testRows);
|
|
}
|
|
|
|
void assertPhonyChanges(params (int Id, double Value)[] phonyChanges) {
|
|
var newDp = bd.ToDataPacket();
|
|
assertDataPacket(newDp, testRows, addedRows: phonyChanges, removedRows: phonyChanges);
|
|
var newTable = bd.ToImmutable();
|
|
assertRows(newTable, testRows);
|
|
}
|
|
}
|
|
|
|
static BlueTableImmutable getTable(params (int Id, double Value)[] rows) =>
|
|
new(schema, ImmutableSortedSet.CreateRange(
|
|
rows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup)))));
|
|
|
|
static MutableBlueRow getBlueRow((int id, double value) tup) =>
|
|
getBlueRow(tup.id, tup.value);
|
|
|
|
static MutableBlueRow getBlueRow(int id, double value) {
|
|
var r = new MutableBlueRow(schema);
|
|
r.setValue("id", id);
|
|
r.setValue("value", value);
|
|
return r;
|
|
}
|
|
|
|
static DataPacket getDataPacket(
|
|
(int Id, double Value)[] newRows,
|
|
(int Id, double Value)[]? addedRows = null,
|
|
(int Id, double Value)[]? removedRows = null) {
|
|
addedRows ??= Array.Empty<(int, double)>();
|
|
removedRows ??= Array.Empty<(int, double)>();
|
|
var kvpComp = new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key };
|
|
return new DataPacket(
|
|
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(newRows))),
|
|
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(addedRows))),
|
|
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(removedRows))));
|
|
|
|
static IEnumerable<KeyValuePair<int, IBlueRow>> GetRowKvps((int Id, double Value)[] rows) =>
|
|
rows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup)));
|
|
}
|
|
|
|
static (int, double) deconstructRow(IBlueRow row) =>
|
|
(row.get<int>("id"), row.get<double>("value"));
|
|
|
|
static void assertRows(BlueTableImmutable table, params (int, double)[] rows) =>
|
|
Assert.That(table.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
|
|
|
|
static void assertDataPacket(
|
|
DataPacket dp,
|
|
(int Id, double Value)[] newRows,
|
|
(int Id, double Value)[]? addedRows = null,
|
|
(int Id, double Value)[]? removedRows = null
|
|
) {
|
|
addedRows ??= Array.Empty<(int, double)>();
|
|
removedRows ??= Array.Empty<(int, double)>();
|
|
if (!newRows.Select(r => r.Id).IsDistinct()) {
|
|
throw new ArgumentException("Duplicate ids", nameof(newRows));
|
|
}
|
|
if (!addedRows.Select(r => r.Id).IsDistinct()) {
|
|
throw new ArgumentException("Duplicate ids", nameof(addedRows));
|
|
}
|
|
if (!removedRows.Select(r => r.Id).IsDistinct()) {
|
|
throw new ArgumentException("Duplicate ids", nameof(removedRows));
|
|
}
|
|
var changes = addedRows.Join(
|
|
removedRows, r => r.Id, r => r.Id, (ar, rr) => new Change<(int Id, double Value)>(rr, ar))
|
|
.ToDictionary(ch => ch.New.Id);
|
|
if (!changes.All(kv => kv.Key == kv.Value.New.Id && kv.Key == kv.Value.Old.Id)) {
|
|
throw new InvalidOperationException("Computed changes are invalid");
|
|
}
|
|
// strip out phony changes (i.e., where no row values actually changed).
|
|
var trueRemoves = removedRows.Where(r => !changes.ContainsKey(r.Id)).ToList();
|
|
if (removedRows.Length != trueRemoves.Count + changes.Count) {
|
|
throw new InvalidOperationException("Computed removes are invalid");
|
|
}
|
|
|
|
assertNewRows(dp, newRows);
|
|
assertAddedRows(dp, addedRows);
|
|
assertRemovedRows(dp, removedRows);
|
|
Assert.Multiple(() => {
|
|
Assert.That(addedRows.All(newRows.Contains));
|
|
Assert.That(!trueRemoves.Any(newRows.Contains));
|
|
});
|
|
}
|
|
|
|
static void assertDataPacketIncremental(
|
|
DataPacket oldDp,
|
|
DataPacket newDp,
|
|
(int Id, double Value)[] newRows,
|
|
(int Id, double Value)[]? addedRows = null,
|
|
(int Id, double Value)[]? removedRows = null) {
|
|
addedRows ??= Array.Empty<(int, double)>();
|
|
removedRows ??= Array.Empty<(int, double)>();
|
|
var removedIds = removedRows.Select(r => r.Id);
|
|
var addedIds = addedRows.Select(r => r.Id);
|
|
var trueAddedIds = addedIds.Except(removedIds);
|
|
var trueRemovedIds = removedIds.Except(addedIds);
|
|
assertDataPacket(newDp, newRows, addedRows, removedRows);
|
|
Assert.Multiple(() => {
|
|
Assert.That(addedIds.All(newDp.newRows.containsID));
|
|
Assert.That(!trueAddedIds.Any(oldDp.newRows.containsID));
|
|
Assert.That(removedIds.All(oldDp.newRows.containsID));
|
|
Assert.That(!trueRemovedIds.Any(newDp.newRows.containsID));
|
|
Assert.That(newDp.newRows.Count, Is.EqualTo(
|
|
oldDp.newRows.Count + addedRows.Length - removedRows.Length));
|
|
});
|
|
}
|
|
|
|
static void assertNewRows(DataPacket dp, params (int, double)[] rows) =>
|
|
Assert.That(dp.newRows.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
|
|
|
|
static void assertAddedRows(DataPacket dp, params (int, double)[] rows) =>
|
|
Assert.That(dp.addedRows.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
|
|
|
|
static void assertRemovedRows(DataPacket dp, params (int, double)[] rows) =>
|
|
Assert.That(dp.deletedRows.Cast<IBlueRow>().Select(deconstructRow).SequenceEqual(rows));
|
|
}
|