From aaca8f0f3a12df8f91d4c8cc89c5843bfcbf809f Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 20 Nov 2024 07:34:36 +0000 Subject: [PATCH] Add BlueColumnTable for pandas integration This will make it easier/more natural to support pandas column expressions --- .../BlueData/BlueColumnTable.cs | 127 ++++++++++++++++++ .../DataStreamTests/BlueColumnTableTests.cs | 114 ++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs create mode 100644 src/DataStreamSuite/DataStreamTests/BlueColumnTableTests.cs diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs new file mode 100644 index 000000000..ff0370a73 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs @@ -0,0 +1,127 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using bts.utils; + +using mbts.utils; + +using Spark.Core; + +using System; + +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +public class BlueColumnTable : IBlueRowCollection { + readonly Variant[][] columns; + readonly List rowIds; + readonly int idColIndex; + + public BlueColumnTable(IBlueTableSchema schema, int capacity = 4) { + ExceptionAssert.Argument.GreaterThanEqualTo(capacity, 0); + rowIds = new(capacity); + columns = new Variant[schema.columnInfo.Length][]; + if (columns.Length == 0) { + throw new ArgumentException("Schema contains no columns", nameof(schema)); + } + if (!schema.columnInfo.Select(c => c.colName).TryFindIndex("id", out idColIndex)) { + throw new ArgumentException("Schema contains no 'id' column", nameof(schema)); + } + this.schema = schema; + for (int i = 0; i < columns.Length; i++) { + columns[schema.columnInfo[i].colNo] = new Variant[capacity]; + } + } + + public int Count => rowIds.Count; + public IBlueTableSchema schema { get; } + + public IBlueRow this[int id] { + get { + var idx = rowIds.BinarySearch(id); + if (idx < 0) { + throw new KeyNotFoundException($"No row with id {id}"); + } + return new MyRow(this, idx); + } + } + + public void Add(Variant[] values) { + var id = values[idColIndex].VInt32; + var idx = rowIds.BinarySearch(id); + if (idx >= 0) { + throw new ArgumentException($"Row with id {id} already exists.", nameof(values)); + } + idx = ~idx; + for (int i = 0; i < columns.Length; i++) { + int len = rowIds.Count; + ArrayHelper.Insert(ref columns[i], idx, values[i], ref len); + } + rowIds.Insert(idx, id); + } + + public void AddOrUpdate(Variant[] values) { + var id = values[idColIndex].VInt32; + var idx = rowIds.BinarySearch(id); + if (idx >= 0) { + rowIds[idx] = id; + for (int i = 0; i < columns.Length; i++) { + columns[idx][i] = values[i]; + } + return; + } + idx = ~idx; + int len; + for (int i = 0; i < columns.Length; i++) { + len = rowIds.Count; + ArrayHelper.Insert(ref columns[i], idx, values[i], ref len); + } + rowIds.Insert(idx, id); + } + + public void Clear() => + rowIds.Clear(); + + public bool Remove(int id) { + var idx = rowIds.BinarySearch(id); + if (idx < 0) { + return false; + } + rowIds.RemoveAt(idx); + for (int i = 0; i < columns.Length; i++) { + ArrayHelper.remove_at(ref columns[i], idx); + } + return true; + } + + public void Trim() { + for (int i = 0; i < columns.Length; i++) { + ArrayHelper.trim(ref columns[i], rowIds.Count); + } + } + + public bool containsID(int id) => + rowIds.BinarySearch(id) >= 0; + + public IEnumerator GetEnumerator() { + var count = rowIds.Count; + for (int i = 0; i < count; i++) { + yield return new MyRow(this, i); + } + } + + class MyRow : IBlueRow { + readonly BlueColumnTable table; + readonly int rowIdx; + + public MyRow(BlueColumnTable table, int rowIdx) { + this.table = table; + this.rowIdx = rowIdx; + } + + public object this[int colNo] => table.columns[colNo][rowIdx].GetValue(); + public IBlueTableSchema schema => table.schema; + } +} diff --git a/src/DataStreamSuite/DataStreamTests/BlueColumnTableTests.cs b/src/DataStreamSuite/DataStreamTests/BlueColumnTableTests.cs new file mode 100644 index 000000000..f695bbb27 --- /dev/null +++ b/src/DataStreamSuite/DataStreamTests/BlueColumnTableTests.cs @@ -0,0 +1,114 @@ +namespace Bts.DataStreamTests; + +using BCG; + +using Bts.DataStreamCore; + +using Spark.Core; + +[TestFixture] +public class BlueColumnTableTests { + static IBlueTableSchema defaultSchema = new BlueTableSchemaBuilder("addTestTable") { + ("id", typeof(int)), + ("name", typeof(string)), + ("value", typeof(double)), + } + .ToSchema(); + + static BlueColumnTable getTable(params Variant[][] rows) { + var bct = new BlueColumnTable(defaultSchema); + for (int i = 0; i < rows.Length; i++) { + bct.Add(rows[i]); + } + return bct; + } + + [Test] + public void TestAdd() { + var rows = new Variant[][] { + new Variant[] { 1, "Joe", 32 }, + new Variant[] { 2, "Angie", 26.5 }, + }; + var bct = getTable(); + Is.EqualTo(bct.Count == 0); + bct.Add(rows[0]); + bct.Add(rows[1]); + Is.EqualTo(bct.Count == 2); + assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]); + assertRowEquals(bct[rows[1][0].GetInt32()], rows[1]); + assertRowSequenceEqual(bct, rows); + rows[0] = new Variant[] { 1, "Mika", 4.2 }; + bct.AddOrUpdate(rows[0]); + Is.EqualTo(bct.Count == 2); + assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]); + } + + [Test] + public void TestRemove() { + var rows = new Variant[][] { + new Variant[] { 5, "Angie", 26.5 }, + new Variant[] { 1, "Joe", 32 }, + new Variant[] { 8, "Myrtle", 103 }, + }; + var bct = getTable(rows); + bct.Remove(rows[1][0].GetInt32()); + assertRowSequenceEqual(bct, new[] { rows[0], rows[2] }); + } + + [Test] + public void TestRemoveReAdd() { + var rows = new Variant[][] { + new Variant[] { 5, "Angie", 26.5 }, + new Variant[] { 1, "Joe", 32 }, + new Variant[] { 8, "Myrtle", 103 }, + }; + var bct = getTable(rows); + bct.Remove(rows[1][0].GetInt32()); + bct.Add(rows[1]); + assertRowSequenceEqual(bct, rows); + } + + [Test] + public void TestClear() { + var rows = new Variant[][] { + new Variant[] { 5, "Angie", 26.5 }, + new Variant[] { 1, "Joe", 32 }, + new Variant[] { 8, "Myrtle", 103 }, + }; + var bct = getTable(rows); + bct.Clear(); + assertRowSequenceEqual(bct, Array.Empty()); + } + + static void assertRowEquals(IBlueRow row, Variant[] expectedValues, IBlueTableSchema? schema = null) { + schema ??= defaultSchema; + for (int i = 0; i < schema.columnInfo.Length; i++) { + Is.EqualTo(Equals(row[schema.columnInfo[i].colNo], expectedValues[i])); + if (!schema.columnInfo.TryGetFirst(c => c!.colNo == i, out var col)) { + throw new InvalidOperationException($"Schema has no column at index {i}"); + } + Is.EqualTo(Equals(row[col.colName], expectedValues[i])); + } + } + + static void assertRowSequenceEqual(BlueColumnTable bct, IEnumerable rowValues) { + if (!bct.schema.columnInfo.TryGetFirst(c => c!.colName == "id", out var idCol)) { + throw new ArgumentException("Schema has no id column", nameof(bct)); + } + Assert.That(bct.Cast().SequenceEqual( + rowValues.OrderBy(r => r[idCol.colNo].GetInt32()).Select(r => new MyRow(r, bct.schema)), + BlueRowEqualityComparer.Instance)); + } + + class MyRow : IBlueRow { + readonly Variant[] values; + + public MyRow(Variant[] values, IBlueTableSchema schema) { + this.values = values; + this.schema = schema; + } + + public IBlueTableSchema schema { get; } + public object? this[int index] => values[index].GetValue(); + } +}