Add BlueColumnTable for pandas integration

This will make it easier/more natural to support pandas column expressions
This commit is contained in:
John Colvin committed 2025-08-24 13:46:53 -04:00
1 parent 59a16677d1
commit aaca8f0f3a
2 files changed
+241

No files matched your search

@@ -0,0 +1,127 @@
namespace Bts.DataStreamCore;
using BCG;
using bts.utils;
using mbts.utils;
using Spark.Core;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
public class BlueColumnTable : IBlueRowCollection {
readonly Variant[][] columns;
readonly List<int> rowIds;
readonly int idColIndex;
public BlueColumnTable(IBlueTableSchema schema, int capacity = 4) {
ExceptionAssert.Argument.GreaterThanEqualTo(capacity, 0);
rowIds = new(capacity);
columns = new Variant[schema.columnInfo.Length][];
if (columns.Length == 0) {
throw new ArgumentException("Schema contains no columns", nameof(schema));
}
if (!schema.columnInfo.Select(c => c.colName).TryFindIndex("id", out idColIndex)) {
throw new ArgumentException("Schema contains no 'id' column", nameof(schema));
}
this.schema = schema;
for (int i = 0; i < columns.Length; i++) {
columns[schema.columnInfo[i].colNo] = new Variant[capacity];
}
}
public int Count => rowIds.Count;
public IBlueTableSchema schema { get; }
public IBlueRow this[int id] {
get {
var idx = rowIds.BinarySearch(id);
if (idx < 0) {
throw new KeyNotFoundException($"No row with id {id}");
}
return new MyRow(this, idx);
}
}
public void Add(Variant[] values) {
var id = values[idColIndex].VInt32;
var idx = rowIds.BinarySearch(id);
if (idx >= 0) {
throw new ArgumentException($"Row with id {id} already exists.", nameof(values));
}
idx = ~idx;
for (int i = 0; i < columns.Length; i++) {
int len = rowIds.Count;
ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
}
rowIds.Insert(idx, id);
}
public void AddOrUpdate(Variant[] values) {
var id = values[idColIndex].VInt32;
var idx = rowIds.BinarySearch(id);
if (idx >= 0) {
rowIds[idx] = id;
for (int i = 0; i < columns.Length; i++) {
columns[idx][i] = values[i];
}
return;
}
idx = ~idx;
int len;
for (int i = 0; i < columns.Length; i++) {
len = rowIds.Count;
ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
}
rowIds.Insert(idx, id);
}
public void Clear() =>
rowIds.Clear();
public bool Remove(int id) {
var idx = rowIds.BinarySearch(id);
if (idx < 0) {
return false;
}
rowIds.RemoveAt(idx);
for (int i = 0; i < columns.Length; i++) {
ArrayHelper.remove_at(ref columns[i], idx);
}
return true;
}
public void Trim() {
for (int i = 0; i < columns.Length; i++) {
ArrayHelper.trim(ref columns[i], rowIds.Count);
}
}
public bool containsID(int id) =>
rowIds.BinarySearch(id) >= 0;
public IEnumerator GetEnumerator() {
var count = rowIds.Count;
for (int i = 0; i < count; i++) {
yield return new MyRow(this, i);
}
}
class MyRow : IBlueRow {
readonly BlueColumnTable table;
readonly int rowIdx;
public MyRow(BlueColumnTable table, int rowIdx) {
this.table = table;
this.rowIdx = rowIdx;
}
public object this[int colNo] => table.columns[colNo][rowIdx].GetValue();
public IBlueTableSchema schema => table.schema;
}
}
@@ -0,0 +1,114 @@
namespace Bts.DataStreamTests;
using BCG;
using Bts.DataStreamCore;
using Spark.Core;
[TestFixture]
public class BlueColumnTableTests {
static IBlueTableSchema defaultSchema = new BlueTableSchemaBuilder("addTestTable") {
("id", typeof(int)),
("name", typeof(string)),
("value", typeof(double)),
}
.ToSchema();
static BlueColumnTable getTable(params Variant[][] rows) {
var bct = new BlueColumnTable(defaultSchema);
for (int i = 0; i < rows.Length; i++) {
bct.Add(rows[i]);
}
return bct;
}
[Test]
public void TestAdd() {
var rows = new Variant[][] {
new Variant[] { 1, "Joe", 32 },
new Variant[] { 2, "Angie", 26.5 },
};
var bct = getTable();
Is.EqualTo(bct.Count == 0);
bct.Add(rows[0]);
bct.Add(rows[1]);
Is.EqualTo(bct.Count == 2);
assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
assertRowEquals(bct[rows[1][0].GetInt32()], rows[1]);
assertRowSequenceEqual(bct, rows);
rows[0] = new Variant[] { 1, "Mika", 4.2 };
bct.AddOrUpdate(rows[0]);
Is.EqualTo(bct.Count == 2);
assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
}
[Test]
public void TestRemove() {
var rows = new Variant[][] {
new Variant[] { 5, "Angie", 26.5 },
new Variant[] { 1, "Joe", 32 },
new Variant[] { 8, "Myrtle", 103 },
};
var bct = getTable(rows);
bct.Remove(rows[1][0].GetInt32());
assertRowSequenceEqual(bct, new[] { rows[0], rows[2] });
}
[Test]
public void TestRemoveReAdd() {
var rows = new Variant[][] {
new Variant[] { 5, "Angie", 26.5 },
new Variant[] { 1, "Joe", 32 },
new Variant[] { 8, "Myrtle", 103 },
};
var bct = getTable(rows);
bct.Remove(rows[1][0].GetInt32());
bct.Add(rows[1]);
assertRowSequenceEqual(bct, rows);
}
[Test]
public void TestClear() {
var rows = new Variant[][] {
new Variant[] { 5, "Angie", 26.5 },
new Variant[] { 1, "Joe", 32 },
new Variant[] { 8, "Myrtle", 103 },
};
var bct = getTable(rows);
bct.Clear();
assertRowSequenceEqual(bct, Array.Empty<Variant[]>());
}
static void assertRowEquals(IBlueRow row, Variant[] expectedValues, IBlueTableSchema? schema = null) {
schema ??= defaultSchema;
for (int i = 0; i < schema.columnInfo.Length; i++) {
Is.EqualTo(Equals(row[schema.columnInfo[i].colNo], expectedValues[i]));
if (!schema.columnInfo.TryGetFirst(c => c!.colNo == i, out var col)) {
throw new InvalidOperationException($"Schema has no column at index {i}");
}
Is.EqualTo(Equals(row[col.colName], expectedValues[i]));
}
}
static void assertRowSequenceEqual(BlueColumnTable bct, IEnumerable<Variant[]> rowValues) {
if (!bct.schema.columnInfo.TryGetFirst(c => c!.colName == "id", out var idCol)) {
throw new ArgumentException("Schema has no id column", nameof(bct));
}
Assert.That(bct.Cast<IBlueRow>().SequenceEqual(
rowValues.OrderBy(r => r[idCol.colNo].GetInt32()).Select(r => new MyRow(r, bct.schema)),
BlueRowEqualityComparer.Instance));
}
class MyRow : IBlueRow {
readonly Variant[] values;
public MyRow(Variant[] values, IBlueTableSchema schema) {
this.values = values;
this.schema = schema;
}
public IBlueTableSchema schema { get; }
public object? this[int index] => values[index].GetValue();
}
}