Add BlueColumnTable for pandas integration
This will make it easier/more natural to support pandas column expressions
This commit is contained in:
1 parent
59a16677d1
commit
aaca8f0f3a
2 files changed
+241
No files matched your search
@@ -0,0 +1,127 @@
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.utils;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core;
|
||||
|
||||
using System;
|
||||
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
public class BlueColumnTable : IBlueRowCollection {
|
||||
readonly Variant[][] columns;
|
||||
readonly List<int> rowIds;
|
||||
readonly int idColIndex;
|
||||
|
||||
public BlueColumnTable(IBlueTableSchema schema, int capacity = 4) {
|
||||
ExceptionAssert.Argument.GreaterThanEqualTo(capacity, 0);
|
||||
rowIds = new(capacity);
|
||||
columns = new Variant[schema.columnInfo.Length][];
|
||||
if (columns.Length == 0) {
|
||||
throw new ArgumentException("Schema contains no columns", nameof(schema));
|
||||
}
|
||||
if (!schema.columnInfo.Select(c => c.colName).TryFindIndex("id", out idColIndex)) {
|
||||
throw new ArgumentException("Schema contains no 'id' column", nameof(schema));
|
||||
}
|
||||
this.schema = schema;
|
||||
for (int i = 0; i < columns.Length; i++) {
|
||||
columns[schema.columnInfo[i].colNo] = new Variant[capacity];
|
||||
}
|
||||
}
|
||||
|
||||
public int Count => rowIds.Count;
|
||||
public IBlueTableSchema schema { get; }
|
||||
|
||||
public IBlueRow this[int id] {
|
||||
get {
|
||||
var idx = rowIds.BinarySearch(id);
|
||||
if (idx < 0) {
|
||||
throw new KeyNotFoundException($"No row with id {id}");
|
||||
}
|
||||
return new MyRow(this, idx);
|
||||
}
|
||||
}
|
||||
|
||||
public void Add(Variant[] values) {
|
||||
var id = values[idColIndex].VInt32;
|
||||
var idx = rowIds.BinarySearch(id);
|
||||
if (idx >= 0) {
|
||||
throw new ArgumentException($"Row with id {id} already exists.", nameof(values));
|
||||
}
|
||||
idx = ~idx;
|
||||
for (int i = 0; i < columns.Length; i++) {
|
||||
int len = rowIds.Count;
|
||||
ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
|
||||
}
|
||||
rowIds.Insert(idx, id);
|
||||
}
|
||||
|
||||
public void AddOrUpdate(Variant[] values) {
|
||||
var id = values[idColIndex].VInt32;
|
||||
var idx = rowIds.BinarySearch(id);
|
||||
if (idx >= 0) {
|
||||
rowIds[idx] = id;
|
||||
for (int i = 0; i < columns.Length; i++) {
|
||||
columns[idx][i] = values[i];
|
||||
}
|
||||
return;
|
||||
}
|
||||
idx = ~idx;
|
||||
int len;
|
||||
for (int i = 0; i < columns.Length; i++) {
|
||||
len = rowIds.Count;
|
||||
ArrayHelper.Insert(ref columns[i], idx, values[i], ref len);
|
||||
}
|
||||
rowIds.Insert(idx, id);
|
||||
}
|
||||
|
||||
public void Clear() =>
|
||||
rowIds.Clear();
|
||||
|
||||
public bool Remove(int id) {
|
||||
var idx = rowIds.BinarySearch(id);
|
||||
if (idx < 0) {
|
||||
return false;
|
||||
}
|
||||
rowIds.RemoveAt(idx);
|
||||
for (int i = 0; i < columns.Length; i++) {
|
||||
ArrayHelper.remove_at(ref columns[i], idx);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Trim() {
|
||||
for (int i = 0; i < columns.Length; i++) {
|
||||
ArrayHelper.trim(ref columns[i], rowIds.Count);
|
||||
}
|
||||
}
|
||||
|
||||
public bool containsID(int id) =>
|
||||
rowIds.BinarySearch(id) >= 0;
|
||||
|
||||
public IEnumerator GetEnumerator() {
|
||||
var count = rowIds.Count;
|
||||
for (int i = 0; i < count; i++) {
|
||||
yield return new MyRow(this, i);
|
||||
}
|
||||
}
|
||||
|
||||
class MyRow : IBlueRow {
|
||||
readonly BlueColumnTable table;
|
||||
readonly int rowIdx;
|
||||
|
||||
public MyRow(BlueColumnTable table, int rowIdx) {
|
||||
this.table = table;
|
||||
this.rowIdx = rowIdx;
|
||||
}
|
||||
|
||||
public object this[int colNo] => table.columns[colNo][rowIdx].GetValue();
|
||||
public IBlueTableSchema schema => table.schema;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
namespace Bts.DataStreamTests;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core;
|
||||
|
||||
[TestFixture]
|
||||
public class BlueColumnTableTests {
|
||||
static IBlueTableSchema defaultSchema = new BlueTableSchemaBuilder("addTestTable") {
|
||||
("id", typeof(int)),
|
||||
("name", typeof(string)),
|
||||
("value", typeof(double)),
|
||||
}
|
||||
.ToSchema();
|
||||
|
||||
static BlueColumnTable getTable(params Variant[][] rows) {
|
||||
var bct = new BlueColumnTable(defaultSchema);
|
||||
for (int i = 0; i < rows.Length; i++) {
|
||||
bct.Add(rows[i]);
|
||||
}
|
||||
return bct;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestAdd() {
|
||||
var rows = new Variant[][] {
|
||||
new Variant[] { 1, "Joe", 32 },
|
||||
new Variant[] { 2, "Angie", 26.5 },
|
||||
};
|
||||
var bct = getTable();
|
||||
Is.EqualTo(bct.Count == 0);
|
||||
bct.Add(rows[0]);
|
||||
bct.Add(rows[1]);
|
||||
Is.EqualTo(bct.Count == 2);
|
||||
assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
|
||||
assertRowEquals(bct[rows[1][0].GetInt32()], rows[1]);
|
||||
assertRowSequenceEqual(bct, rows);
|
||||
rows[0] = new Variant[] { 1, "Mika", 4.2 };
|
||||
bct.AddOrUpdate(rows[0]);
|
||||
Is.EqualTo(bct.Count == 2);
|
||||
assertRowEquals(bct[rows[0][0].GetInt32()], rows[0]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestRemove() {
|
||||
var rows = new Variant[][] {
|
||||
new Variant[] { 5, "Angie", 26.5 },
|
||||
new Variant[] { 1, "Joe", 32 },
|
||||
new Variant[] { 8, "Myrtle", 103 },
|
||||
};
|
||||
var bct = getTable(rows);
|
||||
bct.Remove(rows[1][0].GetInt32());
|
||||
assertRowSequenceEqual(bct, new[] { rows[0], rows[2] });
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestRemoveReAdd() {
|
||||
var rows = new Variant[][] {
|
||||
new Variant[] { 5, "Angie", 26.5 },
|
||||
new Variant[] { 1, "Joe", 32 },
|
||||
new Variant[] { 8, "Myrtle", 103 },
|
||||
};
|
||||
var bct = getTable(rows);
|
||||
bct.Remove(rows[1][0].GetInt32());
|
||||
bct.Add(rows[1]);
|
||||
assertRowSequenceEqual(bct, rows);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestClear() {
|
||||
var rows = new Variant[][] {
|
||||
new Variant[] { 5, "Angie", 26.5 },
|
||||
new Variant[] { 1, "Joe", 32 },
|
||||
new Variant[] { 8, "Myrtle", 103 },
|
||||
};
|
||||
var bct = getTable(rows);
|
||||
bct.Clear();
|
||||
assertRowSequenceEqual(bct, Array.Empty<Variant[]>());
|
||||
}
|
||||
|
||||
static void assertRowEquals(IBlueRow row, Variant[] expectedValues, IBlueTableSchema? schema = null) {
|
||||
schema ??= defaultSchema;
|
||||
for (int i = 0; i < schema.columnInfo.Length; i++) {
|
||||
Is.EqualTo(Equals(row[schema.columnInfo[i].colNo], expectedValues[i]));
|
||||
if (!schema.columnInfo.TryGetFirst(c => c!.colNo == i, out var col)) {
|
||||
throw new InvalidOperationException($"Schema has no column at index {i}");
|
||||
}
|
||||
Is.EqualTo(Equals(row[col.colName], expectedValues[i]));
|
||||
}
|
||||
}
|
||||
|
||||
static void assertRowSequenceEqual(BlueColumnTable bct, IEnumerable<Variant[]> rowValues) {
|
||||
if (!bct.schema.columnInfo.TryGetFirst(c => c!.colName == "id", out var idCol)) {
|
||||
throw new ArgumentException("Schema has no id column", nameof(bct));
|
||||
}
|
||||
Assert.That(bct.Cast<IBlueRow>().SequenceEqual(
|
||||
rowValues.OrderBy(r => r[idCol.colNo].GetInt32()).Select(r => new MyRow(r, bct.schema)),
|
||||
BlueRowEqualityComparer.Instance));
|
||||
}
|
||||
|
||||
class MyRow : IBlueRow {
|
||||
readonly Variant[] values;
|
||||
|
||||
public MyRow(Variant[] values, IBlueTableSchema schema) {
|
||||
this.values = values;
|
||||
this.schema = schema;
|
||||
}
|
||||
|
||||
public IBlueTableSchema schema { get; }
|
||||
public object? this[int index] => values[index].GetValue();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user