Files
spark/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs
T
John Colvin 0d10c2ba5d new: Add GetColumn fn to IBlueRowCollection interface
This allows us to implement python/pandas functionality, much of which is based
on column operations.
2025-11-15 12:38:50 -05:00

385 lines
13 KiB
C#

#nullable enable
namespace Bts.DataStreamCore;
using BCG;
using bts.api.core;
using bts.collections;
using mbts.utils;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
// I would have named this ImmutableBlueTable, but that was already taken.
public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>> rows;
public BlueTableImmutable(IBlueTableSchema schema)
: this(schema, ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Empty) { }
public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<int, IBlueRow>> rows) {
this.schema = schema;
this.rows = rows.WithComparer(new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key });
}
public int Count => rows.Count;
public bool IsSynchronized => false;
public IBlueTableSchema schema { get; }
public object SyncRoot => this;
public IBlueRow this[int id] =>
#nullable disable
rows.TryGetValue(new(id, default), out var kv)
#nullable enable
? kv.Value
: throw new KeyNotFoundException();
public bool containsID(int id) =>
#nullable disable
rows.Contains(new(id, default));
#nullable enable
public void CopyTo(Array array, int index) {
foreach (var kv in rows) {
array.SetValue(kv.Value, index++);
}
}
public IList<object?> GetColumn(int index) =>
rows.WrapSelector(kv => kv.Value[index], null);
public IEnumerator GetEnumerator() {
foreach (var kv in rows) {
yield return kv.Value;
}
}
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder added =
ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder removed =
ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutable table) {
schema = table.schema;
this.table = table.rows.ToBuilder();
}
public void AddOrUpdate(IBlueRow row, IEqualityComparer<IBlueRow>? comparer = null) {
var id = row.Id();
#nullable disable
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
if (comparer?.Equals(row, orig.Value) ?? false) {
// No op: This row was already in the table (according to 'comparer').
// Remove it from 'added' and 'removed'.
added.Remove(new(id, row));
removed.Remove(new(id, orig.Value));
return;
}
// Change op: A row with this id was already in the table,
// but its values have changed (according to 'comparer').
removed.Remove(new(id, orig.Value));
removed.Add(new(id, orig.Value));
}
added.Remove(new(id, row));
added.Add(new(id, row));
}
public void Remove(int id) {
#nullable disable
added.Remove(new(id, default));
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
removed.Remove(new(id, orig.Value));
removed.Add(new(id, orig.Value));
}
}
public DataPacket ToDataPacket(bool reset = true) {
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutable(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
var removedRows = new BlueTableImmutable(schema, removed.ToImmutable());
if (reset) {
added.Clear();
removed.Clear();
}
return new(newRows, addedRows, removedRows);
}
public BlueTableImmutable ToImmutable() =>
new(schema, table.ToImmutable());
}
}
public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
readonly ImmutableSortedSet<KeyValuePair<int, T>> rows;
public BlueTableImmutable(IBlueTableSchema schema)
: this(schema, ImmutableSortedSet<KeyValuePair<int, T>>.Empty) { }
public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<int, T>> rows) {
this.schema = schema;
this.rows = rows;
}
public int Count => rows.Count;
public bool IsSynchronized => false;
public IBlueTableSchema schema { get; }
public object SyncRoot => this;
public T this[int id] =>
#nullable disable
rows.TryGetValue(new(id, default), out var kv)
#nullable enable
? kv.Value
: throw new KeyNotFoundException();
public bool containsID(int id) =>
#nullable disable
rows.Contains(new(id, default));
#nullable enable
public IList<object?> GetColumn(int index) =>
rows.WrapSelector(kv => kv.Value[index], null);
public void CopyTo(Array array, int index) {
foreach (var (k, v) in rows) {
array.SetValue(v, index++);
}
}
public IEnumerator<T> GetEnumerator() {
foreach (var kv in rows) {
yield return kv.Value;
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder added =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder removed =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutable<T> table) {
schema = table.schema;
this.table = table.rows.ToBuilder();
}
public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
var id = row.Id();
#nullable disable
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
if (comparer?.Equals(row, orig.Value) ?? false) {
// No op: This row was already in the table (according to 'comparer').
// Remove it from 'added' and 'removed'.
added.Remove(new(id, row));
removed.Remove(new(id, orig.Value));
return;
}
// Change op: A row with this id was already in the table,
// but its values have changed (according to 'comparer').
removed.Remove(orig);
removed.Add(orig);
}
added.Remove(new(id, row));
added.Add(new(id, row));
}
public void Remove(int id) {
#nullable disable
added.Remove(new(id, default));
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
removed.Remove(orig);
removed.Remove(orig);
}
}
public DataPacket ToDataPacket(bool reset = true) {
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
if (reset) {
added.Clear();
removed.Clear();
}
return new(newRows, addedRows, removedRows);
}
public IDataPacket<T> ToDataPacketT(bool reset = true) {
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
var removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
if (reset) {
added.Clear();
removed.Clear();
}
return new DataPacket<T>(newRows, addedRows, removedRows);
}
public IDataPacket<T> ToDataPacketT(
out BlueTableImmutable<T> newRows,
out BlueTableImmutable<T> addedRows,
out BlueTableImmutable<T> removedRows,
bool reset = true
) {
table.RemoveRange(removed);
table.AddRange(added);
newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
removedRows = new BlueTableImmutable<T>(schema, removed.ToImmutable());
if (reset) {
added.Clear();
removed.Clear();
}
return new DataPacket<T>(newRows, addedRows, removedRows);
}
public BlueTableImmutable<T> ToImmutable() =>
new(schema, table.ToImmutable());
}
}
public class BlueTableImmutableAdapter<T> : IBlueRowCollection<BlueRowAdapter<T>> {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSet<KeyValuePair<int, T>> rows;
public BlueTableImmutableAdapter(BlueTableSchemaAdapter<T> schemaAdapter)
: this(schemaAdapter, ImmutableSortedSet<KeyValuePair<int, T>>.Empty) { }
public BlueTableImmutableAdapter(BlueTableSchemaAdapter<T> schemaAdapter, ImmutableSortedSet<KeyValuePair<int, T>> rows) {
this.schemaAdapter = schemaAdapter;
this.rows = rows;
}
public int Count => rows.Count;
public bool IsSynchronized => false;
public IBlueTableSchema schema => schemaAdapter.Schema;
public object SyncRoot => this;
public BlueRowAdapter<T> this[int id] =>
new(schemaAdapter, rows[id].Value);
public bool containsID(int id) =>
#nullable disable
rows.Contains(new(id, default));
#nullable enable
public IList<object?> GetColumn(int index) =>
rows.WrapSelector(kv => schemaAdapter.GetBlueRow(kv.Value)[index], null);
public void CopyTo(Array array, int index) {
foreach (var kv in rows) {
array.SetValue(new BlueRowAdapter<T>(schemaAdapter, kv.Value), index++);
}
}
public IEnumerator<BlueRowAdapter<T>> GetEnumerator() {
foreach (var kv in rows) {
yield return new BlueRowAdapter<T>(schemaAdapter, kv.Value);
}
}
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class DataPacketBuilder {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder added =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder removed =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutableAdapter<T> table) {
schemaAdapter = table.schemaAdapter;
this.table = table.rows.ToBuilder();
}
public void AddOrUpdate(T rowData, IEqualityComparer<IBlueRow>? comparer = null) {
var row = new BlueRowAdapter<T>(schemaAdapter, rowData);
var id = row.Id();
#nullable disable
if (table.TryGetValue(new(id, default), out var origData)) {
#nullable enable
var orig = new BlueRowAdapter<T>(schemaAdapter, origData.Value);
if (comparer?.Equals(row, orig) ?? false) {
// No op: This row was already in the table (according to 'comparer').
// Remove it from 'added' and 'removed'.
added.Remove(origData);
removed.Remove(origData);
return;
}
// Change op: A row with this id was already in the table,
// but its values have changed (according to 'comparer').
removed.Remove(origData);
removed.Add(origData);
}
removed.Remove(new(id, rowData));
removed.Add(new(id, rowData));
}
public void Remove(int id) {
#nullable disable
added.Remove(new(id, default));
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
removed.Remove(orig);
removed.Add(orig);
}
}
public void Update<TKey>(TableUpdate<T, TKey> tableUpdate) where TKey : IEquatable<TKey> {
foreach (var r in tableUpdate.removed.Concat(tableUpdate.changes.Old())) {
Remove(new BlueRowAdapter<T>(schemaAdapter, r).Id());
}
foreach (var r in tableUpdate.changed.Concat(tableUpdate.added)) {
AddOrUpdate(r);
}
}
public IDataPacket<BlueRowAdapter<T>> ToDataPacket(bool reset = true) {
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutableAdapter<T>(schemaAdapter, table.ToImmutable());
var addedRows = new BlueTableImmutableAdapter<T>(schemaAdapter, added.ToImmutable());
var removedRows = new BlueTableImmutableAdapter<T>(schemaAdapter, removed.ToImmutable());
if (reset) {
added.Clear();
removed.Clear();
}
return new DataPacket<BlueRowAdapter<T>>(newRows, addedRows, removedRows);
}
public BlueTableImmutableAdapter<T> ToImmutable() =>
new(schemaAdapter, table.ToImmutable());
}
}