new: Add GetColumn fn to IBlueRowCollection interface

This allows us to implement python/pandas functionality, much of which is based
on column operations.
This commit is contained in:
John Colvin committed 2025-11-15 12:38:50 -05:00
1 parent 2a4d62a1e9
commit 0d10c2ba5d
24 files changed
+495 -160

No files matched your search

@@ -1,3 +1,5 @@
using bts.collections;
using Bts.DataStreamCore;
using System;
@@ -11,7 +13,7 @@ namespace BCG
/// </summary>
public class GroupByOutputTable : IBlueRowCollection
{
readonly Dictionary<string, IBlueRow> myDict = new();
readonly SortedList<string, IBlueRow> myDict = new();
private BlueTable myTable;
public GroupByOutputTable (IBlueTableSchema schema) {myTable = new BlueTable (schema);}
@@ -58,7 +60,10 @@ namespace BCG
public IBlueRow this[int id] {get{return myTable[id];}}
public IBlueTableSchema schema {get{return myTable.schema;}}
public bool containsID (int id) {return myTable.containsID (id);}
public IList<object?> GetColumn(int index) =>
index < 0 || index >= myTable.schema.columnInfo.Length
? throw new IndexOutOfRangeException()
: myDict.Values.WrapSelector(r => r[index], null);
#endregion
#region ICollection members
@@ -97,6 +97,8 @@ namespace BCG
public IBlueRow this[int id] {get{return myLinearCollection[id];}}
public IBlueTableSchema schema {get{return mySchema;}}
public bool containsID (int id) {return myLinearCollection.containsID (id);}
public IList<object?> GetColumn(int index) =>
myLinearCollection.GetColumn(index);
#endregion
#region ICollection members
@@ -1,14 +1,17 @@
using bts.collections;
using Bts.DataStreamCore;
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
internal class JoinTable : IBlueRowCollection
{
protected JoinTableSchema mySchema;
protected IDictionary myIdIndex = new Hashtable ();
protected SortedList<int, IBlueRow> myIdIndex = new();
protected JoinDeletionIndex myLeftIndex, myRightIndex;
private int myNewRowId;
@@ -20,13 +23,16 @@ namespace BCG
}
public IBlueRow this [int id] =>
myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
myIdIndex.TryGetValue(id, out var r)
? r
: throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
#region JoinTable members
public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp)
{
var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema);
myIdIndex[myNewRowId ++] = newRow;
myIdIndex.Add(myNewRowId, newRow);
myNewRowId++;
((BlueTable)(dp.addedRows)).addRow (newRow);
myLeftIndex.add (newRow);
if (rightRow != null) //in case of left joins when right row is null
@@ -61,7 +67,11 @@ namespace BCG
public object SyncRoot{get{return this;}}
public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);}
public void CopyTo(Array array, int index) {
for (int i = 0; i < myIdIndex.Count; i++) {
array.SetValue(myIdIndex.Values[i], index++);
}
}
#endregion
#region IEnumerable members
@@ -71,6 +81,12 @@ namespace BCG
#region IBlueRowCollection members
public IBlueTableSchema schema {get{return mySchema;}}
public bool containsID (int id) {return myIdIndex[id] != null;}
public IList<object?> GetColumn(int index) {
if (index < 0 || index >= schema.columnInfo.Length) {
throw new IndexOutOfRangeException();
}
return myIdIndex.Values.WrapSelector(r => r[index], null);
}
#endregion
}
@@ -1,5 +1,8 @@
using bts.collections;
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -39,6 +42,11 @@ namespace BCG
public bool containsID (int id) {throw new NotSupportedException ("");}
public IList<object?> GetColumn(int index) =>
index < 0 || index >= schema.columnInfo.Length
? throw new IndexOutOfRangeException()
: myRows.WrapType<IBlueRow>().WrapSelector(r => r[index], null);
public IBlueTableSchema schema {get {return mySchema;}}
#endregion
@@ -1,5 +1,6 @@
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -31,6 +32,13 @@ namespace BCG
public IBlueTableSchema schema {get {return myHelper.schema;}}
public IList<object?> GetColumn(int index) {
if (index < 0 || index >= schema.columnInfo.Length) {
throw new IndexOutOfRangeException();
}
var origIdx = myHelper.colMap[index];
return myRows.GetColumn(origIdx);
}
#endregion
#region ICollection Members
@@ -3,6 +3,8 @@ using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using bts.collections;
using Bts.ColumnSortExpressions;
using Bts.DataStreamCore;
@@ -86,6 +88,11 @@ public class SortedTable : IBlueRowCollection {
public bool containsID(int id) =>
rowIndex[id] is not null;
public IList<object?> GetColumn(int index) =>
index < 0 || index >= schema.columnInfo.Length
? throw new IndexOutOfRangeException()
: rows.WrapSelector(r => r[index], null);
public IEnumerator GetEnumerator() =>
rows.GetEnumerator();
@@ -1,9 +1,17 @@
using bts;
using bts.collections;
using Bts.DataStream;
using Bts.DataStreamCore;
using mbts.utils;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace BCG
{
@@ -14,31 +22,38 @@ namespace BCG
internal class AppendCollection : IBlueRowCollection
{
private int myId = 0;
private IDictionary myLeftIndex = new Hashtable (),
myRightIndex = new Hashtable ();
readonly SortedList<int, IBlueRow> myLeftIndex = new();
readonly SortedList<int, IBlueRow> myRightIndex = new();
readonly Dictionary<int, int> leftIdMap = new();
readonly Dictionary<int, int> rightIdMap = new();
private IBlueTableSchema? mySchema;
int? idColumnIndex;
public AppendCollection () {}
public DataPacket mergeRows (DataPacket dp, bool isLeft)
{
mySchema = mySchema == null ? dp.schema : mySchema;
BlueTable dRows = new BlueTable (mySchema),
aRows = new BlueTable (mySchema);
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
foreach (IBlueRow r in dp.deletedRows)
{
var dr = ExceptionAssert.of_type<IBlueRow>(index[r.Id()]);
dRows.addRow (dr);
index.Remove (r.Id());
ensureInit(dp);
BlueTable dRows = new BlueTable(mySchema),
aRows = new BlueTable(mySchema);
var (index, idMap) = isLeft ? (myLeftIndex, leftIdMap) : (myRightIndex, rightIdMap);
foreach (IBlueRow r in dp.deletedRows) {
int idIn = r.Id();
var dr = ExceptionAssert.of_type<IBlueRow>(index[idIn]);
var id = idMap[idIn];
var maskedRow = new AppendRow(dr, id, mySchema);
dRows.addRow(maskedRow);
index.Remove(r.Id());
}
foreach (IBlueRow r in dp.addedRows)
{
IBlueRow ar = new AppendRow (r, myId++, mySchema);
aRows.addRow (ar);
index.Add(r.Id(), ar);
foreach (IBlueRow r in dp.addedRows) {
int idIn = r.Id();
var id = myId++;
idMap[idIn] = id;
IBlueRow ar = new AppendRow(r, id, mySchema);
aRows.addRow(ar);
index.Add(idIn, ar);
}
return new DataPacket (this, aRows, dRows);
return new DataPacket(this, aRows, dRows);
}
#region IBlueRowCollection Members
@@ -67,8 +82,27 @@ namespace BCG
public object SyncRoot => this;
public IList<object?> GetColumn(int index) {
var leftList = SortedKeyValueList.Create(myLeftIndex).WrapSelector(kv => new KeyValuePair<int, IBlueRow>(kv.Key, leftRowFactory(kv)), null);
var rightList = SortedKeyValueList.Create(myRightIndex).WrapSelector(kv => new KeyValuePair<int, IBlueRow>(kv.Key, rightRowFactory(kv)), null);
var allRows = new MergedSortedList<KeyValuePair<int, IBlueRow>>(leftList, rightList, new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key });
return allRows.as_mutable().WrapSelector(kv => kv.Value[index], null);
}
IBlueRow leftRowFactory(KeyValuePair<int, IBlueRow> kv) =>
new AppendRow(kv.Value, leftIdMap[kv.Key], mySchema ?? kv.Value.schema);
IBlueRow rightRowFactory(KeyValuePair<int, IBlueRow> kv) =>
new AppendRow(kv.Value, rightIdMap[kv.Key], mySchema ?? kv.Value.schema);
#endregion
[MemberNotNull(nameof(mySchema), nameof(idColumnIndex))]
void ensureInit(IDataPacket dp) {
Log.assert(mySchema is null == idColumnIndex is null);
mySchema ??= dp.Schema;
idColumnIndex ??= mySchema.ResolveIndices("id");
}
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator() {
@@ -149,6 +149,9 @@ namespace BCG {
public bool containsID(int id) { return myRows.containsID(id); }
public IList<object?> GetColumn(int index) =>
myRows.GetColumn(index);
#endregion
#region ICollection Members
@@ -2,6 +2,7 @@ namespace Bts.DataStreamCore;
using BCG;
using bts.collections;
using bts.utils;
using mbts.utils;
@@ -139,6 +140,9 @@ public class BlueColumnTable : IBlueRowCollection {
public bool containsID(int id) =>
rowIds.BinarySearch(id) >= 0;
public IList<object> GetColumn(int index) =>
columns[index].WrapSelector(v => v.GetValue(), null);
public IEnumerator GetEnumerator() {
var count = rowIds.Count;
for (int i = 0; i < count; i++) {
@@ -3,6 +3,8 @@ namespace Bts.DataStreamCore;
using BCG;
using bts.collections;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -12,6 +14,7 @@ public class BlueRowCollection : IBlueRowCollection {
readonly ICollection source;
readonly Func<int, IBlueRow> findRow;
readonly Func<int, bool> containsKey;
IList<IBlueRow>? cachedRows;
public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func<int, IBlueRow>? findRow = null, Func<int, bool>? containsKey = null) {
this.source = source;
@@ -51,6 +54,11 @@ public class BlueRowCollection : IBlueRowCollection {
public bool containsID(int id) =>
containsKey(id);
public IList<object?> GetColumn(int index) {
cachedRows ??= source as IList<IBlueRow> ?? source.Cast<IBlueRow>().ToList();
return cachedRows.WrapSelector(r => r[index], null);
}
public IEnumerator GetEnumerator() =>
source.GetEnumerator();
}
@@ -1,6 +1,6 @@
using Bts.DataStreamCore;
using bts.collections;
using Spark.DataStream;
using Bts.DataStreamCore;
using System;
using System.Collections;
@@ -21,7 +21,7 @@ namespace BCG {
/// <summary>
/// Hashtable indexed on "id" to speed up removes
/// </summary>
protected Hashtable myIdIndex = new Hashtable();
protected SortedList<int, IBlueRow> rows = new();
/// <summary>
/// Schema the table adheres to
/// </summary>
@@ -42,7 +42,7 @@ namespace BCG {
mySchema = new BlueTableSchema(cis, table.TableName);
foreach (DataRow row in table.Rows) {
IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
myIdIndex[newRow["id"]] = newRow;
rows[newRow.Id()] = newRow;
}
}
@@ -81,8 +81,7 @@ namespace BCG {
/// <param name="schema"></param>
public BlueTable(IBlueTableSchema schema) { mySchema = schema; }
public ICollection<IBlueRow> Rows =>
GenericCollectionWrapper.Create<IBlueRow>(myIdIndex.Values);
public IList<IBlueRow> Rows => rows.Values;
public static BlueTable Empty(string name, bool includeIdColumn = true) =>
new(BlueTableSchema.Empty(name, includeIdColumn));
@@ -95,7 +94,7 @@ namespace BCG {
/// </summary>
public virtual IBlueRow this[int id] {
get {
object row = myIdIndex[id];
object row = rows[id];
if (row == null)
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
return (IBlueRow)row;
@@ -113,18 +112,16 @@ namespace BCG {
/// </summary>
public virtual IBlueTableSchema schema { get { return mySchema; } }
public virtual bool containsID(int id) { return myIdIndex[id] != null; }
public virtual bool containsID(int id) { return rows[id] != null; }
public virtual bool tryGetValue(int id, out IBlueRow row) {
row = (IBlueRow)myIdIndex[id];
return (row != null);
}
public virtual bool tryGetValue(int id, out IBlueRow row) =>
rows.TryGetValue(id, out row);
#endregion
#region BlueTable members
public virtual void clear() {
myIdIndex.Clear();
rows.Clear();
}
/// <summary>
@@ -132,7 +129,7 @@ namespace BCG {
/// </summary>
/// <param name="r"></param>
public virtual void forceAddRow(IBlueRow r) {
myIdIndex[r["id"]] = r;
rows[r.Id()] = r;
}
/// <summary>
@@ -145,7 +142,7 @@ namespace BCG {
if (id == null || id == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull");
try {
myIdIndex.Add(id, row);
rows.Add((int)id, row);
} catch (ArgumentException) {
throw new BlueDataException($"ID: {id} already exists in the table: {mySchema.tableName}");
}
@@ -157,7 +154,10 @@ namespace BCG {
if (val == null || val == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull");
if (myIdIndex[val] != null) {
if (val is not int id) {
throw new ArgumentException($"Expected id to be of type int (actually {val?.GetType()})", nameof(row));
}
if (rows.ContainsKey(id)) {
string err_msg = $"ID: {val} already exists in the table: {mySchema.tableName} duplication of field ID not allowed";
// temp TODOds remove it
bts.Log.assert(false, err_msg);
@@ -165,7 +165,7 @@ namespace BCG {
}
IBlueRow newRow = new BlueRow(mySchema, row);
myIdIndex[val] = newRow;
rows[id] = newRow;
}
public virtual void addRows(IBlueRowCollection rows) {
@@ -190,12 +190,8 @@ namespace BCG {
/// Removes a row specified by the id. The index helps in making this operation O(1)
/// </summary>
/// <param name="id"></param>
public virtual bool deleteRow(int id) {
if (myIdIndex[id] == null)
return false;
myIdIndex.Remove(id);
return true;
}
public virtual bool deleteRow(int id) =>
rows.Remove(id);
public virtual bool deleteRows(IBlueRowCollection rows) {
bool retVal = true;
@@ -212,7 +208,7 @@ namespace BCG {
}
public virtual void changeRow(IBlueRow r) {
myIdIndex[r["id"]] = r;
rows[r.Id()] = r;
}
public virtual void changeRows(IBlueRowCollection rows) {
@@ -224,17 +220,29 @@ namespace BCG {
public IDictionary<int, IBlueRow> AsDictionary() => this;
public IList<object> GetColumn(int index) {
if (index < 0 || index >= schema.columnInfo.Length) {
throw new IndexOutOfRangeException();
}
return Rows.WrapSelector(r => r[index], null);
}
#region ICollection members
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
{ return myIdIndex.Values.GetEnumerator(); }
public virtual IEnumerator GetEnumerator() =>
rows.Values.GetEnumerator();
public virtual bool IsSynchronized { get { return false; } }
public virtual object SyncRoot { get { return this; } }
public virtual void CopyTo(Array array, int index) { myIdIndex.Values.CopyTo(array, index); }
public virtual void CopyTo(Array array, int index) {
foreach (var r in rows.Values) {
array.SetValue(r, index++);
}
}
public virtual int Count { get { return myIdIndex.Count; } }
public virtual int Count =>
rows.Count;
#endregion
#region IDictionary<int, IBlueRow> members
@@ -262,24 +270,20 @@ namespace BCG {
tryGetValue(item.Key, out var r)
&& BlueRowEqualityComparer.Instance.Equals(r, item.Value);
void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(
KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
foreach (DictionaryEntry de in myIdIndex) {
array[arrayIndex++] = new((int)de.Key, (IBlueRow)de.Value);
void ICollection<KeyValuePair<int, IBlueRow>>.CopyTo(KeyValuePair<int, IBlueRow>[] array, int arrayIndex) {
foreach (var kv in rows) {
array[arrayIndex] = kv;
}
}
bool ICollection<KeyValuePair<int, IBlueRow>>.Remove(KeyValuePair<int, IBlueRow> item) =>
deleteRow(item.Key);
IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() {
foreach (DictionaryEntry de in myIdIndex) {
yield return new((int)de.Key, (IBlueRow)de.Value);
}
}
IEnumerator<KeyValuePair<int, IBlueRow>> IEnumerable<KeyValuePair<int, IBlueRow>>.GetEnumerator() =>
rows.GetEnumerator();
ICollection<int> IDictionary<int, IBlueRow>.Keys =>
GenericCollectionWrapper.Create<int>(myIdIndex.Keys);
rows.Keys;
ICollection<IBlueRow> IDictionary<int, IBlueRow>.Values =>
Rows;
@@ -4,8 +4,11 @@ namespace Bts.DataStreamCore;
using BCG;
using bts.api.core;
using bts.collections;
using Bts.DataStreamCore;
using mbts.utils;
using Spark.Core.Collections;
using System;
using System.Collections;
@@ -15,14 +18,14 @@ using System.Linq;
// I would have named this ImmutableBlueTable, but that was already taken.
public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
readonly ImmutableDictionary<int, IBlueRow> rows;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>> rows;
public BlueTableImmutable(IBlueTableSchema schema)
: this(schema, ImmutableDictionary<int, IBlueRow>.Empty) { }
: this(schema, ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Empty) { }
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, IBlueRow> rows) {
public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<int, IBlueRow>> rows) {
this.schema = schema;
this.rows = rows;
this.rows = rows.WithComparer(new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key });
}
public int Count => rows.Count;
@@ -30,30 +33,43 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
public IBlueTableSchema schema { get; }
public object SyncRoot => this;
public IBlueRow this[int id] => rows[id];
public IBlueRow this[int id] =>
#nullable disable
rows.TryGetValue(new(id, default), out var kv)
#nullable enable
? kv.Value
: throw new KeyNotFoundException();
public bool containsID(int id) =>
rows.ContainsKey(id);
#nullable disable
rows.Contains(new(id, default));
#nullable enable
public void CopyTo(Array array, int index) {
foreach (var r in rows.Values) {
array.SetValue(r, index++);
foreach (var kv in rows) {
array.SetValue(kv.Value, index++);
}
}
public IEnumerator GetEnumerator() =>
rows.Values.GetEnumerator();
public IList<object?> GetColumn(int index) =>
rows.WrapSelector(kv => kv.Value[index], null);
public IEnumerator GetEnumerator() {
foreach (var kv in rows) {
yield return kv.Value;
}
}
public DataPacketBuilder ToDataPacketBuilder() =>
new(this);
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableDictionary<int, IBlueRow>.Builder table;
readonly ImmutableDictionary<int, IBlueRow>.Builder added =
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
readonly ImmutableDictionary<int, IBlueRow>.Builder removed =
ImmutableDictionary<int, IBlueRow>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder table;
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder added =
ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Builder removed =
ImmutableSortedSet<KeyValuePair<int, IBlueRow>>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutable table) {
schema = table.schema;
@@ -62,30 +78,37 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
public void AddOrUpdate(IBlueRow row, IEqualityComparer<IBlueRow>? comparer = null) {
var id = row.Id();
if (table.TryGetValue(id, out var orig)) {
if (comparer?.Equals(row, orig) ?? false) {
#nullable disable
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
if (comparer?.Equals(row, orig.Value) ?? false) {
// No op: This row was already in the table (according to 'comparer').
// Remove it from 'added' and 'removed'.
added.Remove(id);
removed.Remove(id);
added.Remove(new(id, row));
removed.Remove(new(id, orig.Value));
return;
}
// Change op: A row with this id was already in the table,
// but its values have changed (according to 'comparer').
removed[id] = orig;
removed.Remove(new(id, orig.Value));
removed.Add(new(id, orig.Value));
}
added[id] = row;
added.Remove(new(id, row));
added.Add(new(id, row));
}
public void Remove(int id) {
added.Remove(id);
if (table.TryGetValue(id, out var orig)) {
removed[id] = orig;
#nullable disable
added.Remove(new(id, default));
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
removed.Remove(new(id, orig.Value));
removed.Add(new(id, orig.Value));
}
}
public DataPacket ToDataPacket(bool reset = true) {
table.RemoveRange(removed.Keys);
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutable(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable(schema, added.ToImmutable());
@@ -103,12 +126,12 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection {
}
public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
readonly ImmutableDictionary<int, T> rows;
readonly ImmutableSortedSet<KeyValuePair<int, T>> rows;
public BlueTableImmutable(IBlueTableSchema schema)
: this(schema, ImmutableDictionary<int, T>.Empty) { }
: this(schema, ImmutableSortedSet<KeyValuePair<int, T>>.Empty) { }
public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary<int, T> rows) {
public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet<KeyValuePair<int, T>> rows) {
this.schema = schema;
this.rows = rows;
}
@@ -118,19 +141,32 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
public IBlueTableSchema schema { get; }
public object SyncRoot => this;
public T this[int id] => rows[id];
public T this[int id] =>
#nullable disable
rows.TryGetValue(new(id, default), out var kv)
#nullable enable
? kv.Value
: throw new KeyNotFoundException();
public bool containsID(int id) =>
rows.ContainsKey(id);
#nullable disable
rows.Contains(new(id, default));
#nullable enable
public IList<object?> GetColumn(int index) =>
rows.WrapSelector(kv => kv.Value[index], null);
public void CopyTo(Array array, int index) {
foreach (var r in rows.Values) {
array.SetValue(r, index++);
foreach (var (k, v) in rows) {
array.SetValue(v, index++);
}
}
public IEnumerator<T> GetEnumerator() =>
rows.Values.GetEnumerator();
public IEnumerator<T> GetEnumerator() {
foreach (var kv in rows) {
yield return kv.Value;
}
}
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
@@ -140,11 +176,11 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
public class DataPacketBuilder {
readonly IBlueTableSchema schema;
readonly ImmutableDictionary<int, T>.Builder table;
readonly ImmutableDictionary<int, T>.Builder added =
ImmutableDictionary<int, T>.Empty.ToBuilder();
readonly ImmutableDictionary<int, T>.Builder removed =
ImmutableDictionary<int, T>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder added =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder removed =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutable<T> table) {
schema = table.schema;
@@ -153,30 +189,37 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
public void AddOrUpdate(T row, IEqualityComparer<IBlueRow>? comparer = null) {
var id = row.Id();
if (table.TryGetValue(id, out var orig)) {
if (comparer?.Equals(row, orig) ?? false) {
#nullable disable
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
if (comparer?.Equals(row, orig.Value) ?? false) {
// No op: This row was already in the table (according to 'comparer').
// Remove it from 'added' and 'removed'.
added.Remove(id);
removed.Remove(id);
added.Remove(new(id, row));
removed.Remove(new(id, orig.Value));
return;
}
// Change op: A row with this id was already in the table,
// but its values have changed (according to 'comparer').
removed[id] = orig;
removed.Remove(orig);
removed.Add(orig);
}
added[id] = row;
added.Remove(new(id, row));
added.Add(new(id, row));
}
public void Remove(int id) {
added.Remove(id);
if (table.TryGetValue(id, out var orig)) {
removed[id] = orig;
#nullable disable
added.Remove(new(id, default));
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
removed.Remove(orig);
removed.Remove(orig);
}
}
public DataPacket ToDataPacket(bool reset = true) {
table.RemoveRange(removed.Keys);
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
@@ -189,7 +232,7 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
}
public IDataPacket<T> ToDataPacketT(bool reset = true) {
table.RemoveRange(removed.Keys);
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
var addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
@@ -207,7 +250,7 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
out BlueTableImmutable<T> removedRows,
bool reset = true
) {
table.RemoveRange(removed.Keys);
table.RemoveRange(removed);
table.AddRange(added);
newRows = new BlueTableImmutable<T>(schema, table.ToImmutable());
addedRows = new BlueTableImmutable<T>(schema, added.ToImmutable());
@@ -226,12 +269,12 @@ public class BlueTableImmutable<T> : IBlueRowCollection<T> where T : IBlueRow {
public class BlueTableImmutableAdapter<T> : IBlueRowCollection<BlueRowAdapter<T>> {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableDictionary<int, T> rows;
readonly ImmutableSortedSet<KeyValuePair<int, T>> rows;
public BlueTableImmutableAdapter(BlueTableSchemaAdapter<T> schemaAdapter)
: this(schemaAdapter, ImmutableDictionary<int, T>.Empty) { }
: this(schemaAdapter, ImmutableSortedSet<KeyValuePair<int, T>>.Empty) { }
public BlueTableImmutableAdapter(BlueTableSchemaAdapter<T> schemaAdapter, ImmutableDictionary<int, T> rows) {
public BlueTableImmutableAdapter(BlueTableSchemaAdapter<T> schemaAdapter, ImmutableSortedSet<KeyValuePair<int, T>> rows) {
this.schemaAdapter = schemaAdapter;
this.rows = rows;
}
@@ -241,20 +284,26 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection<BlueRowAdapter<T>
public IBlueTableSchema schema => schemaAdapter.Schema;
public object SyncRoot => this;
public BlueRowAdapter<T> this[int id] => new(schemaAdapter, rows[id]);
public BlueRowAdapter<T> this[int id] =>
new(schemaAdapter, rows[id].Value);
public bool containsID(int id) =>
rows.ContainsKey(id);
#nullable disable
rows.Contains(new(id, default));
#nullable enable
public IList<object?> GetColumn(int index) =>
rows.WrapSelector(kv => schemaAdapter.GetBlueRow(kv.Value)[index], null);
public void CopyTo(Array array, int index) {
foreach (var r in rows.Values) {
array.SetValue(new BlueRowAdapter<T>(schemaAdapter, r), index++);
foreach (var kv in rows) {
array.SetValue(new BlueRowAdapter<T>(schemaAdapter, kv.Value), index++);
}
}
public IEnumerator<BlueRowAdapter<T>> GetEnumerator() {
foreach (var row in rows.Values) {
yield return new BlueRowAdapter<T>(schemaAdapter, row);
foreach (var kv in rows) {
yield return new BlueRowAdapter<T>(schemaAdapter, kv.Value);
}
}
@@ -263,11 +312,11 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection<BlueRowAdapter<T>
public class DataPacketBuilder {
readonly BlueTableSchemaAdapter<T> schemaAdapter;
readonly ImmutableDictionary<int, T>.Builder table;
readonly ImmutableDictionary<int, T>.Builder added =
ImmutableDictionary<int, T>.Empty.ToBuilder();
readonly ImmutableDictionary<int, T>.Builder removed =
ImmutableDictionary<int, T>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder table;
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder added =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
readonly ImmutableSortedSet<KeyValuePair<int, T>>.Builder removed =
ImmutableSortedSet<KeyValuePair<int, T>>.Empty.ToBuilder();
public DataPacketBuilder(BlueTableImmutableAdapter<T> table) {
schemaAdapter = table.schemaAdapter;
@@ -277,26 +326,33 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection<BlueRowAdapter<T>
public void AddOrUpdate(T rowData, IEqualityComparer<IBlueRow>? comparer = null) {
var row = new BlueRowAdapter<T>(schemaAdapter, rowData);
var id = row.Id();
if (table.TryGetValue(id, out var origData)) {
var orig = new BlueRowAdapter<T>(schemaAdapter, origData);
#nullable disable
if (table.TryGetValue(new(id, default), out var origData)) {
#nullable enable
var orig = new BlueRowAdapter<T>(schemaAdapter, origData.Value);
if (comparer?.Equals(row, orig) ?? false) {
// No op: This row was already in the table (according to 'comparer').
// Remove it from 'added' and 'removed'.
added.Remove(id);
removed.Remove(id);
added.Remove(origData);
removed.Remove(origData);
return;
}
// Change op: A row with this id was already in the table,
// but its values have changed (according to 'comparer').
removed[id] = origData;
removed.Remove(origData);
removed.Add(origData);
}
added[id] = rowData;
removed.Remove(new(id, rowData));
removed.Add(new(id, rowData));
}
public void Remove(int id) {
added.Remove(id);
if (table.TryGetValue(id, out var orig)) {
removed[id] = orig;
#nullable disable
added.Remove(new(id, default));
if (table.TryGetValue(new(id, default), out var orig)) {
#nullable enable
removed.Remove(orig);
removed.Add(orig);
}
}
@@ -310,7 +366,7 @@ public class BlueTableImmutableAdapter<T> : IBlueRowCollection<BlueRowAdapter<T>
}
public IDataPacket<BlueRowAdapter<T>> ToDataPacket(bool reset = true) {
table.RemoveRange(removed.Keys);
table.RemoveRange(removed);
table.AddRange(added);
var newRows = new BlueTableImmutableAdapter<T>(schemaAdapter, table.ToImmutable());
var addedRows = new BlueTableImmutableAdapter<T>(schemaAdapter, added.ToImmutable());
@@ -31,6 +31,9 @@ public class BlueTableSelectorWrapper<T> : IBlueRowCollection<T> where T : IBlue
public bool containsID(int id) =>
source.containsID(id);
public IList<object?> GetColumn(int index) =>
throw new NotImplementedException();
public IEnumerator<T> GetEnumerator() {
foreach (IBlueRow r in source) {
yield return selector(r);
@@ -6,6 +6,8 @@ using System.Linq;
using BCG;
using bts.collections;
namespace bts {
public class CircularBufferTable : IBlueRowCollection {
private BlueTable _table;
@@ -49,5 +51,8 @@ namespace bts {
public bool IsSynchronized { get { return false; } }
public object SyncRoot { get { return this; } }
public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); }
public IList<object> GetColumn(int index) =>
_ordered_list.WrapSelector(r => r[index], null);
}
}
@@ -16,6 +16,7 @@ namespace BCG
IBlueRow this[int id]{get;}
IBlueTableSchema schema {get;}
bool containsID (int id);
IList<object> GetColumn(int index);
bool ICollection.IsSynchronized => false;
object ICollection.SyncRoot => this;
@@ -35,4 +36,14 @@ namespace BCG
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
public static class BlueRowCollectionExtensions {
public static IList<object> GetColumn(this IBlueRowCollection rows, string colName) {
var idx = Array.IndexOf(rows.schema.columnNames, colName);
if (idx == -1) {
throw new KeyNotFoundException();
}
return rows.GetColumn(idx);
}
}
}
@@ -1,6 +1,7 @@
using bts.collections;
using System;
using BCG;
//using BCG.Utilities;
using System.Collections.Generic;
namespace BCG
{
@@ -87,6 +88,9 @@ namespace BCG
{ return null; }
}
public IList<object> GetColumn(int index) =>
myRows.WrapSelector(r => r[index], null);
#endregion
#region IEnumerable Members
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
namespace BCG
{
@@ -132,6 +133,9 @@ namespace BCG
return getRow(myGetKeyVals(r));
}
public IList<object> GetColumn(int index) =>
throw new NotImplementedException();
public IBlueRow this[params object[] keyVals] {get {return myHash.getValue (keyVals) as IBlueRow;}}
private object[] myGetKeyVals (IBlueRow r) {return BlueDataUtility.getSelectedValues (r, myKeys);}
@@ -1,8 +1,7 @@
using System;
using bts.collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;
namespace BCG
{
@@ -106,6 +105,9 @@ namespace BCG
return this.myIDToRows.Value.ContainsKey(id);
}
public IList<object> GetColumn(int index) =>
mySortedRows.WrapSelector(r => r[index], null);
#endregion
#region ICollection Members
@@ -48,15 +48,16 @@ namespace BCG
public override void addRow(DataRow row)
{
base.addRow(row);
myOrderedRows.Add(myIdIndex[row["id"]]);
myOrderedRows.Add(rows[(int)row["id"]]);
}
public override bool deleteRow(int id)
{
if (myIdIndex[id] == null)
if (!rows.TryGetValue(id, out var value)) {
return false;
myOrderedRows.Remove(myIdIndex[id]);
myIdIndex.Remove(id);
}
myOrderedRows.Remove(value);
rows.Remove(id);
return true;
}
@@ -1,5 +1,7 @@
using bts.collections;
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -42,6 +44,9 @@ namespace BCG
return id > -1 && id < myRows.Length;
}
public IList<object> GetColumn(int index) =>
myRows.WrapSelector(r => r[index], null);
#endregion
#region ICollection Members
@@ -1,6 +1,9 @@
using bts.collections;
using System;
using System.ComponentModel;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
namespace BCG
{
@@ -51,6 +54,9 @@ namespace BCG
public IBlueTableSchema schema {get {return mySchema;}}
public IList<object> GetColumn(int index) =>
myRows.WrapSelector(r => ((IBlueRow)r)[index], null);
#endregion
#region ICollection Members
@@ -1,8 +1,8 @@
using System;
using bts.collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
@@ -37,6 +37,9 @@ namespace BCG
return (id >= 0 && id < _rows.Count && this[id] != null);
}
public IList<object> GetColumn(int index) =>
_rows.WrapSelector(r => r[index], null);
public void CopyTo(Array array, int index) { _rows.CopyTo((IBlueRow[])array, index);}
public int Count { get { return _rows.Count; } }
public bool IsSynchronized {get { return false; }}
@@ -7,6 +7,8 @@ using bts.collections;
using Bts.DataStreamCore;
using mbts.utils;
using NUnit.Framework;
using System.Collections.Immutable;
@@ -184,7 +186,7 @@ public class DataPacketBuilderTests {
}
static BlueTableImmutable getTable(params (int Id, double Value)[] rows) =>
new(schema, ImmutableDictionary.CreateRange(
new(schema, ImmutableSortedSet.CreateRange(
rows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup)))));
static MutableBlueRow getBlueRow((int id, double value) tup) =>
@@ -203,13 +205,14 @@ public class DataPacketBuilderTests {
(int Id, double Value)[]? removedRows = null) {
addedRows ??= Array.Empty<(int, double)>();
removedRows ??= Array.Empty<(int, double)>();
var kvpComp = new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key };
return new DataPacket(
new BlueTableImmutable(schema, ImmutableDictionary.CreateRange(
newRows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup))))),
new BlueTableImmutable(schema, ImmutableDictionary.CreateRange(
addedRows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup))))),
new BlueTableImmutable(schema, ImmutableDictionary.CreateRange(
removedRows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup))))));
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(newRows))),
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(addedRows))),
new BlueTableImmutable(schema, ImmutableSortedSet.CreateRange(kvpComp, GetRowKvps(removedRows))));
static IEnumerable<KeyValuePair<int, IBlueRow>> GetRowKvps((int Id, double Value)[] rows) =>
rows.Select(tup => new KeyValuePair<int, IBlueRow>(tup.Id, getBlueRow(tup)));
}
static (int, double) deconstructRow(IBlueRow row) =>
@@ -0,0 +1,133 @@
namespace Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
public static class SortedKeyValueList {
public static SortedKeyValueList<TKey, T> Create<TKey, T>(SortedList<TKey, T> source) where TKey : notnull =>
new(source);
}
public class SortedKeyValueList<TKey, T> : IList<KeyValuePair<TKey, T>> where TKey : notnull {
readonly SortedList<TKey, T> source;
public SortedKeyValueList(SortedList<TKey, T> source) =>
this.source = source;
public IComparer<TKey> KeyComparer => source.Comparer;
public KeyValuePair<TKey, T> this[int index] {
get {
if (index < 0 || index >= source.Count) {
throw new IndexOutOfRangeException();
}
return new(source.Keys[index], source.Values[index]);
}
set {
if (index < 0 || index >= source.Count) {
throw new IndexOutOfRangeException();
}
if (!canReplace(index, value.Key)) {
throw new ArgumentException("Item key not sorted");
}
source.RemoveAt(index);
source.Add(value.Key, value.Value);
}
}
public int Count => source.Count;
public bool IsReadOnly => false;
public void Add(KeyValuePair<TKey, T> item) =>
source.Add(item.Key, item.Value);
public void Clear() =>
source.Clear();
public bool Contains(KeyValuePair<TKey, T> item) =>
source.TryGetValue(item.Key, out var val) && EqualityComparer<T>.Default.Equals(item.Value, val);
public void CopyTo(KeyValuePair<TKey, T>[] array, int arrayIndex) {
foreach (var kv in source) {
array[arrayIndex++] = kv;
}
}
public IEnumerator<KeyValuePair<TKey, T>> GetEnumerator() =>
source.GetEnumerator();
public int IndexOf(KeyValuePair<TKey, T> item) {
int i = source.IndexOfKey(item.Key);
return i != -1 && !EqualityComparer<T>.Default.Equals(source.GetValueAtIndex(i), item.Value)
? -1
: i;
}
public void Insert(int index, KeyValuePair<TKey, T> item) {
if (index < 0 || index > source.Count) {
throw new IndexOutOfRangeException();
}
if (!canInsert(index, item.Key)) {
throw new ArgumentException("Item key not sorted", nameof(item));
}
source.Add(item.Key, item.Value);
}
public bool Remove(KeyValuePair<TKey, T> item) {
int i = source.IndexOfKey(item.Key);
if (i < 0 || !EqualityComparer<T>.Default.Equals(item.Value, source.GetValueAtIndex(i))) {
return false;
}
source.RemoveAt(i);
return true;
}
public void RemoveAt(int index) =>
source.RemoveAt(index);
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
bool canInsert(int index, TKey key) {
if (index < 0 || index > source.Count) {
return false;
}
if (source.Count == 0) {
return true;
}
if (index < source.Count) {
var upperKey = source.GetKeyAtIndex(index);
if (KeyComparer.Compare(key, upperKey) > 0) {
return false;
}
}
if (index > 0) {
var lowerKey = source.GetKeyAtIndex(index - 1);
if (KeyComparer.Compare(key, lowerKey) < 0) {
return false;
}
}
return true;
}
bool canReplace(int index, TKey key) {
if (index < 0 || index >= source.Count) {
return false;
}
if (index < source.Count - 1) {
var upperKey = source.GetKeyAtIndex(index + 1);
if (KeyComparer.Compare(key, upperKey) > 0) {
return false;
}
}
if (index > 0) {
var lowerKey = source.GetKeyAtIndex(index - 1);
if (KeyComparer.Compare(key, lowerKey) < 0) {
return false;
}
}
return true;
}
}