new: Add GetColumn fn to IBlueRowCollection interface
This allows us to implement python/pandas functionality, much of which is based on column operations.
This commit is contained in:
1 parent
2a4d62a1e9
commit
0d10c2ba5d
24 files changed
+495
-160
No files matched your search
@@ -1,3 +1,5 @@
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
@@ -11,7 +13,7 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class GroupByOutputTable : IBlueRowCollection
|
||||
{
|
||||
readonly Dictionary<string, IBlueRow> myDict = new();
|
||||
readonly SortedList<string, IBlueRow> myDict = new();
|
||||
private BlueTable myTable;
|
||||
|
||||
public GroupByOutputTable (IBlueTableSchema schema) {myTable = new BlueTable (schema);}
|
||||
@@ -58,7 +60,10 @@ namespace BCG
|
||||
public IBlueRow this[int id] {get{return myTable[id];}}
|
||||
public IBlueTableSchema schema {get{return myTable.schema;}}
|
||||
public bool containsID (int id) {return myTable.containsID (id);}
|
||||
|
||||
public IList<object?> GetColumn(int index) =>
|
||||
index < 0 || index >= myTable.schema.columnInfo.Length
|
||||
? throw new IndexOutOfRangeException()
|
||||
: myDict.Values.WrapSelector(r => r[index], null);
|
||||
#endregion
|
||||
|
||||
#region ICollection members
|
||||
|
||||
@@ -97,6 +97,8 @@ namespace BCG
|
||||
public IBlueRow this[int id] {get{return myLinearCollection[id];}}
|
||||
public IBlueTableSchema schema {get{return mySchema;}}
|
||||
public bool containsID (int id) {return myLinearCollection.containsID (id);}
|
||||
public IList<object?> GetColumn(int index) =>
|
||||
myLinearCollection.GetColumn(index);
|
||||
#endregion
|
||||
|
||||
#region ICollection members
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class JoinTable : IBlueRowCollection
|
||||
{
|
||||
protected JoinTableSchema mySchema;
|
||||
protected IDictionary myIdIndex = new Hashtable ();
|
||||
protected SortedList<int, IBlueRow> myIdIndex = new();
|
||||
protected JoinDeletionIndex myLeftIndex, myRightIndex;
|
||||
private int myNewRowId;
|
||||
|
||||
@@ -20,13 +23,16 @@ namespace BCG
|
||||
}
|
||||
|
||||
public IBlueRow this [int id] =>
|
||||
myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
myIdIndex.TryGetValue(id, out var r)
|
||||
? r
|
||||
: throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
|
||||
#region JoinTable members
|
||||
public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp)
|
||||
{
|
||||
var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema);
|
||||
myIdIndex[myNewRowId ++] = newRow;
|
||||
myIdIndex.Add(myNewRowId, newRow);
|
||||
myNewRowId++;
|
||||
((BlueTable)(dp.addedRows)).addRow (newRow);
|
||||
myLeftIndex.add (newRow);
|
||||
if (rightRow != null) //in case of left joins when right row is null
|
||||
@@ -61,7 +67,11 @@ namespace BCG
|
||||
|
||||
public object SyncRoot{get{return this;}}
|
||||
|
||||
public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);}
|
||||
public void CopyTo(Array array, int index) {
|
||||
for (int i = 0; i < myIdIndex.Count; i++) {
|
||||
array.SetValue(myIdIndex.Values[i], index++);
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IEnumerable members
|
||||
@@ -71,6 +81,12 @@ namespace BCG
|
||||
#region IBlueRowCollection members
|
||||
public IBlueTableSchema schema {get{return mySchema;}}
|
||||
public bool containsID (int id) {return myIdIndex[id] != null;}
|
||||
public IList<object?> GetColumn(int index) {
|
||||
if (index < 0 || index >= schema.columnInfo.Length) {
|
||||
throw new IndexOutOfRangeException();
|
||||
}
|
||||
return myIdIndex.Values.WrapSelector(r => r[index], null);
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
||||
+8
@@ -1,5 +1,8 @@
|
||||
using bts.collections;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -39,6 +42,11 @@ namespace BCG
|
||||
|
||||
public bool containsID (int id) {throw new NotSupportedException ("");}
|
||||
|
||||
public IList<object?> GetColumn(int index) =>
|
||||
index < 0 || index >= schema.columnInfo.Length
|
||||
? throw new IndexOutOfRangeException()
|
||||
: myRows.WrapType<IBlueRow>().WrapSelector(r => r[index], null);
|
||||
|
||||
public IBlueTableSchema schema {get {return mySchema;}}
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -31,6 +32,13 @@ namespace BCG
|
||||
|
||||
public IBlueTableSchema schema {get {return myHelper.schema;}}
|
||||
|
||||
public IList<object?> GetColumn(int index) {
|
||||
if (index < 0 || index >= schema.columnInfo.Length) {
|
||||
throw new IndexOutOfRangeException();
|
||||
}
|
||||
var origIdx = myHelper.colMap[index];
|
||||
return myRows.GetColumn(origIdx);
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region ICollection Members
|
||||
|
||||
@@ -3,6 +3,8 @@ using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
using bts.collections;
|
||||
|
||||
using Bts.ColumnSortExpressions;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
@@ -86,6 +88,11 @@ public class SortedTable : IBlueRowCollection {
|
||||
public bool containsID(int id) =>
|
||||
rowIndex[id] is not null;
|
||||
|
||||
public IList<object?> GetColumn(int index) =>
|
||||
index < 0 || index >= schema.columnInfo.Length
|
||||
? throw new IndexOutOfRangeException()
|
||||
: rows.WrapSelector(r => r[index], null);
|
||||
|
||||
public IEnumerator GetEnumerator() =>
|
||||
rows.GetEnumerator();
|
||||
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
using bts;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -14,31 +22,38 @@ namespace BCG
|
||||
internal class AppendCollection : IBlueRowCollection
|
||||
{
|
||||
private int myId = 0;
|
||||
private IDictionary myLeftIndex = new Hashtable (),
|
||||
myRightIndex = new Hashtable ();
|
||||
readonly SortedList<int, IBlueRow> myLeftIndex = new();
|
||||
readonly SortedList<int, IBlueRow> myRightIndex = new();
|
||||
readonly Dictionary<int, int> leftIdMap = new();
|
||||
readonly Dictionary<int, int> rightIdMap = new();
|
||||
private IBlueTableSchema? mySchema;
|
||||
|
||||
int? idColumnIndex;
|
||||
|
||||
public AppendCollection () {}
|
||||
|
||||
public DataPacket mergeRows (DataPacket dp, bool isLeft)
|
||||
{
|
||||
mySchema = mySchema == null ? dp.schema : mySchema;
|
||||
BlueTable dRows = new BlueTable (mySchema),
|
||||
aRows = new BlueTable (mySchema);
|
||||
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
|
||||
foreach (IBlueRow r in dp.deletedRows)
|
||||
{
|
||||
var dr = ExceptionAssert.of_type<IBlueRow>(index[r.Id()]);
|
||||
dRows.addRow (dr);
|
||||
index.Remove (r.Id());
|
||||
ensureInit(dp);
|
||||
BlueTable dRows = new BlueTable(mySchema),
|
||||
aRows = new BlueTable(mySchema);
|
||||
var (index, idMap) = isLeft ? (myLeftIndex, leftIdMap) : (myRightIndex, rightIdMap);
|
||||
foreach (IBlueRow r in dp.deletedRows) {
|
||||
int idIn = r.Id();
|
||||
var dr = ExceptionAssert.of_type<IBlueRow>(index[idIn]);
|
||||
var id = idMap[idIn];
|
||||
var maskedRow = new AppendRow(dr, id, mySchema);
|
||||
dRows.addRow(maskedRow);
|
||||
index.Remove(r.Id());
|
||||
}
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
IBlueRow ar = new AppendRow (r, myId++, mySchema);
|
||||
aRows.addRow (ar);
|
||||
index.Add(r.Id(), ar);
|
||||
foreach (IBlueRow r in dp.addedRows) {
|
||||
int idIn = r.Id();
|
||||
var id = myId++;
|
||||
idMap[idIn] = id;
|
||||
IBlueRow ar = new AppendRow(r, id, mySchema);
|
||||
aRows.addRow(ar);
|
||||
index.Add(idIn, ar);
|
||||
}
|
||||
return new DataPacket (this, aRows, dRows);
|
||||
return new DataPacket(this, aRows, dRows);
|
||||
}
|
||||
|
||||
#region IBlueRowCollection Members
|
||||
@@ -67,8 +82,27 @@ namespace BCG
|
||||
|
||||
public object SyncRoot => this;
|
||||
|
||||
public IList<object?> GetColumn(int index) {
|
||||
var leftList = SortedKeyValueList.Create(myLeftIndex).WrapSelector(kv => new KeyValuePair<int, IBlueRow>(kv.Key, leftRowFactory(kv)), null);
|
||||
var rightList = SortedKeyValueList.Create(myRightIndex).WrapSelector(kv => new KeyValuePair<int, IBlueRow>(kv.Key, rightRowFactory(kv)), null);
|
||||
var allRows = new MergedSortedList<KeyValuePair<int, IBlueRow>>(leftList, rightList, new ComparerBuilder<KeyValuePair<int, IBlueRow>>() { kv => kv.Key });
|
||||
return allRows.as_mutable().WrapSelector(kv => kv.Value[index], null);
|
||||
}
|
||||
|
||||
IBlueRow leftRowFactory(KeyValuePair<int, IBlueRow> kv) =>
|
||||
new AppendRow(kv.Value, leftIdMap[kv.Key], mySchema ?? kv.Value.schema);
|
||||
|
||||
IBlueRow rightRowFactory(KeyValuePair<int, IBlueRow> kv) =>
|
||||
new AppendRow(kv.Value, rightIdMap[kv.Key], mySchema ?? kv.Value.schema);
|
||||
|
||||
#endregion
|
||||
|
||||
[MemberNotNull(nameof(mySchema), nameof(idColumnIndex))]
|
||||
void ensureInit(IDataPacket dp) {
|
||||
Log.assert(mySchema is null == idColumnIndex is null);
|
||||
mySchema ??= dp.Schema;
|
||||
idColumnIndex ??= mySchema.ResolveIndices("id");
|
||||
}
|
||||
#region IEnumerable Members
|
||||
|
||||
public System.Collections.IEnumerator GetEnumerator() {
|
||||
|
||||
@@ -149,6 +149,9 @@ namespace BCG {
|
||||
|
||||
public bool containsID(int id) { return myRows.containsID(id); }
|
||||
|
||||
public IList<object?> GetColumn(int index) =>
|
||||
myRows.GetColumn(index);
|
||||
|
||||
#endregion
|
||||
|
||||
#region ICollection Members
|
||||
|
||||
Reference in new issue
Block a user