diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs index 4446ccc6a..2614e3447 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs @@ -43,7 +43,7 @@ namespace bts.BlueOrb { _last_fire_time = DateTime.Now; IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); - _table.add_row(nr); + _table = _table.Add(nr); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs index 178fc1260..c1d9b7f51 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs @@ -1,9 +1,15 @@ -using System; - using BCG; +using Bts.DataStreamCore; + using mbts.utils; +using Spark.Core.Collections; +using Spark.Core.Threading; + +using System; +using System.Collections.Immutable; + namespace bts.BlueOrb { // Keeps adding changing rows to the stream, hence preserving the history // and creating a time series, for example this can be used to make bbo updates @@ -11,6 +17,8 @@ namespace bts.BlueOrb { [DataStreamUserRespresentation("TimeSeriesStream")] public class TimeSeriesStream : ViewStream { readonly ViewStream _stream; + readonly string timeColumn; + readonly AtomicGate gate = new(); CircularBufferTable? _table; private int _id = 0; private int _size = 0; @@ -34,18 +42,28 @@ namespace bts.BlueOrb { private void _onstream_change(object o) { var dp = ExceptionAssert.Argument.Is(o); DateTime now = DateTime.Now; + using var _ = gate.BeginTransaction(); if (_table == null) { IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName); schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp")); _table = new CircularBufferTable(schema, _size); } + var addedRowsBd = BlueTableImmutable.CreateBuilder(_table.schema); + var deletedRowsBd = BlueTableImmutable.CreateBuilder(_table.schema); foreach (IBlueRow r in dp.addedRows) { MutableBlueRow ar = new MutableBlueRow(_table.schema, r); - ar.setValue("timestamp", now); - ar.setValue("id", _id++); - _table.add_row(ar); + ar.setValue(timeColumn, now); + int id = _id++; + ar.setValue("id", id); + _table = _table.Add(ar, out var oldRow); + addedRowsBd.AddOrUpdate(ar); + if (oldRow is not null) { + deletedRowsBd.AddOrUpdate(oldRow); + } } - var dp_out = BlueDataUtility.build_dp(_table, myLastValue as DataPacket); + var addedRows = addedRowsBd.ToImmutable(); + var deletedRows = deletedRowsBd.ToImmutable(); + var dp_out = new DataPacket(_table, addedRows, deletedRows); myFireEvent(dp_out); } } diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index d9dd5d6f6..e86452aec 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -20,13 +20,13 @@ public static class DataStreamExtensions { public static ViewStream AsViewStream(this IObservable source) => source as ViewStream ?? new ObservableViewStream(source); - public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => - source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger); + // public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => + // source as ViewStream ?? new ObservableViewStream(source, name, logger); - public static IObservable AsObservable(this ViewStream source) { - return source as IObservable ?? Observable.Create(Subscribe); + public static IObservable AsObservable(this ViewStream source) { + return source as IObservable ?? Observable.Create(Subscribe); - IDisposable Subscribe(IObserver observer) { + IDisposable Subscribe(IObserver observer) { source.onChange += OnChange; return Disposable.Create(() => source.onChange -= OnChange); diff --git a/src/DataStreamSuite/DataStream/Misc/PandasStream.cs b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs index 4bf180706..30548dbb9 100644 --- a/src/DataStreamSuite/DataStream/Misc/PandasStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs @@ -114,14 +114,13 @@ import sys executePythonCode(pythonCode.Value, pythonExe); var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema); var rowMap = rows as IDictionary ?? rows.Cast().ToDictionary(r => r.Id()); - var comp = BlueRowEqualityComparer.Instance; var deletedRows = new BlueTable(schema); var addedRows = new BlueTable(schema); if (lastRowMap is not null) { deletedRows.SetItems(lastRowMap.Where(oldRow => - !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value)))); + !(rowMap.TryGetValue(oldRow.Key, out var newRow) && BlueRowEqualityComparer.Equals(newRow, oldRow.Value)))); addedRows.SetItems(rowMap.Where(newRow => - !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value)))); + !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && BlueRowEqualityComparer.Equals(oldRow, newRow.Value)))); } lastRowMap = rowMap; lastFireTime = DateTime.Now; diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs index 33185cd9a..9f69b81f3 100644 --- a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs @@ -93,8 +93,7 @@ namespace BCG { return Task.CompletedTask; } - private void _on_str(DataPacket dp) - { + private void _on_str(IDataPacket dp) { foreach (IBlueRow r in dp.newRows) { string name = r.get("name"); diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs index cbdb0d04b..982cb8e59 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs @@ -3,7 +3,6 @@ using bts.collections; using Bts.DataStreamCore; using System; -using System.Collections; using System.Collections.Generic; namespace BCG @@ -56,6 +55,8 @@ namespace BCG public IBlueRow? getRow(string groupRepString) => myDict.GetValueOrDefault(groupRepString); + public int IndexOf(IBlueRow item) => + myTable.IndexOf(item); #region IBlueRowCollection members public IBlueRow this[int id] {get{return myTable[id];}} public IBlueTableSchema schema {get{return myTable.schema;}} @@ -64,6 +65,8 @@ namespace BCG index < 0 || index >= myTable.schema.columnInfo.Length ? throw new IndexOutOfRangeException() : myDict.Values.WrapSelector(r => r[index], null); + public IBlueRow GetRow(int index) => + myDict.Values[index]; #endregion #region ICollection members @@ -74,7 +77,8 @@ namespace BCG #endregion #region IEnumerable members - public IEnumerator GetEnumerator() {return myTable.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myTable.GetEnumerator (); #endregion diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs index a0334945a..e54bd1c34 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs @@ -99,6 +99,8 @@ namespace BCG public bool containsID (int id) {return myLinearCollection.containsID (id);} public IList GetColumn(int index) => myLinearCollection.GetColumn(index); + public IBlueRow GetRow(int index) => + myLinearCollection.GetRow(index); #endregion #region ICollection members @@ -106,10 +108,13 @@ namespace BCG public bool IsSynchronized {get{return false;}} public object SyncRoot {get{return this;}} public void CopyTo (Array array, int index) {myLinearCollection.CopyTo (array, index);} + public int IndexOf(IBlueRow item) => + myLinearCollection.IndexOf(item); #endregion #region IEnumerable members - public IEnumerator GetEnumerator() {return myLinearCollection.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myLinearCollection.GetEnumerator (); #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs index 985c4b2c7..f9ea293b1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs @@ -3,7 +3,6 @@ using bts.collections; using Bts.DataStreamCore; using System; -using System.Collections; using System.Collections.Generic; namespace BCG @@ -72,10 +71,20 @@ namespace BCG array.SetValue(myIdIndex.Values[i], index++); } } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = myIdIndex.IndexOfKey(id); + if (index != -1 && BlueRowEqualityComparer.Equals(item, myIdIndex.Values[index])) { + return index; + } + return -1; + } #endregion #region IEnumerable members - public IEnumerator GetEnumerator () {return myIdIndex.Values.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myIdIndex.Values.GetEnumerator(); #endregion #region IBlueRowCollection members @@ -87,6 +96,8 @@ namespace BCG } return myIdIndex.Values.WrapSelector(r => r[index], null); } + public IBlueRow GetRow(int index) => + myIdIndex.Values[index]; #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs index 891467434..90a2d7dbb 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections; using System.Collections.Generic; @@ -47,6 +49,12 @@ namespace BCG ? throw new IndexOutOfRangeException() : myRows.WrapType().WrapSelector(r => r[index], null); + public IBlueRow GetRow(int index) { + var rowObj = myRows[index]; + ArgumentNullException.ThrowIfNull(rowObj); + return (IBlueRow)rowObj; + } + public IBlueTableSchema schema {get {return mySchema;}} #endregion @@ -61,11 +69,28 @@ namespace BCG public object SyncRoot {get{return null;}} + public int IndexOf(IBlueRow item) { + int id = item.Id(); + for (int i = 0; i < myRows.Count; i++) { + if (myRows[i] is not IBlueRow r || r.Id() != id) { + continue; + } + if (BlueRowEqualityComparer.Equals(item, r)) { + return i; + } + break; + } + return -1; + } #endregion #region IEnumerable Members - public IEnumerator GetEnumerator() {return myRows.GetEnumerator ();} + public IEnumerator GetEnumerator() { + foreach (IBlueRow r in myRows) { + yield return r; + } + } #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs index 0cf9d6c7d..8697bf23a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs @@ -66,7 +66,7 @@ namespace bts { if (_in_rows.Count > 0) { // fire row IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++); - _out_table.add_row(curr_row); + _out_table = _out_table.Add(curr_row); DataPacket dp = BlueDataUtility.build_dp_inc(_out_table, myLastValue == null ? null : myLastValue as DataPacket, false); myFireEvent(dp); @@ -74,4 +74,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs index 6f4efbce3..6e13f3b19 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs @@ -1,5 +1,8 @@ +using bts; + +using Bts.DataStreamCore; + using System; -using System.Collections; using System.Collections.Generic; namespace BCG @@ -39,63 +42,36 @@ namespace BCG var origIdx = myHelper.colMap[index]; return myRows.GetColumn(origIdx); } + + public IBlueRow GetRow(int index) => + new ProjectedRow(myRows.GetRow(index), myHelper); #endregion #region ICollection Members - public bool IsSynchronized {get {return myRows.IsSynchronized;}} - public int Count {get {return myRows.Count;}} - public void CopyTo (Array array, int index) - { - foreach (IBlueRow r in myRows) - array.SetValue (this[(int)r["id"]], index++); + public int IndexOf(IBlueRow item) { + int id = item.Id(); + var rowIn = myRows[id]; + var projRow = new ProjectedRow(rowIn, myHelper); + if (BlueRowEqualityComparer.Equals(projRow, item)) { + var index = myRows.IndexOf(rowIn); + Log.assert(index >= 0 && index < myRows.Count); + return index; + } + return -1; } - - - public object SyncRoot {get {return myRows.SyncRoot;}} #endregion #region IEnumerable Members - public IEnumerator GetEnumerator () - { - return new MyEnumerator (myRows, myHelper); - } - - private class MyEnumerator : IEnumerator - { - #region IEnumerator Members - - private IEnumerator myMaster; - private ProjectHelper myHelper; - public MyEnumerator (IBlueRowCollection master, ProjectHelper helper) - { - myMaster = master.GetEnumerator (); - myHelper = helper; + public IEnumerator GetEnumerator() { + foreach (var row in myRows) { + yield return new ProjectedRow(row, myHelper); } - - - public void Reset() {myMaster.Reset();} - - public object Current - { - get - { - return myMaster.Current == null ? null : - new ProjectedRow (myMaster.Current as IBlueRow, myHelper); - } - } - - public bool MoveNext() {return myMaster.MoveNext ();} - - #endregion - } - - #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs index 85cfa33e5..59a58a2c1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs @@ -2,12 +2,20 @@ using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; + +using bts.collections; +using bts.utils; using bts.collections; using Bts.ColumnSortExpressions; using Bts.DataStreamCore; +using mbts.utils; + +using Spark.Core.Collections; + namespace BCG; /// @@ -93,7 +101,29 @@ public class SortedTable : IBlueRowCollection { ? throw new IndexOutOfRangeException() : rows.WrapSelector(r => r[index], null); - public IEnumerator GetEnumerator() => + public IBlueRow GetRow(int index) => + rows[index]; + + public int IndexOf(IBlueRow item) { + if (item is not SortRow sortRow) { + return -1; + } + int id = sortRow.Id(); + if (comparer is not null) { + rows.BinarySearch(sortRow, GenericComparerWrapper.Create(comparer)); + } + var colComps = sortCols + .Select(s => { + IComparer colComp = new BlueRowComparer(s.Name); + if (!s.IsAscending) { + colComp = colComp.reverse(); + } + return colComp; + }).ToList(); + return rows.BinarySearch(sortRow, new ComparerBuilder(colComps).to_comparer()); + } + + public IEnumerator GetEnumerator() => rows.GetEnumerator(); public void CopyTo(Array array, int index) { diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs index 82e745c3c..61d3bb966 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs @@ -44,7 +44,7 @@ namespace BCG { _last_fire_time = DateTime.Now; IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); - _table.add_row(nr); + _table = _table.Add(nr); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs index 449ca19af..1a09e192f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs @@ -9,9 +9,9 @@ using mbts.utils; using Spark.Core.Collections; using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; namespace BCG { @@ -89,6 +89,32 @@ namespace BCG return allRows.as_mutable().WrapSelector(kv => kv.Value[index], null); } + public IBlueRow GetRow(int index) { + if (index < 0 || index >= Count) { + throw new IndexOutOfRangeException(); + } + if (index < myLeftIndex.Count) { + return myLeftIndex.Values[index]; + } + return myRightIndex.Values[index - myLeftIndex.Count]; + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = myLeftIndex.IndexOfKey(id); + if (index != -1) { + if (BlueRowEqualityComparer.Equals(myLeftIndex.Values[index], item)) { + return index; + } + } else { + index = myRightIndex.IndexOfKey(id); + if (index != -1 && BlueRowEqualityComparer.Equals(myRightIndex.Values[index], item)) { + return index + myLeftIndex.Count; + } + } + return -1; + } + IBlueRow leftRowFactory(KeyValuePair kv) => new AppendRow(kv.Value, leftIdMap[kv.Key], mySchema ?? kv.Value.schema); @@ -105,40 +131,9 @@ namespace BCG } #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() { - return new MyEnumerator(myLeftIndex, myRightIndex); - } + public IEnumerator GetEnumerator() => + myLeftIndex.Values.Concat(myRightIndex.Values).GetEnumerator(); - private class MyEnumerator : IEnumerator - { - private bool myIsLeft = true; - private IEnumerator myEnum; - private IDictionary myLeft, myRight; - #region IEnumerator Members - - public MyEnumerator (IDictionary left, IDictionary right) - { - myLeft = left; - myRight = right; - myEnum = myLeft.Values.GetEnumerator (); - } - - public void Reset() {myEnum = myLeft.GetEnumerator ();} - - public object Current {get {return myEnum.Current;}} - - public bool MoveNext() - { - bool retVal = myEnum.MoveNext (); - if (!retVal && myIsLeft) - { - myIsLeft = false; - retVal = (myEnum = myRight.Values.GetEnumerator()).MoveNext (); - } - return retVal; - } - #endregion - } #endregion } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs index 840c28328..ebd078aa3 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs @@ -152,6 +152,8 @@ namespace BCG { public IList GetColumn(int index) => myRows.GetColumn(index); + public IBlueRow GetRow(int index) => + myRows.GetRow(index); #endregion #region ICollection Members @@ -164,11 +166,14 @@ namespace BCG { public object SyncRoot { get { return myRows.SyncRoot; } } + public int IndexOf(IBlueRow item) => + myRows.IndexOf(item); #endregion #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); } + public IEnumerator GetEnumerator() => + myRows.GetEnumerator(); #endregion } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs index 662094eec..cf7ace1cd 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs @@ -143,7 +143,18 @@ public class BlueColumnTable : IBlueRowCollection { public IList GetColumn(int index) => columns[index].WrapSelector(v => v.GetValue(), null); - public IEnumerator GetEnumerator() { + public IBlueRow GetRow(int index) => + new MyRow(this, index); + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + var idx = rowIds.BinarySearch(id); + return idx != -1 && BlueRowEqualityComparer.Equals(GetRow(idx), item) + ? idx + : -1; + } + + public IEnumerator GetEnumerator() { var count = rowIds.Count; for (int i = 0; i < count; i++) { yield return new MyRow(this, i); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs index fa8245e45..dcdf2242c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs @@ -14,7 +14,7 @@ public class BlueRowCollection : IBlueRowCollection { readonly ICollection source; readonly Func findRow; readonly Func containsKey; - IList? cachedRows; + IList? cachedRowsBacking; public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func? findRow = null, Func? containsKey = null) { this.source = source; @@ -45,20 +45,41 @@ public class BlueRowCollection : IBlueRowCollection { this.containsKey = containsKey; } + IList cachedRows => + cachedRowsBacking ??= source as IList ?? source.Cast().ToList(); + public IBlueRow this[int id] => findRow(id); public IBlueTableSchema schema { get; } + public IBlueRow GetRow(int index) => + cachedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + for (int i = 0; i < cachedRows.Count; i++) { + var r = cachedRows[i]; + if (r.Id() == id) { + if (BlueRowEqualityComparer.Equals(r, item)) { + return i; + } + break; + } + } + return -1; + } + public int Count => source.Count; public bool containsID(int id) => containsKey(id); - public IList GetColumn(int index) { - cachedRows ??= source as IList ?? source.Cast().ToList(); - return cachedRows.WrapSelector(r => r[index], null); - } + public IList GetColumn(int index) => + cachedRows.WrapSelector(r => r[index], null); - public IEnumerator GetEnumerator() => - source.GetEnumerator(); + public IEnumerator GetEnumerator() { + foreach (IBlueRow row in source) { + yield return row; + } + } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs new file mode 100644 index 000000000..5d937a15e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs @@ -0,0 +1,10 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; + +public static class BlueRowCollectionExtensions { + public static IList AsList(this IBlueRowCollection rowCollection) => + rowCollection; +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs index 29c4c1cbe..ee0355303 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs @@ -93,12 +93,7 @@ namespace BCG { /// Returns the row by the index, interface property /// public virtual IBlueRow this[int id] { - get { - object row = rows[id]; - if (row == null) - throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); - return (IBlueRow)row; - } + get => rows[id]; set { if ((int)value["id"] != id) { throw new ArgumentException("id arg not equal to row's actual id", nameof(id)); @@ -112,7 +107,8 @@ namespace BCG { /// public virtual IBlueTableSchema schema { get { return mySchema; } } - public virtual bool containsID(int id) { return rows[id] != null; } + public virtual bool containsID(int id) => + rows.ContainsKey(id); public virtual bool tryGetValue(int id, out IBlueRow row) => rows.TryGetValue(id, out row); @@ -219,6 +215,7 @@ namespace BCG { #endregion public IDictionary AsDictionary() => this; + public IList AsList() => this; public IList GetColumn(int index) { if (index < 0 || index >= schema.columnInfo.Length) { @@ -227,8 +224,11 @@ namespace BCG { return Rows.WrapSelector(r => r[index], null); } + public IBlueRow GetRow(int index) => + Rows[index]; + #region ICollection members - public virtual IEnumerator GetEnumerator() => + public virtual IEnumerator GetEnumerator() => rows.Values.GetEnumerator(); public virtual bool IsSynchronized { get { return false; } } @@ -243,6 +243,18 @@ namespace BCG { public virtual int Count => rows.Count; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = rows.IndexOfKey(id); + if (index != -1) { + var row = rows.GetValueAtIndex(index); + if (BlueRowEqualityComparer.Equals(row, item)) { + return index; + } + } + return -1; + } #endregion #region IDictionary members @@ -268,7 +280,7 @@ namespace BCG { bool ICollection>.Contains(KeyValuePair item) => tryGetValue(item.Key, out var r) - && BlueRowEqualityComparer.Instance.Equals(r, item.Value); + && BlueRowEqualityComparer.Equals(r, item.Value); void ICollection>.CopyTo(KeyValuePair[] array, int arrayIndex) { foreach (var kv in rows) { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs index f73583d36..60750407e 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs @@ -6,12 +6,9 @@ using BCG; using bts.api.core; using bts.collections; -using mbts.utils; - using Spark.Core.Collections; using System; -using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -25,7 +22,7 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet> rows) { this.schema = schema; - this.rows = rows.WithComparer(new ComparerBuilder>() { kv => kv.Key }); + this.rows = rows; } public int Count => rows.Count; @@ -40,9 +37,31 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { ? kv.Value : throw new KeyNotFoundException(); + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + public static DataPacketBuilder CreateBuilder(IBlueTableSchema schema) => + new(new(schema)); + public static BlueTableImmutable CreateRange(IBlueTableSchema schema, IEnumerable> rows) => new(schema, ImmutableSortedSet.CreateRange(KeyValuePairComparer.KeyComparer(), rows)); + public IBlueRow GetRow(int index) => + rows[index].Value; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (rows.TryGetValue(new(id, item), out var rowKvp)) { + var vals = schema.columnInfo.Select(c => rowKvp.Value[c.colNo]); + var itemVals = item.schema.columnInfo.Select(c => item[c.colNo]); + if (vals.SequenceEqual(itemVals)) { + return rows.IndexOf(rowKvp); + } + } + return -1; + } public bool containsID(int id) => #nullable disable rows.Contains(new(id, default)); @@ -57,7 +76,7 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { public IList GetColumn(int index) => rows.WrapSelector(kv => kv.Value[index], null); - public IEnumerator GetEnumerator() { + public IEnumerator GetEnumerator() { foreach (var kv in rows) { yield return kv.Value; } @@ -70,9 +89,9 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { readonly IBlueTableSchema schema; readonly ImmutableSortedSet>.Builder table; readonly ImmutableSortedSet>.Builder added = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); readonly ImmutableSortedSet>.Builder removed = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); public DataPacketBuilder(BlueTableImmutable table) { schema = table.schema; @@ -264,7 +283,7 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { // } // } -public class BlueTableImmutableAdapter : IBlueRowCollection> { +public class BlueTableImmutableAdapter : IBlueRowCollection { readonly BlueTableSchemaAdapter schemaAdapter; readonly ImmutableSortedSet> rows; @@ -281,8 +300,8 @@ public class BlueTableImmutableAdapter : IBlueRowCollection public IBlueTableSchema schema => schemaAdapter.Schema; public object SyncRoot => this; - public BlueRowAdapter this[int id] => - new(schemaAdapter, rows[id].Value); + public IBlueRow this[int id] => + new BlueRowAdapter(schemaAdapter, rows[id].Value); public bool containsID(int id) => #nullable disable @@ -292,13 +311,30 @@ public class BlueTableImmutableAdapter : IBlueRowCollection public IList GetColumn(int index) => rows.WrapSelector(kv => schemaAdapter.GetBlueRow(kv.Value)[index], null); + public IBlueRow GetRow(int index) => + new BlueRowAdapter(schemaAdapter, rows[index].Value); + public void CopyTo(Array array, int index) { foreach (var kv in rows) { array.SetValue(new BlueRowAdapter(schemaAdapter, kv.Value), index++); } } - public IEnumerator> GetEnumerator() { + public int IndexOf(IBlueRow item) { + int id = item.Id(); +#nullable disable + int index = rows.IndexOf(new(id, default)); + if (index != -1) { + var r = this[index]; + if (BlueRowEqualityComparer.Equals(r, item)) { + return index; + } + } +#nullable enable + return -1; + } + + public IEnumerator GetEnumerator() { foreach (var kv in rows) { yield return new BlueRowAdapter(schemaAdapter, kv.Value); } @@ -309,11 +345,12 @@ public class BlueTableImmutableAdapter : IBlueRowCollection public class DataPacketBuilder { readonly BlueTableSchemaAdapter schemaAdapter; - readonly ImmutableSortedSet>.Builder table; + readonly ImmutableSortedSet>.Builder table = + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); readonly ImmutableSortedSet>.Builder added = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); readonly ImmutableSortedSet>.Builder removed = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); public DataPacketBuilder(BlueTableImmutableAdapter table) { schemaAdapter = table.schemaAdapter; @@ -334,13 +371,10 @@ public class BlueTableImmutableAdapter : IBlueRowCollection removed.Remove(origData); return; } - // Change op: A row with this id was already in the table, - // but its values have changed (according to 'comparer'). - removed.Remove(origData); removed.Add(origData); } - removed.Remove(new(id, rowData)); - removed.Add(new(id, rowData)); + added.Remove(new(id, default!)); + added.Add(new(id, rowData)); } public void Remove(int id) { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs index ed4eacd4d..6333646d3 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs @@ -6,37 +6,37 @@ using BCG; using System; using System.Collections.Generic; -public static class BlueTableSelectorWrapper { - public static BlueTableSelectorWrapper Create( - IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) where T : IBlueRow => - new(source, selector, schema); -} +// public static class BlueTableSelectorWrapper { +// public static BlueTableSelectorWrapper Create( +// IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) where T : IBlueRow => +// new(source, selector, schema); +// } -public class BlueTableSelectorWrapper : IBlueRowCollection where T : IBlueRow { - readonly IBlueRowCollection source; - readonly Func selector; +// public class BlueTableSelectorWrapper : IBlueRowCollection where T : IBlueRow { +// readonly IBlueRowCollection source; +// readonly Func selector; - public BlueTableSelectorWrapper(IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) { - this.source = source; - this.selector = selector; - this.schema = schema ?? source.schema; - } +// public BlueTableSelectorWrapper(IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) { +// this.source = source; +// this.selector = selector; +// this.schema = schema ?? source.schema; +// } - public T this[int id] => selector(source[id]); +// public T this[int id] => selector(source[id]); - public IBlueTableSchema schema { get; } +// public IBlueTableSchema schema { get; } - public int Count => source.Count; +// public int Count => source.Count; - public bool containsID(int id) => - source.containsID(id); +// public bool containsID(int id) => +// source.containsID(id); - public IList GetColumn(int index) => - throw new NotImplementedException(); +// public IList GetColumn(int index) => +// throw new NotImplementedException(); - public IEnumerator GetEnumerator() { - foreach (IBlueRow r in source) { - yield return selector(r); - } - } -} +// public IEnumerator GetEnumerator() { +// foreach (IBlueRow r in source) { +// yield return selector(r); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs index 59c349620..9931ed913 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.Immutable; using System.Data; using System.Linq; @@ -8,51 +9,83 @@ using BCG; using bts.collections; +using Bts.DataStreamCore; + namespace bts { public class CircularBufferTable : IBlueRowCollection { - private BlueTable _table; - private List _ordered_list = new List(); + readonly ImmutableList orderedRows; + readonly ImmutableDictionary rowIndex; private int _buffer_sz; - public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) { - _table = new BlueTable(schema); - _buffer_sz = buffer_sz; + public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) + : this(schema, buffer_sz, ImmutableList.Empty, ImmutableDictionary.Empty) { } + + CircularBufferTable(IBlueTableSchema schema, int bufferSize, ImmutableList orderedRows, ImmutableDictionary rowIndex) { + this.schema = schema; + _buffer_sz = bufferSize; + this.orderedRows = orderedRows; + this.rowIndex = rowIndex; } + public CircularBufferTable Add(IBlueRow row) => + Add(row, out _); + // returns the row that might need to be deleted to keep // the circular buffer intact - public IBlueRow add_row(IBlueRow row) { - _ordered_list.Add(row); - _table.addRow(row); - if (_ordered_list.Count > _buffer_sz) { - IBlueRow rv = _ordered_list.ElementAt(0); - _ordered_list.RemoveAt(0); - _table.deleteRow(rv); - return rv; + public CircularBufferTable Add(IBlueRow row, out IBlueRow oldRow) { + int id = row.Id(); + var rowIndexBd = rowIndex.ToBuilder(); + var orderedRowsBd = orderedRows.ToBuilder(); + if (rowIndexBd.TryGetValue(id, out oldRow)) { + Log.assert(orderedRowsBd.RemoveFirst(r => r.Id() == id)); + } else if (rowIndexBd.Count > _buffer_sz) { + oldRow = orderedRowsBd[0]; + rowIndexBd.Remove(oldRow.Id()); + orderedRowsBd.RemoveAt(0); } - return null; + rowIndexBd[id] = row; + orderedRowsBd.Add(row); + return new(schema, _buffer_sz, orderedRowsBd.ToImmutable(), rowIndexBd.ToImmutable()); } public IBlueRow this[int id] { get { - return _table[id]; - } - } - public IBlueTableSchema schema { - get { - return _table.schema; + return orderedRows[id]; } } + public IBlueTableSchema schema { get; } public bool containsID(int id) { - return _table.containsID(id); + return rowIndex.ContainsKey(id); } - public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); } - public int Count { get { return _table.Count; } } + public void CopyTo(Array array, int index) { + foreach (var row in orderedRows) { + array.SetValue(row, index++); + } + } + public int Count => rowIndex.Count; public bool IsSynchronized { get { return false; } } public object SyncRoot { get { return this; } } - public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); } + public IEnumerator GetEnumerator() => + orderedRows.GetEnumerator(); public IList GetColumn(int index) => - _ordered_list.WrapSelector(r => r[index], null); + orderedRows.WrapSelector(r => r[index], null); + + public IBlueRow GetRow(int index) => + orderedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (rowIndex.TryGetValue(id, out var row) && BlueRowEqualityComparer.Equals(item, row)) { + int i = 0; + foreach (var r in orderedRows) { + if (r.Id() == id) { + return i; + } + i++; + } + } + return -1; + } } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs index 59ccdd910..8b074d42a 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs @@ -1,3 +1,5 @@ +using bts.collections; + using System; using System.Collections; using System.Collections.Generic; @@ -8,30 +10,47 @@ namespace BCG /// Represents a readonly, very primitive form of dotNet DataTable. All it has has an indexer for /// getting the rows /// - public interface IBlueRowCollection : ICollection - { + public interface IBlueRowCollection : IList { /// /// Gets the row corresponding to the id /// - IBlueRow this[int id]{get;} - IBlueTableSchema schema {get;} - bool containsID (int id); - IList GetColumn(int index); + new IBlueRow this[int id] { get; } + IBlueTableSchema schema { get; } + bool containsID(int id); + IList GetColumn(int index) => + this.WrapSelector(r => r[index], null); - bool ICollection.IsSynchronized => false; - object ICollection.SyncRoot => this; + IBlueRow GetRow(int index); - void ICollection.CopyTo(Array array, int index) { - foreach (IBlueRow r in this) { - array.SetValue(r, index++); + bool ICollection.IsReadOnly => true; + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + bool ICollection.Contains(IBlueRow item) => + IndexOf(item) != -1; + + void ICollection.CopyTo(IBlueRow[] array, int arrayIndex) { + foreach (var row in this) { + array[arrayIndex++] = row; } } - } - public interface IBlueRowCollection : IBlueRowCollection, IReadOnlyCollection where T : IBlueRow { - new T this[int id] { get; } + void ICollection.Add(IBlueRow item) => + throw new NotSupportedException(); - IBlueRow IBlueRowCollection.this[int id] => this[id]; + void ICollection.Clear() => + throw new NotSupportedException(); + + bool ICollection.Remove(IBlueRow item) => + throw new NotSupportedException(); + + void IList.RemoveAt(int index) => + throw new NotSupportedException(); + + void IList.Insert(int index, IBlueRow item) => + throw new NotSupportedException(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs index 347db29a7..1a39d9ea5 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; @@ -48,6 +50,11 @@ namespace BCG } } + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + public IBlueTableSchema schema { get @@ -59,6 +66,8 @@ namespace BCG public bool containsID(int id) { return myHash.isPresent(id); } + public IBlueRow GetRow(int index) => + myRows[index]; #endregion #region ICollection Members @@ -82,6 +91,17 @@ namespace BCG myRows.CopyTo(array, index); } + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (myHash.isPresent(id)) { + int index = myHash.find(id); + if (BlueRowEqualityComparer.Equals(item, myRows[index])) { + return index; + } + } + return -1; + } + public object SyncRoot { get @@ -95,8 +115,11 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() - { return myRows.GetEnumerator(); } + public IEnumerator GetEnumerator() { + for (int i = 0; i < myRows.Length; i++) { + yield return myRows[i]; + } + } #endregion diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs b/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs index 381debde7..1e9bc0871 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs @@ -37,11 +37,15 @@ namespace BCG public object SyncRoot {get {return myTable.SyncRoot;}} + public int IndexOf(IBlueRow item) => + myTable.IndexOf(item); + #endregion #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() {return myTable.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myTable.GetEnumerator(); #endregion @@ -136,6 +140,9 @@ namespace BCG public IList GetColumn(int index) => throw new NotImplementedException(); + public IBlueRow GetRow(int index) => + throw new NotImplementedException(); + public IBlueRow this[params object[] keyVals] {get {return myHash.getValue (keyVals) as IBlueRow;}} private object[] myGetKeyVals (IBlueRow r) {return BlueDataUtility.getSelectedValues (r, myKeys);} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs index 6bf9218b6..79abe062d 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; @@ -108,6 +110,17 @@ namespace BCG public IList GetColumn(int index) => mySortedRows.WrapSelector(r => r[index], null); + public IBlueRow GetRow(int index) => + mySortedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (myIDToRows.Value.TryGetValue(id, out var row) && BlueRowEqualityComparer.Equals(item, row)) { + return mySortedRows.BinarySearch(row); + } + return -1; + } + #endregion #region ICollection Members @@ -137,7 +150,7 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() + public IEnumerator GetEnumerator() { return this.mySortedRows.GetEnumerator(); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs index 69a79c538..95d03d4d7 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Data; //using BCG.Utilities; @@ -62,7 +63,11 @@ namespace BCG } #region ICollection members - public override IEnumerator GetEnumerator() { return myOrderedRows.GetEnumerator(); } + public override IEnumerator GetEnumerator() { + foreach (IBlueRow row in myOrderedRows) { + yield return row; + } + } public override void CopyTo(Array array, int index) { myOrderedRows.CopyTo(array, index); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs index b3bdf2892..754a0c6a4 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs @@ -1,7 +1,10 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; +using System.Linq; namespace BCG { @@ -36,6 +39,13 @@ namespace BCG #region IBlueRowCollection Members public IBlueRow this[int id] { get { return myRows[id]; } } + public IBlueRow GetRow(int index) => myRows[index]; + public int IndexOf(IBlueRow item) { + int id = item.Id(); + return id >= 0 && id < myRows.Length && BlueRowEqualityComparer.Equals(myRows[id], item) + ? id + : -1; + } public IBlueTableSchema schema {get {return mySchema;}} @@ -63,7 +73,11 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator ();} + public IEnumerator GetEnumerator() { + for (int i = 0; i < myRows.Length; i++) { + yield return myRows[i]; + } + } #endregion } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs index afa40635b..5a7e7dc08 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections; using System.Collections.Generic; @@ -9,22 +11,22 @@ namespace BCG { public class SimpleBindableCollection : IBlueRowCollection, ITypedList, IList { - private ArrayList myRows; + private List myRows; private IBlueTableSchema mySchema; private PropertyDescriptorCollection myPDColl; - public SimpleBindableCollection (IBlueTableSchema schema, ICollection rows) + public SimpleBindableCollection (IBlueTableSchema schema, ICollection rows) { mySchema = schema; ArrayList temp = new ArrayList (); foreach (ColumnInfo ci in mySchema.columnInfo) temp.Add (new BlueDataPropertyDescriptor (ci)); myPDColl = new PropertyDescriptorCollection (temp.ToArray (typeof(PropertyDescriptor)) as PropertyDescriptor[]); - myRows = new ArrayList (rows); + myRows = new(rows); } public SimpleBindableCollection (IBlueTableSchema schema) - : this (schema, new ArrayList ()) + : this(schema, new List(0)) {} #region IBlueRowCollection Members @@ -57,6 +59,9 @@ namespace BCG public IList GetColumn(int index) => myRows.WrapSelector(r => ((IBlueRow)r)[index], null); + public IBlueRow GetRow(int index) => + (IBlueRow)myRows[index]; + #endregion #region ICollection Members @@ -65,7 +70,25 @@ namespace BCG public int Count {get {return myRows.Count;}} - public void CopyTo(Array array, int index) {myRows.CopyTo (array, index);} + public void CopyTo(Array array, int index) { + for (int i = 0; i < myRows.Count; i++) { + array.SetValue(myRows[i], index++); + } + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + for (int i = 0; i < myRows.Count; i++) { + var r = myRows[i]; + if (r.Id() == id) { + if (BlueRowEqualityComparer.Equals(item, r)) { + return i; + } + break; + } + } + return -1; + } public object SyncRoot { get {return null;}} @@ -73,9 +96,10 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() - { - return myRows.GetEnumerator (); + public IEnumerator GetEnumerator() { + foreach (IBlueRow row in myRows) { + yield return row; + } } #endregion @@ -110,11 +134,13 @@ namespace BCG public void Remove(object value) {throw new ApplicationException ("Collection is read only");} - public bool Contains(object value) {return myRows.Contains (value);} + public bool Contains(object value) => + value is IBlueRow r && myRows.Contains(r); public void Clear() {throw new ApplicationException ("Collection is read only");} - public int IndexOf(object value) {return myRows.IndexOf (value);} + public int IndexOf(object value) => + value is IBlueRow r ? myRows.IndexOf(r) : -1; public int Add(object value) {throw new ApplicationException ("Collection is read only");} @@ -122,6 +148,7 @@ namespace BCG #endregion - public void reset (ICollection rows) {myRows = new ArrayList(rows);} + public void reset(ICollection rows) => + myRows = new(rows); } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs index b0469668a..5893ae655 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs @@ -1,8 +1,9 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; -using System.Linq; namespace BCG { @@ -32,6 +33,17 @@ namespace BCG _rows[id] = value; } } + + public IBlueRow GetRow(int index) => + _rows[index]; + + public int IndexOf(IBlueRow item) { + int index = item.Id(); + return index >= 0 && index < _rows.Count && BlueRowEqualityComparer.Equals(_rows[index], item) + ? index + : -1; + } + public IBlueTableSchema schema { get; private set; } public bool containsID(int id) { return (id >= 0 && id < _rows.Count && this[id] != null); @@ -42,8 +54,13 @@ namespace BCG public void CopyTo(Array array, int index) { _rows.CopyTo((IBlueRow[])array, index);} public int Count { get { return _rows.Count; } } - public bool IsSynchronized {get { return false; }} - public object SyncRoot { get { return this; } } - public System.Collections.IEnumerator GetEnumerator() { return _rows.Where(r => r != null).GetEnumerator(); } + + public IEnumerator GetEnumerator() { + foreach (var r in _rows) { + if (r is not null) { + yield return r; + } + } + } } } diff --git a/src/DataStreamSuite/DataStreamCore/DataPacket.cs b/src/DataStreamSuite/DataStreamCore/DataPacket.cs index eb67db485..1d40def73 100644 --- a/src/DataStreamSuite/DataStreamCore/DataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/DataPacket.cs @@ -1,3 +1,5 @@ +using Spark.Core.Collections; + using System; using System.Collections; using System.Collections.Generic; @@ -57,9 +59,9 @@ namespace BCG { IReadOnlyDictionary IDataPacket.Metadata => meta_data; public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { - new_rows = newRows; - added_rows = addedRows; - deleted_rows = deletedRows; + new_rows = ListNonGenericCollectionWrapper.Create(newRows); + added_rows = ListNonGenericCollectionWrapper.Create(addedRows); + deleted_rows = ListNonGenericCollectionWrapper.Create(deletedRows); } public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) { @@ -69,7 +71,9 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + ICollection ta = CollectionHelpers.difference(addedRowCollection, deletedRowCollection, ed); foreach (IBlueRow r in ta) { truly.addRow(r); @@ -111,7 +115,9 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + ICollection td = CollectionHelpers.difference(deletedRowCollection, addedRowCollection, ed); foreach (IBlueRow r in td) { truly.addRow(r); @@ -128,8 +134,10 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); - ICollection tc = CollectionHelpers.difference(addedRows, ta, ed); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + ICollection ta = CollectionHelpers.difference(addedRowCollection, deletedRowCollection, ed); + ICollection tc = CollectionHelpers.difference(addedRowCollection, ta, ed); foreach (IBlueRow r in tc) { truly.addRow(r); @@ -145,8 +153,10 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); - ICollection tco = CollectionHelpers.difference(deletedRows, td, ed); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + ICollection td = CollectionHelpers.difference(deletedRowCollection, addedRowCollection, ed); + ICollection tco = CollectionHelpers.difference(deletedRowCollection, td, ed); foreach (IBlueRow r in tco) { truly.addRow(r); @@ -223,13 +233,15 @@ namespace BCG { } deleted.AddRange(oldDeleted); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(dp.deletedRows); deleted.AddRange(CollectionHelpers.difference( - CollectionHelpers.difference(dp.deletedRows, oldAdded, ed), + CollectionHelpers.difference(deletedRowCollection, oldAdded, ed), oldDeleted, ed)); - added.AddRange(dp.addedRows); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(dp.addedRows); + added.AddRange(addedRowCollection); added.AddRange(CollectionHelpers.difference( - CollectionHelpers.difference(oldAdded, dp.deletedRows, ed), + CollectionHelpers.difference(oldAdded, deletedRowCollection, ed), added, ed)); nt = dp.newRows; } diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs b/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs index d8506e8cc..fee26852a 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs +++ b/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs @@ -1,4 +1,4 @@ namespace BCG; public delegate void DataStreamChangeDelegate(object newValue); -public delegate void DataStreamChangeDelegate(IDataPacket dp) where T : IBlueRow; +// public delegate void DataStreamChangeDelegate(IDataPacket dp) where T : IBlueRow; diff --git a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs index 900b334b8..953803009 100644 --- a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs @@ -18,22 +18,22 @@ namespace BCG { void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows); } - public interface IDataPacket : IDataPacket where T : IBlueRow { - new IBlueRowCollection newRows { get; } - new IBlueRowCollection addedRows { get; } - new IBlueRowCollection deletedRows { get; } + // public interface IDataPacket : IDataPacket where T : IBlueRow { + // new IBlueRowCollection newRows { get; } + // new IBlueRowCollection addedRows { get; } + // new IBlueRowCollection deletedRows { get; } - IBlueRowCollection IDataPacket.newRows => newRows; - IBlueRowCollection IDataPacket.addedRows => addedRows; - IBlueRowCollection IDataPacket.deletedRows => deletedRows; - IBlueTableSchema IDataPacket.Schema => newRows.schema; + // IBlueRowCollection IDataPacket.newRows => newRows; + // IBlueRowCollection IDataPacket.addedRows => addedRows; + // IBlueRowCollection IDataPacket.deletedRows => deletedRows; + // IBlueTableSchema IDataPacket.Schema => newRows.schema; - void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { - new_rows = newRows; - added_rows = addedRows; - deleted_rows = deletedRows; - } - } + // void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { + // new_rows = newRows; + // added_rows = addedRows; + // deleted_rows = deletedRows; + // } + // } public static class IDataPacketExtensions { public static IDataPacket ToImmutable(this IDataPacket dataPacket) => diff --git a/src/DataStreamSuite/DataStreamCore/IViewStream.cs b/src/DataStreamSuite/DataStreamCore/IViewStream.cs index eae4d3cd2..1eb8c1842 100644 --- a/src/DataStreamSuite/DataStreamCore/IViewStream.cs +++ b/src/DataStreamSuite/DataStreamCore/IViewStream.cs @@ -6,8 +6,8 @@ public interface IViewStream { IBlueRowCollection rows { get; } } -public interface IViewStream : IViewStream where T : IBlueRow { - new event DataStreamChangeDelegate onChange; +// public interface IViewStream : IViewStream where T : IBlueRow { +// new event DataStreamChangeDelegate onChange; - new IBlueRowCollection rows { get; } -} +// new IBlueRowCollection rows { get; } +// } diff --git a/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs b/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs index c18e7ee02..579321b3a 100644 --- a/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs +++ b/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs @@ -58,11 +58,11 @@ public class TestDriverStream : ViewStream, IWorker { if (lastPublishedTable is null) { addedRows.addRows(currentTable); } else { - var comp = BlueRowEqualityComparer.Instance; - deletedRows.AsDictionary().SetItems(lastPublishedTable.Where(oldRow => - !(currentTable.AsDictionary().TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value)))); - addedRows.SetItems(currentTable.Where(newRow => - !(lastPublishedTable.AsDictionary().TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value)))); + deletedRows.AsDictionary().SetItems(lastPublishedTable.AsList().Where(oldRow => + !(currentTable.AsDictionary().TryGetValue(oldRow.Id(), out var newRow) && BlueRowEqualityComparer.Equals(newRow, oldRow))) + .Select(r => new KeyValuePair(r.Id(), r))); + addedRows.SetItems(currentTable.AsList().Where(newRow => + !(lastPublishedTable.AsDictionary().TryGetValue(newRow.Id(), out var oldRow) && BlueRowEqualityComparer.Equals(oldRow, newRow)))); } lastPublishedTable = currentTable; if (deletedRows.Count > 0 || addedRows.Count > 0) { diff --git a/src/client/spark_core/Collections/GenericComparerWrapper.cs b/src/client/spark_core/Collections/GenericComparerWrapper.cs new file mode 100644 index 000000000..1c9494902 --- /dev/null +++ b/src/client/spark_core/Collections/GenericComparerWrapper.cs @@ -0,0 +1,23 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System.Collections; +using System.Collections.Generic; + +public static class GenericComparerWrapper { + public static GenericComparerWrapper Create(IComparer comparer) => + new(comparer); +} + +public class GenericComparerWrapper : IComparer, IComparer { + readonly IComparer comparer; + + public GenericComparerWrapper(IComparer comparer) => + this.comparer = comparer; + + public int Compare(object? x, object? y) => + comparer.Compare(x, y); + + public int Compare(T? x, T? y) => + comparer.Compare(x, y); +} diff --git a/src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs b/src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs new file mode 100644 index 000000000..a0154a8f1 --- /dev/null +++ b/src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs @@ -0,0 +1,72 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; + +public static class ListNonGenericCollectionWrapper { + public static ListGenericCollectionWrapper Create(IList source) => + new(source); +} + +public class ListGenericCollectionWrapper : IList, ICollection { + readonly IList source; + + public ListGenericCollectionWrapper(IList source) { + this.source = source; + } + + public T this[int index] { + get => source[index]; + set => source[index] = value; + } + + public int Count => source.Count; + + public bool IsSynchronized => false; + + public object SyncRoot => this; + + public bool IsReadOnly => source.IsReadOnly; + + public void Add(T item) => + source.Add(item); + + public void Clear() => + source.Clear(); + + public bool Contains(T item) => + source.Contains(item); + + public void CopyTo(Array array, int index) { + if (array is T[] arrayT) { + source.CopyTo(arrayT, index); + return; + } + for (int i = 0; i < source.Count; i++) { + array.SetValue(source[i], index++); + } + } + + public void CopyTo(T[] array, int arrayIndex) => + source.CopyTo(array, arrayIndex); + + public IEnumerator GetEnumerator() => + source.GetEnumerator(); + + public int IndexOf(T item) => + source.IndexOf(item); + + public void Insert(int index, T item) => + source.Insert(index, item); + + public bool Remove(T item) => + source.Remove(item); + + public void RemoveAt(int index) => + source.RemoveAt(index); + + IEnumerator IEnumerable.GetEnumerator() => + source.GetEnumerator(); +} diff --git a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs index 48a028fcc..98ad8f78b 100644 --- a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs @@ -116,7 +116,7 @@ namespace bts.websocket_core { hdlr.dis = vs.AsObservable().Subscribe(OnData); - void OnData(DataPacket dp) { + void OnData(IDataPacket dp) { lock (syncRoot) { removedRows.Clear(); changedRows.Clear();