From d2a1b67717bee82dd67acb65a0b2a875d8c90956 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 17 Sep 2025 23:10:02 -0400 Subject: [PATCH] refactor: Make IBlueRowCollection inherit IList --- .../DataStream/BlueOrb/MillisecondStream.cs | 2 +- .../DataStream/BlueOrb/TimeSeriesStream.cs | 30 +- .../DataStream/DataStreamExtensions.cs | 68 ++-- .../DataStream/Misc/PandasStream.cs | 5 +- .../Offloading/RemoteQueryManager.cs | 3 +- .../Primitives/GroupBy/GroupByOutputTable.cs | 8 +- .../Primitives/GroupBy/RowDictionaryTable.cs | 7 +- .../DataStream/Primitives/Join/JoinTable.cs | 15 +- .../LightForeignJoinSimpleCollection.cs | 27 +- .../Primitives/Misc/SnapshotReplayStream.cs | 4 +- .../Primitives/Project/ProjectedTable.cs | 64 ++-- .../DataStream/Primitives/Sort/SortedTable.cs | 32 +- .../Primitives/Time/MillisecondStream.cs | 2 +- .../Primitives/Union/AppendCollection.cs | 63 ++-- .../Primitives/Union/MutableAppendStream.cs | 7 +- .../BlueData/BlueColumnTable.cs | 13 +- .../BlueData/BlueRowCollection.cs | 35 +- .../BlueData/BlueRowCollectionExtensions.cs | 10 + .../DataStreamCore/BlueData/BlueTable.cs | 30 +- .../BlueData/BlueTableImmutable.cs | 326 ++++++++++-------- .../BlueData/BlueTableSelectorWrapper.cs | 52 +-- .../BlueData/CircularBufferTable.cs | 83 +++-- .../BlueData/IBlueRowCollection.cs | 49 ++- .../BlueData/ImmutableBlueTable.cs | 27 +- .../BlueData/KeysBasedTable2.cs | 9 +- .../BlueData/MovingWindowTable.cs | 15 +- .../BlueData/OrderedBlueTable.cs | 7 +- .../BlueData/SequentialIDBlueTable.cs | 16 +- .../BlueData/SimpleBindableCollection.cs | 49 ++- .../BlueData/SparseVectorBlueTable.cs | 25 +- .../DataStreamCore/DataPacket.cs | 90 ++--- .../DataStreamChangeDelegate.cs | 2 +- .../DataStreamCore/IDataPacket.cs | 28 +- .../DataStreamCore/IViewStream.cs | 8 +- .../DataStreamTests/TestDriverStream.cs | 10 +- .../Collections/GenericComparerWrapper.cs | 23 ++ .../ListNonGenericCollectionWrapper.cs | 72 ++++ src/client/spark_core/Threading/AtomicGate.cs | 156 +++++++++ .../Core/WebsocketSubscriptionHandler.cs | 2 +- 39 files changed, 1035 insertions(+), 439 deletions(-) create mode 100644 src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs create mode 100644 src/client/spark_core/Collections/GenericComparerWrapper.cs create mode 100644 src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs create mode 100644 src/client/spark_core/Threading/AtomicGate.cs diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs index 4446ccc6a..2614e3447 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MillisecondStream.cs @@ -43,7 +43,7 @@ namespace bts.BlueOrb { _last_fire_time = DateTime.Now; IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); - _table.add_row(nr); + _table = _table.Add(nr); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs index 178fc1260..0710f333b 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs @@ -1,9 +1,15 @@ -using System; - using BCG; +using Bts.DataStreamCore; + using mbts.utils; +using Spark.Core.Collections; +using Spark.Core.Threading; + +using System; +using System.Collections.Immutable; + namespace bts.BlueOrb { // Keeps adding changing rows to the stream, hence preserving the history // and creating a time series, for example this can be used to make bbo updates @@ -11,6 +17,8 @@ namespace bts.BlueOrb { [DataStreamUserRespresentation("TimeSeriesStream")] public class TimeSeriesStream : ViewStream { readonly ViewStream _stream; + readonly string timeColumn; + readonly AtomicGate gate = new(); CircularBufferTable? _table; private int _id = 0; private int _size = 0; @@ -34,6 +42,9 @@ namespace bts.BlueOrb { private void _onstream_change(object o) { var dp = ExceptionAssert.Argument.Is(o); DateTime now = DateTime.Now; + var addedRowsBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); + var deletedRowsBd = ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); + using var _ = gate.BeginTransaction(); if (_table == null) { IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName); schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp")); @@ -41,11 +52,18 @@ namespace bts.BlueOrb { } foreach (IBlueRow r in dp.addedRows) { MutableBlueRow ar = new MutableBlueRow(_table.schema, r); - ar.setValue("timestamp", now); - ar.setValue("id", _id++); - _table.add_row(ar); + ar.setValue(timeColumn, now); + int id = _id++; + ar.setValue("id", id); + _table = _table.Add(ar, out var oldRow); + addedRowsBd.Add(new(id, ar)); + if (oldRow is not null) { + deletedRowsBd.Add(new(oldRow.Id(), oldRow)); + } } - var dp_out = BlueDataUtility.build_dp(_table, myLastValue as DataPacket); + var addedRows = new BlueTableImmutable(_table.schema, addedRowsBd.ToImmutable()); + var deletedRows = new BlueTableImmutable(_table.schema, deletedRowsBd.ToImmutable()); + var dp_out = new DataPacket(_table, addedRows, deletedRows); myFireEvent(dp_out); } } diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index 49d9130d6..23d4ff50b 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -19,13 +19,13 @@ public static class DataStreamExtensions { public static ViewStream AsViewStream(this IObservable source) => source as ViewStream ?? new ObservableViewStream(source); - public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => - source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger); + // public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => + // source as ViewStream ?? new ObservableViewStream(source, name, logger); - public static IObservable AsObservable(this ViewStream source) { - return source as IObservable ?? Observable.Create(Subscribe); + public static IObservable AsObservable(this ViewStream source) { + return source as IObservable ?? Observable.Create(Subscribe); - IDisposable Subscribe(IObserver observer) { + IDisposable Subscribe(IObserver observer) { source.onChange += OnChange; return Disposable.Create(() => source.onChange -= OnChange); @@ -35,15 +35,15 @@ public static class DataStreamExtensions { } } - public static IObservable> SelectDataPacket(this IObservable source, Func selector) where T : IBlueRow => - source.Select(dp => { - dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); - return new DataPacket(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows)); + // public static IObservable> SelectDataPacket(this IObservable source, Func selector) where T : IBlueRow => + // source.Select(dp => { + // dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); + // return new DataPacket(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows)); - IBlueRowCollection CastRows(ICollection rows) => - new BlueTableSelectorWrapper( - rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector); - }); + // IBlueRowCollection CastRows(ICollection rows) => + // new BlueTableSelectorWrapper( + // rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector); + // }); public static IObservable> AsTables(this IObservable source) => source.AsTables(r => r, r => r.Id()); @@ -71,9 +71,33 @@ public static class DataStreamExtensions { }); } - public static IObservable> AsDataPackets(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => + public static IObservable AsDataPackets(this IObservable> source) => source.Scan( - (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)DataPacket.Empty(schema)), + (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null), + (res, table) => { + var resTable = res.Table; + if (resTable is null) { + if (table.state.Count == 0) { + return (null, null); + } + resTable = new(table.state.Values.First().schema); + } + var bd = resTable.ToDataPacketBuilder(); + foreach (var r in table.removed) { + bd.Remove(r.Id()); + } + foreach (var r in table.added) { + bd.AddOrUpdate(r); + } + var resDp = bd.ToDataPacket(out var newRows); + return (newRows, resDp); + }) + .Where(t => t.Dp is not null && (t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())) + .Select(t => t.Dp!); + + public static IObservable AsDataPackets(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => + source.Scan( + (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)), (res, table) => { var bd = res.Table.ToDataPacketBuilder(); foreach (var r in table.removed) { @@ -82,22 +106,22 @@ public static class DataStreamExtensions { foreach (var r in table.added) { bd.AddOrUpdate(r); } - var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows); + var resDp = bd.ToDataPacket(out var newRows); return (newRows, resDp); }) .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any()) .Select(t => t.Dp); - public static IObservable>> AsDataPackets(this IObservable> source, BlueTableSchemaAdapter schema) where TKey : IEquatable { - return Observable.Create>>(Subscribe); + public static IObservable AsDataPackets(this IObservable> source, BlueTableSchemaAdapter schema) where TKey : IEquatable { + return Observable.Create(Subscribe); - IDisposable Subscribe(IObserver>> o) { + IDisposable Subscribe(IObserver o) { var adapter = new BlueTableImmutableAdapter(schema); - IDataPacket> initDataPacket = DataPacket>.Empty(schema); + IDataPacket initDataPacket = new DataPacket(schema); return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o); - (BlueTableImmutableAdapter Adapter, IDataPacket> DataPacket) OnUpdate( - (BlueTableImmutableAdapter Adapter, IDataPacket> DataPacket) state, TableUpdate tableUpdate) { + (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) OnUpdate( + (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) state, TableUpdate tableUpdate) { var bd = state.Adapter.ToDataPacketBuilder(); bd.Update(tableUpdate); var dp = bd.ToDataPacket(); diff --git a/src/DataStreamSuite/DataStream/Misc/PandasStream.cs b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs index 4bf180706..30548dbb9 100644 --- a/src/DataStreamSuite/DataStream/Misc/PandasStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/PandasStream.cs @@ -114,14 +114,13 @@ import sys executePythonCode(pythonCode.Value, pythonExe); var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema); var rowMap = rows as IDictionary ?? rows.Cast().ToDictionary(r => r.Id()); - var comp = BlueRowEqualityComparer.Instance; var deletedRows = new BlueTable(schema); var addedRows = new BlueTable(schema); if (lastRowMap is not null) { deletedRows.SetItems(lastRowMap.Where(oldRow => - !(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value)))); + !(rowMap.TryGetValue(oldRow.Key, out var newRow) && BlueRowEqualityComparer.Equals(newRow, oldRow.Value)))); addedRows.SetItems(rowMap.Where(newRow => - !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value)))); + !(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && BlueRowEqualityComparer.Equals(oldRow, newRow.Value)))); } lastRowMap = rowMap; lastFireTime = DateTime.Now; diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs index 33185cd9a..9f69b81f3 100644 --- a/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs @@ -93,8 +93,7 @@ namespace BCG { return Task.CompletedTask; } - private void _on_str(DataPacket dp) - { + private void _on_str(IDataPacket dp) { foreach (IBlueRow r in dp.newRows) { string name = r.get("name"); diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs index cbdb0d04b..982cb8e59 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByOutputTable.cs @@ -3,7 +3,6 @@ using bts.collections; using Bts.DataStreamCore; using System; -using System.Collections; using System.Collections.Generic; namespace BCG @@ -56,6 +55,8 @@ namespace BCG public IBlueRow? getRow(string groupRepString) => myDict.GetValueOrDefault(groupRepString); + public int IndexOf(IBlueRow item) => + myTable.IndexOf(item); #region IBlueRowCollection members public IBlueRow this[int id] {get{return myTable[id];}} public IBlueTableSchema schema {get{return myTable.schema;}} @@ -64,6 +65,8 @@ namespace BCG index < 0 || index >= myTable.schema.columnInfo.Length ? throw new IndexOutOfRangeException() : myDict.Values.WrapSelector(r => r[index], null); + public IBlueRow GetRow(int index) => + myDict.Values[index]; #endregion #region ICollection members @@ -74,7 +77,8 @@ namespace BCG #endregion #region IEnumerable members - public IEnumerator GetEnumerator() {return myTable.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myTable.GetEnumerator (); #endregion diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs index a0334945a..e54bd1c34 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs @@ -99,6 +99,8 @@ namespace BCG public bool containsID (int id) {return myLinearCollection.containsID (id);} public IList GetColumn(int index) => myLinearCollection.GetColumn(index); + public IBlueRow GetRow(int index) => + myLinearCollection.GetRow(index); #endregion #region ICollection members @@ -106,10 +108,13 @@ namespace BCG public bool IsSynchronized {get{return false;}} public object SyncRoot {get{return this;}} public void CopyTo (Array array, int index) {myLinearCollection.CopyTo (array, index);} + public int IndexOf(IBlueRow item) => + myLinearCollection.IndexOf(item); #endregion #region IEnumerable members - public IEnumerator GetEnumerator() {return myLinearCollection.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myLinearCollection.GetEnumerator (); #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs index 985c4b2c7..f9ea293b1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs @@ -3,7 +3,6 @@ using bts.collections; using Bts.DataStreamCore; using System; -using System.Collections; using System.Collections.Generic; namespace BCG @@ -72,10 +71,20 @@ namespace BCG array.SetValue(myIdIndex.Values[i], index++); } } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = myIdIndex.IndexOfKey(id); + if (index != -1 && BlueRowEqualityComparer.Equals(item, myIdIndex.Values[index])) { + return index; + } + return -1; + } #endregion #region IEnumerable members - public IEnumerator GetEnumerator () {return myIdIndex.Values.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myIdIndex.Values.GetEnumerator(); #endregion #region IBlueRowCollection members @@ -87,6 +96,8 @@ namespace BCG } return myIdIndex.Values.WrapSelector(r => r[index], null); } + public IBlueRow GetRow(int index) => + myIdIndex.Values[index]; #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs index 891467434..90a2d7dbb 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections; using System.Collections.Generic; @@ -47,6 +49,12 @@ namespace BCG ? throw new IndexOutOfRangeException() : myRows.WrapType().WrapSelector(r => r[index], null); + public IBlueRow GetRow(int index) { + var rowObj = myRows[index]; + ArgumentNullException.ThrowIfNull(rowObj); + return (IBlueRow)rowObj; + } + public IBlueTableSchema schema {get {return mySchema;}} #endregion @@ -61,11 +69,28 @@ namespace BCG public object SyncRoot {get{return null;}} + public int IndexOf(IBlueRow item) { + int id = item.Id(); + for (int i = 0; i < myRows.Count; i++) { + if (myRows[i] is not IBlueRow r || r.Id() != id) { + continue; + } + if (BlueRowEqualityComparer.Equals(item, r)) { + return i; + } + break; + } + return -1; + } #endregion #region IEnumerable Members - public IEnumerator GetEnumerator() {return myRows.GetEnumerator ();} + public IEnumerator GetEnumerator() { + foreach (IBlueRow r in myRows) { + yield return r; + } + } #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs index 0cf9d6c7d..8697bf23a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/SnapshotReplayStream.cs @@ -66,7 +66,7 @@ namespace bts { if (_in_rows.Count > 0) { // fire row IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++); - _out_table.add_row(curr_row); + _out_table = _out_table.Add(curr_row); DataPacket dp = BlueDataUtility.build_dp_inc(_out_table, myLastValue == null ? null : myLastValue as DataPacket, false); myFireEvent(dp); @@ -74,4 +74,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs index 6f4efbce3..6e13f3b19 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs @@ -1,5 +1,8 @@ +using bts; + +using Bts.DataStreamCore; + using System; -using System.Collections; using System.Collections.Generic; namespace BCG @@ -39,63 +42,36 @@ namespace BCG var origIdx = myHelper.colMap[index]; return myRows.GetColumn(origIdx); } + + public IBlueRow GetRow(int index) => + new ProjectedRow(myRows.GetRow(index), myHelper); #endregion #region ICollection Members - public bool IsSynchronized {get {return myRows.IsSynchronized;}} - public int Count {get {return myRows.Count;}} - public void CopyTo (Array array, int index) - { - foreach (IBlueRow r in myRows) - array.SetValue (this[(int)r["id"]], index++); + public int IndexOf(IBlueRow item) { + int id = item.Id(); + var rowIn = myRows[id]; + var projRow = new ProjectedRow(rowIn, myHelper); + if (BlueRowEqualityComparer.Equals(projRow, item)) { + var index = myRows.IndexOf(rowIn); + Log.assert(index >= 0 && index < myRows.Count); + return index; + } + return -1; } - - - public object SyncRoot {get {return myRows.SyncRoot;}} #endregion #region IEnumerable Members - public IEnumerator GetEnumerator () - { - return new MyEnumerator (myRows, myHelper); - } - - private class MyEnumerator : IEnumerator - { - #region IEnumerator Members - - private IEnumerator myMaster; - private ProjectHelper myHelper; - public MyEnumerator (IBlueRowCollection master, ProjectHelper helper) - { - myMaster = master.GetEnumerator (); - myHelper = helper; + public IEnumerator GetEnumerator() { + foreach (var row in myRows) { + yield return new ProjectedRow(row, myHelper); } - - - public void Reset() {myMaster.Reset();} - - public object Current - { - get - { - return myMaster.Current == null ? null : - new ProjectedRow (myMaster.Current as IBlueRow, myHelper); - } - } - - public bool MoveNext() {return myMaster.MoveNext ();} - - #endregion - } - - #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs index 85cfa33e5..59a58a2c1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs @@ -2,12 +2,20 @@ using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; + +using bts.collections; +using bts.utils; using bts.collections; using Bts.ColumnSortExpressions; using Bts.DataStreamCore; +using mbts.utils; + +using Spark.Core.Collections; + namespace BCG; /// @@ -93,7 +101,29 @@ public class SortedTable : IBlueRowCollection { ? throw new IndexOutOfRangeException() : rows.WrapSelector(r => r[index], null); - public IEnumerator GetEnumerator() => + public IBlueRow GetRow(int index) => + rows[index]; + + public int IndexOf(IBlueRow item) { + if (item is not SortRow sortRow) { + return -1; + } + int id = sortRow.Id(); + if (comparer is not null) { + rows.BinarySearch(sortRow, GenericComparerWrapper.Create(comparer)); + } + var colComps = sortCols + .Select(s => { + IComparer colComp = new BlueRowComparer(s.Name); + if (!s.IsAscending) { + colComp = colComp.reverse(); + } + return colComp; + }).ToList(); + return rows.BinarySearch(sortRow, new ComparerBuilder(colComps).to_comparer()); + } + + public IEnumerator GetEnumerator() => rows.GetEnumerator(); public void CopyTo(Array array, int index) { diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs index 82e745c3c..61d3bb966 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs @@ -44,7 +44,7 @@ namespace BCG { _last_fire_time = DateTime.Now; IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); - _table.add_row(nr); + _table = _table.Add(nr); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); myFireEvent(dp); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs index 449ca19af..1a09e192f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs @@ -9,9 +9,9 @@ using mbts.utils; using Spark.Core.Collections; using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; namespace BCG { @@ -89,6 +89,32 @@ namespace BCG return allRows.as_mutable().WrapSelector(kv => kv.Value[index], null); } + public IBlueRow GetRow(int index) { + if (index < 0 || index >= Count) { + throw new IndexOutOfRangeException(); + } + if (index < myLeftIndex.Count) { + return myLeftIndex.Values[index]; + } + return myRightIndex.Values[index - myLeftIndex.Count]; + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = myLeftIndex.IndexOfKey(id); + if (index != -1) { + if (BlueRowEqualityComparer.Equals(myLeftIndex.Values[index], item)) { + return index; + } + } else { + index = myRightIndex.IndexOfKey(id); + if (index != -1 && BlueRowEqualityComparer.Equals(myRightIndex.Values[index], item)) { + return index + myLeftIndex.Count; + } + } + return -1; + } + IBlueRow leftRowFactory(KeyValuePair kv) => new AppendRow(kv.Value, leftIdMap[kv.Key], mySchema ?? kv.Value.schema); @@ -105,40 +131,9 @@ namespace BCG } #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() { - return new MyEnumerator(myLeftIndex, myRightIndex); - } + public IEnumerator GetEnumerator() => + myLeftIndex.Values.Concat(myRightIndex.Values).GetEnumerator(); - private class MyEnumerator : IEnumerator - { - private bool myIsLeft = true; - private IEnumerator myEnum; - private IDictionary myLeft, myRight; - #region IEnumerator Members - - public MyEnumerator (IDictionary left, IDictionary right) - { - myLeft = left; - myRight = right; - myEnum = myLeft.Values.GetEnumerator (); - } - - public void Reset() {myEnum = myLeft.GetEnumerator ();} - - public object Current {get {return myEnum.Current;}} - - public bool MoveNext() - { - bool retVal = myEnum.MoveNext (); - if (!retVal && myIsLeft) - { - myIsLeft = false; - retVal = (myEnum = myRight.Values.GetEnumerator()).MoveNext (); - } - return retVal; - } - #endregion - } #endregion } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs index 840c28328..ebd078aa3 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs @@ -152,6 +152,8 @@ namespace BCG { public IList GetColumn(int index) => myRows.GetColumn(index); + public IBlueRow GetRow(int index) => + myRows.GetRow(index); #endregion #region ICollection Members @@ -164,11 +166,14 @@ namespace BCG { public object SyncRoot { get { return myRows.SyncRoot; } } + public int IndexOf(IBlueRow item) => + myRows.IndexOf(item); #endregion #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() { return myRows.GetEnumerator(); } + public IEnumerator GetEnumerator() => + myRows.GetEnumerator(); #endregion } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs index 662094eec..cf7ace1cd 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueColumnTable.cs @@ -143,7 +143,18 @@ public class BlueColumnTable : IBlueRowCollection { public IList GetColumn(int index) => columns[index].WrapSelector(v => v.GetValue(), null); - public IEnumerator GetEnumerator() { + public IBlueRow GetRow(int index) => + new MyRow(this, index); + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + var idx = rowIds.BinarySearch(id); + return idx != -1 && BlueRowEqualityComparer.Equals(GetRow(idx), item) + ? idx + : -1; + } + + public IEnumerator GetEnumerator() { var count = rowIds.Count; for (int i = 0; i < count; i++) { yield return new MyRow(this, i); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs index fa8245e45..dcdf2242c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollection.cs @@ -14,7 +14,7 @@ public class BlueRowCollection : IBlueRowCollection { readonly ICollection source; readonly Func findRow; readonly Func containsKey; - IList? cachedRows; + IList? cachedRowsBacking; public BlueRowCollection(ICollection source, IBlueTableSchema schema, Func? findRow = null, Func? containsKey = null) { this.source = source; @@ -45,20 +45,41 @@ public class BlueRowCollection : IBlueRowCollection { this.containsKey = containsKey; } + IList cachedRows => + cachedRowsBacking ??= source as IList ?? source.Cast().ToList(); + public IBlueRow this[int id] => findRow(id); public IBlueTableSchema schema { get; } + public IBlueRow GetRow(int index) => + cachedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + for (int i = 0; i < cachedRows.Count; i++) { + var r = cachedRows[i]; + if (r.Id() == id) { + if (BlueRowEqualityComparer.Equals(r, item)) { + return i; + } + break; + } + } + return -1; + } + public int Count => source.Count; public bool containsID(int id) => containsKey(id); - public IList GetColumn(int index) { - cachedRows ??= source as IList ?? source.Cast().ToList(); - return cachedRows.WrapSelector(r => r[index], null); - } + public IList GetColumn(int index) => + cachedRows.WrapSelector(r => r[index], null); - public IEnumerator GetEnumerator() => - source.GetEnumerator(); + public IEnumerator GetEnumerator() { + foreach (IBlueRow row in source) { + yield return row; + } + } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs new file mode 100644 index 000000000..5d937a15e --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowCollectionExtensions.cs @@ -0,0 +1,10 @@ +namespace Bts.DataStreamCore; + +using BCG; + +using System.Collections.Generic; + +public static class BlueRowCollectionExtensions { + public static IList AsList(this IBlueRowCollection rowCollection) => + rowCollection; +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs index 29c4c1cbe..ee0355303 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTable.cs @@ -93,12 +93,7 @@ namespace BCG { /// Returns the row by the index, interface property /// public virtual IBlueRow this[int id] { - get { - object row = rows[id]; - if (row == null) - throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); - return (IBlueRow)row; - } + get => rows[id]; set { if ((int)value["id"] != id) { throw new ArgumentException("id arg not equal to row's actual id", nameof(id)); @@ -112,7 +107,8 @@ namespace BCG { /// public virtual IBlueTableSchema schema { get { return mySchema; } } - public virtual bool containsID(int id) { return rows[id] != null; } + public virtual bool containsID(int id) => + rows.ContainsKey(id); public virtual bool tryGetValue(int id, out IBlueRow row) => rows.TryGetValue(id, out row); @@ -219,6 +215,7 @@ namespace BCG { #endregion public IDictionary AsDictionary() => this; + public IList AsList() => this; public IList GetColumn(int index) { if (index < 0 || index >= schema.columnInfo.Length) { @@ -227,8 +224,11 @@ namespace BCG { return Rows.WrapSelector(r => r[index], null); } + public IBlueRow GetRow(int index) => + Rows[index]; + #region ICollection members - public virtual IEnumerator GetEnumerator() => + public virtual IEnumerator GetEnumerator() => rows.Values.GetEnumerator(); public virtual bool IsSynchronized { get { return false; } } @@ -243,6 +243,18 @@ namespace BCG { public virtual int Count => rows.Count; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + int index = rows.IndexOfKey(id); + if (index != -1) { + var row = rows.GetValueAtIndex(index); + if (BlueRowEqualityComparer.Equals(row, item)) { + return index; + } + } + return -1; + } #endregion #region IDictionary members @@ -268,7 +280,7 @@ namespace BCG { bool ICollection>.Contains(KeyValuePair item) => tryGetValue(item.Key, out var r) - && BlueRowEqualityComparer.Instance.Equals(r, item.Value); + && BlueRowEqualityComparer.Equals(r, item.Value); void ICollection>.CopyTo(KeyValuePair[] array, int arrayIndex) { foreach (var kv in rows) { diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs index 4d6d35ed8..4916a70b9 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs @@ -6,12 +6,9 @@ using BCG; using bts.api.core; using bts.collections; -using mbts.utils; - using Spark.Core.Collections; using System; -using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -21,11 +18,11 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { readonly ImmutableSortedSet> rows; public BlueTableImmutable(IBlueTableSchema schema) - : this(schema, ImmutableSortedSet>.Empty) { } + : this(schema, ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer())) { } public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet> rows) { this.schema = schema; - this.rows = rows.WithComparer(new ComparerBuilder>() { kv => kv.Key }); + this.rows = rows; } public int Count => rows.Count; @@ -40,6 +37,25 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { ? kv.Value : throw new KeyNotFoundException(); + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + public IBlueRow GetRow(int index) => + rows[index].Value; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (rows.TryGetValue(new(id, item), out var rowKvp)) { + var vals = schema.columnInfo.Select(c => rowKvp.Value[c.colNo]); + var itemVals = item.schema.columnInfo.Select(c => item[c.colNo]); + if (vals.SequenceEqual(itemVals)) { + return rows.IndexOf(rowKvp); + } + } + return -1; + } public bool containsID(int id) => #nullable disable rows.Contains(new(id, default)); @@ -54,7 +70,7 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { public IList GetColumn(int index) => rows.WrapSelector(kv => kv.Value[index], null); - public IEnumerator GetEnumerator() { + public IEnumerator GetEnumerator() { foreach (var kv in rows) { yield return kv.Value; } @@ -67,9 +83,9 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { readonly IBlueTableSchema schema; readonly ImmutableSortedSet>.Builder table; readonly ImmutableSortedSet>.Builder added = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); readonly ImmutableSortedSet>.Builder removed = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); public DataPacketBuilder(BlueTableImmutable table) { schema = table.schema; @@ -128,154 +144,154 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { } } -public class BlueTableImmutable : IBlueRowCollection where T : IBlueRow { - readonly ImmutableSortedSet> rows; +// public class BlueTableImmutable : IBlueRowCollection where T : IBlueRow { +// readonly ImmutableSortedSet> rows; - public BlueTableImmutable(IBlueTableSchema schema) - : this(schema, ImmutableSortedSet>.Empty) { } +// public BlueTableImmutable(IBlueTableSchema schema) +// : this(schema, ImmutableSortedSet>.Empty) { } - public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet> rows) { - this.schema = schema; - this.rows = rows; - } +// public BlueTableImmutable(IBlueTableSchema schema, ImmutableSortedSet> rows) { +// this.schema = schema; +// this.rows = rows; +// } - public int Count => rows.Count; - public bool IsSynchronized => false; - public IBlueTableSchema schema { get; } - public object SyncRoot => this; +// public int Count => rows.Count; +// public bool IsSynchronized => false; +// public IBlueTableSchema schema { get; } +// public object SyncRoot => this; - public T this[int id] => -#nullable disable - rows.TryGetValue(new(id, default), out var kv) -#nullable enable - ? kv.Value - : throw new KeyNotFoundException(); +// public T this[int id] => +// #nullable disable +// rows.TryGetValue(new(id, default), out var kv) +// #nullable enable +// ? kv.Value +// : throw new KeyNotFoundException(); - public bool containsID(int id) => -#nullable disable - rows.Contains(new(id, default)); -#nullable enable +// public bool containsID(int id) => +// #nullable disable +// rows.Contains(new(id, default)); +// #nullable enable - public IList GetColumn(int index) => - rows.WrapSelector(kv => kv.Value[index], null); +// public IList GetColumn(int index) => +// rows.WrapSelector(kv => kv.Value[index], null); - public void CopyTo(Array array, int index) { - foreach (var (k, v) in rows) { - array.SetValue(v, index++); - } - } +// public void CopyTo(Array array, int index) { +// foreach (var (k, v) in rows) { +// array.SetValue(v, index++); +// } +// } - public IEnumerator GetEnumerator() { - foreach (var kv in rows) { - yield return kv.Value; - } - } +// public IEnumerator GetEnumerator() { +// foreach (var kv in rows) { +// yield return kv.Value; +// } +// } - IEnumerator IEnumerable.GetEnumerator() => - GetEnumerator(); +// IEnumerator IEnumerable.GetEnumerator() => +// GetEnumerator(); - public DataPacketBuilder ToDataPacketBuilder() => - new(this); +// public DataPacketBuilder ToDataPacketBuilder() => +// new(this); - public class DataPacketBuilder { - readonly IBlueTableSchema schema; - readonly ImmutableSortedSet>.Builder table; - readonly ImmutableSortedSet>.Builder added = - ImmutableSortedSet>.Empty.ToBuilder(); - readonly ImmutableSortedSet>.Builder removed = - ImmutableSortedSet>.Empty.ToBuilder(); +// public class DataPacketBuilder { +// readonly IBlueTableSchema schema; +// readonly ImmutableSortedSet>.Builder table; +// readonly ImmutableSortedSet>.Builder added = +// ImmutableSortedSet>.Empty.ToBuilder(); +// readonly ImmutableSortedSet>.Builder removed = +// ImmutableSortedSet>.Empty.ToBuilder(); - public DataPacketBuilder(BlueTableImmutable table) { - schema = table.schema; - this.table = table.rows.ToBuilder(); - } +// public DataPacketBuilder(BlueTableImmutable table) { +// schema = table.schema; +// this.table = table.rows.ToBuilder(); +// } - public void AddOrUpdate(T row, IEqualityComparer? comparer = null) { - var id = row.Id(); -#nullable disable - if (table.TryGetValue(new(id, default), out var orig)) { -#nullable enable - if (comparer?.Equals(row, orig.Value) ?? false) { - // No op: This row was already in the table (according to 'comparer'). - // Remove it from 'added' and 'removed'. - added.Remove(new(id, row)); - removed.Remove(new(id, orig.Value)); - return; - } - // Change op: A row with this id was already in the table, - // but its values have changed (according to 'comparer'). - removed.Remove(orig); - removed.Add(orig); - } - added.Remove(new(id, row)); - added.Add(new(id, row)); - } +// public void AddOrUpdate(T row, IEqualityComparer? comparer = null) { +// var id = row.Id(); +// #nullable disable +// if (table.TryGetValue(new(id, default), out var orig)) { +// #nullable enable +// if (comparer?.Equals(row, orig.Value) ?? false) { +// // No op: This row was already in the table (according to 'comparer'). +// // Remove it from 'added' and 'removed'. +// added.Remove(new(id, row)); +// removed.Remove(new(id, orig.Value)); +// return; +// } +// // Change op: A row with this id was already in the table, +// // but its values have changed (according to 'comparer'). +// removed.Remove(orig); +// removed.Add(orig); +// } +// added.Remove(new(id, row)); +// added.Add(new(id, row)); +// } - public void Remove(int id) { -#nullable disable - added.Remove(new(id, default)); - if (table.TryGetValue(new(id, default), out var orig)) { -#nullable enable - removed.Remove(orig); - removed.Remove(orig); - } - } +// public void Remove(int id) { +// #nullable disable +// added.Remove(new(id, default)); +// if (table.TryGetValue(new(id, default), out var orig)) { +// #nullable enable +// removed.Remove(orig); +// removed.Remove(orig); +// } +// } - public DataPacket ToDataPacket(bool reset = true) { - table.RemoveRange(removed); - table.AddRange(added); - var newRows = new BlueTableImmutable(schema, table.ToImmutable()); - var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); - var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); - if (reset) { - added.Clear(); - removed.Clear(); - } - return new(newRows, addedRows, removedRows); - } +// public DataPacket ToDataPacket(bool reset = true) { +// table.RemoveRange(removed); +// table.AddRange(added); +// var newRows = new BlueTableImmutable(schema, table.ToImmutable()); +// var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); +// var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); +// if (reset) { +// added.Clear(); +// removed.Clear(); +// } +// return new(newRows, addedRows, removedRows); +// } - public IDataPacket ToDataPacketT(bool reset = true) { - table.RemoveRange(removed); - table.AddRange(added); - var newRows = new BlueTableImmutable(schema, table.ToImmutable()); - var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); - var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); - if (reset) { - added.Clear(); - removed.Clear(); - } - return new DataPacket(newRows, addedRows, removedRows); - } +// public IDataPacket ToDataPacketT(bool reset = true) { +// table.RemoveRange(removed); +// table.AddRange(added); +// var newRows = new BlueTableImmutable(schema, table.ToImmutable()); +// var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); +// var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); +// if (reset) { +// added.Clear(); +// removed.Clear(); +// } +// return new DataPacket(newRows, addedRows, removedRows); +// } - public IDataPacket ToDataPacketT( - out BlueTableImmutable newRows, - out BlueTableImmutable addedRows, - out BlueTableImmutable removedRows, - bool reset = true - ) { - table.RemoveRange(removed); - table.AddRange(added); - newRows = new BlueTableImmutable(schema, table.ToImmutable()); - addedRows = new BlueTableImmutable(schema, added.ToImmutable()); - removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); - if (reset) { - added.Clear(); - removed.Clear(); - } - return new DataPacket(newRows, addedRows, removedRows); - } +// public IDataPacket ToDataPacketT( +// out BlueTableImmutable newRows, +// out BlueTableImmutable addedRows, +// out BlueTableImmutable removedRows, +// bool reset = true +// ) { +// table.RemoveRange(removed); +// table.AddRange(added); +// newRows = new BlueTableImmutable(schema, table.ToImmutable()); +// addedRows = new BlueTableImmutable(schema, added.ToImmutable()); +// removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); +// if (reset) { +// added.Clear(); +// removed.Clear(); +// } +// return new DataPacket(newRows, addedRows, removedRows); +// } - public BlueTableImmutable ToImmutable() => - new(schema, table.ToImmutable()); - } -} +// public BlueTableImmutable ToImmutable() => +// new(schema, table.ToImmutable()); +// } +// } -public class BlueTableImmutableAdapter : IBlueRowCollection> { +public class BlueTableImmutableAdapter : IBlueRowCollection { readonly BlueTableSchemaAdapter schemaAdapter; readonly ImmutableSortedSet> rows; public BlueTableImmutableAdapter(BlueTableSchemaAdapter schemaAdapter) - : this(schemaAdapter, ImmutableSortedSet>.Empty) { } + : this(schemaAdapter, ImmutableSortedSet.Create(KeyValuePairComparer.KeyComparer())) { } public BlueTableImmutableAdapter(BlueTableSchemaAdapter schemaAdapter, ImmutableSortedSet> rows) { this.schemaAdapter = schemaAdapter; @@ -287,8 +303,8 @@ public class BlueTableImmutableAdapter : IBlueRowCollection public IBlueTableSchema schema => schemaAdapter.Schema; public object SyncRoot => this; - public BlueRowAdapter this[int id] => - new(schemaAdapter, rows[id].Value); + public IBlueRow this[int id] => + new BlueRowAdapter(schemaAdapter, rows[id].Value); public bool containsID(int id) => #nullable disable @@ -298,13 +314,30 @@ public class BlueTableImmutableAdapter : IBlueRowCollection public IList GetColumn(int index) => rows.WrapSelector(kv => schemaAdapter.GetBlueRow(kv.Value)[index], null); + public IBlueRow GetRow(int index) => + new BlueRowAdapter(schemaAdapter, rows[index].Value); + public void CopyTo(Array array, int index) { foreach (var kv in rows) { array.SetValue(new BlueRowAdapter(schemaAdapter, kv.Value), index++); } } - public IEnumerator> GetEnumerator() { + public int IndexOf(IBlueRow item) { + int id = item.Id(); +#nullable disable + int index = rows.IndexOf(new(id, default)); + if (index != -1) { + var r = this[index]; + if (BlueRowEqualityComparer.Equals(r, item)) { + return index; + } + } +#nullable enable + return -1; + } + + public IEnumerator GetEnumerator() { foreach (var kv in rows) { yield return new BlueRowAdapter(schemaAdapter, kv.Value); } @@ -315,11 +348,12 @@ public class BlueTableImmutableAdapter : IBlueRowCollection public class DataPacketBuilder { readonly BlueTableSchemaAdapter schemaAdapter; - readonly ImmutableSortedSet>.Builder table; + readonly ImmutableSortedSet>.Builder table = + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); readonly ImmutableSortedSet>.Builder added = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); readonly ImmutableSortedSet>.Builder removed = - ImmutableSortedSet>.Empty.ToBuilder(); + ImmutableSortedSet.CreateBuilder(KeyValuePairComparer.KeyComparer()); public DataPacketBuilder(BlueTableImmutableAdapter table) { schemaAdapter = table.schemaAdapter; @@ -340,13 +374,10 @@ public class BlueTableImmutableAdapter : IBlueRowCollection removed.Remove(origData); return; } - // Change op: A row with this id was already in the table, - // but its values have changed (according to 'comparer'). - removed.Remove(origData); removed.Add(origData); } - removed.Remove(new(id, rowData)); - removed.Add(new(id, rowData)); + added.Remove(new(id, default!)); + added.Add(new(id, rowData)); } public void Remove(int id) { @@ -367,11 +398,10 @@ public class BlueTableImmutableAdapter : IBlueRowCollection AddOrUpdate(r); } } - - public IDataPacket> ToDataPacket(bool reset = true) => + public IDataPacket ToDataPacket(bool reset = true) => ToDataPacket(out _, reset); - public IDataPacket> ToDataPacket(out BlueTableImmutableAdapter newRows, bool reset = true) { + public IDataPacket ToDataPacket(out IBlueRowCollection newRows, bool reset = true) { table.RemoveRange(removed); table.AddRange(added); newRows = new BlueTableImmutableAdapter(schemaAdapter, table.ToImmutable()); @@ -381,7 +411,7 @@ public class BlueTableImmutableAdapter : IBlueRowCollection added.Clear(); removed.Clear(); } - return new DataPacket>(newRows, addedRows, removedRows); + return new DataPacket(newRows, addedRows, removedRows); } public BlueTableImmutableAdapter ToImmutable() => diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs index ed4eacd4d..6333646d3 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSelectorWrapper.cs @@ -6,37 +6,37 @@ using BCG; using System; using System.Collections.Generic; -public static class BlueTableSelectorWrapper { - public static BlueTableSelectorWrapper Create( - IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) where T : IBlueRow => - new(source, selector, schema); -} +// public static class BlueTableSelectorWrapper { +// public static BlueTableSelectorWrapper Create( +// IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) where T : IBlueRow => +// new(source, selector, schema); +// } -public class BlueTableSelectorWrapper : IBlueRowCollection where T : IBlueRow { - readonly IBlueRowCollection source; - readonly Func selector; +// public class BlueTableSelectorWrapper : IBlueRowCollection where T : IBlueRow { +// readonly IBlueRowCollection source; +// readonly Func selector; - public BlueTableSelectorWrapper(IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) { - this.source = source; - this.selector = selector; - this.schema = schema ?? source.schema; - } +// public BlueTableSelectorWrapper(IBlueRowCollection source, Func selector, IBlueTableSchema? schema = null) { +// this.source = source; +// this.selector = selector; +// this.schema = schema ?? source.schema; +// } - public T this[int id] => selector(source[id]); +// public T this[int id] => selector(source[id]); - public IBlueTableSchema schema { get; } +// public IBlueTableSchema schema { get; } - public int Count => source.Count; +// public int Count => source.Count; - public bool containsID(int id) => - source.containsID(id); +// public bool containsID(int id) => +// source.containsID(id); - public IList GetColumn(int index) => - throw new NotImplementedException(); +// public IList GetColumn(int index) => +// throw new NotImplementedException(); - public IEnumerator GetEnumerator() { - foreach (IBlueRow r in source) { - yield return selector(r); - } - } -} +// public IEnumerator GetEnumerator() { +// foreach (IBlueRow r in source) { +// yield return selector(r); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs index 59c349620..9931ed913 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.Immutable; using System.Data; using System.Linq; @@ -8,51 +9,83 @@ using BCG; using bts.collections; +using Bts.DataStreamCore; + namespace bts { public class CircularBufferTable : IBlueRowCollection { - private BlueTable _table; - private List _ordered_list = new List(); + readonly ImmutableList orderedRows; + readonly ImmutableDictionary rowIndex; private int _buffer_sz; - public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) { - _table = new BlueTable(schema); - _buffer_sz = buffer_sz; + public CircularBufferTable(IBlueTableSchema schema, int buffer_sz) + : this(schema, buffer_sz, ImmutableList.Empty, ImmutableDictionary.Empty) { } + + CircularBufferTable(IBlueTableSchema schema, int bufferSize, ImmutableList orderedRows, ImmutableDictionary rowIndex) { + this.schema = schema; + _buffer_sz = bufferSize; + this.orderedRows = orderedRows; + this.rowIndex = rowIndex; } + public CircularBufferTable Add(IBlueRow row) => + Add(row, out _); + // returns the row that might need to be deleted to keep // the circular buffer intact - public IBlueRow add_row(IBlueRow row) { - _ordered_list.Add(row); - _table.addRow(row); - if (_ordered_list.Count > _buffer_sz) { - IBlueRow rv = _ordered_list.ElementAt(0); - _ordered_list.RemoveAt(0); - _table.deleteRow(rv); - return rv; + public CircularBufferTable Add(IBlueRow row, out IBlueRow oldRow) { + int id = row.Id(); + var rowIndexBd = rowIndex.ToBuilder(); + var orderedRowsBd = orderedRows.ToBuilder(); + if (rowIndexBd.TryGetValue(id, out oldRow)) { + Log.assert(orderedRowsBd.RemoveFirst(r => r.Id() == id)); + } else if (rowIndexBd.Count > _buffer_sz) { + oldRow = orderedRowsBd[0]; + rowIndexBd.Remove(oldRow.Id()); + orderedRowsBd.RemoveAt(0); } - return null; + rowIndexBd[id] = row; + orderedRowsBd.Add(row); + return new(schema, _buffer_sz, orderedRowsBd.ToImmutable(), rowIndexBd.ToImmutable()); } public IBlueRow this[int id] { get { - return _table[id]; - } - } - public IBlueTableSchema schema { - get { - return _table.schema; + return orderedRows[id]; } } + public IBlueTableSchema schema { get; } public bool containsID(int id) { - return _table.containsID(id); + return rowIndex.ContainsKey(id); } - public void CopyTo(Array array, int index) { _table.CopyTo((IBlueRow[])array, index); } - public int Count { get { return _table.Count; } } + public void CopyTo(Array array, int index) { + foreach (var row in orderedRows) { + array.SetValue(row, index++); + } + } + public int Count => rowIndex.Count; public bool IsSynchronized { get { return false; } } public object SyncRoot { get { return this; } } - public System.Collections.IEnumerator GetEnumerator() { return _table.GetEnumerator(); } + public IEnumerator GetEnumerator() => + orderedRows.GetEnumerator(); public IList GetColumn(int index) => - _ordered_list.WrapSelector(r => r[index], null); + orderedRows.WrapSelector(r => r[index], null); + + public IBlueRow GetRow(int index) => + orderedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (rowIndex.TryGetValue(id, out var row) && BlueRowEqualityComparer.Equals(item, row)) { + int i = 0; + foreach (var r in orderedRows) { + if (r.Id() == id) { + return i; + } + i++; + } + } + return -1; + } } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs index 59ccdd910..8b074d42a 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueRowCollection.cs @@ -1,3 +1,5 @@ +using bts.collections; + using System; using System.Collections; using System.Collections.Generic; @@ -8,30 +10,47 @@ namespace BCG /// Represents a readonly, very primitive form of dotNet DataTable. All it has has an indexer for /// getting the rows /// - public interface IBlueRowCollection : ICollection - { + public interface IBlueRowCollection : IList { /// /// Gets the row corresponding to the id /// - IBlueRow this[int id]{get;} - IBlueTableSchema schema {get;} - bool containsID (int id); - IList GetColumn(int index); + new IBlueRow this[int id] { get; } + IBlueTableSchema schema { get; } + bool containsID(int id); + IList GetColumn(int index) => + this.WrapSelector(r => r[index], null); - bool ICollection.IsSynchronized => false; - object ICollection.SyncRoot => this; + IBlueRow GetRow(int index); - void ICollection.CopyTo(Array array, int index) { - foreach (IBlueRow r in this) { - array.SetValue(r, index++); + bool ICollection.IsReadOnly => true; + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + + bool ICollection.Contains(IBlueRow item) => + IndexOf(item) != -1; + + void ICollection.CopyTo(IBlueRow[] array, int arrayIndex) { + foreach (var row in this) { + array[arrayIndex++] = row; } } - } - public interface IBlueRowCollection : IBlueRowCollection, IReadOnlyCollection where T : IBlueRow { - new T this[int id] { get; } + void ICollection.Add(IBlueRow item) => + throw new NotSupportedException(); - IBlueRow IBlueRowCollection.this[int id] => this[id]; + void ICollection.Clear() => + throw new NotSupportedException(); + + bool ICollection.Remove(IBlueRow item) => + throw new NotSupportedException(); + + void IList.RemoveAt(int index) => + throw new NotSupportedException(); + + void IList.Insert(int index, IBlueRow item) => + throw new NotSupportedException(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs index 347db29a7..1a39d9ea5 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ImmutableBlueTable.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; @@ -48,6 +50,11 @@ namespace BCG } } + IBlueRow IList.this[int index] { + get => GetRow(index); + set => throw new NotSupportedException(); + } + public IBlueTableSchema schema { get @@ -59,6 +66,8 @@ namespace BCG public bool containsID(int id) { return myHash.isPresent(id); } + public IBlueRow GetRow(int index) => + myRows[index]; #endregion #region ICollection Members @@ -82,6 +91,17 @@ namespace BCG myRows.CopyTo(array, index); } + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (myHash.isPresent(id)) { + int index = myHash.find(id); + if (BlueRowEqualityComparer.Equals(item, myRows[index])) { + return index; + } + } + return -1; + } + public object SyncRoot { get @@ -95,8 +115,11 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() - { return myRows.GetEnumerator(); } + public IEnumerator GetEnumerator() { + for (int i = 0; i < myRows.Length; i++) { + yield return myRows[i]; + } + } #endregion diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs b/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs index 381debde7..1e9bc0871 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/KeysBasedTable2.cs @@ -37,11 +37,15 @@ namespace BCG public object SyncRoot {get {return myTable.SyncRoot;}} + public int IndexOf(IBlueRow item) => + myTable.IndexOf(item); + #endregion #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() {return myTable.GetEnumerator ();} + public IEnumerator GetEnumerator() => + myTable.GetEnumerator(); #endregion @@ -136,6 +140,9 @@ namespace BCG public IList GetColumn(int index) => throw new NotImplementedException(); + public IBlueRow GetRow(int index) => + throw new NotImplementedException(); + public IBlueRow this[params object[] keyVals] {get {return myHash.getValue (keyVals) as IBlueRow;}} private object[] myGetKeyVals (IBlueRow r) {return BlueDataUtility.getSelectedValues (r, myKeys);} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs index 6bf9218b6..79abe062d 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/MovingWindowTable.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; @@ -108,6 +110,17 @@ namespace BCG public IList GetColumn(int index) => mySortedRows.WrapSelector(r => r[index], null); + public IBlueRow GetRow(int index) => + mySortedRows[index]; + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + if (myIDToRows.Value.TryGetValue(id, out var row) && BlueRowEqualityComparer.Equals(item, row)) { + return mySortedRows.BinarySearch(row); + } + return -1; + } + #endregion #region ICollection Members @@ -137,7 +150,7 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() + public IEnumerator GetEnumerator() { return this.mySortedRows.GetEnumerator(); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs index 69a79c538..95d03d4d7 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/OrderedBlueTable.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Data; //using BCG.Utilities; @@ -62,7 +63,11 @@ namespace BCG } #region ICollection members - public override IEnumerator GetEnumerator() { return myOrderedRows.GetEnumerator(); } + public override IEnumerator GetEnumerator() { + foreach (IBlueRow row in myOrderedRows) { + yield return row; + } + } public override void CopyTo(Array array, int index) { myOrderedRows.CopyTo(array, index); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs index b3bdf2892..754a0c6a4 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SequentialIDBlueTable.cs @@ -1,7 +1,10 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; +using System.Linq; namespace BCG { @@ -36,6 +39,13 @@ namespace BCG #region IBlueRowCollection Members public IBlueRow this[int id] { get { return myRows[id]; } } + public IBlueRow GetRow(int index) => myRows[index]; + public int IndexOf(IBlueRow item) { + int id = item.Id(); + return id >= 0 && id < myRows.Length && BlueRowEqualityComparer.Equals(myRows[id], item) + ? id + : -1; + } public IBlueTableSchema schema {get {return mySchema;}} @@ -63,7 +73,11 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator ();} + public IEnumerator GetEnumerator() { + for (int i = 0; i < myRows.Length; i++) { + yield return myRows[i]; + } + } #endregion } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs index afa40635b..5a7e7dc08 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SimpleBindableCollection.cs @@ -1,5 +1,7 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections; using System.Collections.Generic; @@ -9,22 +11,22 @@ namespace BCG { public class SimpleBindableCollection : IBlueRowCollection, ITypedList, IList { - private ArrayList myRows; + private List myRows; private IBlueTableSchema mySchema; private PropertyDescriptorCollection myPDColl; - public SimpleBindableCollection (IBlueTableSchema schema, ICollection rows) + public SimpleBindableCollection (IBlueTableSchema schema, ICollection rows) { mySchema = schema; ArrayList temp = new ArrayList (); foreach (ColumnInfo ci in mySchema.columnInfo) temp.Add (new BlueDataPropertyDescriptor (ci)); myPDColl = new PropertyDescriptorCollection (temp.ToArray (typeof(PropertyDescriptor)) as PropertyDescriptor[]); - myRows = new ArrayList (rows); + myRows = new(rows); } public SimpleBindableCollection (IBlueTableSchema schema) - : this (schema, new ArrayList ()) + : this(schema, new List(0)) {} #region IBlueRowCollection Members @@ -57,6 +59,9 @@ namespace BCG public IList GetColumn(int index) => myRows.WrapSelector(r => ((IBlueRow)r)[index], null); + public IBlueRow GetRow(int index) => + (IBlueRow)myRows[index]; + #endregion #region ICollection Members @@ -65,7 +70,25 @@ namespace BCG public int Count {get {return myRows.Count;}} - public void CopyTo(Array array, int index) {myRows.CopyTo (array, index);} + public void CopyTo(Array array, int index) { + for (int i = 0; i < myRows.Count; i++) { + array.SetValue(myRows[i], index++); + } + } + + public int IndexOf(IBlueRow item) { + int id = item.Id(); + for (int i = 0; i < myRows.Count; i++) { + var r = myRows[i]; + if (r.Id() == id) { + if (BlueRowEqualityComparer.Equals(item, r)) { + return i; + } + break; + } + } + return -1; + } public object SyncRoot { get {return null;}} @@ -73,9 +96,10 @@ namespace BCG #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() - { - return myRows.GetEnumerator (); + public IEnumerator GetEnumerator() { + foreach (IBlueRow row in myRows) { + yield return row; + } } #endregion @@ -110,11 +134,13 @@ namespace BCG public void Remove(object value) {throw new ApplicationException ("Collection is read only");} - public bool Contains(object value) {return myRows.Contains (value);} + public bool Contains(object value) => + value is IBlueRow r && myRows.Contains(r); public void Clear() {throw new ApplicationException ("Collection is read only");} - public int IndexOf(object value) {return myRows.IndexOf (value);} + public int IndexOf(object value) => + value is IBlueRow r ? myRows.IndexOf(r) : -1; public int Add(object value) {throw new ApplicationException ("Collection is read only");} @@ -122,6 +148,7 @@ namespace BCG #endregion - public void reset (ICollection rows) {myRows = new ArrayList(rows);} + public void reset(ICollection rows) => + myRows = new(rows); } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs index b0469668a..5893ae655 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SparseVectorBlueTable.cs @@ -1,8 +1,9 @@ using bts.collections; +using Bts.DataStreamCore; + using System; using System.Collections.Generic; -using System.Linq; namespace BCG { @@ -32,6 +33,17 @@ namespace BCG _rows[id] = value; } } + + public IBlueRow GetRow(int index) => + _rows[index]; + + public int IndexOf(IBlueRow item) { + int index = item.Id(); + return index >= 0 && index < _rows.Count && BlueRowEqualityComparer.Equals(_rows[index], item) + ? index + : -1; + } + public IBlueTableSchema schema { get; private set; } public bool containsID(int id) { return (id >= 0 && id < _rows.Count && this[id] != null); @@ -42,8 +54,13 @@ namespace BCG public void CopyTo(Array array, int index) { _rows.CopyTo((IBlueRow[])array, index);} public int Count { get { return _rows.Count; } } - public bool IsSynchronized {get { return false; }} - public object SyncRoot { get { return this; } } - public System.Collections.IEnumerator GetEnumerator() { return _rows.Where(r => r != null).GetEnumerator(); } + + public IEnumerator GetEnumerator() { + foreach (var r in _rows) { + if (r is not null) { + yield return r; + } + } + } } } diff --git a/src/DataStreamSuite/DataStreamCore/DataPacket.cs b/src/DataStreamSuite/DataStreamCore/DataPacket.cs index 171d4be66..1d40def73 100644 --- a/src/DataStreamSuite/DataStreamCore/DataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/DataPacket.cs @@ -1,12 +1,10 @@ +using Spark.Core.Collections; + using System; using System.Collections; using System.Collections.Generic; using System.Data; -using BCG.Utilities.General; - -using Bts.DataStreamCore; - namespace BCG { [Serializable] public class DataPacket : ICloneable, IDataPacket { @@ -45,7 +43,11 @@ namespace BCG { public IBlueRowCollection newRows, addedRows, deletedRows; public Dictionary meta_data = null; public DataPacket() { } - public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { + + public DataPacket(IBlueTableSchema schema) + : this(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema)) { } + + public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { this.newRows = newRows; this.addedRows = addedRows; this.deletedRows = deletedRows; @@ -57,9 +59,9 @@ namespace BCG { IReadOnlyDictionary IDataPacket.Metadata => meta_data; public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { - new_rows = newRows; - added_rows = addedRows; - deleted_rows = deletedRows; + new_rows = ListNonGenericCollectionWrapper.Create(newRows); + added_rows = ListNonGenericCollectionWrapper.Create(addedRows); + deleted_rows = ListNonGenericCollectionWrapper.Create(deletedRows); } public IBlueRowCollection getTrulyAddedRows(params string[] idColumns) { @@ -69,7 +71,9 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + ICollection ta = CollectionHelpers.difference(addedRowCollection, deletedRowCollection, ed); foreach (IBlueRow r in ta) { truly.addRow(r); @@ -111,7 +115,9 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + ICollection td = CollectionHelpers.difference(deletedRowCollection, addedRowCollection, ed); foreach (IBlueRow r in td) { truly.addRow(r); @@ -128,8 +134,10 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed); - ICollection tc = CollectionHelpers.difference(addedRows, ta, ed); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + ICollection ta = CollectionHelpers.difference(addedRowCollection, deletedRowCollection, ed); + ICollection tc = CollectionHelpers.difference(addedRowCollection, ta, ed); foreach (IBlueRow r in tc) { truly.addRow(r); @@ -145,8 +153,10 @@ namespace BCG { CollectionHelpers.EqualDelegate ed = new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator); - ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed); - ICollection tco = CollectionHelpers.difference(deletedRows, td, ed); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(deletedRows); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(addedRows); + ICollection td = CollectionHelpers.difference(deletedRowCollection, addedRowCollection, ed); + ICollection tco = CollectionHelpers.difference(deletedRowCollection, td, ed); foreach (IBlueRow r in tco) { truly.addRow(r); @@ -223,13 +233,15 @@ namespace BCG { } deleted.AddRange(oldDeleted); + var deletedRowCollection = ListNonGenericCollectionWrapper.Create(dp.deletedRows); deleted.AddRange(CollectionHelpers.difference( - CollectionHelpers.difference(dp.deletedRows, oldAdded, ed), + CollectionHelpers.difference(deletedRowCollection, oldAdded, ed), oldDeleted, ed)); - added.AddRange(dp.addedRows); + var addedRowCollection = ListNonGenericCollectionWrapper.Create(dp.addedRows); + added.AddRange(addedRowCollection); added.AddRange(CollectionHelpers.difference( - CollectionHelpers.difference(oldAdded, dp.deletedRows, ed), + CollectionHelpers.difference(oldAdded, deletedRowCollection, ed), added, ed)); nt = dp.newRows; } @@ -287,30 +299,30 @@ namespace BCG { } } - public class DataPacket : IDataPacket where T : IBlueRow { - public DataPacket( - IBlueRowCollection newRows, - IBlueRowCollection addedRows, - IBlueRowCollection deletedRows, - IReadOnlyDictionary metadata = null - ) { - this.newRows = newRows; - this.addedRows = addedRows; - this.deletedRows = deletedRows; - Metadata = metadata ?? DictionaryHelper.empty(); - } + // public class DataPacket : IDataPacket where T : IBlueRow { + // public DataPacket( + // IBlueRowCollection newRows, + // IBlueRowCollection addedRows, + // IBlueRowCollection deletedRows, + // IReadOnlyDictionary metadata = null + // ) { + // this.newRows = newRows; + // this.addedRows = addedRows; + // this.deletedRows = deletedRows; + // Metadata = metadata ?? DictionaryHelper.empty(); + // } - public IBlueRowCollection newRows { get; } - public IBlueRowCollection addedRows { get; } - public IBlueRowCollection deletedRows { get; } - public IReadOnlyDictionary Metadata { get; } + // public IBlueRowCollection newRows { get; } + // public IBlueRowCollection addedRows { get; } + // public IBlueRowCollection deletedRows { get; } + // public IReadOnlyDictionary Metadata { get; } - public static DataPacket Empty(IBlueTableSchema schema) => - new( - new BlueTableImmutable(schema), - new BlueTableImmutable(schema), - new BlueTableImmutable(schema)); - } + // public static DataPacket Empty(IBlueTableSchema schema) => + // new( + // new BlueTableImmutable(schema), + // new BlueTableImmutable(schema), + // new BlueTableImmutable(schema)); + // } public static class DataPacketExtensions { public static bool HasChanges(this DataPacket dataPacket) => diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs b/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs index d8506e8cc..fee26852a 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs +++ b/src/DataStreamSuite/DataStreamCore/DataStreamChangeDelegate.cs @@ -1,4 +1,4 @@ namespace BCG; public delegate void DataStreamChangeDelegate(object newValue); -public delegate void DataStreamChangeDelegate(IDataPacket dp) where T : IBlueRow; +// public delegate void DataStreamChangeDelegate(IDataPacket dp) where T : IBlueRow; diff --git a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs index 900b334b8..953803009 100644 --- a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs @@ -18,22 +18,22 @@ namespace BCG { void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows); } - public interface IDataPacket : IDataPacket where T : IBlueRow { - new IBlueRowCollection newRows { get; } - new IBlueRowCollection addedRows { get; } - new IBlueRowCollection deletedRows { get; } + // public interface IDataPacket : IDataPacket where T : IBlueRow { + // new IBlueRowCollection newRows { get; } + // new IBlueRowCollection addedRows { get; } + // new IBlueRowCollection deletedRows { get; } - IBlueRowCollection IDataPacket.newRows => newRows; - IBlueRowCollection IDataPacket.addedRows => addedRows; - IBlueRowCollection IDataPacket.deletedRows => deletedRows; - IBlueTableSchema IDataPacket.Schema => newRows.schema; + // IBlueRowCollection IDataPacket.newRows => newRows; + // IBlueRowCollection IDataPacket.addedRows => addedRows; + // IBlueRowCollection IDataPacket.deletedRows => deletedRows; + // IBlueTableSchema IDataPacket.Schema => newRows.schema; - void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { - new_rows = newRows; - added_rows = addedRows; - deleted_rows = deletedRows; - } - } + // void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { + // new_rows = newRows; + // added_rows = addedRows; + // deleted_rows = deletedRows; + // } + // } public static class IDataPacketExtensions { public static IDataPacket ToImmutable(this IDataPacket dataPacket) => diff --git a/src/DataStreamSuite/DataStreamCore/IViewStream.cs b/src/DataStreamSuite/DataStreamCore/IViewStream.cs index eae4d3cd2..1eb8c1842 100644 --- a/src/DataStreamSuite/DataStreamCore/IViewStream.cs +++ b/src/DataStreamSuite/DataStreamCore/IViewStream.cs @@ -6,8 +6,8 @@ public interface IViewStream { IBlueRowCollection rows { get; } } -public interface IViewStream : IViewStream where T : IBlueRow { - new event DataStreamChangeDelegate onChange; +// public interface IViewStream : IViewStream where T : IBlueRow { +// new event DataStreamChangeDelegate onChange; - new IBlueRowCollection rows { get; } -} +// new IBlueRowCollection rows { get; } +// } diff --git a/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs b/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs index c18e7ee02..579321b3a 100644 --- a/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs +++ b/src/DataStreamSuite/DataStreamTests/TestDriverStream.cs @@ -58,11 +58,11 @@ public class TestDriverStream : ViewStream, IWorker { if (lastPublishedTable is null) { addedRows.addRows(currentTable); } else { - var comp = BlueRowEqualityComparer.Instance; - deletedRows.AsDictionary().SetItems(lastPublishedTable.Where(oldRow => - !(currentTable.AsDictionary().TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value)))); - addedRows.SetItems(currentTable.Where(newRow => - !(lastPublishedTable.AsDictionary().TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value)))); + deletedRows.AsDictionary().SetItems(lastPublishedTable.AsList().Where(oldRow => + !(currentTable.AsDictionary().TryGetValue(oldRow.Id(), out var newRow) && BlueRowEqualityComparer.Equals(newRow, oldRow))) + .Select(r => new KeyValuePair(r.Id(), r))); + addedRows.SetItems(currentTable.AsList().Where(newRow => + !(lastPublishedTable.AsDictionary().TryGetValue(newRow.Id(), out var oldRow) && BlueRowEqualityComparer.Equals(oldRow, newRow)))); } lastPublishedTable = currentTable; if (deletedRows.Count > 0 || addedRows.Count > 0) { diff --git a/src/client/spark_core/Collections/GenericComparerWrapper.cs b/src/client/spark_core/Collections/GenericComparerWrapper.cs new file mode 100644 index 000000000..1c9494902 --- /dev/null +++ b/src/client/spark_core/Collections/GenericComparerWrapper.cs @@ -0,0 +1,23 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System.Collections; +using System.Collections.Generic; + +public static class GenericComparerWrapper { + public static GenericComparerWrapper Create(IComparer comparer) => + new(comparer); +} + +public class GenericComparerWrapper : IComparer, IComparer { + readonly IComparer comparer; + + public GenericComparerWrapper(IComparer comparer) => + this.comparer = comparer; + + public int Compare(object? x, object? y) => + comparer.Compare(x, y); + + public int Compare(T? x, T? y) => + comparer.Compare(x, y); +} diff --git a/src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs b/src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs new file mode 100644 index 000000000..a0154a8f1 --- /dev/null +++ b/src/client/spark_core/Collections/ListNonGenericCollectionWrapper.cs @@ -0,0 +1,72 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System; +using System.Collections; +using System.Collections.Generic; + +public static class ListNonGenericCollectionWrapper { + public static ListGenericCollectionWrapper Create(IList source) => + new(source); +} + +public class ListGenericCollectionWrapper : IList, ICollection { + readonly IList source; + + public ListGenericCollectionWrapper(IList source) { + this.source = source; + } + + public T this[int index] { + get => source[index]; + set => source[index] = value; + } + + public int Count => source.Count; + + public bool IsSynchronized => false; + + public object SyncRoot => this; + + public bool IsReadOnly => source.IsReadOnly; + + public void Add(T item) => + source.Add(item); + + public void Clear() => + source.Clear(); + + public bool Contains(T item) => + source.Contains(item); + + public void CopyTo(Array array, int index) { + if (array is T[] arrayT) { + source.CopyTo(arrayT, index); + return; + } + for (int i = 0; i < source.Count; i++) { + array.SetValue(source[i], index++); + } + } + + public void CopyTo(T[] array, int arrayIndex) => + source.CopyTo(array, arrayIndex); + + public IEnumerator GetEnumerator() => + source.GetEnumerator(); + + public int IndexOf(T item) => + source.IndexOf(item); + + public void Insert(int index, T item) => + source.Insert(index, item); + + public bool Remove(T item) => + source.Remove(item); + + public void RemoveAt(int index) => + source.RemoveAt(index); + + IEnumerator IEnumerable.GetEnumerator() => + source.GetEnumerator(); +} diff --git a/src/client/spark_core/Threading/AtomicGate.cs b/src/client/spark_core/Threading/AtomicGate.cs new file mode 100644 index 000000000..2103031b3 --- /dev/null +++ b/src/client/spark_core/Threading/AtomicGate.cs @@ -0,0 +1,156 @@ +#nullable enable +namespace Spark.Core.Threading; + +using bts; + +using Spark.Core.Disposables; + +using System; +using System.Threading; +using System.Threading.Tasks; + +public class AtomicGate { + const int MAX_ATTEMPTS_BEFORE_SLEEP = 32; + static readonly int[] increasingSleeps = new[] { 1, 10, 100, 200, 500, 1000 }; + static int id_counter = 0; + + readonly string name; + readonly int id = Interlocked.Increment(ref id_counter); + readonly object syncRoot = new(); + int versionIn = 0; + int versionOut = 0; + + public AtomicGate(string? name = null) { + this.name = name ?? string.Empty; + } + + public bool IsBusy => Volatile.Read(ref versionIn) != Volatile.Read(ref versionOut); + + public void Enter() { + int n; + for (n = 0; n < MAX_ATTEMPTS_BEFORE_SLEEP; n++) { + if (TryEnter()) { + return; + } + Thread.Yield(); + } + for (n = 0; n < increasingSleeps.Length; n++) { + if (TryEnter()) { + return; + } + sleep(increasingSleeps[n]); + } + while (!TryEnter()) { + sleep(increasingSleeps[^1]); + } + } + + public bool TryEnter() { + var vin = Volatile.Read(ref versionIn); + var vout = Volatile.Read(ref versionOut); + Log.DataFlow.diag($"[atomic_gate_try_entering.json]{{ \"id\": {id}, \"name\": \"{name}\", \"vin\": {vin}, \"vout\": {vout}, \"ts\": \"{DateTime.Now}\" }}"); + bool res = Impl(); + vin = Volatile.Read(ref versionIn); + vout = Volatile.Read(ref versionOut); + Log.DataFlow.diag($"[atomic_gate_try_entered.json]{{ \"id\": {id}, \"name\": \"{name}\", \"vin\": {vin}, \"vout\": {vout}, \"res\": {res}, \"ts\": \"{DateTime.Now}\" }}"); + return res; + + bool Impl() { + lock (syncRoot) { + if (versionIn == versionOut) { + versionIn++; + return true; + } + } + return false; + } + } + + public void Execute(Action action) { + Enter(); + try { + action(); + } finally { + Exit(); + } + } + + public bool TryExecute(Action action) { + if (!TryEnter()) { + return false; + } + try { + action(); + } finally { + Exit(); + } + return true; + } + + public async Task TryExecuteAsync(Func asyncAction) { + if (!TryEnter()) { + return false; + } + try { + await asyncAction(); + } finally { + Exit(); + } + return true; + } + + public async Task<(bool Executed, T? Result)> TryExecuteAsync(Func> asyncAction) { + if (!TryEnter()) { + return (false, default); + } + T? res; + try { + res = await asyncAction(); + } finally { + Exit(); + } + return (true, res); + } + + public IDisposable BeginTransaction() { + int version = Interlocked.Increment(ref versionIn) - 1; + int n = 0; + var end = Disposable.Create(EndTransaction); + for (n = 0; n < MAX_ATTEMPTS_BEFORE_SLEEP; n++) { + if (Volatile.Read(ref versionOut) == version) { + return end; + } + Thread.Yield(); + } + for (n = 0; n < increasingSleeps.Length; n++) { + if (Volatile.Read(ref versionOut) == version) { + return end; + } + sleep(increasingSleeps[n], version); + } + while (Volatile.Read(ref versionOut) != version) { + sleep(increasingSleeps[^1], version); + } + return end; + + void EndTransaction() => + Log.assert(Interlocked.CompareExchange(ref versionOut, version + 1, version) == version); + } + + public Task BeginTransactionAsync() => + Task.Run(BeginTransaction); + + public void Exit() { + lock (syncRoot) { + Log.assert(versionIn == versionOut + 1); + versionOut = versionIn; + } + } + + void sleep(int sleepMillis, int? version = null) { + var vout = Volatile.Read(ref versionOut); + var vin = Volatile.Read(ref versionIn); + Log.DataFlow.warn($"[atomic_gate_sleep.json]{{ \"id\": {id}, \"name\": \"{name}\", \"vin\": {vin}, \"vout\": {vout}, \"v\": {version}, \"dur\": {sleepMillis}ms, \"ts\": \"{DateTime.Now}\" }}"); + Thread.Sleep(sleepMillis); + } +} diff --git a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs index 48a028fcc..98ad8f78b 100644 --- a/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs +++ b/src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs @@ -116,7 +116,7 @@ namespace bts.websocket_core { hdlr.dis = vs.AsObservable().Subscribe(OnData); - void OnData(DataPacket dp) { + void OnData(IDataPacket dp) { lock (syncRoot) { removedRows.Clear(); changedRows.Clear();