diff --git a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs index 58a8f9d76..9b6bdfab2 100644 --- a/src/DataStreamSuite/DataStream/DataStreamExtensions.cs +++ b/src/DataStreamSuite/DataStream/DataStreamExtensions.cs @@ -5,19 +5,19 @@ using BCG; using bts.api.core; using Bts.DataStreamCore; +using Bts.PubSub; using Microsoft.Extensions.Logging; using Spark.Core.Disposables; +using Spark.Core.Threading; using System; -using System.Collections; -using System.Collections.Generic; using System.Linq; using System.Reactive.Linq; public static class DataStreamExtensions { - public static ViewStream AsViewStream(this IObservable source) => + public static ViewStream AsViewStream(this IObservable source) => source as ViewStream ?? new ObservableViewStream(source); public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => @@ -36,15 +36,15 @@ public static class DataStreamExtensions { } } - public static IObservable> SelectDataPacket(this IObservable source, Func selector) where T : IBlueRow => - source.Select(dp => { - dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); - return new DataPacket(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows)); + // public static IObservable> SelectDataPacket(this IObservable source, Func selector) where T : IBlueRow => + // source.Select(dp => { + // dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); + // return new DataPacket(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows)); - IBlueRowCollection CastRows(ICollection rows) => - new BlueTableSelectorWrapper( - rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector); - }); + // IBlueRowCollection CastRows(ICollection rows) => + // new BlueTableSelectorWrapper( + // rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector); + // }); public static IObservable> AsTables(this IObservable source) => source.AsTables(r => r, r => r.Id()); @@ -72,9 +72,33 @@ public static class DataStreamExtensions { }); } - public static IObservable> AsDataPackets(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => + public static IObservable AsDataPackets(this IObservable> source) => source.Scan( - (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)DataPacket.Empty(schema)), + (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null), + (res, table) => { + var resTable = res.Table; + if (resTable is null) { + if (table.state.Count == 0) { + return (null, null); + } + resTable = new(table.state.Values.First().schema); + } + var bd = resTable.ToDataPacketBuilder(); + foreach (var r in table.removed) { + bd.Remove(r.Id()); + } + foreach (var r in table.added) { + bd.AddOrUpdate(r); + } + var resDp = bd.ToDataPacket(out var newRows); + return (newRows, resDp); + }) + .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)) + .Select(t => t.Dp!); + + public static IObservable AsDataPackets(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => + source.Scan( + (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)), (res, table) => { var bd = res.Table.ToDataPacketBuilder(); foreach (var r in table.removed) { @@ -83,12 +107,30 @@ public static class DataStreamExtensions { foreach (var r in table.added) { bd.AddOrUpdate(r); } - var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows); + var resDp = bd.ToDataPacket(out var newRows); return (newRows, resDp); }) - .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any()) + .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0) .Select(t => t.Dp); + public static IObservable AsDataPackets(this IObservable> source, BlueTableSchemaAdapter schema) where TKey : IEquatable { + return Observable.Create(Subscribe); + + IDisposable Subscribe(IObserver o) { + var adapter = new BlueTableImmutableAdapter(schema); + IDataPacket initDataPacket = new DataPacket(schema); + return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o); + + (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) OnUpdate( + (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) state, TableUpdate tableUpdate) { + var bd = state.Adapter.ToDataPacketBuilder(); + bd.Update(tableUpdate); + var newRows = bd.ToImmutable(out var dp); + return (newRows, dp); + } + } + } + public static ViewStream AsViewStream(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => source.AsDataPackets(schema).AsViewStream(); @@ -131,4 +173,46 @@ public static class DataStreamExtensions { } return indices; } + + public static IObservable OnWorkScheduler(this IObservable source, IWorkScheduler scheduler) => + Observable.Create(o => { + var inputGate = new AtomicGate(); + var outputGate = new AtomicGate(); + BlueTableImmutable? lastTable = null; + BlueTableImmutable.DataPacketBuilder? bd = null; + IDataPacket? pendingDpOut = null; + return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork)); + + void OnNext(IDataPacket value) { + using (inputGate.BeginTransaction()) { + lastTable ??= new(value.newRows.schema); + bd ??= lastTable.ToDataPacketBuilder(); + if (pendingDpOut is null) { + // The last outgoing datapacket was just fired. Don't merge this update with the + // previous one (otherwise its adds/removes would be fired twice). + bd.Reset(); + } + foreach (IBlueRow r in value.deletedRows) { + bd.Remove(r.Id()); + } + foreach (IBlueRow r in value.addedRows) { + bd.AddOrUpdate(r); + } + pendingDpOut = bd.ToDataPacket(out lastTable); + } + } + + void DoWork() { + IDataPacket? dpOut = null; + if (outputGate.TryEnter()) { + using var exit = Disposable.Create(outputGate.Exit); + (dpOut, pendingDpOut) = (pendingDpOut, dpOut); + if (dpOut is not null) { + o.OnNext(dpOut); + } + } + } + }); + + } diff --git a/src/DataStreamSuite/DataStream/ObservableViewStream.cs b/src/DataStreamSuite/DataStream/ObservableViewStream.cs index b9a90ebce..1dc3c618d 100644 --- a/src/DataStreamSuite/DataStream/ObservableViewStream.cs +++ b/src/DataStreamSuite/DataStream/ObservableViewStream.cs @@ -8,13 +8,13 @@ using Microsoft.Extensions.Logging; using System; -public class ObservableViewStream : ViewStream, IObservable, IDisposable { - readonly IObservable source; +public class ObservableViewStream : ViewStream, IObservable, IDisposable { + readonly IObservable source; readonly string? name; readonly ILogger? logger; IDisposable? subscription; - public ObservableViewStream(IObservable source, string? name = null, ILogger? logger = null) { + public ObservableViewStream(IObservable source, string? name = null, ILogger? logger = null) { this.source = source; this.name = name; this.logger = logger; @@ -45,6 +45,6 @@ public class ObservableViewStream : ViewStream, IObservable, IDispos subscription = null; } - IDisposable IObservable.Subscribe(IObserver observer) => + IDisposable IObservable.Subscribe(IObserver observer) => source.Subscribe(observer); } diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs index 81389f068..38842a5f6 100644 --- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs @@ -8,8 +8,8 @@ using System; using System.Collections.Generic; public static class MongoSchemaAdaptor { - public static BlueTableSchemaAdaptor InferSchema(IEnumerable docs) { - var schemaAdaptor = new BlueTableSchemaAdaptor("MongoCollectionStream"); + public static BlueTableSchemaAdapter InferSchema(IEnumerable docs) { + var schemaAdaptor = new BlueTableSchemaAdapter("MongoCollectionStream"); foreach (var doc in docs) { foreach (var el in doc) { if (schemaAdaptor.Contains(el.Name)) { diff --git a/src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs b/src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs deleted file mode 100644 index d27a23bf4..000000000 --- a/src/DataStreamSuite/DataStream/TableUpdateDataPacket.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace Bts.DataStream; - -using BCG; - -using bts.api.core; - -using Bts.DataStreamCore; - -using System; -using System.Collections; -using System.Collections.Generic; - - -public class TableUpdateDataPacket : IDataPacket where T : IBlueRow { - readonly TableUpdate table; - readonly IBlueTableSchema schema; - - public TableUpdateDataPacket(TableUpdate table, IBlueTableSchema schema) { - this.table = table; - this.schema = schema; - } - - public IBlueRowCollection newRows { - get { - var rv = new BlueTableImmutable(schema); - return rv; - } - } - - public IBlueRowCollection addedRows => throw new NotImplementedException(); - - public IBlueRowCollection deletedRows => throw new NotImplementedException(); - - class MyRowCollection : IBlueRowCollection { - readonly TableUpdate table; - readonly IEnumerable items; - Lazy> itemList; - - public MyRowCollection(IEnumerable items, IBlueTableSchema schema) { - this.items = items; - this.schema = schema; - itemList = new(() => { - var rv = new SortedList(); - foreach (var item in this.items) { - rv.Add(item.Id(), item); - } - return rv; - }); - } - - public int Count => itemList.Value.Count; - public bool IsSynchronized => false; - public object SyncRoot => this; - - public T this[int id] => itemList.Value[id]; - - public IBlueTableSchema schema { get; } - - public bool containsID(int id) => - itemList.Value.ContainsKey(id); - - public void CopyTo(Array array, int index) { - } - - public IEnumerator GetEnumerator() { - throw new NotImplementedException(); - } - - IEnumerator IEnumerable.GetEnumerator() { - throw new NotImplementedException(); - } - } -} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowAdapter.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowAdapter.cs new file mode 100644 index 000000000..f846b5bba --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowAdapter.cs @@ -0,0 +1,23 @@ +#nullable enable +namespace Bts.DataStreamCore; + +using BCG; + +using Bts.DataStream; + +public class BlueRowAdapter : IBlueRow { + readonly BlueTableSchemaAdapter adapter; + readonly T data; + + public BlueRowAdapter(BlueTableSchemaAdapter adapter, T data) { + this.adapter = adapter; + this.data = data; + } + + public T Data => data; + public IBlueTableSchema Schema => adapter; + IBlueTableSchema IBlueRow.schema => adapter.Schema; + + public object? this[int columnIndex] => adapter[columnIndex][data]; + public object? this[string columnName] => adapter[columnName][data]; +} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs index 349549cfd..1ef16a3a9 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableImmutable.cs @@ -3,10 +3,16 @@ namespace Bts.DataStreamCore; using BCG; +using bts.api.core; + +using Bts.DataStream; +using Bts.DataStreamCore; + using System; using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; +using System.Linq; // I would have named this ImmutableBlueTable, but that was already taken. public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { @@ -79,10 +85,24 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { } } - public DataPacket ToDataPacket(bool reset = true) { + public void Clear() { + added.Clear(); + removed.Clear(); + removed.AddRange(table); + } + + public void Reset() { + added.Clear(); + removed.Clear(); + } + + public DataPacket ToDataPacket(bool reset = true) => + ToDataPacket(out _, reset); + + public DataPacket ToDataPacket(out BlueTableImmutable newRows, bool reset = true) { table.RemoveRange(removed.Keys); table.AddRange(added); - var newRows = new BlueTableImmutable(schema, table.ToImmutable()); + newRows = new BlueTableImmutable(schema, table.ToImmutable()); var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); if (reset) { @@ -97,58 +117,183 @@ public class BlueTableImmutable : IBlueRowCollection, IImmutableRowCollection { } } -public class BlueTableImmutable : IBlueRowCollection where T : IBlueRow { +// public class BlueTableImmutable : IBlueRowCollection where T : IBlueRow { +// readonly ImmutableDictionary rows; + +// public BlueTableImmutable(IBlueTableSchema schema) +// : this(schema, ImmutableDictionary.Empty) { } + +// public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary rows) { +// this.schema = schema; +// this.rows = rows; +// } + +// public int Count => rows.Count; +// public bool IsSynchronized => false; +// public IBlueTableSchema schema { get; } +// public object SyncRoot => this; + +// public T this[int id] => rows[id]; + +// public bool containsID(int id) => +// rows.ContainsKey(id); + +// public void CopyTo(Array array, int index) { +// foreach (var r in rows.Values) { +// array.SetValue(r, index++); +// } +// } + +// public IEnumerator GetEnumerator() => +// rows.Values.GetEnumerator(); + +// IEnumerator IEnumerable.GetEnumerator() => +// GetEnumerator(); + +// public DataPacketBuilder ToDataPacketBuilder() => +// new(this); + +// public class DataPacketBuilder { +// readonly IBlueTableSchema schema; +// readonly ImmutableDictionary.Builder table; +// readonly ImmutableDictionary.Builder added = +// ImmutableDictionary.Empty.ToBuilder(); +// readonly ImmutableDictionary.Builder removed = +// ImmutableDictionary.Empty.ToBuilder(); + +// public DataPacketBuilder(BlueTableImmutable table) { +// schema = table.schema; +// this.table = table.rows.ToBuilder(); +// } + +// public void AddOrUpdate(T row, IEqualityComparer? comparer = null) { +// var id = row.Id(); +// if (table.TryGetValue(id, out var orig)) { +// if (comparer?.Equals(row, orig) ?? false) { +// // No op: This row was already in the table (according to 'comparer'). +// // Remove it from 'added' and 'removed'. +// added.Remove(id); +// removed.Remove(id); +// return; +// } +// // Change op: A row with this id was already in the table, +// // but its values have changed (according to 'comparer'). +// removed[id] = orig; +// } +// added[id] = row; +// } + +// public void Remove(int id) { +// added.Remove(id); +// if (table.TryGetValue(id, out var orig)) { +// removed[id] = orig; +// } +// } + +// public DataPacket ToDataPacket(bool reset = true) { +// table.RemoveRange(removed.Keys); +// table.AddRange(added); +// var newRows = new BlueTableImmutable(schema, table.ToImmutable()); +// var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); +// var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); +// if (reset) { +// added.Clear(); +// removed.Clear(); +// } +// return new(newRows, addedRows, removedRows); +// } + +// public IDataPacket ToDataPacketT(bool reset = true) { +// table.RemoveRange(removed.Keys); +// table.AddRange(added); +// var newRows = new BlueTableImmutable(schema, table.ToImmutable()); +// var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); +// var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); +// if (reset) { +// added.Clear(); +// removed.Clear(); +// } +// return new DataPacket(newRows, addedRows, removedRows); +// } + +// public IDataPacket ToDataPacketT( +// out BlueTableImmutable newRows, +// out BlueTableImmutable addedRows, +// out BlueTableImmutable removedRows, +// bool reset = true +// ) { +// table.RemoveRange(removed.Keys); +// table.AddRange(added); +// newRows = new BlueTableImmutable(schema, table.ToImmutable()); +// addedRows = new BlueTableImmutable(schema, added.ToImmutable()); +// removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); +// if (reset) { +// added.Clear(); +// removed.Clear(); +// } +// return new DataPacket(newRows, addedRows, removedRows); +// } + +// public BlueTableImmutable ToImmutable() => +// new(schema, table.ToImmutable()); +// } +// } + +public class BlueTableImmutableAdapter : IBlueRowCollection> { + readonly BlueTableSchemaAdapter schemaAdapter; readonly ImmutableDictionary rows; - public BlueTableImmutable(IBlueTableSchema schema) - : this(schema, ImmutableDictionary.Empty) { } + public BlueTableImmutableAdapter(BlueTableSchemaAdapter schemaAdapter) + : this(schemaAdapter, ImmutableDictionary.Empty) { } - public BlueTableImmutable(IBlueTableSchema schema, ImmutableDictionary rows) { - this.schema = schema; + public BlueTableImmutableAdapter(BlueTableSchemaAdapter schemaAdapter, ImmutableDictionary rows) { + this.schemaAdapter = schemaAdapter; this.rows = rows; } public int Count => rows.Count; public bool IsSynchronized => false; - public IBlueTableSchema schema { get; } + public IBlueTableSchema schema => schemaAdapter.Schema; public object SyncRoot => this; - public T this[int id] => rows[id]; + public BlueRowAdapter this[int id] => new(schemaAdapter, rows[id]); public bool containsID(int id) => rows.ContainsKey(id); public void CopyTo(Array array, int index) { foreach (var r in rows.Values) { - array.SetValue(r, index++); + array.SetValue(new BlueRowAdapter(schemaAdapter, r), index++); } } - public IEnumerator GetEnumerator() => - rows.Values.GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() => - GetEnumerator(); + public IEnumerator> GetEnumerator() { + foreach (var row in rows.Values) { + yield return new BlueRowAdapter(schemaAdapter, row); + } + } public DataPacketBuilder ToDataPacketBuilder() => new(this); public class DataPacketBuilder { - readonly IBlueTableSchema schema; + readonly BlueTableSchemaAdapter schemaAdapter; readonly ImmutableDictionary.Builder table; readonly ImmutableDictionary.Builder added = ImmutableDictionary.Empty.ToBuilder(); readonly ImmutableDictionary.Builder removed = ImmutableDictionary.Empty.ToBuilder(); - public DataPacketBuilder(BlueTableImmutable table) { - schema = table.schema; + public DataPacketBuilder(BlueTableImmutableAdapter table) { + schemaAdapter = table.schemaAdapter; this.table = table.rows.ToBuilder(); } - public void AddOrUpdate(T row, IEqualityComparer? comparer = null) { + public void AddOrUpdate(T rowData, IEqualityComparer? comparer = null) { + var row = new BlueRowAdapter(schemaAdapter, rowData); var id = row.Id(); - if (table.TryGetValue(id, out var orig)) { + if (table.TryGetValue(id, out var origData)) { + var orig = new BlueRowAdapter(schemaAdapter, origData); if (comparer?.Equals(row, orig) ?? false) { // No op: This row was already in the table (according to 'comparer'). // Remove it from 'added' and 'removed'. @@ -158,9 +303,9 @@ public class BlueTableImmutable : IBlueRowCollection where T : IBlueRow { } // Change op: A row with this id was already in the table, // but its values have changed (according to 'comparer'). - removed[id] = orig; + removed[id] = origData; } - added[id] = row; + added[id] = rowData; } public void Remove(int id) { @@ -170,51 +315,27 @@ public class BlueTableImmutable : IBlueRowCollection where T : IBlueRow { } } - public DataPacket ToDataPacket(bool reset = true) { + public void Update(TableUpdate tableUpdate) where TKey : IEquatable { + foreach (var r in tableUpdate.removed.Concat(tableUpdate.changes.Old())) { + Remove(new BlueRowAdapter(schemaAdapter, r).Id()); + } + foreach (var r in tableUpdate.changed.Concat(tableUpdate.added)) { + AddOrUpdate(r); + } + } + + public BlueTableImmutableAdapter ToImmutable(out IDataPacket dp, bool reset = true) { table.RemoveRange(removed.Keys); table.AddRange(added); - var newRows = new BlueTableImmutable(schema, table.ToImmutable()); - var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); - var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); + var newRows = new BlueTableImmutableAdapter(schemaAdapter, table.ToImmutable()); + var addedRows = new BlueTableImmutableAdapter(schemaAdapter, added.ToImmutable()); + var removedRows = new BlueTableImmutableAdapter(schemaAdapter, removed.ToImmutable()); if (reset) { added.Clear(); removed.Clear(); } - return new(newRows, addedRows, removedRows); + dp = new DataPacket(newRows, addedRows, removedRows); + return newRows; } - - public IDataPacket ToDataPacketT(bool reset = true) { - table.RemoveRange(removed.Keys); - table.AddRange(added); - var newRows = new BlueTableImmutable(schema, table.ToImmutable()); - var addedRows = new BlueTableImmutable(schema, added.ToImmutable()); - var removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); - if (reset) { - added.Clear(); - removed.Clear(); - } - return new DataPacket(newRows, addedRows, removedRows); - } - - public IDataPacket ToDataPacketT( - out BlueTableImmutable newRows, - out BlueTableImmutable addedRows, - out BlueTableImmutable removedRows, - bool reset = true - ) { - table.RemoveRange(removed.Keys); - table.AddRange(added); - newRows = new BlueTableImmutable(schema, table.ToImmutable()); - addedRows = new BlueTableImmutable(schema, added.ToImmutable()); - removedRows = new BlueTableImmutable(schema, removed.ToImmutable()); - if (reset) { - added.Clear(); - removed.Clear(); - } - return new DataPacket(newRows, addedRows, removedRows); - } - - public BlueTableImmutable ToImmutable() => - new(schema, table.ToImmutable()); } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs similarity index 52% rename from src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs rename to src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs index c7bd05612..819f14bd2 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdaptor.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaAdapter.cs @@ -7,21 +7,30 @@ using bts.collections; using System; using System.Collections.Generic; +using System.IO; using System.Linq; -public class BlueTableSchemaAdaptor : IMutableBlueRowFactory { +public class BlueTableSchemaAdapter : IMutableBlueRowFactory, IBlueTableSchema { readonly string name; - readonly List columns = new(); + readonly List columns = new(); + readonly Dictionary columnNameIndex = new(); + ColumnInfo[]? columnInfos; + string[]? columnNames; BlueTableSchema? schema; - public BlueTableSchemaAdaptor(string name) => + public BlueTableSchemaAdapter(string name) => this.name = name; + public BlueTableSchemaAdapter(string name, Func getId) { + this.name = name; + AddColumn("id", getId); + } + public IBlueTableSchema Schema => schema ??= new BlueTableSchema(columns.WrapSelectorRO(c => c.Info).ToArray(), name); public bool Contains(string name) => - columns.Any(c => c.Info.colName == name); + columnNameIndex.ContainsKey(name); public void AddColumn(string name, Func getValue) => AddColumn(name, r => getValue(r), typeof(int)); @@ -57,9 +66,27 @@ public class BlueTableSchemaAdaptor : IMutableBlueRowFactory { } // Invalidate cached schema (set it to null) schema = null; - columns.Add(new ColumnAdaptor(new(name, columns.Count, dataType), r => getValue(r))); + var adapter = new ColumnAdapter(new(name, columns.Count, dataType), r => getValue(r)); + columns.Add(adapter); + columnNameIndex[name] = adapter; + columnInfos = null; + columnNames = null; } + public ColumnAdapter this[int columnIndex] => + columns[columnIndex]; + + public ColumnAdapter this[string columnName] => + columnNameIndex[columnName]; + + public object? this[int columnIndex, T row] => columns[columnIndex][row]; + public object? this[string columnName, T row] => columnNameIndex[columnName][row]; + ColumnInfo IBlueTableSchema.this[int columnIndex] => columns[columnIndex].Info; + ColumnInfo IBlueTableSchema.this[string columnName] => columnNameIndex[columnName].Info; + string[] IBlueTableSchema.columnNames => columnNames ??= columns.Select(c => c.Info.colName).ToArray(); + ColumnInfo[] IBlueTableSchema.columnInfo => columnInfos ??= columns.Select(c => c.Info).ToArray(); + string IBlueTableSchema.tableName => name; + public IBlueRow GetBlueRow(T item) => GetMutableBlueRow(item); @@ -71,10 +98,28 @@ public class BlueTableSchemaAdaptor : IMutableBlueRowFactory { return mbr; } - class ColumnAdaptor { + void ISerializedType.serialize(BinaryWriter bw) { + bw.Write(columns.Count); + bw.Write(name); + foreach (var col in columns) { + bw.Write(col.Info.colName); + bw.Write(col.Info.colType.ToString()); + bw.Write(false); + } + } + + void ISerializedType.deserialze(BinaryReader br) => + // We can't reconstruct this class from serialized form. The column lambdas + // (i.e., to get cell values from T) are lost during serialization. + throw new NotSupportedException(); + + int ISerializedType.getSerializedSize() => + throw new NotImplementedException(); + + public class ColumnAdapter { readonly Func getValue; - public ColumnAdaptor(ColumnInfo info, Func getValue) { + public ColumnAdapter(ColumnInfo info, Func getValue) { Info = info; this.getValue = getValue; } @@ -83,5 +128,8 @@ public class BlueTableSchemaAdaptor : IMutableBlueRowFactory { public object? GetValue(T row) => getValue(row); + + public object? this[T row] => + getValue(row); } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs index 26c49a572..07807483a 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchemaBuilder.cs @@ -4,21 +4,28 @@ namespace Bts.DataStreamCore; using BCG; using System; +using System.Collections; using System.Collections.Generic; -public class BlueTableSchemaBuilder { +public class BlueTableSchemaBuilder : IEnumerable<(string, Type)> { readonly string? tableName; readonly List<(string Name, Type Type)> colDefs = new(); public BlueTableSchemaBuilder(string? tableName = null) => this.tableName = tableName; + public void Add((string Name, Type Type) tup) => + Add(tup.Name, tup.Type); + public void Add(string Name) => colDefs.Add((Name, typeof(T))); public void Add(string Name, Type Type) => colDefs.Add((Name, Type)); + public IEnumerator<(string, Type)> GetEnumerator() => + colDefs.GetEnumerator(); + public IBlueTableSchema ToSchema(string? tableName = null) { var colInfos = new ColumnInfo[colDefs.Count]; for (int i = 0; i < colInfos.Length; i++) { @@ -26,4 +33,7 @@ public class BlueTableSchemaBuilder { } return new BlueTableSchema(colInfos, tableName ?? this.tableName); } + + IEnumerator IEnumerable.GetEnumerator() => + colDefs.GetEnumerator(); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs index c56cdc841..9f279dc9f 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/IBlueTableSchema.cs @@ -30,4 +30,9 @@ namespace BCG /// string tableName {get;} } + + public interface IBlueTableSchema : IBlueTableSchema { + object this[int colIndex, T row] { get; } + object this[string colName, T row] { get; } + } } diff --git a/src/DataStreamSuite/DataStreamCore/DataPacket.cs b/src/DataStreamSuite/DataStreamCore/DataPacket.cs index c51b62a50..adc3dbed4 100644 --- a/src/DataStreamSuite/DataStreamCore/DataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/DataPacket.cs @@ -1,12 +1,10 @@ +using Spark.Core.Collections; + using System; using System.Collections; using System.Collections.Generic; using System.Data; -using BCG.Utilities.General; - -using Bts.DataStreamCore; - namespace BCG { [Serializable] public class DataPacket : ICloneable, IDataPacket { @@ -45,12 +43,20 @@ namespace BCG { public IBlueRowCollection newRows, addedRows, deletedRows; public IDictionary meta_data = null; public DataPacket() { } - public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { + + public DataPacket(IBlueTableSchema schema) + : this(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema)) { } + + public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { this.newRows = newRows; this.addedRows = addedRows; this.deletedRows = deletedRows; } + IBlueRowCollection IDataPacket.newRows => newRows; + IBlueRowCollection IDataPacket.addedRows => addedRows; + IBlueRowCollection IDataPacket.deletedRows => deletedRows; + public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { new_rows = newRows; added_rows = addedRows; @@ -282,23 +288,30 @@ namespace BCG { } } - public class DataPacket : IDataPacket where T : IBlueRow { - public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { - this.newRows = newRows; - this.addedRows = addedRows; - this.deletedRows = deletedRows; - } + // public class DataPacket : IDataPacket where T : IBlueRow { + // public DataPacket( + // IBlueRowCollection newRows, + // IBlueRowCollection addedRows, + // IBlueRowCollection deletedRows, + // IReadOnlyDictionary metadata = null + // ) { + // this.newRows = newRows; + // this.addedRows = addedRows; + // this.deletedRows = deletedRows; + // Metadata = metadata ?? DictionaryHelper.empty(); + // } - public IBlueRowCollection newRows { get; } - public IBlueRowCollection addedRows { get; } - public IBlueRowCollection deletedRows { get; } + // public IBlueRowCollection newRows { get; } + // public IBlueRowCollection addedRows { get; } + // public IBlueRowCollection deletedRows { get; } + // public IReadOnlyDictionary Metadata { get; } - public static DataPacket Empty(IBlueTableSchema schema) => - new( - new BlueTableImmutable(schema), - new BlueTableImmutable(schema), - new BlueTableImmutable(schema)); - } + // public static DataPacket Empty(IBlueTableSchema schema) => + // new( + // new BlueTableImmutable(schema), + // new BlueTableImmutable(schema), + // new BlueTableImmutable(schema)); + // } public static class DataPacketExtensions { public static bool HasChanges(this DataPacket dataPacket) => diff --git a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs index f2fef8402..a61f31b66 100644 --- a/src/DataStreamSuite/DataStreamCore/IDataPacket.cs +++ b/src/DataStreamSuite/DataStreamCore/IDataPacket.cs @@ -1,24 +1,30 @@ -using System; +using Bts.DataStreamCore; + +using System; using System.Collections; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; -namespace BCG -{ - public interface IDataPacket - { +namespace BCG { + public interface IDataPacket { IBlueTableSchema Schema { get; } + IBlueRowCollection newRows { get; } + IBlueRowCollection addedRows { get; } + IBlueRowCollection deletedRows { get; } //this is an ICollection of IDictionary void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows); } public interface IDataPacket : IDataPacket where T : IBlueRow { - IBlueRowCollection newRows { get; } - IBlueRowCollection addedRows { get; } - IBlueRowCollection deletedRows { get; } + new IBlueRowCollection newRows { get; } + new IBlueRowCollection addedRows { get; } + new IBlueRowCollection deletedRows { get; } + IBlueRowCollection IDataPacket.newRows => newRows; + IBlueRowCollection IDataPacket.addedRows => addedRows; + IBlueRowCollection IDataPacket.deletedRows => deletedRows; IBlueTableSchema IDataPacket.Schema => newRows.schema; void IDataPacket.get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) { @@ -27,4 +33,35 @@ namespace BCG deleted_rows = deletedRows; } } + + public static class IDataPacketExtensions { + public static IDataPacket ToImmutable(this IDataPacket dataPacket) => + new DataPacket( + dataPacket.newRows is IImmutableRowCollection ? dataPacket.newRows : new ImmutableBlueTable(dataPacket.newRows), + dataPacket.addedRows, + dataPacket.deletedRows); + + public static string GetIncrementalIdString(this IDataPacket dataPacket) { + var incIdSerializer = new DataPacketIncrementalIdsJsonSerializer(); + return incIdSerializer.Serialize(dataPacket); + // var sb = new StringBuilder(dataPacket.Schema.tableName); + // sb.Append("{ \"deleted\": ["); + // AddIds(dataPacket.deletedRows); + // sb.Append("], \"added\": ["); + // AddIds(dataPacket.addedRows); + // sb.Append("] }"); + // return sb.ToString(); + + // void AddIds(IBlueRowCollection rows) { + // var en = rows.GetEnumerator(); + // if (en.MoveNext()) { + // sb.Append(((IBlueRow)en.Current).Id()); + // while (en.MoveNext()) { + // sb.Append(','); + // sb.Append(((IBlueRow)en.Current).Id()); + // } + // } + // } + } + } } diff --git a/src/DataStreamSuite/DataStreamCore/ViewStream.cs b/src/DataStreamSuite/DataStreamCore/ViewStream.cs index 246bdf010..03baf677a 100644 --- a/src/DataStreamSuite/DataStreamCore/ViewStream.cs +++ b/src/DataStreamSuite/DataStreamCore/ViewStream.cs @@ -72,7 +72,7 @@ namespace BCG protected override void myFireEvent(object o) { - DataPacket dp = (DataPacket) o; + var dp = o as IDataPacket; // BlueTrace.WriteViewStreamFireEvent(this.ID, // dp.newRows == null ? 0 : dp.newRows.Count, @@ -91,8 +91,7 @@ namespace BCG } // the view already garantees that added & deleted are immutable. // this garantess that new is also immutable - base.myFireEvent(myUseUncopiedNewRowsInFireEvent ? dp : dp.createImmutable()); - + base.myFireEvent(myUseUncopiedNewRowsInFireEvent ? dp : dp.ToImmutable()); } private void myDummy (object o)