namespace Bts.DataStream; using BCG; using bts; using bts.api.core; using bts.BlueOrb; using Bts.DataStreamCore; using Bts.PubSub; using Bts.SharedStreams.StreamFactories; using mbts.utils; using Spark.Core.Disposables; using System; using System.Linq; using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; public static class DataStreamExtensions { public static ViewStream AsViewStream(this IObservable source) => source as ViewStream ?? new ObservableViewStream(source); // public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => // source as ViewStream ?? new ObservableViewStream(source, name, logger); public static IObservable AsObservable(this ViewStream source) { return source as IObservable ?? Observable.Create(Subscribe); IDisposable Subscribe(IObserver observer) { source.onChange += OnChange; return Disposable.Create(() => source.onChange -= OnChange); void OnChange(object? newValue) => observer.OnNext(newValue as DataPacket ?? throw new ArgumentException( $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue))); } } // public static IObservable> SelectDataPacket(this IObservable source, Func selector) where T : IBlueRow => // source.Select(dp => { // dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); // return new DataPacket(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows)); // IBlueRowCollection CastRows(ICollection rows) => // new BlueTableSelectorWrapper( // rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector); // }); public static IObservable> AsTables(this IObservable source) => source.AsTables(r => r, r => r.Id()); public static IObservable> AsTables(this IObservable source, Func valueSelector) where T : IBlueRow => source.AsTables(valueSelector, r => r.Id()); public static IObservable> AsTables( this IObservable source, Func valueSelector, Func keySelector) => source.AsTables(valueSelector, keySelector); public static IObservable> AsTables( this IObservable source, Func valueSelector, Func keySelector, Func? rowKeySelector = null) where TKey : IEquatable { rowKeySelector ??= r => keySelector(valueSelector(r)); return source.Scan(TableUpdate.mk_empty(keySelector), (res, dp) => { var bd = res.to_builder(); dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); foreach (IBlueRow r in addedRows) { bd.add_or_update(valueSelector(r)); } foreach (IBlueRow r in deletedRows) { bd.remove(rowKeySelector(r)); } return bd.to_table(); }); } public static IObservable AsDataPackets(this IObservable> source) => source.Scan( (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null), (res, table) => { var resTable = res.Table; if (resTable is null) { if (table.state.Count == 0) { return (null, null); } resTable = new(table.state.Values.First().schema); } var bd = resTable.ToDataPacketBuilder(); foreach (var r in table.removed) { bd.Remove(r.Id()); } foreach (var r in table.added) { bd.AddOrUpdate(r); } var resDp = bd.ToDataPacket(out var newRows); return (newRows, resDp); }) .Where(t => t.Dp is not null && (t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())) .Select(t => t.Dp!); public static IObservable AsDataPackets(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => source.Scan( (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)), (res, table) => { var bd = res.Table.ToDataPacketBuilder(); foreach (var r in table.removed) { bd.Remove(r.Id()); } foreach (var r in table.added) { bd.AddOrUpdate(r); } var resDp = bd.ToDataPacket(out var newRows); return (newRows, resDp); }) .Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any()) .Select(t => t.Dp); public static IObservable AsDataPackets(this IObservable> source, BlueTableSchemaAdapter schema) where TKey : IEquatable { return Observable.Create(Subscribe); IDisposable Subscribe(IObserver o) { var adapter = new BlueTableImmutableAdapter(schema); IDataPacket initDataPacket = new DataPacket(schema); return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o); (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) OnUpdate( (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) state, TableUpdate tableUpdate) { var bd = state.Adapter.ToDataPacketBuilder(); bd.Update(tableUpdate); var dp = bd.ToDataPacket(); var tableAdapter = bd.ToImmutable(); return (bd.ToImmutable(), dp); } } } public static ViewStream AsViewStream(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => source.AsDataPackets(schema).AsViewStream(); public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func apply) { var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns); var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, FunctionEvaluator.Create(apply)); return applyTables.AsDataPackets().AsViewStream(); } public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func apply) { var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns); var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, new Func(ApplyWrapper)); return applyTables.AsDataPackets().AsViewStream(); IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) => apply(groupKey.Values, groupRows); } public static ViewStream RenameColumns(this ViewStream viewStream, bool includeUnmapped, params (string From, string To)[] columnMap) => new ColumnMaskStream(viewStream, includeUnmapped, columnMap); public static IObservable TimeSeries(this IObservable source, int bufferSize, string timeColumn) => source.AsViewStream().TimeSeries(bufferSize, timeColumn).AsObservable(); public static ViewStream TimeSeries(this ViewStream source, int bufferSize, string timeColumn) => new TimeSeriesStream(source, bufferSize, timeColumn); public static IDisposable Subscribe(this ViewStream viewStream, Action onChange) { viewStream.onChange += OnChange; return Disposable.Create(Dispose); void OnChange(object? dpObj) => onChange(ExceptionAssert.Argument.Is(dpObj)); void Dispose() => viewStream.onChange -= OnChange; } public static IObservable NonIncrementalAsyncTransform(this IObservable source, Func> transformAsync, IWorkScheduler workScheduler) { const int STATE_IDLE = 0; const int STATE_BUSY = 1; const int STATE_DISPOSED = 2; return Observable.Create(Subscribe); IDisposable Subscribe(IObserver o) { var syncRoot = new object(); IBlueRowCollection? lastTableIn = null; BlueTableImmutable? lastTableOut = null; int nConcurrentDoWorks = 0; int state = STATE_IDLE; IDisposable? dispose = null; dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork)); return Disposable.Create(Dispose); void Dispose() { lock (syncRoot) { switch (state) { case STATE_DISPOSED: return; case STATE_BUSY: state = STATE_DISPOSED; return; } state = STATE_DISPOSED; } Disposable.dispose(ref dispose); } void OnNext(IDataPacket value) => Volatile.Write(ref lastTableIn, value.newRows); async void DoWork() { lock (syncRoot) { nConcurrentDoWorks++; switch (state) { case STATE_DISPOSED: case STATE_BUSY: return; case STATE_IDLE: break; default: Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called"); return; } state = STATE_BUSY; } var dp = await PublishTable(); bool isDisposed = false; lock (syncRoot) { switch (state) { case STATE_DISPOSED: isDisposed = true; break; case STATE_BUSY: break; default: Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)"); return; } } if (isDisposed) { Disposable.dispose(ref dispose); return; } if (dp is not null) { o.OnNext(dp); } lock (syncRoot) { nConcurrentDoWorks = 0; switch (state) { case STATE_DISPOSED: isDisposed = true; break; case STATE_BUSY: state = STATE_IDLE; break; default: Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table"); return; } } if (isDisposed) { Disposable.dispose(ref dispose); } async Task PublishTable() { var tableIn = Interlocked.Exchange(ref lastTableIn, null); if (tableIn is null) { return null; } var tableOut = await transformAsync(tableIn); if (tableOut is null) { return null; } lastTableOut ??= new(tableOut.schema); var bd = lastTableOut.ToDataPacketBuilder(); bd.Clear(); foreach (var r in tableOut) { bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance); } return bd.ToDataPacket(out lastTableOut); } } static string StateName(int state) => state switch { STATE_IDLE => "IDLE", STATE_BUSY => "BUSY", STATE_DISPOSED => "DISPOSED", _ => $"STATE_{state}", }; } } public static ViewStream Diff(this ViewStream source, int period = 1) => new bts.BlueOrb.DiffStream(source, period); }