namespace Bts.DataStream; using BCG; using bts.api.core; using Bts.DataStreamCore; using Bts.PubSub; using Microsoft.Extensions.Logging; using Spark.Core.Disposables; using Spark.Core.Threading; using System; using System.Linq; using System.Reactive.Linq; public static class DataStreamExtensions { public static ViewStream AsViewStream(this IObservable source) => source as ViewStream ?? new ObservableViewStream(source); public static ViewStream AsViewStream(this IObservable> source, string? name = null, ILogger? logger = null) where T : IBlueRow => source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger); public static IObservable AsObservable(this ViewStream source) { return source as IObservable ?? Observable.Create(Subscribe); IDisposable Subscribe(IObserver observer) { source.onChange += OnChange; return Disposable.Create(() => source.onChange -= OnChange); void OnChange(object? newValue) => observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException( $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue))); } } // public static IObservable> SelectDataPacket(this IObservable source, Func selector) where T : IBlueRow => // source.Select(dp => { // dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); // return new DataPacket(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows)); // IBlueRowCollection CastRows(ICollection rows) => // new BlueTableSelectorWrapper( // rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector); // }); public static IObservable> AsTables(this IObservable source) => source.AsTables(r => r, r => r.Id()); public static IObservable> AsTables(this IObservable source, Func valueSelector) where T : IBlueRow => source.AsTables(valueSelector, r => r.Id()); public static IObservable> AsTables( this IObservable source, Func valueSelector, Func keySelector) => source.AsTables(valueSelector, keySelector); public static IObservable> AsTables( this IObservable source, Func valueSelector, Func keySelector, Func? rowKeySelector = null) where TKey : IEquatable { rowKeySelector ??= r => keySelector(valueSelector(r)); return source.Scan(TableUpdate.Empty(keySelector), (res, dp) => { var bd = res.ToBuilder(); dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows); foreach (IBlueRow r in addedRows) { bd.AddOrUpdate(valueSelector(r)); } foreach (IBlueRow r in deletedRows) { bd.Remove(rowKeySelector(r)); } return bd.ToImmutable(); }); } public static IObservable AsDataPackets(this IObservable> source) => source.Scan( (Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null), (res, table) => { var resTable = res.Table; if (resTable is null) { if (table.Index.Count == 0) { return (null, null); } resTable = new(table.Index.Values.First().schema); } var bd = resTable.ToDataPacketBuilder(); foreach (var r in table.RemovedItems) { bd.Remove(r.Id()); } foreach (var r in table.AddedItems) { bd.AddOrUpdate(r); } var resDp = bd.ToDataPacket(out var newRows); return (newRows, resDp); }) .Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)) .Select(t => t.Dp!); public static IObservable AsDataPackets(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => source.Scan( (Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)), (res, table) => { var bd = res.Table.ToDataPacketBuilder(); foreach (var r in table.RemovedItems) { bd.Remove(r.Id()); } foreach (var r in table.AddedItems) { bd.AddOrUpdate(r); } var resDp = bd.ToDataPacket(out var newRows); return (newRows, resDp); }) .Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0) .Select(t => t.Dp); public static IObservable AsDataPackets(this IObservable> source, BlueTableSchemaAdapter schema) where TKey : IEquatable { return Observable.Create(Subscribe); IDisposable Subscribe(IObserver o) { var adapter = new BlueTableImmutableAdapter(schema); IDataPacket initDataPacket = new DataPacket(schema); return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o); (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) OnUpdate( (BlueTableImmutableAdapter Adapter, IDataPacket DataPacket) state, TableUpdate tableUpdate) { var bd = state.Adapter.ToDataPacketBuilder(); bd.Update(tableUpdate); var newRows = bd.ToImmutable(out var dp); return (newRows, dp); } } } public static ViewStream AsViewStream(this IObservable> source, IBlueTableSchema schema) where T : IBlueRow => source.AsDataPackets(schema).AsViewStream(); public static int ResolveIndices(this IBlueTableSchema schema, string name) { var indices = ResolveIndices(schema, new[] { name }); return indices[0]; } public static (int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2) { var indices = ResolveIndices(schema, new[] { name1, name2 }); return (indices[0], indices[1]); } public static (int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3) { var indices = ResolveIndices(schema, new[] { name1, name2, name3 }); return (indices[0], indices[1], indices[2]); } public static (int, int, int, int) ResolveIndices(this IBlueTableSchema schema, string name1, string name2, string name3, string name4) { var indices = ResolveIndices(schema, new[] { name1, name2, name3, name4 }); return (indices[0], indices[1], indices[2], indices[3]); } public static int[] ResolveIndices(this IBlueTableSchema schema, string[] columnNames) { var columns = schema.columnInfo; var indices = new int[columnNames.Length]; int nResolved = 0; for (int i = 0; i < columnNames.Length; i++) { int origNResolved = nResolved; for (int j = 0; j < columns.Length; j++) { if (columns[j].colName == columnNames[i]) { indices[i] = j; nResolved++; break; } } if (nResolved == origNResolved) { throw new ArgumentException($"No column named '{columnNames[i]}'", nameof(schema)); } } return indices; } public static IObservable OnWorkScheduler(this IObservable source, IWorkScheduler scheduler) => Observable.Create(o => { var inputGate = new AtomicGate(); var outputGate = new AtomicGate(); BlueTableImmutable? lastTable = null; BlueTableImmutable.DataPacketBuilder? bd = null; IDataPacket? pendingDpOut = null; return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork)); void OnNext(IDataPacket value) { using (inputGate.BeginTransaction()) { lastTable ??= new(value.newRows.schema); bd ??= lastTable.ToDataPacketBuilder(); if (pendingDpOut is null) { // The last outgoing datapacket was just fired. Don't merge this update with the // previous one (otherwise its adds/removes would be fired twice). bd.Reset(); } foreach (IBlueRow r in value.deletedRows) { bd.Remove(r.Id()); } foreach (IBlueRow r in value.addedRows) { bd.AddOrUpdate(r); } pendingDpOut = bd.ToDataPacket(out lastTable); } } void DoWork() { IDataPacket? dpOut = null; if (outputGate.TryEnter()) { using var exit = Disposable.Create(outputGate.Exit); (dpOut, pendingDpOut) = (pendingDpOut, dpOut); if (dpOut is not null) { o.OnNext(dpOut); } } } }); }