Files
spark/src/DataStreamSuite/DataStream/DataStreamExtensions.cs
T

294 lines
12 KiB
C#

namespace Bts.DataStream;
using BCG;
using bts;
using bts.api.core;
using bts.BlueOrb;
using Bts.DataStreamCore;
using Bts.PubSub;
using Bts.SharedStreams.StreamFactories;
using mbts.utils;
using Spark.Core.Disposables;
using System;
using System.Linq;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
public static class DataStreamExtensions {
public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
source as ViewStream ?? new ObservableViewStream(source);
// public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
// source as ViewStream ?? new ObservableViewStream(source, name, logger);
public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
IDisposable Subscribe(IObserver<IDataPacket> observer) {
source.onChange += OnChange;
return Disposable.Create(() => source.onChange -= OnChange);
void OnChange(object? newValue) =>
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
}
}
// public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
// source.Select(dp => {
// dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
// return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
// IBlueRowCollection<T> CastRows(ICollection rows) =>
// new BlueTableSelectorWrapper<T>(
// rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
// });
public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
source.AsTables<IBlueRow, int>(r => r, r => r.Id());
public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
source.AsTables<T, int>(valueSelector, r => r.Id());
public static IObservable<TableUpdate<T, int>> AsTables<T>(
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
source.AsTables<T, int>(valueSelector, keySelector);
public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
rowKeySelector ??= r => keySelector(valueSelector(r));
return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
var bd = res.to_builder();
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
foreach (IBlueRow r in addedRows) {
bd.add_or_update(valueSelector(r));
}
foreach (IBlueRow r in deletedRows) {
bd.remove(rowKeySelector(r));
}
return bd.to_table();
});
}
public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
source.Scan(
(Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
(res, table) => {
var resTable = res.Table;
if (resTable is null) {
if (table.state.Count == 0) {
return (null, null);
}
resTable = new(table.state.Values.First().schema);
}
var bd = resTable.ToDataPacketBuilder();
foreach (var r in table.removed) {
bd.Remove(r.Id());
}
foreach (var r in table.added) {
bd.AddOrUpdate(r);
}
var resDp = bd.ToDataPacket(out var newRows);
return (newRows, resDp);
})
.Where(t => t.Dp is not null && (t.Dp.addedRows.Any() || t.Dp.deletedRows.Any()))
.Select(t => t.Dp!);
public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
source.Scan(
(Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
(res, table) => {
var bd = res.Table.ToDataPacketBuilder();
foreach (var r in table.removed) {
bd.Remove(r.Id());
}
foreach (var r in table.added) {
bd.AddOrUpdate(r);
}
var resDp = bd.ToDataPacket(out var newRows);
return (newRows, resDp);
})
.Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
.Select(t => t.Dp);
public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
return Observable.Create<IDataPacket>(Subscribe);
IDisposable Subscribe(IObserver<IDataPacket> o) {
var adapter = new BlueTableImmutableAdapter<T>(schema);
IDataPacket initDataPacket = new DataPacket(schema);
return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
var bd = state.Adapter.ToDataPacketBuilder();
bd.Update(tableUpdate);
var dp = bd.ToDataPacket();
var tableAdapter = bd.ToImmutable();
return (bd.ToImmutable(), dp);
}
}
}
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
source.AsDataPackets(schema).AsViewStream();
public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func<IBlueRowCollection, IBlueRowCollection> apply) {
var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns);
var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, FunctionEvaluator.Create(apply));
return applyTables.AsDataPackets().AsViewStream();
}
public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func<object?[], IBlueRowCollection, IBlueRowCollection?> apply) {
var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns);
var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, new Func<EquatableKeyArrayWrapper, IBlueRowCollection, IBlueRowCollection?>(ApplyWrapper));
return applyTables.AsDataPackets().AsViewStream();
IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) =>
apply(groupKey.Values, groupRows);
}
public static ViewStream RenameColumns(this ViewStream viewStream, bool includeUnmapped, params (string From, string To)[] columnMap) =>
new ColumnMaskStream(viewStream, includeUnmapped, columnMap);
public static IObservable<IDataPacket> TimeSeries(this IObservable<IDataPacket> source, int bufferSize, string timeColumn) =>
source.AsViewStream().TimeSeries(bufferSize, timeColumn).AsObservable();
public static ViewStream TimeSeries(this ViewStream source, int bufferSize, string timeColumn) =>
new TimeSeriesStream(source, bufferSize, timeColumn);
public static IDisposable Subscribe(this ViewStream viewStream, Action<IDataPacket> onChange) {
viewStream.onChange += OnChange;
return Disposable.Create(Dispose);
void OnChange(object? dpObj) =>
onChange(ExceptionAssert.Argument.Is<IDataPacket>(dpObj));
void Dispose() =>
viewStream.onChange -= OnChange;
}
public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
const int STATE_IDLE = 0;
const int STATE_BUSY = 1;
const int STATE_DISPOSED = 2;
return Observable.Create<IDataPacket>(Subscribe);
IDisposable Subscribe(IObserver<IDataPacket> o) {
var syncRoot = new object();
IBlueRowCollection? lastTableIn = null;
BlueTableImmutable? lastTableOut = null;
int nConcurrentDoWorks = 0;
int state = STATE_IDLE;
IDisposable? dispose = null;
dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
return Disposable.Create(Dispose);
void Dispose() {
lock (syncRoot) {
switch (state) {
case STATE_DISPOSED:
return;
case STATE_BUSY:
state = STATE_DISPOSED;
return;
}
state = STATE_DISPOSED;
}
Disposable.dispose(ref dispose);
}
void OnNext(IDataPacket value) =>
Volatile.Write(ref lastTableIn, value.newRows);
async void DoWork() {
lock (syncRoot) {
nConcurrentDoWorks++;
switch (state) {
case STATE_DISPOSED:
case STATE_BUSY:
return;
case STATE_IDLE:
break;
default:
Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
return;
}
state = STATE_BUSY;
}
var dp = await PublishTable();
bool isDisposed = false;
lock (syncRoot) {
switch (state) {
case STATE_DISPOSED:
isDisposed = true;
break;
case STATE_BUSY:
break;
default:
Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
return;
}
}
if (isDisposed) {
Disposable.dispose(ref dispose);
return;
}
if (dp is not null) {
o.OnNext(dp);
}
lock (syncRoot) {
nConcurrentDoWorks = 0;
switch (state) {
case STATE_DISPOSED:
isDisposed = true;
break;
case STATE_BUSY:
state = STATE_IDLE;
break;
default:
Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
return;
}
}
if (isDisposed) {
Disposable.dispose(ref dispose);
}
async Task<DataPacket?> PublishTable() {
var tableIn = Interlocked.Exchange(ref lastTableIn, null);
if (tableIn is null) {
return null;
}
var tableOut = await transformAsync(tableIn);
if (tableOut is null) {
return null;
}
lastTableOut ??= new(tableOut.schema);
var bd = lastTableOut.ToDataPacketBuilder();
bd.Clear();
foreach (var r in tableOut) {
bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
}
return bd.ToDataPacket(out lastTableOut);
}
}
static string StateName(int state) =>
state switch {
STATE_IDLE => "IDLE",
STATE_BUSY => "BUSY",
STATE_DISPOSED => "DISPOSED",
_ => $"STATE_{state}",
};
}
}
public static ViewStream Diff(this ViewStream source, int period = 1) =>
new bts.BlueOrb.DiffStream(source, period);
}