294 lines
12 KiB
C#
294 lines
12 KiB
C#
namespace Bts.DataStream;
|
|
|
|
using BCG;
|
|
|
|
using bts;
|
|
using bts.api.core;
|
|
using bts.BlueOrb;
|
|
|
|
using Bts.DataStreamCore;
|
|
using Bts.PubSub;
|
|
using Bts.SharedStreams.StreamFactories;
|
|
|
|
using mbts.utils;
|
|
|
|
using Spark.Core.Disposables;
|
|
|
|
using System;
|
|
using System.Linq;
|
|
using System.Reactive.Linq;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
public static class DataStreamExtensions {
|
|
public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
|
|
source as ViewStream ?? new ObservableViewStream(source);
|
|
|
|
// public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
|
|
// source as ViewStream ?? new ObservableViewStream(source, name, logger);
|
|
|
|
public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
|
|
return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
|
|
|
|
IDisposable Subscribe(IObserver<IDataPacket> observer) {
|
|
source.onChange += OnChange;
|
|
return Disposable.Create(() => source.onChange -= OnChange);
|
|
|
|
void OnChange(object? newValue) =>
|
|
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
|
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
|
}
|
|
}
|
|
|
|
// public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
|
|
// source.Select(dp => {
|
|
// dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
|
// return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
|
|
|
|
// IBlueRowCollection<T> CastRows(ICollection rows) =>
|
|
// new BlueTableSelectorWrapper<T>(
|
|
// rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
|
|
// });
|
|
|
|
public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
|
|
source.AsTables<IBlueRow, int>(r => r, r => r.Id());
|
|
|
|
public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
|
|
source.AsTables<T, int>(valueSelector, r => r.Id());
|
|
|
|
public static IObservable<TableUpdate<T, int>> AsTables<T>(
|
|
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
|
|
source.AsTables<T, int>(valueSelector, keySelector);
|
|
|
|
public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
|
|
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
|
|
rowKeySelector ??= r => keySelector(valueSelector(r));
|
|
return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
|
|
var bd = res.to_builder();
|
|
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
|
foreach (IBlueRow r in addedRows) {
|
|
bd.add_or_update(valueSelector(r));
|
|
}
|
|
foreach (IBlueRow r in deletedRows) {
|
|
bd.remove(rowKeySelector(r));
|
|
}
|
|
return bd.to_table();
|
|
});
|
|
}
|
|
|
|
public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
|
|
source.Scan(
|
|
(Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
|
|
(res, table) => {
|
|
var resTable = res.Table;
|
|
if (resTable is null) {
|
|
if (table.state.Count == 0) {
|
|
return (null, null);
|
|
}
|
|
resTable = new(table.state.Values.First().schema);
|
|
}
|
|
var bd = resTable.ToDataPacketBuilder();
|
|
foreach (var r in table.removed) {
|
|
bd.Remove(r.Id());
|
|
}
|
|
foreach (var r in table.added) {
|
|
bd.AddOrUpdate(r);
|
|
}
|
|
var resDp = bd.ToDataPacket(out var newRows);
|
|
return (newRows, resDp);
|
|
})
|
|
.Where(t => t.Dp is not null && (t.Dp.addedRows.Any() || t.Dp.deletedRows.Any()))
|
|
.Select(t => t.Dp!);
|
|
|
|
public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
|
source.Scan(
|
|
(Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
|
|
(res, table) => {
|
|
var bd = res.Table.ToDataPacketBuilder();
|
|
foreach (var r in table.removed) {
|
|
bd.Remove(r.Id());
|
|
}
|
|
foreach (var r in table.added) {
|
|
bd.AddOrUpdate(r);
|
|
}
|
|
var resDp = bd.ToDataPacket(out var newRows);
|
|
return (newRows, resDp);
|
|
})
|
|
.Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
|
|
.Select(t => t.Dp);
|
|
|
|
public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
|
|
return Observable.Create<IDataPacket>(Subscribe);
|
|
|
|
IDisposable Subscribe(IObserver<IDataPacket> o) {
|
|
var adapter = new BlueTableImmutableAdapter<T>(schema);
|
|
IDataPacket initDataPacket = new DataPacket(schema);
|
|
return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
|
|
|
|
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
|
|
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
|
|
var bd = state.Adapter.ToDataPacketBuilder();
|
|
bd.Update(tableUpdate);
|
|
var dp = bd.ToDataPacket();
|
|
var tableAdapter = bd.ToImmutable();
|
|
return (bd.ToImmutable(), dp);
|
|
}
|
|
}
|
|
}
|
|
|
|
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
|
source.AsDataPackets(schema).AsViewStream();
|
|
|
|
public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func<IBlueRowCollection, IBlueRowCollection> apply) {
|
|
var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns);
|
|
var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, FunctionEvaluator.Create(apply));
|
|
return applyTables.AsDataPackets().AsViewStream();
|
|
}
|
|
|
|
public static ViewStream GroupByApply(this ViewStream viewStream, string[] columns, Func<object?[], IBlueRowCollection, IBlueRowCollection?> apply) {
|
|
var groupByTables = GroupByApplyStream.CreateGroupByTable(viewStream, columns);
|
|
var applyTables = GroupByApplyStream.CreateApplyTable(groupByTables, new Func<EquatableKeyArrayWrapper, IBlueRowCollection, IBlueRowCollection?>(ApplyWrapper));
|
|
return applyTables.AsDataPackets().AsViewStream();
|
|
|
|
IBlueRowCollection? ApplyWrapper(EquatableKeyArrayWrapper groupKey, IBlueRowCollection groupRows) =>
|
|
apply(groupKey.Values, groupRows);
|
|
}
|
|
public static ViewStream RenameColumns(this ViewStream viewStream, bool includeUnmapped, params (string From, string To)[] columnMap) =>
|
|
new ColumnMaskStream(viewStream, includeUnmapped, columnMap);
|
|
|
|
public static IObservable<IDataPacket> TimeSeries(this IObservable<IDataPacket> source, int bufferSize, string timeColumn) =>
|
|
source.AsViewStream().TimeSeries(bufferSize, timeColumn).AsObservable();
|
|
|
|
public static ViewStream TimeSeries(this ViewStream source, int bufferSize, string timeColumn) =>
|
|
new TimeSeriesStream(source, bufferSize, timeColumn);
|
|
|
|
public static IDisposable Subscribe(this ViewStream viewStream, Action<IDataPacket> onChange) {
|
|
viewStream.onChange += OnChange;
|
|
return Disposable.Create(Dispose);
|
|
|
|
void OnChange(object? dpObj) =>
|
|
onChange(ExceptionAssert.Argument.Is<IDataPacket>(dpObj));
|
|
|
|
void Dispose() =>
|
|
viewStream.onChange -= OnChange;
|
|
}
|
|
|
|
public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
|
|
const int STATE_IDLE = 0;
|
|
const int STATE_BUSY = 1;
|
|
const int STATE_DISPOSED = 2;
|
|
return Observable.Create<IDataPacket>(Subscribe);
|
|
IDisposable Subscribe(IObserver<IDataPacket> o) {
|
|
var syncRoot = new object();
|
|
IBlueRowCollection? lastTableIn = null;
|
|
BlueTableImmutable? lastTableOut = null;
|
|
int nConcurrentDoWorks = 0;
|
|
int state = STATE_IDLE;
|
|
IDisposable? dispose = null;
|
|
dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
|
|
return Disposable.Create(Dispose);
|
|
|
|
void Dispose() {
|
|
lock (syncRoot) {
|
|
switch (state) {
|
|
case STATE_DISPOSED:
|
|
return;
|
|
case STATE_BUSY:
|
|
state = STATE_DISPOSED;
|
|
return;
|
|
}
|
|
state = STATE_DISPOSED;
|
|
}
|
|
Disposable.dispose(ref dispose);
|
|
}
|
|
|
|
void OnNext(IDataPacket value) =>
|
|
Volatile.Write(ref lastTableIn, value.newRows);
|
|
|
|
async void DoWork() {
|
|
lock (syncRoot) {
|
|
nConcurrentDoWorks++;
|
|
switch (state) {
|
|
case STATE_DISPOSED:
|
|
case STATE_BUSY:
|
|
return;
|
|
case STATE_IDLE:
|
|
break;
|
|
default:
|
|
Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
|
|
return;
|
|
}
|
|
state = STATE_BUSY;
|
|
}
|
|
var dp = await PublishTable();
|
|
bool isDisposed = false;
|
|
lock (syncRoot) {
|
|
switch (state) {
|
|
case STATE_DISPOSED:
|
|
isDisposed = true;
|
|
break;
|
|
case STATE_BUSY:
|
|
break;
|
|
default:
|
|
Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
|
|
return;
|
|
}
|
|
}
|
|
if (isDisposed) {
|
|
Disposable.dispose(ref dispose);
|
|
return;
|
|
}
|
|
if (dp is not null) {
|
|
o.OnNext(dp);
|
|
}
|
|
lock (syncRoot) {
|
|
nConcurrentDoWorks = 0;
|
|
switch (state) {
|
|
case STATE_DISPOSED:
|
|
isDisposed = true;
|
|
break;
|
|
case STATE_BUSY:
|
|
state = STATE_IDLE;
|
|
break;
|
|
default:
|
|
Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
|
|
return;
|
|
}
|
|
}
|
|
if (isDisposed) {
|
|
Disposable.dispose(ref dispose);
|
|
}
|
|
|
|
async Task<DataPacket?> PublishTable() {
|
|
var tableIn = Interlocked.Exchange(ref lastTableIn, null);
|
|
if (tableIn is null) {
|
|
return null;
|
|
}
|
|
var tableOut = await transformAsync(tableIn);
|
|
if (tableOut is null) {
|
|
return null;
|
|
}
|
|
lastTableOut ??= new(tableOut.schema);
|
|
var bd = lastTableOut.ToDataPacketBuilder();
|
|
bd.Clear();
|
|
foreach (var r in tableOut) {
|
|
bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
|
|
}
|
|
return bd.ToDataPacket(out lastTableOut);
|
|
}
|
|
}
|
|
|
|
static string StateName(int state) =>
|
|
state switch {
|
|
STATE_IDLE => "IDLE",
|
|
STATE_BUSY => "BUSY",
|
|
STATE_DISPOSED => "DISPOSED",
|
|
_ => $"STATE_{state}",
|
|
};
|
|
}
|
|
}
|
|
|
|
public static ViewStream Diff(this ViewStream source, int period = 1) =>
|
|
new bts.BlueOrb.DiffStream(source, period);
|
|
}
|