refactor: Replace DataPacket w/ IDataPacket in stream handlers
This commit also renames BlueTableSchemaAdaptor to BlueTableSchemaAdapter, and makes it implement IBlueTableSchema.
This commit is contained in:
1 parent
770e14752a
commit
5080960b97
12 files changed
+461
-194
No files matched your search
@@ -5,19 +5,19 @@ using BCG;
|
||||
using bts.api.core;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.PubSub;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
using Spark.Core.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
|
||||
public static class DataStreamExtensions {
|
||||
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
||||
public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
|
||||
source as ViewStream ?? new ObservableViewStream(source);
|
||||
|
||||
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
|
||||
@@ -36,15 +36,15 @@ public static class DataStreamExtensions {
|
||||
}
|
||||
}
|
||||
|
||||
public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
|
||||
source.Select(dp => {
|
||||
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
||||
return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
|
||||
// public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
|
||||
// source.Select(dp => {
|
||||
// dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
||||
// return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
|
||||
|
||||
IBlueRowCollection<T> CastRows(ICollection rows) =>
|
||||
new BlueTableSelectorWrapper<T>(
|
||||
rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
|
||||
});
|
||||
// IBlueRowCollection<T> CastRows(ICollection rows) =>
|
||||
// new BlueTableSelectorWrapper<T>(
|
||||
// rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
|
||||
// });
|
||||
|
||||
public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
|
||||
source.AsTables<IBlueRow, int>(r => r, r => r.Id());
|
||||
@@ -72,9 +72,33 @@ public static class DataStreamExtensions {
|
||||
});
|
||||
}
|
||||
|
||||
public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
||||
public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
|
||||
source.Scan(
|
||||
(Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
|
||||
(Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
|
||||
(res, table) => {
|
||||
var resTable = res.Table;
|
||||
if (resTable is null) {
|
||||
if (table.state.Count == 0) {
|
||||
return (null, null);
|
||||
}
|
||||
resTable = new(table.state.Values.First().schema);
|
||||
}
|
||||
var bd = resTable.ToDataPacketBuilder();
|
||||
foreach (var r in table.removed) {
|
||||
bd.Remove(r.Id());
|
||||
}
|
||||
foreach (var r in table.added) {
|
||||
bd.AddOrUpdate(r);
|
||||
}
|
||||
var resDp = bd.ToDataPacket(out var newRows);
|
||||
return (newRows, resDp);
|
||||
})
|
||||
.Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
|
||||
.Select(t => t.Dp!);
|
||||
|
||||
public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
||||
source.Scan(
|
||||
(Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
|
||||
(res, table) => {
|
||||
var bd = res.Table.ToDataPacketBuilder();
|
||||
foreach (var r in table.removed) {
|
||||
@@ -83,12 +107,30 @@ public static class DataStreamExtensions {
|
||||
foreach (var r in table.added) {
|
||||
bd.AddOrUpdate(r);
|
||||
}
|
||||
var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
|
||||
var resDp = bd.ToDataPacket(out var newRows);
|
||||
return (newRows, resDp);
|
||||
})
|
||||
.Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
|
||||
.Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
|
||||
.Select(t => t.Dp);
|
||||
|
||||
public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
|
||||
return Observable.Create<IDataPacket>(Subscribe);
|
||||
|
||||
IDisposable Subscribe(IObserver<IDataPacket> o) {
|
||||
var adapter = new BlueTableImmutableAdapter<T>(schema);
|
||||
IDataPacket initDataPacket = new DataPacket(schema);
|
||||
return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
|
||||
|
||||
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
|
||||
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
|
||||
var bd = state.Adapter.ToDataPacketBuilder();
|
||||
bd.Update(tableUpdate);
|
||||
var newRows = bd.ToImmutable(out var dp);
|
||||
return (newRows, dp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
||||
source.AsDataPackets(schema).AsViewStream();
|
||||
|
||||
@@ -131,4 +173,46 @@ public static class DataStreamExtensions {
|
||||
}
|
||||
return indices;
|
||||
}
|
||||
|
||||
public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
|
||||
Observable.Create<IDataPacket>(o => {
|
||||
var inputGate = new AtomicGate();
|
||||
var outputGate = new AtomicGate();
|
||||
BlueTableImmutable? lastTable = null;
|
||||
BlueTableImmutable.DataPacketBuilder? bd = null;
|
||||
IDataPacket? pendingDpOut = null;
|
||||
return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
|
||||
|
||||
void OnNext(IDataPacket value) {
|
||||
using (inputGate.BeginTransaction()) {
|
||||
lastTable ??= new(value.newRows.schema);
|
||||
bd ??= lastTable.ToDataPacketBuilder();
|
||||
if (pendingDpOut is null) {
|
||||
// The last outgoing datapacket was just fired. Don't merge this update with the
|
||||
// previous one (otherwise its adds/removes would be fired twice).
|
||||
bd.Reset();
|
||||
}
|
||||
foreach (IBlueRow r in value.deletedRows) {
|
||||
bd.Remove(r.Id());
|
||||
}
|
||||
foreach (IBlueRow r in value.addedRows) {
|
||||
bd.AddOrUpdate(r);
|
||||
}
|
||||
pendingDpOut = bd.ToDataPacket(out lastTable);
|
||||
}
|
||||
}
|
||||
|
||||
void DoWork() {
|
||||
IDataPacket? dpOut = null;
|
||||
if (outputGate.TryEnter()) {
|
||||
using var exit = Disposable.Create(outputGate.Exit);
|
||||
(dpOut, pendingDpOut) = (pendingDpOut, dpOut);
|
||||
if (dpOut is not null) {
|
||||
o.OnNext(dpOut);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user