refactor: Replace DataPacket w/ IDataPacket in stream handlers

This commit also renames BlueTableSchemaAdaptor to BlueTableSchemaAdapter,
and makes it implement IBlueTableSchema.
This commit is contained in:
John Colvin committed 2025-11-19 00:36:56 -05:00
1 parent 770e14752a
commit 5080960b97
12 files changed
+461 -194

No files matched your search

@@ -5,19 +5,19 @@ using BCG;
using bts.api.core;
using Bts.DataStreamCore;
using Bts.PubSub;
using Microsoft.Extensions.Logging;
using Spark.Core.Disposables;
using Spark.Core.Threading;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Reactive.Linq;
public static class DataStreamExtensions {
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
public static ViewStream AsViewStream(this IObservable<IDataPacket> source) =>
source as ViewStream ?? new ObservableViewStream(source);
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
@@ -36,15 +36,15 @@ public static class DataStreamExtensions {
}
}
public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
source.Select(dp => {
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
// public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
// source.Select(dp => {
// dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
// return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
IBlueRowCollection<T> CastRows(ICollection rows) =>
new BlueTableSelectorWrapper<T>(
rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
});
// IBlueRowCollection<T> CastRows(ICollection rows) =>
// new BlueTableSelectorWrapper<T>(
// rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
// });
public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
source.AsTables<IBlueRow, int>(r => r, r => r.Id());
@@ -72,9 +72,33 @@ public static class DataStreamExtensions {
});
}
public static IObservable<IDataPacket<T>> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
source.Scan(
(Table: new BlueTableImmutable<T>(schema), Dp: (IDataPacket<T>)DataPacket<T>.Empty(schema)),
(Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
(res, table) => {
var resTable = res.Table;
if (resTable is null) {
if (table.state.Count == 0) {
return (null, null);
}
resTable = new(table.state.Values.First().schema);
}
var bd = resTable.ToDataPacketBuilder();
foreach (var r in table.removed) {
bd.Remove(r.Id());
}
foreach (var r in table.added) {
bd.AddOrUpdate(r);
}
var resDp = bd.ToDataPacket(out var newRows);
return (newRows, resDp);
})
.Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
.Select(t => t.Dp!);
public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
source.Scan(
(Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
(res, table) => {
var bd = res.Table.ToDataPacketBuilder();
foreach (var r in table.removed) {
@@ -83,12 +107,30 @@ public static class DataStreamExtensions {
foreach (var r in table.added) {
bd.AddOrUpdate(r);
}
var resDp = bd.ToDataPacketT(out var newRows, out var addedRows, out var deletedRows);
var resDp = bd.ToDataPacket(out var newRows);
return (newRows, resDp);
})
.Where(t => t.Dp.addedRows.Any() || t.Dp.deletedRows.Any())
.Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
.Select(t => t.Dp);
public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
return Observable.Create<IDataPacket>(Subscribe);
IDisposable Subscribe(IObserver<IDataPacket> o) {
var adapter = new BlueTableImmutableAdapter<T>(schema);
IDataPacket initDataPacket = new DataPacket(schema);
return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
var bd = state.Adapter.ToDataPacketBuilder();
bd.Update(tableUpdate);
var newRows = bd.ToImmutable(out var dp);
return (newRows, dp);
}
}
}
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
source.AsDataPackets(schema).AsViewStream();
@@ -131,4 +173,46 @@ public static class DataStreamExtensions {
}
return indices;
}
public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
Observable.Create<IDataPacket>(o => {
var inputGate = new AtomicGate();
var outputGate = new AtomicGate();
BlueTableImmutable? lastTable = null;
BlueTableImmutable.DataPacketBuilder? bd = null;
IDataPacket? pendingDpOut = null;
return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
void OnNext(IDataPacket value) {
using (inputGate.BeginTransaction()) {
lastTable ??= new(value.newRows.schema);
bd ??= lastTable.ToDataPacketBuilder();
if (pendingDpOut is null) {
// The last outgoing datapacket was just fired. Don't merge this update with the
// previous one (otherwise its adds/removes would be fired twice).
bd.Reset();
}
foreach (IBlueRow r in value.deletedRows) {
bd.Remove(r.Id());
}
foreach (IBlueRow r in value.addedRows) {
bd.AddOrUpdate(r);
}
pendingDpOut = bd.ToDataPacket(out lastTable);
}
}
void DoWork() {
IDataPacket? dpOut = null;
if (outputGate.TryEnter()) {
using var exit = Disposable.Create(outputGate.Exit);
(dpOut, pendingDpOut) = (pendingDpOut, dpOut);
if (dpOut is not null) {
o.OnNext(dpOut);
}
}
}
});
}