Files
spark/src/client/spark_api/api/Update.cs
T

461 lines
18 KiB
C#

using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Reactive.Concurrency;
using System.Reactive.Disposables;
using System.Reactive.Linq;
namespace bts.api.core {
class RawUpdate<T> {
// RawUpdate uses List<T> instead of IReadonlyList to avoid virtual dispatch on a type not likely to be see in a public api surface
// There is a single stack allocated RawUpdate per subscription
public RawUpdate() {
is_reset = false;
added_items = new List<T>();
added_keys = new List<int>();
changed_items = new List<T>();
changed_keys = new List<int>();
removed_keys = new List<int>();
}
public RawUpdate<T> MarkAsReset() {
Clear();
is_reset = true;
return this;
}
public void Clear() {
is_reset = false;
added_items.Clear();
added_keys.Clear();
changed_items.Clear();
changed_keys.Clear();
removed_keys.Clear();
}
public bool is_reset { get; private set; }
public List<T> added_items { get; }
public List<int> added_keys { get; }
public List<T> changed_items { get; }
public List<int> changed_keys { get; }
public List<int> removed_keys { get; }
#if DEBUG
public override string ToString() {
var added = string.Join(", ", added_items);
var changed = string.Join(", ", changed_items);
var removed = string.Join(", ", removed_keys);
return $"added [{added}] changed [{changed}] removed [{removed}]";
}
#endif
}
public readonly struct Change<T> {
public Change(T o, T n) {
Old = o;
New = n;
}
public readonly T Old;
public readonly T New;
public Change<T> merge(Change<T> newer) {
return new Change<T>(Old, newer.New);
}
public void Deconstruct(out T Old, out T New) {
Old = this.Old;
New = this.New;
}
public override string ToString() {
return $"{{{Old}=>{New}}}";
}
}
public class Update<T, TKey> where TKey : IEquatable<TKey> {
readonly Func<T, TKey> _key;
internal Update(
IEnumerable<T> all,
IEnumerable<T> added,
IEnumerable<Change<T>> changes,
IEnumerable<T> removed,
Func<T, TKey> key_extractor) {
this.added = added ?? new T[0];
this.removed = removed ?? new T[0];
this.changes = changes ?? new Change<T>[0];
this.all = all;
_key = key_extractor;
}
public static Update<T, TKey> empty(Func<T, TKey> key_extractor) =>
new Update<T, TKey>(new T[0], new T[0], new Change<T>[0], new T[0], key_extractor);
public Update<T,TKey> merge(Update<T, TKey> next) {
var comp = new RowEqualityComparer<T, TKey>(_key);
var added = this.added as HashSet<T> ?? new HashSet<T>(this.added, comp);
var changes = this.changed as ChangeDict<T,TKey> ?? new ChangeDict<T,TKey>(this.changes, _key);
var removed = this.removed as HashSet<T> ?? new HashSet<T>(this.removed, comp);
// Add
added.UnionWith(next.added);
// Change
foreach (var c in next.changes) {
if (added.Contains(c.Old)) {
added.Remove(c.Old);
added.Add(c.New);
}
else {
changes.AddOrMerge(c);
}
}
// Remove
foreach (var r in next.removed) {
if (!added.Remove(r)) {
removed.Add(r);
}
changes.Remove(_key(r));
}
return new Update<T, TKey>(
next.all,
added, changes, removed,
_key
);
}
public IEnumerable<T> all { get; }
public IEnumerable<T> added { get; }
public IEnumerable<Change<T>> changes { get; }
public IEnumerable<T> changed => changes.Select(c => c.New);
public IEnumerable<T> removed { get; }
public TKey key(T t) { return _key(t); }
internal Func<T, TKey> key_extractor => _key;
#if DEBUG
public override string ToString() {
string key(T t) => _key(t).ToString().Trim();
var a = string.Join(", ", added.Select(key));
var c = string.Join(", ", changed.Select(key));
var r = string.Join(", ", removed.Select(key));
return $"{{added [{a}] changed [{c}] removed [{r}]}}";
}
#endif
}
class ChangeDict<T, TKey> : IEnumerable<Change<T>> {
Dictionary<TKey, Change<T>> _changes;
Func<Change<T>, TKey> _key;
public ChangeDict(IEnumerable<Change<T>> changes, Func<T, TKey> key) {
_key = x => key(x.New);
_changes = changes.ToDictionary(_key);
}
public Change<T>? AddOrMerge(Change<T> newer) {
var key = _key(newer);
if (_changes.TryGetValue(key, out Change<T> old)) {
_changes[key] = old.merge(newer);
return old;
}
_changes.Add(key, newer);
return null;
}
public void Remove(TKey key) {
_changes.Remove(key);
}
#region IEnumerable
public IEnumerator<Change<T>> GetEnumerator() {
return _changes.Values.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator() {
return _changes.Values.GetEnumerator();
}
#endregion
}
struct RowEqualityComparer<T,TKey> : IEqualityComparer<T>, IEqualityComparer<Change<T>>
where TKey : IEquatable<TKey> {
readonly Func<T,TKey> _key;
public RowEqualityComparer(Func<T,TKey> key) {
_key = key;
}
public bool Equals(T x, T y) {
return _key(x).Equals(_key(y));
}
public bool Equals(Change<T> x, Change<T> y) {
return Equals(x.New, y.New);
}
public int GetHashCode(T obj) {
return _key(obj).GetHashCode();
}
public int GetHashCode(Change<T> c) {
return GetHashCode(c.New);
}
}
public class TableUpdate<T, TKey> where TKey : IEquatable<TKey> {
readonly ImmutableDictionary<TKey, T> _state;
internal TableUpdate(
ImmutableDictionary<TKey, T> init,
IEnumerable<T> added,
IEnumerable<Change<T>> changed,
IEnumerable<T> removed,
Func<T,TKey> maybe_key_extractor) {
_state = init;
this.added = added;
this.changes = changed;
this.removed = removed;
this.key_extractor = maybe_key_extractor;
}
public IReadOnlyDictionary<TKey, T> state { get { return _state; } }
public IEnumerable<T> added { get; }
public IEnumerable<Change<T>> changes { get; }
public IEnumerable<T> changed => changes.Select(c => c.New);
public IEnumerable<T> removed { get; }
internal static TableUpdate<T, TKey> update(TableUpdate<T,TKey> table, Update<T, TKey> update) {
var builder = table._state.ToBuilder();
builder.RemoveRange(update.removed.Select(i => update.key(i)));
builder.AddRange(update.added.Select(i => new KeyValuePair<TKey, T>(update.key(i), i)));
foreach (var i in update.changed) {
builder[update.key(i)] = i;
}
return new TableUpdate<T, TKey>(builder.ToImmutableDictionary(), update.added, update.changes, update.removed, update.key_extractor);
}
internal TableUpdate<T, TKey> as_snapshot() {
return new TableUpdate<T, TKey>(_state, _state.Values.ToList(), new Change<T>[0], new T[0], key_extractor);
}
internal Func<T,TKey> key_extractor { get; private set; }
}
internal class LegacyTableUpdate<T, TKey> where TKey : IEquatable<TKey> {
public LegacyTableUpdate(
TableUpdate<T, TKey> update) {
this.state = update.state;
this.added = update.added.Concat(update.changes.Select(t => t.New));
this.removed = update.removed.Concat(update.changes.Select(t => t.Old));
}
public IReadOnlyDictionary<TKey, T> state { get; }
public IEnumerable<T> added { get; }
public IEnumerable<T> removed { get; }
}
public static class ObservableUpdateExtensions {
internal static IObservable<Update<T, TKey>> reconstitute<T, TKey>(this IObservable<RawUpdate<T>> source, Func<T, TKey> key_extractor) where TKey : IEquatable<TKey> {
var state = new Dictionary<int, T>();
return source.Select(raw => {
if (raw.is_reset) {
var removed = state.Values.ToList();
state.Clear();
return new Update<T, TKey>(new T[0], new T[0], new Change<T>[0], removed, key_extractor);
}
else {
var removed = new List<T>(raw.removed_keys.Count);
foreach (var r in raw.removed_keys) {
T temp;
if (state.TryGetValue(r, out temp)) {
removed.Add(temp);
state.Remove(r);
}
}
for (int x = 0; x < raw.added_items.Count; ++x) {
state[raw.added_keys[x]] = raw.added_items[x];
}
var changes = new List<Change<T>>(raw.changed_items.Count);
for (int x = 0; x < raw.changed_items.Count; ++x) {
var old = state[raw.changed_keys[x]];
state[raw.changed_keys[x]] = raw.changed_items[x];
changes.Add(new Change<T>(old, raw.changed_items[x]));
}
return new Update<T, TKey>(state.Values.ToList(), raw.added_items.ToList(), changes, removed, key_extractor);
}
});
}
public static IObservable<TableUpdate<T,TKey>> Table<T,TKey>(this IObservable<Update<T,TKey>> source) where TKey : IEquatable<TKey> =>
source.Scan(
new TableUpdate<T,TKey>(ImmutableDictionary<TKey,T>.Empty, new T[0], new Change<T>[0], new T[0], null),
(table, update) => TableUpdate<T,TKey>.update(table,update));
// Fold updates and emit at max 1/interval
public static IObservable<Update<T,TKey>> Naggle<T, TKey>(this IObservable<Update<T,TKey>> source, TimeSpan interval, IScheduler scheduler) where TKey : IEquatable<TKey> {
// fold updates together and emit on TimeSpan (max) interval. Emit on threadpool
return Observable.Create<Update<T,TKey>>(observer => {
object guard = new object();
Update<T, TKey> cache = null;
bool finished = false;
Log.DataFlow.diag($"Create Naggle {typeof(T).Name}");
var d1 = source.Subscribe(update => {
Log.DataFlow.diag($"Naggle source updated {typeof(T).Name}");
// subscribe to the source, merge updates (Quickbus thread)
lock (guard) {
if (cache == null) cache = update;
else cache = cache.merge(update);
}
},
() => { finished = true; });
// publish on interval
var d2 = Observable.Interval(interval, scheduler).Subscribe(_ => {
Update<T, TKey> update;
lock (guard) {
update = cache;
cache = null;
}
if (update != null) {
Log.DataFlow.diag($"Naggle fire {typeof(T).Name}");
observer.OnNext(update);
}
if (finished) {
observer.OnCompleted();
}
});
return new CompositeDisposable(d1, d2);
});
}
public static IObservable<TableUpdate<T,TKey>> Cache<T, TKey>(this IObservable<TableUpdate<T, TKey>> source) where TKey : IEquatable<TKey> {
var inner = source.Replay(1);
Log.DataFlow.diag($"Create Cache {typeof(T).Name}");
return Observable.Create<TableUpdate<T,TKey>>(o => {
inner.Connect();
bool first = true;
Log.DataFlow.diag($"Cache Subscribe {typeof(T).Name}");
return inner.Subscribe(upd => {
Log.DataFlow.diag($"Cache Inner: {typeof(T).Name}");
if (first) {
o.OnNext(upd.as_snapshot());
first = false;
}
else {
o.OnNext(upd);
}
});
});
}
internal static IObservable<LegacyTableUpdate<T, TKey>> AddRemoveOnly<T, TKey>(this IObservable<TableUpdate<T, TKey>> source) where TKey : IEquatable<TKey> {
return source.Select(t => new LegacyTableUpdate<T, TKey>(t));
}
public static IObservable<Update<T, TKey>> Filter<T, TKey>(this IObservable<Update<T, TKey>> source, IObservable<Func<T, bool>> predicate) where TKey : IEquatable<TKey>
{
return Observable.Create<Update<T, TKey>>(observer =>
{
Update<T, TKey> unfiltered_cache = null, filtered_cache = null;
Func<T, bool> pred_cache = null;
Log.DataFlow.diag($"Create Filter {typeof(T).Name}");
return source.CombineLatest(predicate, (upd, pred) => (upd, pred)).Subscribe( tup =>
{
bool pred_changed = !object.ReferenceEquals(tup.pred, pred_cache) && (pred_cache != null);
pred_cache = tup.pred;
unfiltered_cache = tup.upd;
if (pred_changed && filtered_cache != null)
{
//clear out existing
var clear = new Update<T, TKey>(
(IReadOnlyCollection<T>)Array.Empty<T>(),
Enumerable.Empty<T>(),
Enumerable.Empty<Change<T>>(),
filtered_cache.all, unfiltered_cache.key_extractor);
observer.OnNext(clear);
var refilter = unfiltered_cache.all.Where(tup.pred).ToList();
var readd = new Update<T, TKey>(
refilter, refilter,
Enumerable.Empty<Change<T>>(),
Enumerable.Empty<T>(), unfiltered_cache.key_extractor);
observer.OnNext(readd);
filtered_cache = readd;
}
else
{
IReadOnlyCollection<T> all; IEnumerable<T> added, removed; IEnumerable<Change<T>> changed;
apply_predicate(tup.pred, tup.upd, out all, out added, out changed, out removed);
var next = new Update<T, TKey>(all, added, changed, removed, tup.upd.key_extractor);
observer.OnNext(next);
filtered_cache = next;
}
},
() => observer.OnCompleted());
});
}
private static void apply_predicate<T, TKey>(Func<T, bool> predicate, Update<T, TKey> cache,
out IReadOnlyCollection<T> all, out IEnumerable<T> added,
out IEnumerable<Change<T>> changed, out IEnumerable<T> removed) where TKey : IEquatable<TKey>
{
var adds = new List<T>();
adds.AddRange(cache.added.Where(predicate));
var rems = new List<T>();
rems.AddRange(cache.removed.Where(predicate));
var diffs = new List<Change<T>>();
foreach (var change in cache.changes)
{
if(!predicate(change.Old) && predicate(change.New))
adds.Add(change.New);
else if(predicate(change.Old) && !predicate(change.New))
rems.Add(change.Old);
else if(predicate(change.Old) && predicate(change.New))
diffs.Add(change);
}
all = cache.all.Where(predicate).ToList();
added = adds;
changed = diffs;
removed = rems;
}
// This method does everything lazily so it's cheap until the collections are enumerated
public static IObservable<Update<T2, TKey>>
Select<T1, T2, TKey>(this IObservable<Update<T1, TKey>> src, Func<T1, T2> map, Func<T2, TKey> key) where TKey : IEquatable<TKey> {
return src.Select(update => {
return new Update<T2, TKey>(
update.all.Select(map),
update.added.Select(map),
update.changes.Select(ch => new Change<T2>(map(ch.Old), map(ch.New))),
update.removed.Select(map),
key);
});
}
// Constructs a new dictionary so not as cheap as Select on the Update<> type
public static IObservable<TableUpdate<T2, TKey>>
Select<T1, T2, TKey>(this IObservable<TableUpdate<T1, TKey>> src, Func<T1, T2> map, Func<T2, TKey> key) where TKey : IEquatable<TKey> {
var cache = ImmutableDictionary<TKey, T2>.Empty;
return src.Select(update => {
var added = update.added.Select(map);
var removed = update.removed.Select(map);
var changes = update.changes.Select(ch => new Change<T2>(map(ch.Old), map(ch.New)));
var builder = cache.ToBuilder();
builder.RemoveRange(removed.Select(key));
builder.AddRange(added.Select(t => new KeyValuePair<TKey,T2>(key(t), t)));
foreach (var (o,n) in changes) {
Debug.Assert(key(o).Equals(key(n)));
builder[key(o)] = n;
}
cache = builder.ToImmutable();
return new TableUpdate<T2, TKey>(cache, added, changes, removed, key);
});
}
}
}