Files
spark/src/client/spark_api/api/Update.cs
T

941 lines
47 KiB
C#

using bts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Reactive.Concurrency;
using System.Reactive.Disposables;
using System.Reactive.Linq;
using System.Text;
namespace bts.api.core {
class RawUpdate<T> {
// RawUpdate uses List<T> instead of IReadonlyList to avoid virtual dispatch on a type not likely to be see in a public api surface
// There is a single stack allocated RawUpdate per subscription
public RawUpdate() {
is_reset = false;
added_items = new List<T>();
added_keys = new List<int>();
changed_items = new List<T>();
changed_keys = new List<int>();
removed_keys = new List<int>();
}
public RawUpdate<T> MarkAsReset() {
Clear();
is_reset = true;
return this;
}
public void Clear() {
is_reset = false;
added_items.Clear();
added_keys.Clear();
changed_items.Clear();
changed_keys.Clear();
removed_keys.Clear();
}
public bool is_reset { get; private set; }
public List<T> added_items { get; }
public List<int> added_keys { get; }
public List<T> changed_items { get; }
public List<int> changed_keys { get; }
public List<int> removed_keys { get; }
#if DEBUG
public override string ToString() {
var added = string.Join(", ", added_items);
var changed = string.Join(", ", changed_items);
var removed = string.Join(", ", removed_keys);
return $"added [{added}] changed [{changed}] removed [{removed}]";
}
#endif
}
public readonly struct Change<T> {
public Change(T o, T n) {
Old = o;
New = n;
}
public readonly T Old;
public readonly T New;
public Change<T> merge(Change<T> newer) {
return new Change<T>(Old, newer.New);
}
public void Deconstruct(out T Old, out T New) {
Old = this.Old;
New = this.New;
}
public override string ToString() {
return $"{{{Old}=>{New}}}";
}
}
public class Update<T, TKey> where TKey : IEquatable<TKey> {
readonly Func<T, TKey> _key;
internal Update(
IEnumerable<T> all,
IEnumerable<T> added,
IEnumerable<Change<T>> changes,
IEnumerable<T> removed,
Func<T, TKey> key_extractor) {
this.added = added ?? new T[0];
this.removed = removed ?? new T[0];
this.changes = changes ?? new Change<T>[0];
this.all = all;
_key = key_extractor;
}
public static Update<T, TKey> empty(Func<T, TKey> key_extractor) =>
new Update<T, TKey>(new T[0], new T[0], new Change<T>[0], new T[0], key_extractor);
public Update<T,TKey> merge(Update<T, TKey> next) {
var added = this.added as KeyedValueSet<TKey, T> ?? new KeyedValueSet<TKey, T>(this.added, _key);
var changes = this.changed as ChangeDict<T,TKey> ?? new ChangeDict<T,TKey>(this.changes, _key);
var removed = this.removed as KeyedValueSet<TKey, T> ?? new KeyedValueSet<TKey, T>(this.removed, _key);
// Add
foreach (var v in next.added) {
var k = _key(v);
if (removed.Remove(k, out var oldval)) {
changes.AddOrMerge(new Change<T>(oldval, v));
} else {
added[k] = v;
}
}
// Change
foreach (var c in next.changes) {
var k = _key(c.Old);
if (!added.Replace(k, c.New))
changes.AddOrMerge(c);
}
// Remove
foreach (var r in next.removed) {
var k = _key(r);
if (!added.Remove(k))
removed[k] = changes.TryRemove(k, out var ch) ? ch.Old : r;
}
return new Update<T, TKey>(
next.all,
added, changes, removed,
_key);
}
public IEnumerable<T> all { get; }
public IEnumerable<T> added { get; }
public IEnumerable<Change<T>> changes { get; }
public IEnumerable<T> changed => changes.Select(c => c.New);
public IEnumerable<T> removed { get; }
public TKey key(T t) { return _key(t); }
internal Func<T, TKey> key_extractor => _key;
#if DEBUG
public override string ToString() {
string key(T t) => _key(t).ToString().Trim();
var a = string.Join(", ", added.Select(key));
var c = string.Join(", ", changed.Select(key));
var r = string.Join(", ", removed.Select(key));
return $"{{added [{a}] changed [{c}] removed [{r}]}}";
}
#endif
}
class KeyedValueSet<TKey, TValue> : IEnumerable<TValue> {
readonly Dictionary<TKey, TValue> _cache = new Dictionary<TKey, TValue>();
public KeyedValueSet(IEnumerable<TValue> value, Func<TValue, TKey> key) {
foreach (var v in value)
_cache[key(v)] = v;
}
public TValue this[TKey key] {
set => _cache[key] = value;
}
public bool Remove(TKey key) =>
_cache.Remove(key);
public bool Remove(TKey key, out TValue value) =>
_cache.TryGetValue(key, out value) && _cache.Remove(key);
public bool Replace(TKey key, TValue value) {
if (_cache.ContainsKey(key)) {
_cache[key] = value;
return true;
}
return false;
}
public IEnumerator<TValue> GetEnumerator() =>
_cache.Values.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() =>
GetEnumerator();
}
class ChangeDict<T, TKey> : IEnumerable<Change<T>> {
Dictionary<TKey, Change<T>> _changes;
Func<Change<T>, TKey> _key;
public ChangeDict(IEnumerable<Change<T>> changes, Func<T, TKey> key) {
_key = x => key(x.New);
_changes = changes.ToDictionary(_key);
}
public Change<T>? AddOrMerge(Change<T> newer) {
var key = _key(newer);
if (_changes.TryGetValue(key, out Change<T> old)) {
_changes[key] = old.merge(newer);
return old;
}
_changes.Add(key, newer);
return null;
}
public bool Remove(TKey key) =>
_changes.Remove(key);
public bool TryRemove(TKey key, out Change<T> result) =>
_changes.TryGetValue(key, out result) && _changes.Remove(key);
#region IEnumerable
public IEnumerator<Change<T>> GetEnumerator() {
return _changes.Values.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator() {
return _changes.Values.GetEnumerator();
}
#endregion
}
public class TableUpdate<T, TKey> where TKey : IEquatable<TKey> {
// warning: this does not have a key_extractor
public static readonly TableUpdate<T, TKey> empty = new TableUpdate<T, TKey>(ImmutableDictionary<TKey, T>.Empty, Array.Empty<T>(), Array.Empty<Change<T>>(), Array.Empty<T>(), null);
readonly ImmutableDictionary<TKey, T> _state;
public static TableUpdate<T, TKey> mk_empty(Func<T, TKey> key_extractor) => new TableUpdate<T, TKey>(ImmutableDictionary<TKey, T>.Empty, Array.Empty<T>(), Array.Empty<Change<T>>(), Array.Empty<T>(), key_extractor);
internal TableUpdate(
ImmutableDictionary<TKey, T> init,
IEnumerable<T> added,
IEnumerable<Change<T>> changed,
IEnumerable<T> removed,
Func<T,TKey> maybe_key_extractor) {
_state = init;
this.added = added;
this.changes = changed;
this.removed = removed;
this.key_extractor = maybe_key_extractor;
}
public IReadOnlyDictionary<TKey, T> state { get { return _state; } }
public IEnumerable<T> added { get; }
public IEnumerable<Change<T>> changes { get; }
public IEnumerable<T> changed => changes.Select(c => c.New);
public IEnumerable<T> removed { get; }
internal Func<T,TKey> key_extractor { get; private set; }
internal static TableUpdate<T, TKey> update(TableUpdate<T, TKey> table, Update<T, TKey> update) =>
TableUpdate<T, TKey>.update(table, update.added, update.changes, update.removed, update.key);
internal static TableUpdate<T, TKey> update(TableUpdate<T, TKey> table, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key) {
var builder = table._state.ToBuilder();
builder.RemoveRange(removed.Select(i => key(i)));
builder.AddRange(added.Select(i => new KeyValuePair<TKey, T>(key(i), i)));
foreach (var i in changes) {
builder[key(i.New)] = i.New;
}
return new TableUpdate<T, TKey>(builder.ToImmutableDictionary(), added, changes, removed, key);
}
internal static TableUpdate<T, TKey> update(TableUpdate<T,TKey> table, IReadOnlyDictionary<TKey, T> new_state, IEqualityComparer<T> value_comparer) {
var builder = table._state.ToBuilder();
// removed/changes
var removed = new List<T>();
var changes = new List<Change<T>>();
foreach (var kv in table._state) {
if (new_state.TryGetValue(kv.Key, out T new_val)) {
if (!value_comparer.Equals(kv.Value, new_val)) {
changes.Add(new Change<T>(kv.Value, new_val));
builder[kv.Key] = new_val;
}
} else {
removed.Add(kv.Value);
builder.Remove(kv.Key);
}
}
// added
var added = new List<T>();
foreach (var kv in new_state) {
if (!table._state.ContainsKey(kv.Key)) {
added.Add(kv.Value);
builder.Add(kv);
}
}
return new TableUpdate<T, TKey>(builder.ToImmutableDictionary(), added, changes, removed, table.key_extractor);
}
internal TableUpdate<T, TKey> as_snapshot() {
return new TableUpdate<T, TKey>(_state, _state.Values.ToList(), new Change<T>[0], new T[0], key_extractor);
}
internal Builder to_builder() =>
new Builder(this);
public static class FilterHelper {
internal static TableUpdate<T, TKey> on_source_changed(TableUpdate<T, TKey> filtered_target, IEnumerable<T> raw_added, IEnumerable<Change<T>> raw_changes, IEnumerable<T> raw_removed, Func<T, bool> predicate) {
var builder = filtered_target._state.ToBuilder();
List<T> added, removed;
List<Change<T>> changes;
if (predicate == null) {
added = raw_added.ToList();
removed = raw_removed.ToList();
changes = raw_changes.ToList();
} else {
added = new List<T>(raw_added.Where(predicate));
removed = new List<T>(raw_removed.Where(predicate));
changes = new List<Change<T>>();
foreach (var ch in raw_changes) {
if (predicate(ch.Old)) {
if (predicate(ch.New)) {
changes.Add(ch);
} else {
removed.Add(ch.Old);
}
} else if (predicate(ch.New)) {
added.Add(ch.New);
}
}
}
var key = filtered_target.key_extractor;
builder.RemoveRange(removed.Select(key));
foreach (var ch in changes)
builder[key(ch.New)] = ch.New;
builder.AddRange(added.Select(v => new KeyValuePair<TKey, T>(key(v), v)));
return new TableUpdate<T, TKey>(builder.ToImmutable(), added, changes, removed, key);
}
internal static TableUpdate<T, TKey> on_predicate_changed(TableUpdate<T, TKey> filtered_target, IReadOnlyDictionary<TKey, T> unfiltered_state, Func<T, bool> old_predicate, Func<T, bool> new_predicate) {
var added = new List<T>();
var removed = new List<T>();
if (new_predicate == old_predicate) {
// nothing changed - this method probably shouldn't be called when nothing changes. Having said that, it doesn't really hurt anything,
// so don't throw a metaphorical wrench into the gears by throwing a literal exception. Just return the table as is.
if (filtered_target.has_changes())
filtered_target = new TableUpdate<T, TKey>(filtered_target._state, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), filtered_target.key_extractor);
return filtered_target;
}
if (old_predicate == null) {
removed.AddRange(filtered_target.state.Values.Where(v => !new_predicate(v)));
} else if (new_predicate == null) {
added.AddRange(unfiltered_state.Values.Where(v => !old_predicate(v)));
} else {
foreach (var kv in unfiltered_state) {
if (old_predicate(kv.Value)) {
if (new_predicate(kv.Value)) {
continue;
} else {
removed.Add(kv.Value);
}
} else if (new_predicate(kv.Value)) {
added.Add(kv.Value);
}
}
}
var builder = filtered_target._state.ToBuilder();
var key = filtered_target.key_extractor;
builder.RemoveRange(removed.Select(key));
builder.AddRange(added.Select(v => new KeyValuePair<TKey, T>(key(v), v)));
return new TableUpdate<T, TKey>(builder.ToImmutable(), added, Array.Empty<Change<T>>(), removed, key);
}
}
/// <summary>
/// This class is used to aggregate a series of changes into a single update.
/// </summary>
internal class Builder {
readonly TableUpdate<T, TKey> _state;
readonly Dictionary<TKey, T> _added = new Dictionary<TKey, T>(), _removed = new Dictionary<TKey, T>();
readonly Dictionary<TKey, Change<T>> _changes = new Dictionary<TKey, Change<T>>();
public Builder(TableUpdate<T, TKey> state) =>
_state = state;
public void add_or_update(T value) =>
add_or_update(_state.key_extractor(value), value);
public void add_or_update(TKey key, T value) {
if (_state.state.TryGetValue(key, out T curr)) {
_removed.Remove(key);
_changes[key] = new Change<T>(curr, value);
} else {
_added[key] = value;
}
}
public void remove(TKey key) {
if (_state.state.TryGetValue(key, out T orig)) {
_removed[key] = orig;
_changes.Remove(key);
} else {
_added.Remove(key);
}
}
public void merge_table(TableUpdate<T, TKey> table, Func<T, TKey> _key) {
foreach (T v in table.removed)
remove(_key(v));
foreach (T v in table.added.Concat(table.changed))
add_or_update(_key(v), v);
}
public void clear() {
_added.Clear();
_changes.Clear();
foreach (var kv in _state.state)
_removed[kv.Key] = kv.Value;
}
public TableUpdate<T, TKey> to_table() {
var builder = _state._state.ToBuilder();
builder.RemoveRange(_removed.Keys);
foreach (var kv in _changes)
builder[kv.Key] = kv.Value.New;
builder.AddRange(_added);
return new TableUpdate<T, TKey>(builder.ToImmutableDictionary(), _added.Values, _changes.Values, _removed.Values, _state.key_extractor);
}
public TableUpdate<T, TKey> to_table_checked(IEqualityComparer<T> comparer) {
var builder = _state._state.ToBuilder();
builder.RemoveRange(_removed.Keys);
foreach (var kv in _changes) {
if (!comparer.Equals(kv.Value.Old, kv.Value.New))
builder[kv.Key] = kv.Value.New;
}
builder.AddRange(_added);
return new TableUpdate<T, TKey>(builder.ToImmutableDictionary(), _added.Values, _changes.Values, _removed.Values, _state.key_extractor);
}
}
}
internal class LegacyTableUpdate<T, TKey> where TKey : IEquatable<TKey> {
public LegacyTableUpdate(
TableUpdate<T, TKey> update) {
this.state = update.state;
this.added = update.added.Concat(update.changes.Select(t => t.New));
this.removed = update.removed.Concat(update.changes.Select(t => t.Old));
}
public IReadOnlyDictionary<TKey, T> state { get; }
public IEnumerable<T> added { get; }
public IEnumerable<T> removed { get; }
}
public static class ObservableUpdateExtensions {
public static IEnumerable<T> Old<T>(this IEnumerable<Change<T>> changes) {
foreach (var ch in changes)
yield return ch.Old;
}
public static IEnumerable<T> New<T>(this IEnumerable<Change<T>> changes) {
foreach (var ch in changes)
yield return ch.New;
}
public static IEnumerable<Change<T>> Actual<T>(this IEnumerable<Change<T>> changes) where T : IEquatable<T> {
foreach (var ch in changes) {
if (!ch.New.Equals(ch.Old))
yield return ch;
}
}
public static IEnumerable<Change<T>> Actual<T>(this IEnumerable<Change<T>> changes, IEqualityComparer<T> comparer) {
foreach (var ch in changes) {
if (!comparer.Equals(ch.New, ch.Old))
yield return ch;
}
}
internal static bool has_changes<T, TKey>(this TableUpdate<T, TKey> table) where TKey : IEquatable<TKey> =>
table.added.Any() || table.changes.Any() || table.removed.Any();
internal static IObservable<Update<T, TKey>> reconstitute<T, TKey>(this IObservable<RawUpdate<T>> source, Func<T, TKey> key_extractor) where TKey : IEquatable<TKey> {
var state = new Dictionary<int, T>();
return source.Select(raw => {
if (raw.is_reset) {
var removed = state.Values.ToList();
state.Clear();
return new Update<T, TKey>(new T[0], new T[0], new Change<T>[0], removed, key_extractor);
}
else {
var removed = new List<T>(raw.removed_keys.Count);
foreach (var r in raw.removed_keys) {
T temp;
if (state.TryGetValue(r, out temp)) {
removed.Add(temp);
state.Remove(r);
}
}
for (int x = 0; x < raw.added_items.Count; ++x) {
state[raw.added_keys[x]] = raw.added_items[x];
}
var changes = new List<Change<T>>(raw.changed_items.Count);
for (int x = 0; x < raw.changed_items.Count; ++x) {
var old = state[raw.changed_keys[x]];
state[raw.changed_keys[x]] = raw.changed_items[x];
changes.Add(new Change<T>(old, raw.changed_items[x]));
}
return new Update<T, TKey>(state.Values.ToList(), raw.added_items.ToList(), changes, removed, key_extractor);
}
});
}
public static IObservable<TableUpdate<T,TKey>> Table<T,TKey>(this IObservable<Update<T,TKey>> source) where TKey : IEquatable<TKey> =>
source.Scan(
new TableUpdate<T,TKey>(ImmutableDictionary<TKey,T>.Empty, new T[0], new Change<T>[0], new T[0], null),
(table, update) => TableUpdate<T,TKey>.update(table,update));
public static IObservable<TableUpdate<T, TKey>> Table<T, TKey>(this IObservable<IReadOnlyDictionary<TKey, T>> source, Func<T, TKey> key) where TKey : IEquatable<TKey> =>
source.Table(key, EqualityComparer<T>.Default);
public static IObservable<TableUpdate<T, TKey>> Table<T, TKey>(this IObservable<IReadOnlyDictionary<TKey, T>> source, Func<T, TKey> key, IEqualityComparer<T> value_comparer) where TKey : IEquatable<TKey> =>
source.Scan(
TableUpdate<T, TKey>.mk_empty(key),
(table, new_state) => TableUpdate<T, TKey>.update(table, new_state, value_comparer));
public static IObservable<TableUpdate<T, TKey>> TableChecked<T, TKey>(this IObservable<Update<T, TKey>> source, IEqualityComparer<T> value_comparer = null) where TKey : IEquatable<TKey> =>
source.Scan(
new TableUpdate<T, TKey>(ImmutableDictionary<TKey, T>.Empty, new T[0], new Change<T>[0], new T[0], null),
(table, update) => {
Log.assert(!update.added.Select(update.key).Any(table.state.ContainsKey), "Update - duplicate keys added.");
Log.assert(update.removed.Select(update.key).All(table.state.ContainsKey), "Update - unknown key removed.");
Log.assert(update.all.Count() + update.removed.Count() - update.added.Count() == table.state.Count, "Table is out of sync with Update");
return TableUpdate<T, TKey>.update(table, update.added, value_comparer == null ? update.changes : update.changes.Actual(value_comparer), update.removed, update.key);
});
public static IObservable<TableUpdate<T, TKey>> TableCheckedAutoCorrect<T, TKey>(this IObservable<Update<T, TKey>> source, IEqualityComparer<T> value_comparer = null) where TKey : IEquatable<TKey> =>
source.Scan(
new TableUpdate<T, TKey>(ImmutableDictionary<TKey, T>.Empty, new T[0], new Change<T>[0], new T[0], null),
(table, update) => {
bool has_errors = false;
var sb = new StringBuilder();
if (update.added.Select(update.key).Any(table.state.ContainsKey)) {
sb.Append("\tUpdate - duplicate keys added.");
has_errors = true;
}
if (!update.removed.Select(update.key).All(table.state.ContainsKey)) {
if (has_errors)
sb.AppendLine();
sb.Append("\tUpdate - unknown key removed.");
has_errors = true;
}
if (update.all.Count() + update.removed.Count() - update.added.Count() != table.state.Count) {
if (has_errors)
sb.AppendLine();
sb.Append("\tTable is out of sync with Update.");
has_errors = true;
}
if (has_errors) {
sb.Insert(0, $"TableUpdate unable to apply incremental update ({TableUpdateTypeDesc<T, TKey>.Value}). Will attempt to get back on track by performing a full reset. The following errors were detected:\r\n");
Log.DataFlow.warn(sb.ToString());
return TableUpdate<T, TKey>.update(table, update.all.ToDictionary(update.key), value_comparer ?? EqualityComparer<T>.Default);
}
return TableUpdate<T, TKey>.update(table, update.added, value_comparer == null ? update.changes : update.changes.Actual(value_comparer), update.removed, update.key);
});
public static IObservable<TableUpdate<T, TKey>> AssertIncremental<T, TKey>(this IObservable<TableUpdate<T, TKey>> source) where TKey : IEquatable<TKey> {
return Observable.Create<TableUpdate<T, TKey>>(o => {
var last = TableUpdate<T, TKey>.empty;
return source.Subscribe(update => {
Log.assert(!update.added.Select(update.key_extractor).Any(last.state.ContainsKey), "TableUpdate - duplicate keys added.");
Log.assert(update.removed.Select(update.key_extractor).All(last.state.ContainsKey), "TableUpdate - unknown key removed.");
Log.assert(update.state.Count + update.removed.Count() - update.added.Count() == last.state.Count, "TableUpdate - table is out of sync with Update");
last = update;
o.OnNext(update);
});
});
}
public static IObservable<TableUpdate<T, TKey>> AssertIncremental<T, TKey, TOtherKey>(this IObservable<TableUpdate<T, TKey>> source, Func<T, TOtherKey> convert_key) where TKey : IEquatable<TKey> where TOtherKey : IEquatable<TOtherKey> {
return Observable.Create<TableUpdate<T, TKey>>(o => {
var last = TableUpdate<T, TKey>.empty;
return source.Subscribe(update => {
Log.assert(!update.added.Select(update.key_extractor).Any(last.state.ContainsKey), "TableUpdate - duplicate keys added.");
Log.assert(update.removed.Select(update.key_extractor).All(last.state.ContainsKey), "TableUpdate - unknown key removed.");
int total_count = update.state.Count;
int removed_count = update.removed.Count();
int added_count = update.added.Count();
int last_count = last.state.Count;
Log.assert(total_count + removed_count - added_count == last_count, "TableUpdate - table is out of sync with Update");
//
Log.assert(update.added.Select(convert_key).Distinct().Count() == added_count, "TableUpdate - duplicate converted keys added.");
Log.assert(update.removed.Select(convert_key).Distinct().Count() == removed_count, "TableUpdate - unknown converted key removed.");
Log.assert(update.state.Values.Select(convert_key).Distinct().Count() == update.state.Count, "TableUpdate - table (converted keys) is out of sync with Update");
//
last = update;
o.OnNext(update);
});
});
}
public static IObservable<TableUpdate<T, TKey>> BufferTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, TimeSpan throttle) where TKey : IEquatable<TKey> =>
source.Buffer(throttle)
.Where(updates => updates.Count > 0)
.Scan(TableUpdate<T, TKey>.empty,
(table, updates) => updates.Aggregate(table, (agg, t) => TableUpdate<T, TKey>.update(agg, t.added, t.changes, t.removed, t.key_extractor)));
// Fold updates and emit at max 1/interval
public static IObservable<Update<T,TKey>> Naggle<T, TKey>(this IObservable<Update<T,TKey>> source, TimeSpan interval, IScheduler scheduler) where TKey : IEquatable<TKey> {
// fold updates together and emit on TimeSpan (max) interval. Emit on threadpool
return Observable.Create<Update<T,TKey>>(observer => {
object guard = new object();
Update<T, TKey> cache = null;
bool finished = false;
Log.DataFlow.diag($"Create Naggle {typeof(T).Name}");
var d1 = source.Subscribe(update => {
Log.DataFlow.diag($"Naggle source updated {typeof(T).Name}");
// subscribe to the source, merge updates (Quickbus thread)
lock (guard) {
if (cache == null) cache = update;
else cache = cache.merge(update);
}
},
() => { finished = true; });
// publish on interval
var d2 = Observable.Interval(interval, scheduler).Subscribe(_ => {
Update<T, TKey> update;
lock (guard) {
update = cache;
cache = null;
}
if (update != null) {
Log.DataFlow.diag($"Naggle fire {typeof(T).Name}");
observer.OnNext(update);
}
if (finished) {
observer.OnCompleted();
}
});
return new CompositeDisposable(d1, d2);
});
}
public static IObservable<TableUpdate<T,TKey>> Cache<T, TKey>(this IObservable<TableUpdate<T, TKey>> source) where TKey : IEquatable<TKey> {
var inner = source.Replay(1);
Log.DataFlow.diag($"Create Cache {typeof(T).Name}");
return Observable.Create<TableUpdate<T,TKey>>(o => {
inner.Connect();
bool first = true;
Log.DataFlow.diag($"Cache Subscribe {typeof(T).Name}");
return inner.Subscribe(upd => {
Log.DataFlow.diag($"Cache Inner: {typeof(T).Name}");
if (first) {
o.OnNext(upd.as_snapshot());
first = false;
}
else {
o.OnNext(upd);
}
});
});
}
internal static IObservable<LegacyTableUpdate<T, TKey>> AddRemoveOnly<T, TKey>(this IObservable<TableUpdate<T, TKey>> source) where TKey : IEquatable<TKey> {
return source.Select(t => new LegacyTableUpdate<T, TKey>(t));
}
public static IObservable<Update<T, TKey>> Filter<T, TKey>(this IObservable<Update<T, TKey>> source, Func<T, bool> predicate) where TKey : IEquatable<TKey> =>
source.Select(u => {
apply_predicate(predicate, u, out IReadOnlyCollection<T> all, out IEnumerable<T> added, out IEnumerable<Change<T>> changes, out IEnumerable<T> removed);
return new Update<T, TKey>(all, added, changes, removed, u.key_extractor);
});
public static IObservable<Update<T, TKey>> Filter<T, TKey>(this IObservable<Update<T, TKey>> source, IObservable<Func<T, bool>> predicate) where TKey : IEquatable<TKey>
{
return Observable.Create<Update<T, TKey>>(observer =>
{
Update<T, TKey> unfiltered_cache = null, filtered_cache = null;
Func<T, bool> pred_cache = null;
Log.DataFlow.diag($"Create Filter {typeof(T).Name}");
return source.CombineLatest(predicate, (upd, pred) => (upd, pred)).Subscribe( tup =>
{
bool pred_changed = !object.ReferenceEquals(tup.pred, pred_cache) && (pred_cache != null);
pred_cache = tup.pred;
unfiltered_cache = tup.upd;
if (pred_changed && filtered_cache != null)
{
//clear out existing
var clear = new Update<T, TKey>(
(IReadOnlyCollection<T>)Array.Empty<T>(),
Enumerable.Empty<T>(),
Enumerable.Empty<Change<T>>(),
filtered_cache.all, unfiltered_cache.key_extractor);
observer.OnNext(clear);
var refilter = unfiltered_cache.all.Where(tup.pred).ToList();
var readd = new Update<T, TKey>(
refilter, refilter,
Enumerable.Empty<Change<T>>(),
Enumerable.Empty<T>(), unfiltered_cache.key_extractor);
observer.OnNext(readd);
filtered_cache = readd;
}
else
{
IReadOnlyCollection<T> all; IEnumerable<T> added, removed; IEnumerable<Change<T>> changed;
apply_predicate(tup.pred, tup.upd, out all, out added, out changed, out removed);
var next = new Update<T, TKey>(all, added, changed, removed, tup.upd.key_extractor);
observer.OnNext(next);
filtered_cache = next;
}
},
() => observer.OnCompleted());
});
}
public static IObservable<TableUpdate<T, TKey>> Filter<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, Func<T, bool> predicate) where TKey : IEquatable<TKey> =>
source.Scan((TableUpdate<T, TKey>)null, (src, tgt) => _apply_predicate(src ?? TableUpdate<T, TKey>.mk_empty(tgt.key_extractor), tgt, predicate, tgt.key_extractor));
public static IObservable<TableUpdate<T, TKey>> Filter<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, Func<T, bool> predicate, Func<T, TKey> key) where TKey : IEquatable<TKey> =>
source.Scan(TableUpdate<T, TKey>.mk_empty(key), (src, tgt) => _apply_predicate(src, tgt, predicate, key));
public static IObservable<TableUpdate<T, TKey>> Filter<T, TKey>(this IObservable<TableUpdate<T, TKey>> table_source, IObservable<Func<T, bool>> predicate_source, Func<T, TKey> key) where TKey : IEquatable<TKey> =>
Filter(table_source, predicate_source, key, _ => true);
public static IObservable<TableUpdate<T, TKey>> Filter<T, TKey>(this IObservable<TableUpdate<T, TKey>> table_source, IObservable<Func<T, bool>> predicate_source, Func<T, TKey> key, Func<T, bool> init_predicate) where TKey : IEquatable<TKey> =>
table_source.AggregateLatest(predicate_source,
(tout, tin, pred) => TableUpdate<T, TKey>.FilterHelper.on_source_changed(tout, tin.New.added, tin.New.changes, tin.New.removed, pred),
(tout, tin, pred) => TableUpdate<T, TKey>.FilterHelper.on_predicate_changed(tout, tin.state, pred.Old, pred.New),
() => TableUpdate<T, TKey>.mk_empty(key),
TableUpdate<T, TKey>.mk_empty(key),
init_predicate);
public static IObservable<(TableUpdate<T, TKey> filtered, TableUpdate<T, TKey> raw)> FilterWithRaw<T, TKey>(this IObservable<TableUpdate<T, TKey>> table_source, IObservable<Func<T, bool>> predicate_source, Func<T, TKey> key, Func<T, bool> init_predicate) where TKey : IEquatable<TKey> {
var empty = TableUpdate<T, TKey>.mk_empty(key);
return table_source.AggregateLatest<TableUpdate<T, TKey>, Func<T, bool>, (TableUpdate<T, TKey> filtered, TableUpdate<T, TKey> raw)>(predicate_source,
(tout, tin, pred) => {
var filtered = TableUpdate<T, TKey>.FilterHelper.on_source_changed(tout.filtered, tin.New.added, tin.New.changes, tin.New.removed, pred);
return (filtered, tin.New);
},
(tout, tin, pred) => {
var filtered = TableUpdate<T, TKey>.FilterHelper.on_predicate_changed(tout.filtered, tin.state, pred.Old, pred.New);
return (filtered, tin);
},
() => (empty, empty),
empty,
init_predicate);
}
public static IObservable<TAggregate> AggregateLatest<TLeft, TRight, TAggregate>(this IObservable<TLeft> source, IObservable<TRight> right,
Func<TAggregate, Change<TLeft>, TRight, TAggregate> update_left, Func<TAggregate, TLeft, Change<TRight>, TAggregate> update_right, Func<TAggregate> seed_factory, TLeft init_left = default, TRight init_right = default) =>
Observable.Create<TAggregate>(o => {
Log.DataFlow.diag($"Create AggregateLatest (Left: {typeof(TLeft)}, Right: {typeof(TRight)}).");
var aggregator = new ConcurrentAggregator<TLeft, TRight, TAggregate>(update_left, update_right, o.OnNext, seed_factory(), init_left, init_right);
return new CompositeDisposable(source.Subscribe(aggregator.on_left), right.Subscribe(aggregator.on_right));
});
public static IObservable<TResult> AggregateLatest<TLeft, TRight, TAggregate, TResult>(
this IObservable<TLeft> source,
IObservable<TRight> right,
Func<TAggregate, Change<TLeft>, TRight, TAggregate> update_left,
Func<TAggregate, TLeft, Change<TRight>, TAggregate> update_right,
Func<TAggregate> seed_factory,
Func<TAggregate, TResult> result_selector,
TLeft init_left = default,
TRight init_right = default) =>
//
Observable.Create<TResult>(o => {
Log.DataFlow.diag($"Create AggregateLatest (Left: {typeof(TLeft)}, Right: {typeof(TRight)}).");
var aggregator = new ConcurrentAggregator<TLeft, TRight, TAggregate>(update_left, update_right, agg => o.OnNext(result_selector(agg)), seed_factory(), init_left, init_right);
return new CompositeDisposable(source.Subscribe(aggregator.on_left), right.Subscribe(aggregator.on_right));
});
public static IObservable<TableUpdate<T, TKey>> Operate<T, TKey, TParam>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TParam> param_source, Func<T, TParam, T> operate, TParam init_param = default) where TKey : IEquatable<TKey> {
return AggregateLatest(source, param_source, on_left, on_right, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, init_param);
//
TableUpdate<T, TKey> on_left(TableUpdate<T, TKey> last_output, Change<TableUpdate<T, TKey>> change, TParam last_param) {
return TableUpdate<T, TKey>.update(
last_output,
change.New.added.Select(get_value),
change.New.changes.Select(get_change),
change.New.removed.Select(get_value),
change.New.key_extractor);
T get_value(T v) =>
operate(v, last_param);
Change<T> get_change(Change<T> ch) =>
new Change<T>(get_value(ch.Old), get_value(ch.New));
}
TableUpdate<T, TKey> on_right(TableUpdate<T, TKey> last_output, TableUpdate<T, TKey> last_input, Change<TParam> change) {
var param = change.New;
return TableUpdate<T, TKey>.update(
last_output,
Enumerable.Empty<T>(),
last_output.state.Values.Select(convert_value),
Enumerable.Empty<T>(),
last_output.key_extractor);
Change<T> convert_value(T v) =>
new Change<T>(v, operate(v, param));
}
}
static TableUpdate<T, TKey> _apply_predicate<T, TKey>(TableUpdate<T, TKey> tgt_table, TableUpdate<T, TKey> src_table, Func<T, bool> predicate, Func<T, TKey> key) where TKey : IEquatable<TKey> {
var added = src_table.added.Where(predicate).ToList();
var removed = src_table.removed.Where(predicate).ToList();
var changes = new List<Change<T>>();
foreach (var ch in src_table.changes) {
if (predicate(ch.Old)) {
if (predicate(ch.New)) {
changes.Add(ch);
} else {
removed.Add(ch.Old);
}
} else if (predicate(ch.New))
added.Add(ch.New);
}
return TableUpdate<T, TKey>.update(tgt_table, added, changes, removed, key);
}
private static void apply_predicate<T, TKey>(Func<T, bool> predicate, Update<T, TKey> cache,
out IReadOnlyCollection<T> all, out IEnumerable<T> added,
out IEnumerable<Change<T>> changed, out IEnumerable<T> removed) where TKey : IEquatable<TKey>
{
var adds = new List<T>();
adds.AddRange(cache.added.Where(predicate));
var rems = new List<T>();
rems.AddRange(cache.removed.Where(predicate));
var diffs = new List<Change<T>>();
foreach (var change in cache.changes)
{
if(!predicate(change.Old) && predicate(change.New))
adds.Add(change.New);
else if(predicate(change.Old) && !predicate(change.New))
rems.Add(change.Old);
else if(predicate(change.Old) && predicate(change.New))
diffs.Add(change);
}
all = cache.all.Where(predicate).ToList();
added = adds;
changed = diffs;
removed = rems;
}
// This method does everything lazily so it's cheap until the collections are enumerated
public static IObservable<Update<T2, TKey>>
Select<T1, T2, TKey>(this IObservable<Update<T1, TKey>> src, Func<T1, T2> map, Func<T2, TKey> key) where TKey : IEquatable<TKey> {
return src.Select(update => {
return new Update<T2, TKey>(
update.all.Select(map),
update.added.Select(map),
update.changes.Select(ch => new Change<T2>(map(ch.Old), map(ch.New))),
update.removed.Select(map),
key);
});
}
// This method does everything lazily so it's cheap until the collections are enumerated
public static IObservable<Update<T2, TKey2>>
SelectKeyValue<T1, T2, TKey, TKey2>(this IObservable<Update<T1, TKey>> src, Func<T1, T2> map, Func<T2, TKey2> key) where TKey : IEquatable<TKey> where TKey2 : IEquatable<TKey2> {
return src.Select(update => {
return new Update<T2, TKey2>(
update.all.Select(map),
update.added.Select(map),
update.changes.Select(ch => new Change<T2>(map(ch.Old), map(ch.New))),
update.removed.Select(map),
key);
});
}
// Constructs a new dictionary so not as cheap as Select on the Update<> type
public static IObservable<TableUpdate<T2, TKey>>
Select<T1, T2, TKey>(this IObservable<TableUpdate<T1, TKey>> src, Func<T1, T2> map, Func<T2, TKey> key) where TKey : IEquatable<TKey> =>
Observable.Create<TableUpdate<T2, TKey>>(observer => {
var cache = ImmutableDictionary<TKey, T2>.Empty;
return src.Subscribe(update => {
var added = update.added.Select(map);
var removed = update.removed.Select(map);
var changes = update.changes.Select(ch => new Change<T2>(map(ch.Old), map(ch.New)));
var builder = cache.ToBuilder();
builder.RemoveRange(removed.Select(key));
builder.AddRange(added.Select(t => new KeyValuePair<TKey, T2>(key(t), t)));
foreach (var (o, n) in changes) {
Debug.Assert(key(o).Equals(key(n)));
builder[key(o)] = n;
}
cache = builder.ToImmutable();
observer.OnNext(new TableUpdate<T2, TKey>(cache, added, changes, removed, key));
},
err => { observer.OnError(err); },
() => { observer.OnCompleted(); });
});
// Constructs a new dictionary so not as cheap as Select on the Update<> type
public static IObservable<TableUpdate<T2, TKey2>>
SelectKeyValue<T1, T2, TKey, TKey2>(this IObservable<TableUpdate<T1, TKey>> src, Func<T1, T2> map, Func<T2, TKey2> key) where TKey : IEquatable<TKey> where TKey2 : IEquatable<TKey2> =>
Observable.Create<TableUpdate<T2, TKey2>>(observer => {
var cache = ImmutableDictionary<TKey2, T2>.Empty;
return src.Subscribe(update => {
var added = update.added.Select(map);
var removed = update.removed.Select(map);
var changes = update.changes.Select(ch => new Change<T2>(map(ch.Old), map(ch.New)));
var builder = cache.ToBuilder();
builder.RemoveRange(removed.Select(key));
builder.AddRange(added.Select(t => new KeyValuePair<TKey2, T2>(key(t), t)));
foreach (var (o, n) in changes) {
Debug.Assert(key(o).Equals(key(n)));
builder[key(o)] = n;
}
cache = builder.ToImmutable();
observer.OnNext(new TableUpdate<T2, TKey2>(cache, added, changes, removed, key));
},
err => { observer.OnError(err); },
() => { observer.OnCompleted(); });
});
public static IObservable<TableUpdate<(TLeft left, TRight right), TKey>> LeftJoin<TLeft, TRight, TKey>(this IObservable<TableUpdate<TLeft, TKey>> left_source, IObservable<TableUpdate<TRight, TKey>> right_source,
Func<TLeft, TKey> left_key, Func<TRight, TKey> right_key, TRight right_default = default)
where TKey : IEquatable<TKey> =>
LeftJoin<TLeft, TKey, TRight, TKey, (TLeft left, TRight right)>(left_source, right_source, left_key, right_key, left_key, (lv, lk, rv, rk) => (lv, rv), (lv, lk, rk) => (lv, right_default), t => left_key(t.left));
public static IObservable<TableUpdate<TResult, TKey>> LeftJoin<TLeft, TRight, TResult, TKey>(this IObservable<TableUpdate<TLeft, TKey>> left_source, IObservable<TableUpdate<TRight, TKey>> right_source,
Func<TLeft, TKey> left_key, Func<TRight, TKey> right_key, Func<TLeft, TKey, TRight, TResult> result_selector, Func<TLeft, TKey, TResult> absent_right_result_selector, Func<TResult, TKey> result_key)
where TKey : IEquatable<TKey> =>
LeftJoin(left_source, right_source, left_key, right_key, left_key, (lv, lk, rv, rk) => result_selector(lv, lk, rv), (lv, lk, rk) => absent_right_result_selector(lv, lk), result_key);
public static IObservable<TableUpdate<TResult, TLeftKey>> LeftJoin<TLeft, TLeftKey, TRight, TRightKey, TResult>(this IObservable<TableUpdate<TLeft, TLeftKey>> left_source, IObservable<TableUpdate<TRight, TRightKey>> right_source,
Func<TLeft, TLeftKey> left_key, Func<TRight, TRightKey> right_key, Func<TLeft, TRightKey> join_key, Func<TLeft, TLeftKey, TRight, TRightKey, TResult> result_selector, Func<TLeft, TLeftKey, TRightKey, TResult> absent_right_result_selector, Func<TResult, TLeftKey> result_key)
where TLeftKey : IEquatable<TLeftKey>
where TRightKey : IEquatable<TRightKey> =>
TableUpdateLeftJoin.Create(left_source, right_source, left_key, right_key, join_key, result_selector, absent_right_result_selector, result_key);
public static IObservable<TableUpdate<TResult, TKey>> FullJoin<TLeft, TKey, TRight, TResult>(this IObservable<TableUpdate<TLeft, TKey>> left_source, IObservable<TableUpdate<TRight, TKey>> right_source,
Func<TLeft, TKey> left_key, Func<TRight, TKey> right_key, Func<TLeft, TRight, TKey, TResult> result_selector, Func<TLeft, TKey, TResult> absent_right_result_selector, Func<TRight, TKey, TResult> absent_left_result_selector, Func<TResult, TKey> result_key)
where TKey : IEquatable<TKey> =>
TableUpdateFullJoin.Create(left_source, right_source, left_key, right_key, result_selector, absent_right_result_selector, absent_left_result_selector, result_key);
public static IObservable<TableUpdate<TableUpdateGrouping<T, TKey, TGroupKey>, TGroupKey>> GroupTable<T, TKey, TGroupKey>(this IObservable<TableUpdate<T, TKey>> source, Func<T, TKey> key, Func<T, TGroupKey> group_key)
where TKey : IEquatable<TKey>
where TGroupKey : IEquatable<TGroupKey> =>
TableUpdateGroupTable.Create(source, key, group_key);
static class TableUpdateTypeDesc<T, TKey> {
public static readonly string Value =
$"T is '{typeof(T)}', TKey is '{typeof(TKey)}'";
}
}
}