using bts.utils; using System; using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Disposables; using System.Reactive.Linq; using System.Text; namespace bts.api.core { class RawUpdate { // RawUpdate uses List instead of IReadonlyList to avoid virtual dispatch on a type not likely to be see in a public api surface // There is a single stack allocated RawUpdate per subscription public RawUpdate() { is_reset = false; added_items = new List(); added_keys = new List(); changed_items = new List(); changed_keys = new List(); removed_keys = new List(); } public RawUpdate MarkAsReset() { Clear(); is_reset = true; return this; } public void Clear() { is_reset = false; added_items.Clear(); added_keys.Clear(); changed_items.Clear(); changed_keys.Clear(); removed_keys.Clear(); } public bool is_reset { get; private set; } public List added_items { get; } public List added_keys { get; } public List changed_items { get; } public List changed_keys { get; } public List removed_keys { get; } #if DEBUG public override string ToString() { var added = string.Join(", ", added_items); var changed = string.Join(", ", changed_items); var removed = string.Join(", ", removed_keys); return $"added [{added}] changed [{changed}] removed [{removed}]"; } #endif } public readonly struct Change { public Change(T o, T n) { Old = o; New = n; } public readonly T Old; public readonly T New; public Change merge(Change newer) { return new Change(Old, newer.New); } public void Deconstruct(out T Old, out T New) { Old = this.Old; New = this.New; } public override string ToString() { return $"{{{Old}=>{New}}}"; } } public class Update where TKey : IEquatable { readonly Func _key; internal Update( IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor) { this.added = added ?? new T[0]; this.removed = removed ?? new T[0]; this.changes = changes ?? new Change[0]; this.all = all; _key = key_extractor; } public static Update empty(Func key_extractor) => new Update(new T[0], new T[0], new Change[0], new T[0], key_extractor); public Update merge(Update next) { var added = this.added as KeyedValueSet ?? new KeyedValueSet(this.added, _key); var changes = this.changed as ChangeDict ?? new ChangeDict(this.changes, _key); var removed = this.removed as KeyedValueSet ?? new KeyedValueSet(this.removed, _key); // Add foreach (var v in next.added) { var k = _key(v); if (removed.Remove(k, out var oldval)) { changes.AddOrMerge(new Change(oldval, v)); } else { added[k] = v; } } // Change foreach (var c in next.changes) { var k = _key(c.Old); if (!added.Replace(k, c.New)) changes.AddOrMerge(c); } // Remove foreach (var r in next.removed) { var k = _key(r); if (!added.Remove(k)) removed[k] = changes.TryRemove(k, out var ch) ? ch.Old : r; } return new Update( next.all, added, changes, removed, _key); } public IEnumerable all { get; } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); public IEnumerable removed { get; } public TKey key(T t) { return _key(t); } internal Func key_extractor => _key; #if DEBUG public override string ToString() { string key(T t) => _key(t).ToString().Trim(); var a = string.Join(", ", added.Select(key)); var c = string.Join(", ", changed.Select(key)); var r = string.Join(", ", removed.Select(key)); return $"{{added [{a}] changed [{c}] removed [{r}]}}"; } #endif } class KeyedValueSet : IEnumerable { readonly Dictionary _cache = new Dictionary(); public KeyedValueSet(IEnumerable value, Func key) { foreach (var v in value) _cache[key(v)] = v; } public TValue this[TKey key] { set => _cache[key] = value; } public bool Remove(TKey key) => _cache.Remove(key); public bool Remove(TKey key, out TValue value) => _cache.TryGetValue(key, out value) && _cache.Remove(key); public bool Replace(TKey key, TValue value) { if (_cache.ContainsKey(key)) { _cache[key] = value; return true; } return false; } public IEnumerator GetEnumerator() => _cache.Values.GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } class ChangeDict : IEnumerable> { Dictionary> _changes; Func, TKey> _key; public ChangeDict(IEnumerable> changes, Func key) { _key = x => key(x.New); _changes = changes.ToDictionary(_key); } public Change? AddOrMerge(Change newer) { var key = _key(newer); if (_changes.TryGetValue(key, out Change old)) { _changes[key] = old.merge(newer); return old; } _changes.Add(key, newer); return null; } public bool Remove(TKey key) => _changes.Remove(key); public bool TryRemove(TKey key, out Change result) => _changes.TryGetValue(key, out result) && _changes.Remove(key); #region IEnumerable public IEnumerator> GetEnumerator() { return _changes.Values.GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return _changes.Values.GetEnumerator(); } #endregion } public class TableUpdate where TKey : IEquatable { // warning: this does not have a key_extractor public static readonly TableUpdate empty = new TableUpdate(ImmutableDictionary.Empty, Array.Empty(), Array.Empty>(), Array.Empty(), null); readonly ImmutableDictionary _state; public static TableUpdate mk_empty(Func key_extractor) => new TableUpdate(ImmutableDictionary.Empty, Array.Empty(), Array.Empty>(), Array.Empty(), key_extractor); internal TableUpdate( ImmutableDictionary init, IEnumerable added, IEnumerable> changed, IEnumerable removed, Func maybe_key_extractor) { _state = init; this.added = added; this.changes = changed; this.removed = removed; this.key_extractor = maybe_key_extractor; } public IReadOnlyDictionary state { get { return _state; } } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); public IEnumerable removed { get; } internal Func key_extractor { get; private set; } internal static TableUpdate update(TableUpdate table, Update update) => TableUpdate.update(table, update.added, update.changes, update.removed, update.key); internal static TableUpdate update(TableUpdate table, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key) { var builder = table._state.ToBuilder(); builder.RemoveRange(removed.Select(i => key(i))); builder.AddRange(added.Select(i => new KeyValuePair(key(i), i))); foreach (var i in changes) { builder[key(i.New)] = i.New; } return new TableUpdate(builder.ToImmutableDictionary(), added, changes, removed, key); } internal static TableUpdate update(TableUpdate table, IReadOnlyDictionary new_state, IEqualityComparer value_comparer) { var builder = table._state.ToBuilder(); // removed/changes var removed = new List(); var changes = new List>(); foreach (var kv in table._state) { if (new_state.TryGetValue(kv.Key, out T new_val)) { if (!value_comparer.Equals(kv.Value, new_val)) { changes.Add(new Change(kv.Value, new_val)); builder[kv.Key] = new_val; } } else { removed.Add(kv.Value); builder.Remove(kv.Key); } } // added var added = new List(); foreach (var kv in new_state) { if (!table._state.ContainsKey(kv.Key)) { added.Add(kv.Value); builder.Add(kv); } } return new TableUpdate(builder.ToImmutableDictionary(), added, changes, removed, table.key_extractor); } internal TableUpdate as_snapshot() { return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0], key_extractor); } internal Builder to_builder() => new Builder(this); public static class FilterHelper { internal static TableUpdate on_source_changed(TableUpdate filtered_target, IEnumerable raw_added, IEnumerable> raw_changes, IEnumerable raw_removed, Func predicate) { var builder = filtered_target._state.ToBuilder(); List added, removed; List> changes; if (predicate == null) { added = raw_added.ToList(); removed = raw_removed.ToList(); changes = raw_changes.ToList(); } else { added = new List(raw_added.Where(predicate)); removed = new List(raw_removed.Where(predicate)); changes = new List>(); foreach (var ch in raw_changes) { if (predicate(ch.Old)) { if (predicate(ch.New)) { changes.Add(ch); } else { removed.Add(ch.Old); } } else if (predicate(ch.New)) { added.Add(ch.New); } } } var key = filtered_target.key_extractor; builder.RemoveRange(removed.Select(key)); foreach (var ch in changes) builder[key(ch.New)] = ch.New; builder.AddRange(added.Select(v => new KeyValuePair(key(v), v))); return new TableUpdate(builder.ToImmutable(), added, changes, removed, key); } internal static TableUpdate on_predicate_changed(TableUpdate filtered_target, IReadOnlyDictionary unfiltered_state, Func old_predicate, Func new_predicate) { var added = new List(); var removed = new List(); if (new_predicate == old_predicate) { // nothing changed - this method probably shouldn't be called when nothing changes. Having said that, it doesn't really hurt anything, // so don't throw a metaphorical wrench into the gears by throwing a literal exception. Just return the table as is. if (filtered_target.has_changes()) filtered_target = new TableUpdate(filtered_target._state, Enumerable.Empty(), Enumerable.Empty>(), Enumerable.Empty(), filtered_target.key_extractor); return filtered_target; } if (old_predicate == null) { removed.AddRange(filtered_target.state.Values.Where(v => !new_predicate(v))); } else if (new_predicate == null) { added.AddRange(unfiltered_state.Values.Where(v => !old_predicate(v))); } else { foreach (var kv in unfiltered_state) { if (old_predicate(kv.Value)) { if (new_predicate(kv.Value)) { continue; } else { removed.Add(kv.Value); } } else if (new_predicate(kv.Value)) { added.Add(kv.Value); } } } var builder = filtered_target._state.ToBuilder(); var key = filtered_target.key_extractor; builder.RemoveRange(removed.Select(key)); builder.AddRange(added.Select(v => new KeyValuePair(key(v), v))); return new TableUpdate(builder.ToImmutable(), added, Array.Empty>(), removed, key); } } /// /// This class is used to aggregate a series of changes into a single update. /// internal class Builder { readonly TableUpdate _state; readonly Dictionary _added = new Dictionary(), _removed = new Dictionary(); readonly Dictionary> _changes = new Dictionary>(); public Builder(TableUpdate state) => _state = state; public void add_or_update(T value) => add_or_update(_state.key_extractor(value), value); public void add_or_update(TKey key, T value) { if (_state.state.TryGetValue(key, out T curr)) { _removed.Remove(key); _changes[key] = new Change(curr, value); } else { _added[key] = value; } } public void remove(TKey key) { if (_state.state.TryGetValue(key, out T orig)) { _removed[key] = orig; _changes.Remove(key); } else { _added.Remove(key); } } public void merge_table(TableUpdate table, Func _key) { foreach (T v in table.removed) remove(_key(v)); foreach (T v in table.added.Concat(table.changed)) add_or_update(_key(v), v); } public void clear() { _added.Clear(); _changes.Clear(); foreach (var kv in _state.state) _removed[kv.Key] = kv.Value; } public TableUpdate to_table() { var builder = _state._state.ToBuilder(); builder.RemoveRange(_removed.Keys); foreach (var kv in _changes) builder[kv.Key] = kv.Value.New; builder.AddRange(_added); return new TableUpdate(builder.ToImmutableDictionary(), _added.Values, _changes.Values, _removed.Values, _state.key_extractor); } public TableUpdate to_table_checked(IEqualityComparer comparer) { var builder = _state._state.ToBuilder(); builder.RemoveRange(_removed.Keys); foreach (var kv in _changes) { if (!comparer.Equals(kv.Value.Old, kv.Value.New)) builder[kv.Key] = kv.Value.New; } builder.AddRange(_added); return new TableUpdate(builder.ToImmutableDictionary(), _added.Values, _changes.Values, _removed.Values, _state.key_extractor); } } } internal class LegacyTableUpdate where TKey : IEquatable { public LegacyTableUpdate( TableUpdate update) { this.state = update.state; this.added = update.added.Concat(update.changes.Select(t => t.New)); this.removed = update.removed.Concat(update.changes.Select(t => t.Old)); } public IReadOnlyDictionary state { get; } public IEnumerable added { get; } public IEnumerable removed { get; } } public static class ObservableUpdateExtensions { public static IEnumerable Old(this IEnumerable> changes) { foreach (var ch in changes) yield return ch.Old; } public static IEnumerable New(this IEnumerable> changes) { foreach (var ch in changes) yield return ch.New; } public static IEnumerable> Actual(this IEnumerable> changes) where T : IEquatable { foreach (var ch in changes) { if (!ch.New.Equals(ch.Old)) yield return ch; } } public static IEnumerable> Actual(this IEnumerable> changes, IEqualityComparer comparer) { foreach (var ch in changes) { if (!comparer.Equals(ch.New, ch.Old)) yield return ch; } } internal static bool has_changes(this TableUpdate table) where TKey : IEquatable => table.added.Any() || table.changes.Any() || table.removed.Any(); internal static IObservable> reconstitute(this IObservable> source, Func key_extractor) where TKey : IEquatable { var state = new Dictionary(); return source.Select(raw => { if (raw.is_reset) { var removed = state.Values.ToList(); state.Clear(); return new Update(new T[0], new T[0], new Change[0], removed, key_extractor); } else { var removed = new List(raw.removed_keys.Count); foreach (var r in raw.removed_keys) { T temp; if (state.TryGetValue(r, out temp)) { removed.Add(temp); state.Remove(r); } } for (int x = 0; x < raw.added_items.Count; ++x) { state[raw.added_keys[x]] = raw.added_items[x]; } var changes = new List>(raw.changed_items.Count); for (int x = 0; x < raw.changed_items.Count; ++x) { var old = state[raw.changed_keys[x]]; state[raw.changed_keys[x]] = raw.changed_items[x]; changes.Add(new Change(old, raw.changed_items[x])); } return new Update(state.Values.ToList(), raw.added_items.ToList(), changes, removed, key_extractor); } }); } public static IObservable> Table(this IObservable> source) where TKey : IEquatable => source.Scan( new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => TableUpdate.update(table,update)); public static IObservable> Table(this IObservable> source, Func key) where TKey : IEquatable => source.Table(key, EqualityComparer.Default); public static IObservable> Table(this IObservable> source, Func key, IEqualityComparer value_comparer) where TKey : IEquatable => source.Scan( TableUpdate.mk_empty(key), (table, new_state) => TableUpdate.update(table, new_state, value_comparer)); public static IObservable> TableChecked(this IObservable> source, IEqualityComparer value_comparer = null) where TKey : IEquatable => source.Scan( new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => { Log.assert(!update.added.Select(update.key).Any(table.state.ContainsKey), "Update - duplicate keys added."); Log.assert(update.removed.Select(update.key).All(table.state.ContainsKey), "Update - unknown key removed."); Log.assert(update.all.Count() + update.removed.Count() - update.added.Count() == table.state.Count, "Table is out of sync with Update"); return TableUpdate.update(table, update.added, value_comparer == null ? update.changes : update.changes.Actual(value_comparer), update.removed, update.key); }); public static IObservable> TableCheckedAutoCorrect(this IObservable> source, IEqualityComparer value_comparer = null) where TKey : IEquatable => source.Scan( new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => { bool has_errors = false; var sb = new StringBuilder(); if (update.added.Select(update.key).Any(table.state.ContainsKey)) { sb.Append("\tUpdate - duplicate keys added."); has_errors = true; } if (!update.removed.Select(update.key).All(table.state.ContainsKey)) { if (has_errors) sb.AppendLine(); sb.Append("\tUpdate - unknown key removed."); has_errors = true; } if (update.all.Count() + update.removed.Count() - update.added.Count() != table.state.Count) { if (has_errors) sb.AppendLine(); sb.Append("\tTable is out of sync with Update."); has_errors = true; } if (has_errors) { sb.Insert(0, $"TableUpdate unable to apply incremental update ({TableUpdateTypeDesc.Value}). Will attempt to get back on track by performing a full reset. The following errors were detected:\r\n"); Log.DataFlow.warn(sb.ToString()); return TableUpdate.update(table, update.all.ToDictionary(update.key), value_comparer ?? EqualityComparer.Default); } return TableUpdate.update(table, update.added, value_comparer == null ? update.changes : update.changes.Actual(value_comparer), update.removed, update.key); }); public static IObservable> AssertIncremental(this IObservable> source) where TKey : IEquatable { return Observable.Create>(o => { var last = TableUpdate.empty; return source.Subscribe(update => { Log.assert(!update.added.Select(update.key_extractor).Any(last.state.ContainsKey), "TableUpdate - duplicate keys added."); Log.assert(update.removed.Select(update.key_extractor).All(last.state.ContainsKey), "TableUpdate - unknown key removed."); Log.assert(update.state.Count + update.removed.Count() - update.added.Count() == last.state.Count, "TableUpdate - table is out of sync with Update"); last = update; o.OnNext(update); }); }); } public static IObservable> AssertIncremental(this IObservable> source, Func convert_key) where TKey : IEquatable where TOtherKey : IEquatable { return Observable.Create>(o => { var last = TableUpdate.empty; return source.Subscribe(update => { Log.assert(!update.added.Select(update.key_extractor).Any(last.state.ContainsKey), "TableUpdate - duplicate keys added."); Log.assert(update.removed.Select(update.key_extractor).All(last.state.ContainsKey), "TableUpdate - unknown key removed."); int total_count = update.state.Count; int removed_count = update.removed.Count(); int added_count = update.added.Count(); int last_count = last.state.Count; Log.assert(total_count + removed_count - added_count == last_count, "TableUpdate - table is out of sync with Update"); // Log.assert(update.added.Select(convert_key).Distinct().Count() == added_count, "TableUpdate - duplicate converted keys added."); Log.assert(update.removed.Select(convert_key).Distinct().Count() == removed_count, "TableUpdate - unknown converted key removed."); Log.assert(update.state.Values.Select(convert_key).Distinct().Count() == update.state.Count, "TableUpdate - table (converted keys) is out of sync with Update"); // last = update; o.OnNext(update); }); }); } public static IObservable> BufferTable(this IObservable> source, TimeSpan throttle) where TKey : IEquatable => source.Buffer(throttle) .Where(updates => updates.Count > 0) .Scan(TableUpdate.empty, (table, updates) => updates.Aggregate(table, (agg, t) => TableUpdate.update(agg, t.added, t.changes, t.removed, t.key_extractor))); // Fold updates and emit at max 1/interval public static IObservable> Naggle(this IObservable> source, TimeSpan interval, IScheduler scheduler) where TKey : IEquatable { // fold updates together and emit on TimeSpan (max) interval. Emit on threadpool return Observable.Create>(observer => { object guard = new object(); Update cache = null; bool finished = false; Log.DataFlow.diag($"Create Naggle {typeof(T).Name}"); var d1 = source.Subscribe(update => { Log.DataFlow.diag($"Naggle source updated {typeof(T).Name}"); // subscribe to the source, merge updates (Quickbus thread) lock (guard) { if (cache == null) cache = update; else cache = cache.merge(update); } }, () => { finished = true; }); // publish on interval var d2 = Observable.Interval(interval, scheduler).Subscribe(_ => { Update update; lock (guard) { update = cache; cache = null; } if (update != null) { Log.DataFlow.diag($"Naggle fire {typeof(T).Name}"); observer.OnNext(update); } if (finished) { observer.OnCompleted(); } }); return new CompositeDisposable(d1, d2); }); } public static IObservable> Cache(this IObservable> source) where TKey : IEquatable { var inner = source.Replay(1); Log.DataFlow.diag($"Create Cache {typeof(T).Name}"); return Observable.Create>(o => { inner.Connect(); bool first = true; Log.DataFlow.diag($"Cache Subscribe {typeof(T).Name}"); return inner.Subscribe(upd => { Log.DataFlow.diag($"Cache Inner: {typeof(T).Name}"); if (first) { o.OnNext(upd.as_snapshot()); first = false; } else { o.OnNext(upd); } }); }); } internal static IObservable> AddRemoveOnly(this IObservable> source) where TKey : IEquatable { return source.Select(t => new LegacyTableUpdate(t)); } public static IObservable> Filter(this IObservable> source, Func predicate) where TKey : IEquatable => source.Select(u => { apply_predicate(predicate, u, out IReadOnlyCollection all, out IEnumerable added, out IEnumerable> changes, out IEnumerable removed); return new Update(all, added, changes, removed, u.key_extractor); }); public static IObservable> Filter(this IObservable> source, IObservable> predicate) where TKey : IEquatable { return Observable.Create>(observer => { Update unfiltered_cache = null, filtered_cache = null; Func pred_cache = null; Log.DataFlow.diag($"Create Filter {typeof(T).Name}"); return source.CombineLatest(predicate, (upd, pred) => (upd, pred)).Subscribe( tup => { bool pred_changed = !object.ReferenceEquals(tup.pred, pred_cache) && (pred_cache != null); pred_cache = tup.pred; unfiltered_cache = tup.upd; if (pred_changed && filtered_cache != null) { //clear out existing var clear = new Update( (IReadOnlyCollection)Array.Empty(), Enumerable.Empty(), Enumerable.Empty>(), filtered_cache.all, unfiltered_cache.key_extractor); observer.OnNext(clear); var refilter = unfiltered_cache.all.Where(tup.pred).ToList(); var readd = new Update( refilter, refilter, Enumerable.Empty>(), Enumerable.Empty(), unfiltered_cache.key_extractor); observer.OnNext(readd); filtered_cache = readd; } else { IReadOnlyCollection all; IEnumerable added, removed; IEnumerable> changed; apply_predicate(tup.pred, tup.upd, out all, out added, out changed, out removed); var next = new Update(all, added, changed, removed, tup.upd.key_extractor); observer.OnNext(next); filtered_cache = next; } }, () => observer.OnCompleted()); }); } public static IObservable> Filter(this IObservable> source, Func predicate) where TKey : IEquatable => source.Scan((TableUpdate)null, (src, tgt) => _apply_predicate(src ?? TableUpdate.mk_empty(tgt.key_extractor), tgt, predicate, tgt.key_extractor)); public static IObservable> Filter(this IObservable> source, Func predicate, Func key) where TKey : IEquatable => source.Scan(TableUpdate.mk_empty(key), (src, tgt) => _apply_predicate(src, tgt, predicate, key)); public static IObservable> Filter(this IObservable> table_source, IObservable> predicate_source, Func key) where TKey : IEquatable => Filter(table_source, predicate_source, key, _ => true); public static IObservable> Filter(this IObservable> table_source, IObservable> predicate_source, Func key, Func init_predicate) where TKey : IEquatable => table_source.AggregateLatest(predicate_source, (tout, tin, pred) => TableUpdate.FilterHelper.on_source_changed(tout, tin.New.added, tin.New.changes, tin.New.removed, pred), (tout, tin, pred) => TableUpdate.FilterHelper.on_predicate_changed(tout, tin.state, pred.Old, pred.New), () => TableUpdate.mk_empty(key), TableUpdate.mk_empty(key), init_predicate); public static IObservable<(TableUpdate filtered, TableUpdate raw)> FilterWithRaw(this IObservable> table_source, IObservable> predicate_source, Func key, Func init_predicate) where TKey : IEquatable { var empty = TableUpdate.mk_empty(key); return table_source.AggregateLatest, Func, (TableUpdate filtered, TableUpdate raw)>(predicate_source, (tout, tin, pred) => { var filtered = TableUpdate.FilterHelper.on_source_changed(tout.filtered, tin.New.added, tin.New.changes, tin.New.removed, pred); return (filtered, tin.New); }, (tout, tin, pred) => { var filtered = TableUpdate.FilterHelper.on_predicate_changed(tout.filtered, tin.state, pred.Old, pred.New); return (filtered, tin); }, () => (empty, empty), empty, init_predicate); } public static IObservable AggregateLatest(this IObservable source, IObservable right, Func, TRight, TAggregate> update_left, Func, TAggregate> update_right, Func seed_factory, TLeft init_left = default, TRight init_right = default) => Observable.Create(o => { Log.DataFlow.diag($"Create AggregateLatest (Left: {typeof(TLeft)}, Right: {typeof(TRight)})."); var aggregator = new ConcurrentAggregator(update_left, update_right, o.OnNext, seed_factory(), init_left, init_right); return new CompositeDisposable(source.Subscribe(aggregator.on_left), right.Subscribe(aggregator.on_right)); }); public static IObservable AggregateLatest( this IObservable source, IObservable right, Func, TRight, TAggregate> update_left, Func, TAggregate> update_right, Func seed_factory, Func result_selector, TLeft init_left = default, TRight init_right = default) => // Observable.Create(o => { Log.DataFlow.diag($"Create AggregateLatest (Left: {typeof(TLeft)}, Right: {typeof(TRight)})."); var aggregator = new ConcurrentAggregator(update_left, update_right, agg => o.OnNext(result_selector(agg)), seed_factory(), init_left, init_right); return new CompositeDisposable(source.Subscribe(aggregator.on_left), right.Subscribe(aggregator.on_right)); }); public static IObservable> Operate(this IObservable> source, IObservable param_source, Func operate, TParam init_param = default) where TKey : IEquatable { return AggregateLatest(source, param_source, on_left, on_right, () => TableUpdate.empty, TableUpdate.empty, init_param); // TableUpdate on_left(TableUpdate last_output, Change> change, TParam last_param) { return TableUpdate.update( last_output, change.New.added.Select(get_value), change.New.changes.Select(get_change), change.New.removed.Select(get_value), change.New.key_extractor); T get_value(T v) => operate(v, last_param); Change get_change(Change ch) => new Change(get_value(ch.Old), get_value(ch.New)); } TableUpdate on_right(TableUpdate last_output, TableUpdate last_input, Change change) { var param = change.New; return TableUpdate.update( last_output, Enumerable.Empty(), last_output.state.Values.Select(convert_value), Enumerable.Empty(), last_output.key_extractor); Change convert_value(T v) => new Change(v, operate(v, param)); } } static TableUpdate _apply_predicate(TableUpdate tgt_table, TableUpdate src_table, Func predicate, Func key) where TKey : IEquatable { var added = src_table.added.Where(predicate).ToList(); var removed = src_table.removed.Where(predicate).ToList(); var changes = new List>(); foreach (var ch in src_table.changes) { if (predicate(ch.Old)) { if (predicate(ch.New)) { changes.Add(ch); } else { removed.Add(ch.Old); } } else if (predicate(ch.New)) added.Add(ch.New); } return TableUpdate.update(tgt_table, added, changes, removed, key); } private static void apply_predicate(Func predicate, Update cache, out IReadOnlyCollection all, out IEnumerable added, out IEnumerable> changed, out IEnumerable removed) where TKey : IEquatable { var adds = new List(); adds.AddRange(cache.added.Where(predicate)); var rems = new List(); rems.AddRange(cache.removed.Where(predicate)); var diffs = new List>(); foreach (var change in cache.changes) { if(!predicate(change.Old) && predicate(change.New)) adds.Add(change.New); else if(predicate(change.Old) && !predicate(change.New)) rems.Add(change.Old); else if(predicate(change.Old) && predicate(change.New)) diffs.Add(change); } all = cache.all.Where(predicate).ToList(); added = adds; changed = diffs; removed = rems; } // This method does everything lazily so it's cheap until the collections are enumerated public static IObservable> Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { return src.Select(update => { return new Update( update.all.Select(map), update.added.Select(map), update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), update.removed.Select(map), key); }); } // This method does everything lazily so it's cheap until the collections are enumerated public static IObservable> SelectKeyValue(this IObservable> src, Func map, Func key) where TKey : IEquatable where TKey2 : IEquatable { return src.Select(update => { return new Update( update.all.Select(map), update.added.Select(map), update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), update.removed.Select(map), key); }); } // Constructs a new dictionary so not as cheap as Select on the Update<> type public static IObservable> Select(this IObservable> src, Func map, Func key) where TKey : IEquatable => Observable.Create>(observer => { var cache = ImmutableDictionary.Empty; return src.Subscribe(update => { var added = update.added.Select(map); var removed = update.removed.Select(map); var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); var builder = cache.ToBuilder(); builder.RemoveRange(removed.Select(key)); builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); foreach (var (o, n) in changes) { Debug.Assert(key(o).Equals(key(n))); builder[key(o)] = n; } cache = builder.ToImmutable(); observer.OnNext(new TableUpdate(cache, added, changes, removed, key)); }, err => { observer.OnError(err); }, () => { observer.OnCompleted(); }); }); // Constructs a new dictionary so not as cheap as Select on the Update<> type public static IObservable> SelectKeyValue(this IObservable> src, Func map, Func key) where TKey : IEquatable where TKey2 : IEquatable => Observable.Create>(observer => { var cache = ImmutableDictionary.Empty; return src.Subscribe(update => { var added = update.added.Select(map); var removed = update.removed.Select(map); var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); var builder = cache.ToBuilder(); builder.RemoveRange(removed.Select(key)); builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); foreach (var (o, n) in changes) { Debug.Assert(key(o).Equals(key(n))); builder[key(o)] = n; } cache = builder.ToImmutable(); observer.OnNext(new TableUpdate(cache, added, changes, removed, key)); }, err => { observer.OnError(err); }, () => { observer.OnCompleted(); }); }); public static IObservable> LeftJoin(this IObservable> left_source, IObservable> right_source, Func left_key, Func right_key, TRight right_default = default) where TKey : IEquatable => LeftJoin(left_source, right_source, left_key, right_key, left_key, (lv, lk, rv, rk) => (lv, rv), (lv, lk, rk) => (lv, right_default), t => left_key(t.left)); public static IObservable> LeftJoin(this IObservable> left_source, IObservable> right_source, Func left_key, Func right_key, Func result_selector, Func absent_right_result_selector, Func result_key) where TKey : IEquatable => LeftJoin(left_source, right_source, left_key, right_key, left_key, (lv, lk, rv, rk) => result_selector(lv, lk, rv), (lv, lk, rk) => absent_right_result_selector(lv, lk), result_key); public static IObservable> LeftJoin(this IObservable> left_source, IObservable> right_source, Func left_key, Func right_key, Func join_key, Func result_selector, Func absent_right_result_selector, Func result_key) where TLeftKey : IEquatable where TRightKey : IEquatable => TableUpdateLeftJoin.Create(left_source, right_source, left_key, right_key, join_key, result_selector, absent_right_result_selector, result_key); public static IObservable> FullJoin(this IObservable> left_source, IObservable> right_source, Func left_key, Func right_key, Func result_selector, Func absent_right_result_selector, Func absent_left_result_selector, Func result_key) where TKey : IEquatable => TableUpdateFullJoin.Create(left_source, right_source, left_key, right_key, result_selector, absent_right_result_selector, absent_left_result_selector, result_key); public static IObservable, TGroupKey>> GroupTable(this IObservable> source, Func key, Func group_key) where TKey : IEquatable where TGroupKey : IEquatable => TableUpdateGroupTable.Create(source, key, group_key); static class TableUpdateTypeDesc { public static readonly string Value = $"T is '{typeof(T)}', TKey is '{typeof(TKey)}'"; } } }