using System; using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Disposables; using System.Reactive.Linq; namespace bts.api.core { class RawUpdate { // RawUpdate uses List instead of IReadonlyList to avoid virtual dispatch on a type not likely to be see in a public api surface // There is a single stack allocated RawUpdate per subscription public RawUpdate() { is_reset = false; added_items = new List(); added_keys = new List(); changed_items = new List(); changed_keys = new List(); removed_keys = new List(); } public RawUpdate MarkAsReset() { Clear(); is_reset = true; return this; } public void Clear() { is_reset = false; added_items.Clear(); added_keys.Clear(); changed_items.Clear(); changed_keys.Clear(); removed_keys.Clear(); } public bool is_reset { get; private set; } public List added_items { get; } public List added_keys { get; } public List changed_items { get; } public List changed_keys { get; } public List removed_keys { get; } #if DEBUG public override string ToString() { var added = string.Join(", ", added_items); var changed = string.Join(", ", changed_items); var removed = string.Join(", ", removed_keys); return $"added [{added}] changed [{changed}] removed [{removed}]"; } #endif } public readonly struct Change { public Change(T o, T n) { Old = o; New = n; } public readonly T Old; public readonly T New; public Change merge(Change newer) { return new Change(Old, newer.New); } public void Deconstruct(out T Old, out T New) { Old = this.Old; New = this.New; } public override string ToString() { return $"{{{Old}=>{New}}}"; } } public class Update where TKey : IEquatable { readonly Func _key; internal Update( IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor) { this.added = added ?? new T[0]; this.removed = removed ?? new T[0]; this.changes = changes ?? new Change[0]; this.all = all; _key = key_extractor; } public static Update empty(Func key_extractor) => new Update(new T[0], new T[0], new Change[0], new T[0], key_extractor); public Update merge(Update next) { var comp = new RowEqualityComparer(_key); var added = this.added as HashSet ?? new HashSet(this.added, comp); var changes = this.changed as ChangeDict ?? new ChangeDict(this.changes, _key); var removed = this.removed as HashSet ?? new HashSet(this.removed, comp); // Add added.UnionWith(next.added); // Change foreach (var c in next.changes) { if (added.Contains(c.Old)) { added.Remove(c.Old); added.Add(c.New); } else { changes.AddOrMerge(c); } } // Remove foreach (var r in next.removed) { if (!added.Remove(r)) { removed.Add(r); } changes.Remove(_key(r)); } return new Update( next.all, added, changes, removed, _key ); } public IEnumerable all { get; } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); public IEnumerable removed { get; } public TKey key(T t) { return _key(t); } internal Func key_extractor => _key; #if DEBUG public override string ToString() { string key(T t) => _key(t).ToString().Trim(); var a = string.Join(", ", added.Select(key)); var c = string.Join(", ", changed.Select(key)); var r = string.Join(", ", removed.Select(key)); return $"{{added [{a}] changed [{c}] removed [{r}]}}"; } #endif } class ChangeDict : IEnumerable> { Dictionary> _changes; Func, TKey> _key; public ChangeDict(IEnumerable> changes, Func key) { _key = x => key(x.New); _changes = changes.ToDictionary(_key); } public Change? AddOrMerge(Change newer) { var key = _key(newer); if (_changes.TryGetValue(key, out Change old)) { _changes[key] = old.merge(newer); return old; } _changes.Add(key, newer); return null; } public void Remove(TKey key) { _changes.Remove(key); } #region IEnumerable public IEnumerator> GetEnumerator() { return _changes.Values.GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return _changes.Values.GetEnumerator(); } #endregion } struct RowEqualityComparer : IEqualityComparer, IEqualityComparer> where TKey : IEquatable { readonly Func _key; public RowEqualityComparer(Func key) { _key = key; } public bool Equals(T x, T y) { return _key(x).Equals(_key(y)); } public bool Equals(Change x, Change y) { return Equals(x.New, y.New); } public int GetHashCode(T obj) { return _key(obj).GetHashCode(); } public int GetHashCode(Change c) { return GetHashCode(c.New); } } public class TableUpdate where TKey : IEquatable { readonly ImmutableDictionary _state; internal TableUpdate( ImmutableDictionary init, IEnumerable added, IEnumerable> changed, IEnumerable removed, Func maybe_key_extractor) { _state = init; this.added = added; this.changes = changed; this.removed = removed; this.key_extractor = maybe_key_extractor; } public IReadOnlyDictionary state { get { return _state; } } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); public IEnumerable removed { get; } internal static TableUpdate update(TableUpdate table, Update update) { var builder = table._state.ToBuilder(); builder.RemoveRange(update.removed.Select(i => update.key(i))); builder.AddRange(update.added.Select(i => new KeyValuePair(update.key(i), i))); foreach (var i in update.changed) { builder[update.key(i)] = i; } return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed, update.key_extractor); } internal TableUpdate as_snapshot() { return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0], key_extractor); } internal Func key_extractor { get; private set; } } internal class LegacyTableUpdate where TKey : IEquatable { public LegacyTableUpdate( TableUpdate update) { this.state = update.state; this.added = update.added.Concat(update.changes.Select(t => t.New)); this.removed = update.removed.Concat(update.changes.Select(t => t.Old)); } public IReadOnlyDictionary state { get; } public IEnumerable added { get; } public IEnumerable removed { get; } } public static class ObservableUpdateExtensions { internal static IObservable> reconstitute(this IObservable> source, Func key_extractor) where TKey : IEquatable { var state = new Dictionary(); return source.Select(raw => { if (raw.is_reset) { var removed = state.Values.ToList(); state.Clear(); return new Update(new T[0], new T[0], new Change[0], removed, key_extractor); } else { var removed = new List(raw.removed_keys.Count); foreach (var r in raw.removed_keys) { T temp; if (state.TryGetValue(r, out temp)) { removed.Add(temp); state.Remove(r); } } for (int x = 0; x < raw.added_items.Count; ++x) { state[raw.added_keys[x]] = raw.added_items[x]; } var changes = new List>(raw.changed_items.Count); for (int x = 0; x < raw.changed_items.Count; ++x) { var old = state[raw.changed_keys[x]]; state[raw.changed_keys[x]] = raw.changed_items[x]; changes.Add(new Change(old, raw.changed_items[x])); } return new Update(state.Values.ToList(), raw.added_items.ToList(), changes, removed, key_extractor); } }); } public static IObservable> Table(this IObservable> source) where TKey : IEquatable => source.Scan( new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => TableUpdate.update(table,update)); // Fold updates and emit at max 1/interval public static IObservable> Naggle(this IObservable> source, TimeSpan interval, IScheduler scheduler) where TKey : IEquatable { // fold updates together and emit on TimeSpan (max) interval. Emit on threadpool return Observable.Create>(observer => { object guard = new object(); Update cache = null; bool finished = false; Log.DataFlow.diag($"Create Naggle {typeof(T).Name}"); var d1 = source.Subscribe(update => { Log.DataFlow.diag($"Naggle source updated {typeof(T).Name}"); // subscribe to the source, merge updates (Quickbus thread) lock (guard) { if (cache == null) cache = update; else cache = cache.merge(update); } }, () => { finished = true; }); // publish on interval var d2 = Observable.Interval(interval, scheduler).Subscribe(_ => { Update update; lock (guard) { update = cache; cache = null; } if (update != null) { Log.DataFlow.diag($"Naggle fire {typeof(T).Name}"); observer.OnNext(update); } if (finished) { observer.OnCompleted(); } }); return new CompositeDisposable(d1, d2); }); } public static IObservable> Cache(this IObservable> source) where TKey : IEquatable { var inner = source.Replay(1); Log.DataFlow.diag($"Create Cache {typeof(T).Name}"); return Observable.Create>(o => { inner.Connect(); bool first = true; Log.DataFlow.diag($"Cache Subscribe {typeof(T).Name}"); return inner.Subscribe(upd => { Log.DataFlow.diag($"Cache Inner: {typeof(T).Name}"); if (first) { o.OnNext(upd.as_snapshot()); first = false; } else { o.OnNext(upd); } }); }); } internal static IObservable> AddRemoveOnly(this IObservable> source) where TKey : IEquatable { return source.Select(t => new LegacyTableUpdate(t)); } public static IObservable> Filter(this IObservable> source, IObservable> predicate) where TKey : IEquatable { return Observable.Create>(observer => { Update unfiltered_cache = null, filtered_cache = null; Func pred_cache = null; Log.DataFlow.diag($"Create Filter {typeof(T).Name}"); return source.CombineLatest(predicate, (upd, pred) => (upd, pred)).Subscribe( tup => { bool pred_changed = !object.ReferenceEquals(tup.pred, pred_cache) && (pred_cache != null); pred_cache = tup.pred; unfiltered_cache = tup.upd; if (pred_changed && filtered_cache != null) { //clear out existing var clear = new Update( (IReadOnlyCollection)Array.Empty(), Enumerable.Empty(), Enumerable.Empty>(), filtered_cache.all, unfiltered_cache.key_extractor); observer.OnNext(clear); var refilter = unfiltered_cache.all.Where(tup.pred).ToList(); var readd = new Update( refilter, refilter, Enumerable.Empty>(), Enumerable.Empty(), unfiltered_cache.key_extractor); observer.OnNext(readd); filtered_cache = readd; } else { IReadOnlyCollection all; IEnumerable added, removed; IEnumerable> changed; apply_predicate(tup.pred, tup.upd, out all, out added, out changed, out removed); var next = new Update(all, added, changed, removed, tup.upd.key_extractor); observer.OnNext(next); filtered_cache = next; } }, () => observer.OnCompleted()); }); } private static void apply_predicate(Func predicate, Update cache, out IReadOnlyCollection all, out IEnumerable added, out IEnumerable> changed, out IEnumerable removed) where TKey : IEquatable { var adds = new List(); adds.AddRange(cache.added.Where(predicate)); var rems = new List(); rems.AddRange(cache.removed.Where(predicate)); var diffs = new List>(); foreach (var change in cache.changes) { if(!predicate(change.Old) && predicate(change.New)) adds.Add(change.New); else if(predicate(change.Old) && !predicate(change.New)) rems.Add(change.Old); else if(predicate(change.Old) && predicate(change.New)) diffs.Add(change); } all = cache.all.Where(predicate).ToList(); added = adds; changed = diffs; removed = rems; } // This method does everything lazily so it's cheap until the collections are enumerated public static IObservable> Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { return src.Select(update => { return new Update( update.all.Select(map), update.added.Select(map), update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), update.removed.Select(map), key); }); } // Constructs a new dictionary so not as cheap as Select on the Update<> type public static IObservable> Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { var cache = ImmutableDictionary.Empty; return src.Select(update => { var added = update.added.Select(map); var removed = update.removed.Select(map); var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); var builder = cache.ToBuilder(); builder.RemoveRange(removed.Select(key)); builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); foreach (var (o,n) in changes) { Debug.Assert(key(o).Equals(key(n))); builder[key(o)] = n; } cache = builder.ToImmutable(); return new TableUpdate(cache, added, changes, removed, key); }); } } }