using bts.api.core; using bts.collections; using bts.utils.proto; using Google.Protobuf; using mbts; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reactive.Linq; namespace bts.utils { public static class UpdateExtensions { public static IObservable> AppendTable(this IObservable> source, IEnumerable append_items) where TKey : IEquatable { HashSet append_keys = null; Func append_key_filter = null; return Observable.Create>(o => { TableUpdate output = null; return source.Subscribe(table => { if (output == null) { // first update - initialize output, and (maybe) append_keys/append_key_filter // Note: append_keys/filter can be shared among all subscriptions because, and only needs to be initialized once; // once initialized, the append_keys/filter will not change. output = table; if (append_keys == null) { append_keys = new HashSet(append_items.Select(table.key_extractor)); append_key_filter = v => !append_keys.Contains(table.key_extractor(v)); } // filter append_items out of the normal source (ensures that they will be immune to removes/changes). output = TableUpdate.FilterHelper.on_predicate_changed(output, table.state, null, append_key_filter); // add the append_items to the output table (only need to do this once) output = TableUpdate.update(output, append_items, Enumerable.Empty>(), Enumerable.Empty(), table.key_extractor); output = output.as_snapshot(); } else { output = TableUpdate.FilterHelper.on_source_changed( output, table.added, table.changes, table.removed, append_key_filter); } o.OnNext(output); }); }); } public static void FilterIncremental(this TableUpdate table, Func predicate, out IEnumerable added, out IEnumerable> changes, out IEnumerable removed) where TKey : IEquatable { // added added = table.added.Where(predicate).Concat( table.changes.Where(ch => predicate(ch.New) && !predicate(ch.Old)).Select(ch => ch.New)); // changes changes = table.changes.Where(ch => predicate(ch.New) && predicate(ch.Old)); // removed removed = table.removed.Where(predicate).Concat( table.changes.Where(ch => !predicate(ch.New) && predicate(ch.Old)).Select(ch => ch.Old)); } public static TableUpdate with_added(this TableUpdate table, IEnumerable added) where TKey : IEquatable => TableUpdate.update(table, added, Enumerable.Empty>(), Enumerable.Empty(), table.key_extractor); public static TableUpdate with_removed(this TableUpdate table, IEnumerable removed) where TKey : IEquatable => TableUpdate.update(table, Enumerable.Empty(), Enumerable.Empty>(), removed, table.key_extractor); public static TableUpdate with_changed(this TableUpdate table, IEnumerable changed) where TKey : IEquatable => TableUpdate.update(table, Enumerable.Empty(), changed.Select(v => new Change(table.state[table.key_extractor(v)], v)), Enumerable.Empty(), table.key_extractor); public static TableUpdate with_changes(this TableUpdate table, IEnumerable> changes) where TKey : IEquatable => TableUpdate.update(table, Enumerable.Empty(), changes, Enumerable.Empty(), table.key_extractor); public static IObservable> RootDependent( this IObservable> source, Func key, Func parent_key, Func with_root, Func without_root, Func result_key) where TKey : IEquatable => RootDependent(source, key, parent_key, with_root, without_root, result_key, default); // TODO: unit tests public static IObservable> RootDependent( this IObservable> source, Func key, Func parent_key, Func get_result, Func get_result_no_parent, Func result_key, TKey no_parent_key) where TKey : IEquatable => Observable.Create>(observer => { var state = DictionaryHelper.empty(); var output = TableUpdate.mk_empty(result_key); var parent_to_children = new SetMap(); return source.Subscribe(update); // void update(TableUpdate input) { state = input.state; var builder = output.to_builder(); // execute all parent_to_children changes first, so descendent-propagation will be efficient // (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added). foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) { var k = input.key_extractor(v); parent_to_children.remove(parent_key(v), v); } foreach (var v in input.added.Concat(input.changed)) { var pk = parent_key(v); if (!pk.Equals(no_parent_key)) parent_to_children.add(pk, v); } foreach (var v in input.removed) { var k = input.key_extractor(v); builder.remove(k); if (parent_to_children.try_lookup(k, out ISet children)) { foreach (var child in children) { var ck = key(child); builder.add_or_update(ck, get_result_no_parent(child)); foreach (var d in descendents(ck)) builder.add_or_update(key(d), get_result(d, child)); } } } foreach (var v in input.changed) { var k = input.key_extractor(v); var root = get_root(v); TResult result; if (key(root).Equals(k)) { result = get_result_no_parent(v); foreach (var d in descendents(k)) builder.add_or_update(key(d), get_result(d, v)); } else { result = get_result(v, root); } builder.add_or_update(k, result); } foreach (var v in input.added) { var k = input.key_extractor(v); var pk = parent_key(v); TResult result; if (pk.Equals(no_parent_key)) { result = get_result_no_parent(v); foreach (var d in descendents(k)) builder.add_or_update(key(d), get_result(d, v)); } else { parent_to_children.add(pk, v); var root = get_root(v); if (key(root).Equals(k)) { result = get_result_no_parent(v); foreach (var d in descendents(k)) builder.add_or_update(key(d), get_result(d, v)); } else { result = get_result(v, root); } } builder.add_or_update(k, result); } observer.OnNext(output = builder.to_table()); } T get_root(T value) { var pk = parent_key(value); while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) { value = parent; pk = parent_key(value); } return value; } IEnumerable descendents(TKey k) { if (parent_to_children.try_lookup(k, out ISet children)) { foreach (var child in children) { yield return child; foreach (var d in descendents(key(child))) yield return d; } } } }); public static IObservable> MergeTable(this IObservable> source, IObservable> other) where TKey : IEquatable => source.AggregateLatest(other, _Merge.source_change_del, _Merge.other_change_del, () => TableUpdate.empty, TableUpdate.empty, TableUpdate.empty); public static IObservable, TKey>> ParentDependent( this IObservable> source, Func key, Func parent_key, Func get_result, Func get_result_no_parent) where TResult : IEquatable where TKey : IEquatable => ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer.Default, default); public static IObservable, TKey>> ParentDependent( this IObservable> source, Func key, Func parent_key, Func get_result, Func get_result_no_parent, IEqualityComparer result_comparer, TKey no_parent_key) where TKey : IEquatable => Observable.Create, TKey>>(observer => { var value_comparer = EqualityComparer.Create, TResult>(kv => kv.Value, result_comparer); var state = DictionaryHelper.empty(); var output = TableUpdate, TKey>.mk_empty(kv => kv.Key); var parent_to_children = new SetMap(); return source.Subscribe(update); // void update(TableUpdate input) { state = input.state; var builder = output.to_builder(); foreach (var v in input.removed) { var k = input.key_extractor(v); builder.remove(k); parent_to_children.remove(parent_key(v), v); if (parent_to_children.try_lookup(k, out ISet children)) { foreach (var child in children) builder.add_or_update(new KeyValuePair(key(child), get_result_no_parent(child))); } } foreach (var v in input.changed) { var k = key(v); builder.add_or_update(new KeyValuePair(k, lookup_result(v))); parent_to_children.set(parent_key(v), v); if (parent_to_children.try_lookup(k, out ISet children)) { foreach (var child in children) builder.add_or_update(new KeyValuePair(key(child), get_result(child, v))); } } foreach (var v in input.added) { var k = key(v); builder.add_or_update(new KeyValuePair(k, lookup_result(v))); parent_to_children.add(parent_key(v), v); if (parent_to_children.try_lookup(k, out ISet children)) { foreach (var child in children) builder.add_or_update(new KeyValuePair(key(child), get_result(child, v))); } } var result = builder.to_table_checked(value_comparer); if (result.has_changes()) observer.OnNext(output = result); } TResult lookup_result(T child) { var pk = parent_key(child); return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ? get_result(child, parent) : get_result_no_parent(child); } }); static class _Merge where TKey : IEquatable { public static readonly Func, Change>, TableUpdate, TableUpdate> source_change_del = _source_changed_impl; public static readonly Func, TableUpdate, Change>, TableUpdate> other_change_del = _other_changed_impl; static TableUpdate _source_changed_impl(TableUpdate target, Change> source_change, TableUpdate other_table) => TableUpdate.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor); static TableUpdate _other_changed_impl(TableUpdate target, TableUpdate source_table, Change> other_change) => TableUpdate.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor); } } } namespace bts.utils.diagnostics { public static class DiagnosticUpdateExtensions { // Hard to imagine why this util would be needed outside of diagnosig bugs. // The #if DEBUG directive is an extra precaution to avoid accidentally writing // files like this on a client's machine. #if DEBUG static readonly string _proto_capture_ext = "proto_capture"; public static IObservable> ProtoBufCapture(this IObservable> source, Func to_message) where TKey : IEquatable where TProto : class, IMessage => ProtoBufCapture(source, FilePathHelper.GetUniquePathWithDate(Path.Combine(Directory.GetCurrentDirectory(), typeof(TProto).Name), _proto_capture_ext), to_message); public static IObservable> ProtoBufCapture(this IObservable> source, string path, Func to_message) where TKey : IEquatable where TProto : class, IMessage => Observable.Create>(o => { // write the state on dispose, and/or on error IReadOnlyDictionary state = DictionaryHelper.empty(); var sub = source.Subscribe(table => { state = table.state; o.OnNext(table); }, ex => { save(true); o.OnError(ex); }, o.OnCompleted); return Disposable.Create(() => { save(false); Disposable.dispose(ref sub); }); void save(bool has_error) { if (state.Count > 0) FileStreamHelper.WriteProtos(has_error ? $"{path}(error)" : path, state.Values.Select(to_message).ExceptNull()); } }); #endif } }