Files
spark/src/client/spark_core/utils/UpdateExtensions.cs
T

295 lines
16 KiB
C#

using bts.api.core;
using bts.collections;
using bts.utils.proto;
using Google.Protobuf;
using mbts;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reactive.Linq;
namespace bts.utils {
public static class UpdateExtensions {
public static IObservable<TableUpdate<T, TKey>> AppendTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IEnumerable<T> append_items) where TKey : IEquatable<TKey> {
HashSet<TKey> append_keys = null;
Func<T, bool> append_key_filter = null;
return Observable.Create<TableUpdate<T, TKey>>(o => {
TableUpdate<T, TKey> output = null;
return source.Subscribe(table => {
if (output == null) {
// first update - initialize output, and (maybe) append_keys/append_key_filter
// Note: append_keys/filter can be shared among all subscriptions because, and only needs to be initialized once;
// once initialized, the append_keys/filter will not change.
output = table;
if (append_keys == null) {
append_keys = new HashSet<TKey>(append_items.Select(table.key_extractor));
append_key_filter = v => !append_keys.Contains(table.key_extractor(v));
}
// filter append_items out of the normal source (ensures that they will be immune to removes/changes).
output = TableUpdate<T, TKey>.FilterHelper.on_predicate_changed(output, table.state, null, append_key_filter);
// add the append_items to the output table (only need to do this once)
output = TableUpdate<T, TKey>.update(output, append_items, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
output = output.as_snapshot();
} else {
output = TableUpdate<T, TKey>.FilterHelper.on_source_changed(
output, table.added, table.changes, table.removed, append_key_filter);
}
o.OnNext(output);
});
});
}
public static void FilterIncremental<T, TKey>(this TableUpdate<T, TKey> table, Func<T, bool> predicate, out IEnumerable<T> added, out IEnumerable<Change<T>> changes, out IEnumerable<T> removed) where TKey : IEquatable<TKey> {
// added
added = table.added.Where(predicate).Concat(
table.changes.Where(ch => predicate(ch.New) && !predicate(ch.Old)).Select(ch => ch.New));
// changes
changes = table.changes.Where(ch => predicate(ch.New) && predicate(ch.Old));
// removed
removed = table.removed.Where(predicate).Concat(
table.changes.Where(ch => !predicate(ch.New) && predicate(ch.Old)).Select(ch => ch.Old));
}
public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
this IObservable<TableUpdate<T, TKey>> source,
Func<T, TKey> key,
Func<T, TKey> parent_key,
Func<T, T, TResult> with_root,
Func<T, TResult> without_root,
Func<TResult, TKey> result_key)
where TKey : IEquatable<TKey> =>
RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
// TODO: unit tests
public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
this IObservable<TableUpdate<T, TKey>> source,
Func<T, TKey> key,
Func<T, TKey> parent_key,
Func<T, T, TResult> get_result,
Func<T, TResult> get_result_no_parent,
Func<TResult, TKey> result_key,
TKey no_parent_key)
where TKey : IEquatable<TKey> =>
Observable.Create<TableUpdate<TResult, TKey>>(observer => {
var state = DictionaryHelper.empty<TKey, T>();
var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
var parent_to_children = new SetMap<TKey, T>();
return source.Subscribe(update);
//
void update(TableUpdate<T, TKey> input) {
state = input.state;
var builder = output.to_builder();
// execute all parent_to_children changes first, so descendent-propagation will be efficient
// (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
var k = input.key_extractor(v);
parent_to_children.remove(parent_key(v), v);
}
foreach (var v in input.added.Concat(input.changed)) {
var pk = parent_key(v);
if (!pk.Equals(no_parent_key))
parent_to_children.add(pk, v);
}
foreach (var v in input.removed) {
var k = input.key_extractor(v);
builder.remove(k);
if (parent_to_children.try_lookup(k, out ISet<T> children)) {
foreach (var child in children) {
var ck = key(child);
builder.add_or_update(ck, get_result_no_parent(child));
foreach (var d in descendents(ck))
builder.add_or_update(key(d), get_result(d, child));
}
}
}
foreach (var v in input.changed) {
var k = input.key_extractor(v);
var root = get_root(v);
TResult result;
if (key(root).Equals(k)) {
result = get_result_no_parent(v);
foreach (var d in descendents(k))
builder.add_or_update(key(d), get_result(d, v));
} else {
result = get_result(v, root);
}
builder.add_or_update(k, result);
}
foreach (var v in input.added) {
var k = input.key_extractor(v);
var pk = parent_key(v);
TResult result;
if (pk.Equals(no_parent_key)) {
result = get_result_no_parent(v);
foreach (var d in descendents(k))
builder.add_or_update(key(d), get_result(d, v));
} else {
parent_to_children.add(pk, v);
var root = get_root(v);
if (key(root).Equals(k)) {
result = get_result_no_parent(v);
foreach (var d in descendents(k))
builder.add_or_update(key(d), get_result(d, v));
} else {
result = get_result(v, root);
}
}
builder.add_or_update(k, result);
}
observer.OnNext(output = builder.to_table());
}
T get_root(T value) {
var pk = parent_key(value);
while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
value = parent;
pk = parent_key(value);
}
return value;
}
IEnumerable<T> descendents(TKey k) {
if (parent_to_children.try_lookup(k, out ISet<T> children)) {
foreach (var child in children) {
yield return child;
foreach (var d in descendents(key(child)))
yield return d;
}
}
}
});
public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
this IObservable<TableUpdate<T, TKey>> source,
Func<T, TKey> key,
Func<T, TKey> parent_key,
Func<T, T, TResult> get_result,
Func<T, TResult> get_result_no_parent)
where TResult : IEquatable<TResult>
where TKey : IEquatable<TKey> =>
ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
this IObservable<TableUpdate<T, TKey>> source,
Func<T, TKey> key,
Func<T, TKey> parent_key,
Func<T, T, TResult> get_result,
Func<T, TResult> get_result_no_parent,
IEqualityComparer<TResult> result_comparer,
TKey no_parent_key)
where TKey : IEquatable<TKey> =>
Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
var state = DictionaryHelper.empty<TKey, T>();
var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
var parent_to_children = new SetMap<TKey, T>();
return source.Subscribe(update);
//
void update(TableUpdate<T, TKey> input) {
state = input.state;
var builder = output.to_builder();
foreach (var v in input.removed) {
var k = input.key_extractor(v);
builder.remove(k);
parent_to_children.remove(parent_key(v), v);
if (parent_to_children.try_lookup(k, out ISet<T> children)) {
foreach (var child in children)
builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
}
}
foreach (var v in input.changed) {
var k = key(v);
builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
parent_to_children.set(parent_key(v), v);
if (parent_to_children.try_lookup(k, out ISet<T> children)) {
foreach (var child in children)
builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
}
}
foreach (var v in input.added) {
var k = key(v);
builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
parent_to_children.add(parent_key(v), v);
if (parent_to_children.try_lookup(k, out ISet<T> children)) {
foreach (var child in children)
builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
}
}
var result = builder.to_table_checked(value_comparer);
if (result.has_changes())
observer.OnNext(output = result);
}
TResult lookup_result(T child) {
var pk = parent_key(child);
return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ? get_result(child, parent) : get_result_no_parent(child);
}
});
static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
}
}
}
namespace bts.utils.diagnostics {
public static class DiagnosticUpdateExtensions {
// Hard to imagine why this util would be needed outside of diagnosig bugs.
// The #if DEBUG directive is an extra precaution to avoid accidentally writing
// files like this on a client's machine.
#if DEBUG
static readonly string _proto_capture_ext = "proto_capture";
public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, Func<T, TProto> to_message)
where TKey : IEquatable<TKey>
where TProto : class, IMessage<TProto> =>
ProtoBufCapture(source,
FilePathHelper.GetUniquePathWithDate(Path.Combine(Directory.GetCurrentDirectory(), typeof(TProto).Name), _proto_capture_ext),
to_message);
public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, string path, Func<T, TProto> to_message)
where TKey : IEquatable<TKey>
where TProto : class, IMessage<TProto> =>
Observable.Create<TableUpdate<T, TKey>>(o => {
// write the state on dispose, and/or on error
IReadOnlyDictionary<TKey, T> state = DictionaryHelper.empty<TKey, T>();
var sub = source.Subscribe(table => {
state = table.state;
o.OnNext(table);
}, ex => {
save(true);
o.OnError(ex);
},
o.OnCompleted);
return Disposable.Create(() => {
save(false);
Disposable.dispose(ref sub);
});
void save(bool has_error) {
if (state.Count > 0)
FileStreamHelper.WriteProtos(has_error ? $"{path}(error)" : path, state.Values.Select(to_message).ExceptNull());
}
});
#endif
}
}