From 9bcd8bcbdbda1debdf1913b5e94dbfab5c814ef6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 24 Jan 2019 15:38:39 -0500 Subject: [PATCH] Add tests for IObservable.Select methods --- src/client/spark_api/api/Update.cs | 38 ++++ src/client/spark_test/api/ApiUpdateTests.cs | 193 +++++++++++++++----- 2 files changed, 184 insertions(+), 47 deletions(-) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index d5f1b04fd..b2585f8ad 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -64,6 +64,10 @@ namespace bts.api.core { public Change merge(Change newer) { return new Change(Old, newer.New); } + public void Deconstruct(out T Old, out T New) { + Old = this.Old; + New = this.New; + } public override string ToString() { return $"{{{Old}=>{New}}}"; } @@ -415,5 +419,39 @@ namespace bts.api.core { changed = diffs; removed = rems; } + + // This method does everything lazily so it's cheap until the collections are enumerated + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + return src.Select(update => { + return new Update( + update.all.Select(map), + update.added.Select(map), + update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), + update.removed.Select(map), + key); + }); + } + + // Constructs a new dictionary so not as cheap as Select on the Update<> type + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + var cache = ImmutableDictionary.Empty; + return src.Select(update => { + var added = update.added.Select(map); + var removed = update.removed.Select(map); + var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); + var builder = cache.ToBuilder(); + builder.RemoveRange(removed.Select(key)); + builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); + foreach (var (o,n) in changes) { + Debug.Assert(key(o).Equals(key(n))); + builder[key(o)] = n; + } + cache = builder.ToImmutable(); + + return new TableUpdate(cache, added, changes, removed); + }); + } } } \ No newline at end of file diff --git a/src/client/spark_test/api/ApiUpdateTests.cs b/src/client/spark_test/api/ApiUpdateTests.cs index 791d9a773..e1f6f08b7 100644 --- a/src/client/spark_test/api/ApiUpdateTests.cs +++ b/src/client/spark_test/api/ApiUpdateTests.cs @@ -1,12 +1,12 @@ using bts.api.core; -using System.Collections.Generic; using bts.test; using System; -using System.Reactive.Linq; -using System.Threading.Tasks; +using System.Collections.Generic; using System.Linq; using System.Reactive.Concurrency; +using System.Reactive.Linq; using System.Reactive.Subjects; +using System.Threading.Tasks; namespace mbts.api.Test { [TestGroup] @@ -44,7 +44,7 @@ namespace mbts.api.Test { static int id(Item i) => i.id; Item mkitem() { - return new Item(++_id, Guid.NewGuid().ToString().Substring(0,8)); + return new Item(++_id, Guid.NewGuid().ToString().Substring(0, 8)); } IObservable> validate_incremental(IObservable> source) { @@ -68,7 +68,7 @@ namespace mbts.api.Test { // This will mostly be collapsing to add() for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -80,7 +80,7 @@ namespace mbts.api.Test { // Do it again with TableUpdate for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -94,35 +94,32 @@ namespace mbts.api.Test { // collapse rolling window var update_list = raw.reconstitute(id).ToEnumerable().ToList(); var window = 10; - for (int x = 0; x+window < total; x += 5) { + for (int x = 0; x + window < total; x += 5) { var start = update_list[x]; var end = update_list[x + window]; - var super_update = update_list.Skip(x).Take(window+1).Aggregate((u1, u2) => u1.merge(u2)); + var super_update = update_list.Skip(x).Take(window + 1).Aggregate((u1, u2) => u1.merge(u2)); start.merge(super_update); Assert.SequenceEqual(end.all, super_update.all); } } - void validate_filter(IObservable> raw, IObservable> filtered, Func pred) - { + void validate_filter(IObservable> raw, IObservable> filtered, Func pred) { var total = raw.ToEnumerable().Count(); // Validate that update is right at each point - for (var count = 1; count < total; ++count) - { - var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); + for (var count = 1; count < total; ++count) { + var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.all.All(pred)); Assert.That(next.added.All(pred)); Assert.That(next.removed.All(pred)); - Assert.That(next.changed.All(pred)); + Assert.That(next.changed.All(pred)); } // Do it again with TableUpdate - for (var count = 1; count < total; ++count) - { + for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.added.All(pred)); @@ -132,36 +129,32 @@ namespace mbts.api.Test { } - void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) - { - raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => - { + void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) { + raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => { Assert.That(tup.Item2.all.All(pred.Value)); Assert.That(tup.Item2.added.All(pred.Value)); - Assert.That(!tup.Item2.removed.Any(pred.Value)); + Assert.That(!tup.Item2.removed.Any(pred.Value)); Assert.That(tup.Item2.changed.All(pred.Value)); Dictionary rawd = tup.Item1.added.ToDictionary(x => x.id); Dictionary fd = tup.Item2.added.ToDictionary(x => x.id); - foreach (var kv in rawd) - { - if(pred.Value(kv.Value)) + foreach (var kv in rawd) { + if (pred.Value(kv.Value)) { Assert.That(fd.ContainsKey(kv.Key)); - else + } + else { Assert.That(!fd.ContainsKey(kv.Key)); + } } }); } [Test] - public void test_filter() - { - var raw = Observable.Create>(observer => - { + public void test_filter() { + var raw = Observable.Create>(observer => { RawUpdate rup = new RawUpdate(); - for (int x = 0; x < 50; ++x) - { + for (int x = 0; x < 50; ++x) { rup.Clear(); var item = _item_pool[x]; rup.added_items.Add(item); @@ -169,39 +162,35 @@ namespace mbts.api.Test { observer.OnNext(rup); } var rand = new Random(); - for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) - { + for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) { rup.Clear(); rup.removed_keys.Add(_item_pool[r].id); observer.OnNext(rup); } rup.Clear(); - for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) - { + for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) { var ch = new Item(_item_pool[c].id, _item_pool[c].name + "_change"); rup.changed_items.Add(ch); - rup.changed_keys.Add(ch.id); + rup.changed_keys.Add(ch.id); } observer.OnNext(rup); - observer.OnCompleted(); + observer.OnCompleted(); return Disposable.empty; }); Func evens = (x => x.id % 2 == 0); Func odds = (x => x.id % 2 == 1); - var predeven = Observable.Create>(observer => - { - observer.OnNext(evens); + var predeven = Observable.Create>(observer => { + observer.OnNext(evens); observer.OnCompleted(); return Disposable.empty; }); - var filtered = raw.reconstitute(x=> x.id).Filter(predeven); + var filtered = raw.reconstitute(x => x.id).Filter(predeven); validate_filter(raw, filtered, evens); - var predodd = Observable.Create>(observer => - { + var predodd = Observable.Create>(observer => { observer.OnNext(odds); observer.OnCompleted(); return Disposable.empty; @@ -212,13 +201,13 @@ namespace mbts.api.Test { - var sub= new BehaviorSubject>(evens); + var sub = new BehaviorSubject>(evens); filtered = raw.reconstitute(x => x.id).Filter(sub); validate_filter_sub(raw, filtered, sub); sub.OnNext(odds); sub.OnCompleted(); - } + } [Test] public async Task test_add() { @@ -249,7 +238,7 @@ namespace mbts.api.Test { rup.added_keys.AddRange(_item_pool.Take(20).Select(id)); observer.OnNext(rup); - for (int x = 0; x < 50; x+=2) { + for (int x = 0; x < 50; x += 2) { rup.Clear(); var item = _item_pool[x]; rup.removed_keys.Add(item.id); @@ -347,11 +336,121 @@ namespace mbts.api.Test { public void test_merge_begin_end_state() { int key(string s) => s[0]; var empty = new string[0]; - var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); + var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); var a3 = new Update(new[] { "a3" }, empty, new[] { new Change("a2", "a3") }, empty, key); var merged = a2.merge(a3); Assert.SequenceEqual(new[] { new Change("a1", "a3") }, merged.changes); } + T id(T t) => t; + + [Test,Ignore("SPARK-1988")] + public void test_reconstitute_twice() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + var input = + Enumerable.Range(1, 10) + .Select(create) + .ToObservable() + .reconstitute(id); + + var expected = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i)).SelectMany(id); + var a = input.ToEnumerable().SelectMany(u => u.all).ToList(); + var b = input.ToEnumerable().SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected, a); + Assert.SequenceEqual(expected, b); + } + + [Test] + public void test_select_update() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + } + + [Test] + public void test_select_tableupdate() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Table() + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.state.Values).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + + } + + //public class Box { + // public int value; + // public override string ToString() { + // return value.ToString(); + // } + //} + + //[Test] + //public void test_scan() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan( + // new Box { value = 0 }, + // (box, i) => { + // box.value += i; + // return box; + // }); + // var a = obs.Select(box => box.value).ToEnumerable(); + // var b = obs.Select(box => box.value).ToEnumerable(); + // Assert.SequenceEqual(a, b); + //} + + //[Test] + //public void test_scan2() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan>( + // () => new Box { value = 0 }, + // (getbox, i) => { + // var box = getbox(); + // box.value += i; + // return () => box; + // }); + // var a = obs.Select(box => box().value).ToEnumerable().ToList(); + // var b = obs.Select(box => box().value).ToEnumerable().ToList(); + // Assert.SequenceEqual(a, b); + //} } }