From 2dcfad979c2c1b57e7366e18c28e440efb69ea26 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 6 May 2020 12:11:03 -0500 Subject: [PATCH 1/4] move FileStreamHelper.cs to spark_core --- .../utils}/FileStreamHelper.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/client/{spark/core/Utility/ProtoBufDictionaryServer => spark_core/utils}/FileStreamHelper.cs (100%) diff --git a/src/client/spark/core/Utility/ProtoBufDictionaryServer/FileStreamHelper.cs b/src/client/spark_core/utils/FileStreamHelper.cs similarity index 100% rename from src/client/spark/core/Utility/ProtoBufDictionaryServer/FileStreamHelper.cs rename to src/client/spark_core/utils/FileStreamHelper.cs From c573b5fbb21232b1b266686f4eb6dc82aae27264 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 6 May 2020 17:30:00 -0500 Subject: [PATCH 2/4] add ProtoBufCapture extension (writes TableUpdate state to file when subscription is disposed) --- src/client/spark_core/utils/FilePathHelper.cs | 21 + .../spark_core/utils/UpdateExtensions.cs | 461 ++++++++++-------- 2 files changed, 274 insertions(+), 208 deletions(-) create mode 100644 src/client/spark_core/utils/FilePathHelper.cs diff --git a/src/client/spark_core/utils/FilePathHelper.cs b/src/client/spark_core/utils/FilePathHelper.cs new file mode 100644 index 000000000..58c27b85f --- /dev/null +++ b/src/client/spark_core/utils/FilePathHelper.cs @@ -0,0 +1,21 @@ +using System; +using System.IO; + +namespace bts.utils { + public static class FilePathHelper { + public static string GetUniquePathWithDate(string path_root, string ext) => + GetUniquePath($"{path_root}_{DateTime.Now:MM - dd - yyyy_hh - mm}", ext); + + public static string GetUniquePath(string path_root, string ext) { + var path = $"{path_root}.{ext}"; + const int max_copies = 2 ^ 14; + int i = 1; + while (File.Exists(path)) { + path = $"{path_root}({i++}).{ext}"; + if (i == max_copies) + throw new InvalidOperationException("Unable to make path unique by concating a number."); + } + return path; + } + } +} diff --git a/src/client/spark_core/utils/UpdateExtensions.cs b/src/client/spark_core/utils/UpdateExtensions.cs index eefce32fc..389803a8d 100644 --- a/src/client/spark_core/utils/UpdateExtensions.cs +++ b/src/client/spark_core/utils/UpdateExtensions.cs @@ -1,208 +1,253 @@ -using bts.api.core; -using bts.collections; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reactive.Linq; - -namespace bts.utils { - public static class UpdateExtensions { - public static TableUpdate with_added(this TableUpdate table, IEnumerable added) where TKey : IEquatable => - TableUpdate.update(table, added, Enumerable.Empty>(), Enumerable.Empty(), table.key_extractor); - - public static TableUpdate with_removed(this TableUpdate table, IEnumerable removed) where TKey : IEquatable => - TableUpdate.update(table, Enumerable.Empty(), Enumerable.Empty>(), removed, table.key_extractor); - - public static TableUpdate with_changed(this TableUpdate table, IEnumerable changed) where TKey : IEquatable => - TableUpdate.update(table, Enumerable.Empty(), changed.Select(v => new Change(table.state[table.key_extractor(v)], v)), Enumerable.Empty(), table.key_extractor); - - public static TableUpdate with_changes(this TableUpdate table, IEnumerable> changes) where TKey : IEquatable => - TableUpdate.update(table, Enumerable.Empty(), changes, Enumerable.Empty(), table.key_extractor); - - public static IObservable> RootDependent( - this IObservable> source, - Func key, - Func parent_key, - Func with_root, - Func without_root, - Func result_key) - where TKey : IEquatable => - RootDependent(source, key, parent_key, with_root, without_root, result_key, default); - - // TODO: unit tests - public static IObservable> RootDependent( - this IObservable> source, - Func key, - Func parent_key, - Func get_result, - Func get_result_no_parent, - Func result_key, - TKey no_parent_key) - where TKey : IEquatable => - Observable.Create>(observer => { - var state = DictionaryHelper.empty(); - var output = TableUpdate.mk_empty(result_key); - var parent_to_children = new SetMap(); - return source.Subscribe(update); - // - void update(TableUpdate input) { - state = input.state; - var builder = output.to_builder(); - // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added). - foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) { - var k = input.key_extractor(v); - parent_to_children.remove(parent_key(v), v); - } - foreach (var v in input.added.Concat(input.changed)) { - var pk = parent_key(v); - if (!pk.Equals(no_parent_key)) - parent_to_children.add(pk, v); - } - foreach (var v in input.removed) { - var k = input.key_extractor(v); - builder.remove(k); - if (parent_to_children.try_lookup(k, out ISet children)) { - foreach (var child in children) { - var ck = key(child); - builder.add_or_update(ck, get_result_no_parent(child)); - foreach (var d in descendents(ck)) - builder.add_or_update(key(d), get_result(d, child)); - } - } - } - foreach (var v in input.changed) { - var k = input.key_extractor(v); - var root = get_root(v); - TResult result; - if (key(root).Equals(k)) { - result = get_result_no_parent(v); - foreach (var d in descendents(k)) - builder.add_or_update(key(d), get_result(d, v)); - } else { - result = get_result(v, root); - } - builder.add_or_update(k, result); - } - foreach (var v in input.added) { - var k = input.key_extractor(v); - var pk = parent_key(v); - TResult result; - if (pk.Equals(no_parent_key)) { - result = get_result_no_parent(v); - foreach (var d in descendents(k)) - builder.add_or_update(key(d), get_result(d, v)); - } else { - parent_to_children.add(pk, v); - var root = get_root(v); - if (key(root).Equals(k)) { - result = get_result_no_parent(v); - foreach (var d in descendents(k)) - builder.add_or_update(key(d), get_result(d, v)); - } else { - result = get_result(v, root); - } - } - builder.add_or_update(k, result); - } - observer.OnNext(output = builder.to_table()); - } - T get_root(T value) { - var pk = parent_key(value); - while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) { - value = parent; - pk = parent_key(value); - } - return value; - } - IEnumerable descendents(TKey k) { - if (parent_to_children.try_lookup(k, out ISet children)) { - foreach (var child in children) { - yield return child; - foreach (var d in descendents(key(child))) - yield return d; - } - } - } - }); - - public static IObservable> MergeTable(this IObservable> source, IObservable> other) where TKey : IEquatable => - source.AggregateLatest(other, _Merge.source_change_del, _Merge.other_change_del, () => TableUpdate.empty, TableUpdate.empty, TableUpdate.empty); - - public static IObservable, TKey>> ParentDependent( - this IObservable> source, - Func key, - Func parent_key, - Func get_result, - Func get_result_no_parent) - where TResult : IEquatable - where TKey : IEquatable => - ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer.Default, default); - - public static IObservable, TKey>> ParentDependent( - this IObservable> source, - Func key, - Func parent_key, - Func get_result, - Func get_result_no_parent, - IEqualityComparer result_comparer, - TKey no_parent_key) - where TKey : IEquatable => - Observable.Create, TKey>>(observer => { - var value_comparer = EqualityComparer.Create, TResult>(kv => kv.Value, result_comparer); - var state = DictionaryHelper.empty(); - var output = TableUpdate, TKey>.mk_empty(kv => kv.Key); - var parent_to_children = new SetMap(); - return source.Subscribe(update); - // - void update(TableUpdate input) { - state = input.state; - var builder = output.to_builder(); - foreach (var v in input.removed) { - var k = input.key_extractor(v); - builder.remove(k); - parent_to_children.remove(parent_key(v), v); - if (parent_to_children.try_lookup(k, out ISet children)) { - foreach (var child in children) - builder.add_or_update(new KeyValuePair(key(child), get_result_no_parent(child))); - } - } - foreach (var v in input.changed) { - var k = key(v); - builder.add_or_update(new KeyValuePair(k, lookup_result(v))); - parent_to_children.set(parent_key(v), v); - if (parent_to_children.try_lookup(k, out ISet children)) { - foreach (var child in children) - builder.add_or_update(new KeyValuePair(key(child), get_result(child, v))); - } - } - foreach (var v in input.added) { - var k = key(v); - builder.add_or_update(new KeyValuePair(k, lookup_result(v))); - parent_to_children.add(parent_key(v), v); - if (parent_to_children.try_lookup(k, out ISet children)) { - foreach (var child in children) - builder.add_or_update(new KeyValuePair(key(child), get_result(child, v))); - } - } - var result = builder.to_table_checked(value_comparer); - if (result.has_changes()) - observer.OnNext(output = result); - } - TResult lookup_result(T child) { - var pk = parent_key(child); - return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ? get_result(child, parent) : get_result_no_parent(child); - } - }); - - static class _Merge where TKey : IEquatable { - public static readonly Func, Change>, TableUpdate, TableUpdate> source_change_del = _source_changed_impl; - public static readonly Func, TableUpdate, Change>, TableUpdate> other_change_del = _other_changed_impl; - - static TableUpdate _source_changed_impl(TableUpdate target, Change> source_change, TableUpdate other_table) => - TableUpdate.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor); - - static TableUpdate _other_changed_impl(TableUpdate target, TableUpdate source_table, Change> other_change) => - TableUpdate.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor); - } - } -} +using bts.api.core; +using bts.collections; +using bts.utils.proto; +using Google.Protobuf; +using mbts; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reactive.Linq; + +namespace bts.utils { + public static class UpdateExtensions { + public static TableUpdate with_added(this TableUpdate table, IEnumerable added) where TKey : IEquatable => + TableUpdate.update(table, added, Enumerable.Empty>(), Enumerable.Empty(), table.key_extractor); + + public static TableUpdate with_removed(this TableUpdate table, IEnumerable removed) where TKey : IEquatable => + TableUpdate.update(table, Enumerable.Empty(), Enumerable.Empty>(), removed, table.key_extractor); + + public static TableUpdate with_changed(this TableUpdate table, IEnumerable changed) where TKey : IEquatable => + TableUpdate.update(table, Enumerable.Empty(), changed.Select(v => new Change(table.state[table.key_extractor(v)], v)), Enumerable.Empty(), table.key_extractor); + + public static TableUpdate with_changes(this TableUpdate table, IEnumerable> changes) where TKey : IEquatable => + TableUpdate.update(table, Enumerable.Empty(), changes, Enumerable.Empty(), table.key_extractor); + + public static IObservable> RootDependent( + this IObservable> source, + Func key, + Func parent_key, + Func with_root, + Func without_root, + Func result_key) + where TKey : IEquatable => + RootDependent(source, key, parent_key, with_root, without_root, result_key, default); + + // TODO: unit tests + public static IObservable> RootDependent( + this IObservable> source, + Func key, + Func parent_key, + Func get_result, + Func get_result_no_parent, + Func result_key, + TKey no_parent_key) + where TKey : IEquatable => + Observable.Create>(observer => { + var state = DictionaryHelper.empty(); + var output = TableUpdate.mk_empty(result_key); + var parent_to_children = new SetMap(); + return source.Subscribe(update); + // + void update(TableUpdate input) { + state = input.state; + var builder = output.to_builder(); + // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added). + foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) { + var k = input.key_extractor(v); + parent_to_children.remove(parent_key(v), v); + } + foreach (var v in input.added.Concat(input.changed)) { + var pk = parent_key(v); + if (!pk.Equals(no_parent_key)) + parent_to_children.add(pk, v); + } + foreach (var v in input.removed) { + var k = input.key_extractor(v); + builder.remove(k); + if (parent_to_children.try_lookup(k, out ISet children)) { + foreach (var child in children) { + var ck = key(child); + builder.add_or_update(ck, get_result_no_parent(child)); + foreach (var d in descendents(ck)) + builder.add_or_update(key(d), get_result(d, child)); + } + } + } + foreach (var v in input.changed) { + var k = input.key_extractor(v); + var root = get_root(v); + TResult result; + if (key(root).Equals(k)) { + result = get_result_no_parent(v); + foreach (var d in descendents(k)) + builder.add_or_update(key(d), get_result(d, v)); + } else { + result = get_result(v, root); + } + builder.add_or_update(k, result); + } + foreach (var v in input.added) { + var k = input.key_extractor(v); + var pk = parent_key(v); + TResult result; + if (pk.Equals(no_parent_key)) { + result = get_result_no_parent(v); + foreach (var d in descendents(k)) + builder.add_or_update(key(d), get_result(d, v)); + } else { + parent_to_children.add(pk, v); + var root = get_root(v); + if (key(root).Equals(k)) { + result = get_result_no_parent(v); + foreach (var d in descendents(k)) + builder.add_or_update(key(d), get_result(d, v)); + } else { + result = get_result(v, root); + } + } + builder.add_or_update(k, result); + } + observer.OnNext(output = builder.to_table()); + } + T get_root(T value) { + var pk = parent_key(value); + while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) { + value = parent; + pk = parent_key(value); + } + return value; + } + IEnumerable descendents(TKey k) { + if (parent_to_children.try_lookup(k, out ISet children)) { + foreach (var child in children) { + yield return child; + foreach (var d in descendents(key(child))) + yield return d; + } + } + } + }); + + public static IObservable> MergeTable(this IObservable> source, IObservable> other) where TKey : IEquatable => + source.AggregateLatest(other, _Merge.source_change_del, _Merge.other_change_del, () => TableUpdate.empty, TableUpdate.empty, TableUpdate.empty); + + public static IObservable, TKey>> ParentDependent( + this IObservable> source, + Func key, + Func parent_key, + Func get_result, + Func get_result_no_parent) + where TResult : IEquatable + where TKey : IEquatable => + ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer.Default, default); + + public static IObservable, TKey>> ParentDependent( + this IObservable> source, + Func key, + Func parent_key, + Func get_result, + Func get_result_no_parent, + IEqualityComparer result_comparer, + TKey no_parent_key) + where TKey : IEquatable => + Observable.Create, TKey>>(observer => { + var value_comparer = EqualityComparer.Create, TResult>(kv => kv.Value, result_comparer); + var state = DictionaryHelper.empty(); + var output = TableUpdate, TKey>.mk_empty(kv => kv.Key); + var parent_to_children = new SetMap(); + return source.Subscribe(update); + // + void update(TableUpdate input) { + state = input.state; + var builder = output.to_builder(); + foreach (var v in input.removed) { + var k = input.key_extractor(v); + builder.remove(k); + parent_to_children.remove(parent_key(v), v); + if (parent_to_children.try_lookup(k, out ISet children)) { + foreach (var child in children) + builder.add_or_update(new KeyValuePair(key(child), get_result_no_parent(child))); + } + } + foreach (var v in input.changed) { + var k = key(v); + builder.add_or_update(new KeyValuePair(k, lookup_result(v))); + parent_to_children.set(parent_key(v), v); + if (parent_to_children.try_lookup(k, out ISet children)) { + foreach (var child in children) + builder.add_or_update(new KeyValuePair(key(child), get_result(child, v))); + } + } + foreach (var v in input.added) { + var k = key(v); + builder.add_or_update(new KeyValuePair(k, lookup_result(v))); + parent_to_children.add(parent_key(v), v); + if (parent_to_children.try_lookup(k, out ISet children)) { + foreach (var child in children) + builder.add_or_update(new KeyValuePair(key(child), get_result(child, v))); + } + } + var result = builder.to_table_checked(value_comparer); + if (result.has_changes()) + observer.OnNext(output = result); + } + TResult lookup_result(T child) { + var pk = parent_key(child); + return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ? get_result(child, parent) : get_result_no_parent(child); + } + }); + + static class _Merge where TKey : IEquatable { + public static readonly Func, Change>, TableUpdate, TableUpdate> source_change_del = _source_changed_impl; + public static readonly Func, TableUpdate, Change>, TableUpdate> other_change_del = _other_changed_impl; + + static TableUpdate _source_changed_impl(TableUpdate target, Change> source_change, TableUpdate other_table) => + TableUpdate.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor); + + static TableUpdate _other_changed_impl(TableUpdate target, TableUpdate source_table, Change> other_change) => + TableUpdate.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor); + } + } +} +namespace bts.utils.diagnostics { + public static class DiagnosticUpdateExtensions { + // Hard to imagine why this util would be needed outside of diagnosig bugs. + // The #if DEBUG directive is an extra precaution to avoid accidentally writing + // files like this on a client's machine. +#if DEBUG + static readonly string _proto_capture_ext = "proto_capture"; + + public static IObservable> ProtoBufCapture(this IObservable> source, Func to_message) + where TKey : IEquatable + where TProto : class, IMessage => + ProtoBufCapture(source, + FilePathHelper.GetUniquePathWithDate(Path.Combine(Directory.GetCurrentDirectory(), typeof(TProto).Name), _proto_capture_ext), + to_message); + + public static IObservable> ProtoBufCapture(this IObservable> source, string path, Func to_message) + where TKey : IEquatable + where TProto : class, IMessage => + Observable.Create>(o => { + // write the state on dispose, and/or on error + IReadOnlyDictionary state = DictionaryHelper.empty(); + var sub = source.Subscribe(table => { + state = table.state; + o.OnNext(table); + }, ex => { + save(true); + o.OnError(ex); + }, + o.OnCompleted); + return Disposable.Create(() => { + save(false); + Disposable.dispose(ref sub); + }); + void save(bool has_error) { + if (state.Count > 0) + FileStreamHelper.WriteProtos(has_error ? $"{path}(error)" : path, state.Values.Select(to_message).ExceptNull()); + } + }); +#endif + } +} From 742af36b938feebfe6527ad4d9dc5109375cae26 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 6 May 2020 17:32:33 -0500 Subject: [PATCH 3/4] Security comparison unit test (addresses bug that caused Krandall OrderBook crash) --- .../KrandallOrderBookCrash.security_proto | Bin 0 -> 29544 bytes .../spark_test/Resources/ResourceHelper.cs | 13 +++++++ .../spark_test/Security/SecurityTests.cs | 30 ++++++++++++---- .../spark_test/Utils/ComparisonTests.cs | 34 ++++++++++++++++++ src/client/spark_test/spark_test.csproj | 4 ++- 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 src/client/spark_test/Resources/KrandallOrderBookCrash.security_proto create mode 100644 src/client/spark_test/Resources/ResourceHelper.cs create mode 100644 src/client/spark_test/Utils/ComparisonTests.cs diff --git a/src/client/spark_test/Resources/KrandallOrderBookCrash.security_proto b/src/client/spark_test/Resources/KrandallOrderBookCrash.security_proto new file mode 100644 index 0000000000000000000000000000000000000000..65b1a3f2ae4372af4bef803e028d1d0a85238bcb GIT binary patch literal 29544 zcmeHQYitzP6=r75g6$1K0n!vqvxdA#v}#N%Rm~q0tc4A%t?OMAq3MqRl^QCwf1*+~ zNu>mcscF=xlvf}n50^GTiAyLnN~mBlU{EP2V4#304du}$k?r*sC}18v_uRR&_s$Nx z^&Rgl#`X_nA zivqq50wUpQD4^em>@EmDzBVh4cy=KVtx`Y;O7vLB*=+Drj+$d>yJUnE?oXarwE{~E<;lhLRRhS$hesEdFV=8&zf zL<~;oWmLp&Q-o{=6!Bg1Cs!iC(qo$gV@dY{oh?H)g}IE6T*&}iBT^e5Wg=u_m?BQO z5&HvMAz+PA*9W2de_sTlK*DN?s3$B~8lzUoG}xY!>O19_Cfh z|5*<~9vewnVSfzVF&vLzL`mh)duCyktW0VXm-yKGla;ZTVzcvJiixr-(z2TaLb}P~EOjLRhHgxkS6o$`I9j7jL3$!J-t^)?-le71N?J6bZAat`;lH zR-If>LiMud`O*Z3iTxy&?}{}GsBLdTLIhB&DX6T%l0e;6MD~A`6(?Fs`&KRA1-z%- z!k^ApGIs{a%5*q<=F^fsU-OV|UrA{s9IOuhXzmO1f;Gd78?|;lHefPfHjX>bWM0xdjt(&{Y5vOdPERzI#CAclWHQEO4UJ1yUPy7d zr-&RL*9&oZ!&fp)h|9++CeYCQQjZPTj7Fwh*kzB*oHi4!EksA7=CP*l8rlWJfX3)KqtCC#NnagIPxrG2>Wr>HbCdfXjlhaDUd4+A| zy-Z*B)S2%-wP-F|biQ+PXgWGs!;o(p&`E&vt|-B3cJsV+27&S@-4+nlH`*AgwUqN5 zPz7gQe1Fg9Hz~jAtX?k7?yUJ9d1P_cP0MIIk0J`NvevWYtN`l@H?VrSG&`*Mp7UgZ z^~=o+R)fOIMnn>uf+ZaxnC18~-<6@V(6i%F=S(8+$4 zSxm}*jqRiJK2{SKC*n#s@p&J(iNd9vHIB4QgRP7!i%qx-uv8K51zPO;q(glnmdW8%1|IQQ-3qNV zYV(=dZ7$(Gp6uRFJ8%d!1)M!>`8Ngpb02xk(2Xi~kgBKNz0enXCH6hyQ+6 zKm|^2dmI&DCl7glNm78$@&#{e4m?awezQ>h^jc}j5llOY(1u*bQn@$qvyfwD%` z)C=@}XlmQ32|8mxc!lCi8TL&;NKZ66+J=2pUwep>ahW9vI`?Utg3AMn9G{H zB&9)b_2vgn;ue_J_v6WP>atnjL?`Ssq1wWDa$qfP0SrYeV0M7}4}cE`;@y=8xxhc| zC%^3tGF#yDw*3a% z&0wO=!eUrFauyqu02Uhn-?jVa0{^T|=DBup&pk{i@o|%*t1bbzlJxQv+$F(#qn)ph z!L4LOy4#d}D;Xr`r(vFknVM0>o|Jk|0P>GjHB9-<-9^DcE>c~B1v2k*{$=|;2=eqE zf&9U|G|22pJIKRTHMBtX+>z{_TxgIFPC<|(72wGZAj3fDFf{g!DwhMH{pv5Vx<7m% zw2uyiKqI+pUPrQY&2b>~*Iv+p4}{K`I>?Rl3>~EUejM2cSG-G6b3ofZ5Tg3SK&Vw+ z%k)5~z;NS0=z#h{ER(}MV7OL3mV9^>dOFnR@|++$-s8!~^szMHa)$yR2EdP~>zM|; z0KO$M$(;iD`fDcOxPvvV#aSBx2SWL9ST7JtfdNNC*{Hq{%ftDd+x7{}!CAe`2VvPA zd>l!@M+B&UGq~ki8=vja$!2ve(Z37kTcU&CA$VUk@y4x9ode#-eB^CfQ$+cvI`~Ij zx-|Zqx#LMJ5I*^m@?X^P&S|^W^M@^8;^o%bwWDac6_pdD0pE+c?THC`lB?5=Y(7!$ zI58S!%ZX8vZ66v?*{c;mv)0YNpi)LwEG$Ra@^NTMwr#)M9C(B@{Q{b9!;ENpnk-+h z!{_1f;p+-2T4y@@gIyg41RsBb4NdUGkP)?wkEuHXtn2zJtX{gTR7aMRKi1PToixA! zR%dVB&{ttyqoBCO1bhI}`Jcy;3FLAyg9&V~u6{#+RW24e33MoEcSn%GQ3b^b53fftvh*#x+EhS?aJtoIdAOJz-ErjT^9)?>FJf^#lG09TI_jpSos~+ zO8NA7)$WIBlRqvyaYi>BdC-&2h|`gY6NCQ`SHnBS literal 0 HcmV?d00001 diff --git a/src/client/spark_test/Resources/ResourceHelper.cs b/src/client/spark_test/Resources/ResourceHelper.cs new file mode 100644 index 000000000..28d870d55 --- /dev/null +++ b/src/client/spark_test/Resources/ResourceHelper.cs @@ -0,0 +1,13 @@ +using System.IO; +using System.Reflection; + +namespace bts.resources { + class ResourceHelper { + const string _default_namespace = "spark_test.Resources"; + + public static Stream LoadResource(string name) { + var ass = Assembly.GetExecutingAssembly(); + return ass.GetManifestResourceStream(name) ?? ass.GetManifestResourceStream($"{_default_namespace}.{name}"); + } + } +} diff --git a/src/client/spark_test/Security/SecurityTests.cs b/src/client/spark_test/Security/SecurityTests.cs index f564bd858..4e06581f9 100644 --- a/src/client/spark_test/Security/SecurityTests.cs +++ b/src/client/spark_test/Security/SecurityTests.cs @@ -1,13 +1,11 @@ -using System; +using bts.api; +using bts.resources; +using bts.test; +using bts.utils.proto; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -using bts.api; -using bts.test; -namespace bts.mcore -{ +namespace bts.mcore { [TestGroup] class SecurityTests { [Test] @@ -28,5 +26,23 @@ namespace bts.mcore Assert.That(dict[b] == 1); Assert.That(dict[c] == 2); } + + [Test] + public void test_security_compare() { + const string securities_resource = "KrandallOrderBookCrash.security_proto"; + // verify that the sample size doesn't shrink for some reason. + // if it grows a bit that's fine; the generic comparison test + // asserts against collections that are too large. + const int min_sample_size = 91; + + var securities = StreamHelper.ReadProtos( + ResourceHelper.LoadResource(securities_resource)) + .Select(s => new SecurityImmutable(s)) + .ToList(); + Assert.IsTrue(securities.Count >= min_sample_size); + + ComparisonTests.transitive_comparison(securities, SecurityHelper.Comparer.InstanceImmutable); + ComparisonTests.identity_comparison(securities, SecurityHelper.Comparer.InstanceImmutable); + } } } diff --git a/src/client/spark_test/Utils/ComparisonTests.cs b/src/client/spark_test/Utils/ComparisonTests.cs new file mode 100644 index 000000000..7dfa4db81 --- /dev/null +++ b/src/client/spark_test/Utils/ComparisonTests.cs @@ -0,0 +1,34 @@ +using bts.test; +using System.Collections.Generic; +using System.Linq; + +namespace bts.mcore { + static class ComparisonTests { + public static void transitive_comparison(IEnumerable items, IComparer comparer) { + // Verify that the sample size isn't too big. + // This test compares each item to every other item + // (technically twice; lhs/rhs is flipped the second time), + // so performance is O(n^2). + const int max_sample_size = 4092; + var sorted_items = items.ToList(); + Assert.That(sorted_items.Count <= max_sample_size); + sorted_items.Sort(comparer); + for (int i = 0; i < sorted_items.Count - 1; i++) { + for (int j = i + 1; j < sorted_items.Count; j++) { + assert_comp(i, j); + } + } + void assert_comp(int lhs, int rhs) { + int pos_comp = lhs.CompareTo(rhs); + // hit both sides for good measure + Assert.That(comparer.Compare(sorted_items[lhs], sorted_items[rhs]) == pos_comp); + Assert.That(comparer.Compare(sorted_items[rhs], sorted_items[lhs]) == -pos_comp); + } + } + + public static void identity_comparison(IEnumerable items, IComparer comparer) { + foreach (var v in items) + Assert.That(comparer.Compare(v, v) == 0); + } + } +} diff --git a/src/client/spark_test/spark_test.csproj b/src/client/spark_test/spark_test.csproj index 308bc8928..1b423bc89 100644 --- a/src/client/spark_test/spark_test.csproj +++ b/src/client/spark_test/spark_test.csproj @@ -14,5 +14,7 @@ - + + + From f3ed3075cf3e493464c5db0bee01c52443b0aadf Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 6 May 2020 17:33:10 -0500 Subject: [PATCH 4/4] fix Security comparison impl (its transitive now) --- .../spark_api/security/SecurityHelper.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/client/spark_api/security/SecurityHelper.cs b/src/client/spark_api/security/SecurityHelper.cs index 905d34846..b72e43913 100644 --- a/src/client/spark_api/security/SecurityHelper.cs +++ b/src/client/spark_api/security/SecurityHelper.cs @@ -83,22 +83,24 @@ namespace bts.api { } public static int Compare(SecurityImmutable x, SecurityImmutable y) { - if (x.ContractCase == ContractOneofCase.Future) return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1; - else if (y.ContractCase == ContractOneofCase.Future) return 1; + if (x.ContractCase == ContractOneofCase.Future) { + return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1; + } else if (y.ContractCase == ContractOneofCase.Future) { + return 1; + } var xlegs = get_flat_future_legs(x).ToList(); var ylegs = get_flat_future_legs(y).ToList(); int max = Math.Min(xlegs.Count, ylegs.Count); for (int i = 0; i < max; i++) { int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future); - if (rv != 0) return rv; - } - for (int i = 0; i < max; i++) { - int rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity); - if (rv != 0) return rv; - } - for (int i = 0; i < max; i++) { - int rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange); - if (rv != 0) return rv; + if (rv != 0) + return rv; + rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity); + if (rv != 0) + return rv; + rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange); + if (rv != 0) + return rv; } return xlegs.Count.CompareTo(ylegs.Count); }