From 4e68c98cfb557cb32f5756dd2ba8d6c999ea09b5 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 8 Jun 2018 22:48:52 -0400 Subject: [PATCH] fix FillTupleSource --- .../spark/Modules/Trading/Fills/FillsVM.cs | 4 +- src/client/spark/Orders/FillTuple.cs | 32 ++- src/client/spark/Orders/FillTupleSource.cs | 267 ++++++++---------- src/client/spark/Orders/OrdersModel.cs | 2 +- .../core/Utility/AnonymousEqualityComparer.cs | 14 + .../core/Utility/TableUpdateTransform.cs | 3 + 6 files changed, 171 insertions(+), 151 deletions(-) diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index b3feae79a..fe296df5c 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -24,7 +24,7 @@ namespace mbts.spark { public FillsVM(IBus bus, OrdersModel model, StartupData state) : this(bus, model, _get_observable(bus, model, state), state.sorting_criteria, state.filters) { } - FillsVM(IBus bus, OrdersModel model, IObservable> fills_source, IReadOnlyCollection sorting_criteria, FilterStates filters) + FillsVM(IBus bus, OrdersModel model, IObservable> fills_source, IReadOnlyCollection sorting_criteria, FilterStates filters) : base(RXCollection.CreateIncrementalUpdate(fills_source), sorting_criteria) { _model = model; account_filter = DiscreteValuesFilterVM.Create(_model.factory.subscribe_accounts(), r => r.account, nameof(Row.account)); @@ -126,7 +126,7 @@ namespace mbts.spark { protected override Row row_factory(FillTuple row_model) => new Row(row_model, _model); protected override IEnumerable get_default_sorting() { yield return new SortingCriteria(nameof(Row.fill_time), false); } - static IObservable> _get_observable(IBus bus, OrdersModel model, StartupData state) { + static IObservable> _get_observable(IBus bus, OrdersModel model, StartupData state) { if (state.orders != null && state.orders.Length == 0) { var orders_hash = new HashSet(state.orders); return model.on_fill_tuples.filter(f => orders_hash.Contains(f.fill.OrderKey)); diff --git a/src/client/spark/Orders/FillTuple.cs b/src/client/spark/Orders/FillTuple.cs index c0a9e0d68..1eb57a16b 100644 --- a/src/client/spark/Orders/FillTuple.cs +++ b/src/client/spark/Orders/FillTuple.cs @@ -1,5 +1,6 @@ using bts.api; using bts.api.orderman; +using mbts.utils; using System; namespace mbts.spark @@ -8,7 +9,7 @@ namespace mbts.spark public readonly FillImmutable fill; public readonly OrderRecordImmutable? order; public readonly SecurityImmutable? security; - public readonly string key; + public readonly FillTupleKey key; public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityImmutable? security) { this.fill = fill; @@ -16,9 +17,7 @@ namespace mbts.spark this.security = security; // a FillTuple that DOES NOT HAVE associated order/security needs to be distinguishable // from a FillTuple (of the same underlying FillImmutable) that DOES HAVE associated order/security - key = fill.FillKey; - if (!order.HasValue) key += "!o"; - if (!security.HasValue) key += "!s"; + key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security.HasValue ? security.Value.Id : 0u); } public bool Equals(FillTuple other) => other != null @@ -29,4 +28,29 @@ namespace mbts.spark public override bool Equals(object obj) => Equals(obj as FillTuple); public override int GetHashCode() => key.GetHashCode(); } + + struct FillTupleKey : IEquatable { + public readonly string fill_key; + public readonly uint order_key, security_key; + + public FillTupleKey(string fill_key, uint order_key, uint security_key) { + this.fill_key = fill_key; + this.order_key = order_key; + this.security_key = security_key; + } + + public bool Equals(FillTupleKey other) => + other.order_key == order_key && + other.security_key == security_key && + other.fill_key == fill_key; + + public override bool Equals(object obj) => + obj is FillTupleKey k && Equals(k); + + public override int GetHashCode() => + EqualityComparer.aggregate_hash( + fill_key.GetHashCode(), + (int)order_key, + (int)security_key); + } } diff --git a/src/client/spark/Orders/FillTupleSource.cs b/src/client/spark/Orders/FillTupleSource.cs index ce2067564..60ddfbee9 100644 --- a/src/client/spark/Orders/FillTupleSource.cs +++ b/src/client/spark/Orders/FillTupleSource.cs @@ -11,20 +11,21 @@ using System.Reactive.Subjects; using System.Threading; namespace mbts.spark { - class FillTupleSource : IDisposable { - public static IObservable> create_observable( + class FillTupleSource : IObservable>, IDisposable { + // returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] FillTupleSource. + // (initializes when the first subscription is made, and destroys when the last subscription is disposed). + public static IObservable> create_observable( IObservable> securities, IObservable> orders, - IObservable> fills - ) { + IObservable> fills) { FillTupleSource source = null; int ref_count = 0; - return Observable.Create>(observer => { + return Observable.Create>(observer => { if (Interlocked.Increment(ref ref_count) == 1) { SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak.")); source = new FillTupleSource(securities, orders, fills); } - IDisposable sub = source.fill_tuples.Subscribe(observer); + IDisposable sub = source.Subscribe(observer); return Disposable.Create(() => { Disposable.dispose_once(ref sub); if (Interlocked.Decrement(ref ref_count) == 0) @@ -34,186 +35,164 @@ namespace mbts.spark { string fmt_assert_msg(string msg) => $"{nameof(FillTupleSource)}.{nameof(create_observable)}: {msg}"; } + readonly Subject> _subject = new Subject>(); + readonly Dictionary _state = new Dictionary(); + readonly SetMap _order_fills = new SetMap(); + readonly SetMap _security_fills = new SetMap(); + IReadOnlyDictionary _fills = new Dictionary(0); + IReadOnlyDictionary _orders = new Dictionary(0); + IReadOnlyDictionary _securities = new Dictionary(0); + IncrementalUpdate _last_update = new IncrementalUpdate(); + // _n_write_ops should never be greater than 1, bc all 3 input observables fire 'OnDispatcher' + int _n_read_ops, _n_write_ops; IDisposable _dispose; FillTupleSource( IObservable> securities, IObservable> orders, IObservable> fills) { - State state = new State(); - fill_tuples = Observable.Create>(observer => { - observer.OnNext(new IncrementalUpdate(state.full_state.ToDictionary(), new Dictionary(0))); - return _subject.Subscribe(observer); - }); + // _dispose = Disposable.Create( - securities.OnDispatcher().Subscribe(s => { - state.update_securities(s); - _subject.OnNext(state.last_update); - }), - orders.OnDispatcher().Subscribe(o => { - state.update_orders(o); - _subject.OnNext(state.last_update); - }), - fills.OnDispatcher().Subscribe(f => { - state.update_fills(f); - _subject.OnNext(state.last_update); - })); + securities.OnDispatcher().Subscribe(_update_securities), + orders.OnDispatcher().Subscribe(_update_orders), + fills.OnDispatcher().Subscribe(_update_fills)); } - readonly Subject> _subject = new Subject>(); + public IncrementalUpdate last_update => _last_update; + public IReadOnlyDictionary full_state => _state; - IObservable> fill_tuples { get; } - - public void Dispose() => - Disposable.dispose(ref _dispose); - - class State { - IReadOnlyDictionary _fills; - IReadOnlyDictionary _orders; - IReadOnlyDictionary _securities; - IncrementalUpdate _last_update; - readonly Dictionary _state; - readonly SetMap _order_fills; - readonly SetMap _security_fills; - - public State() : this( - new Dictionary(), - new Dictionary(), - new Dictionary(), - new IncrementalUpdate(new Dictionary(), new Dictionary()), - new Dictionary(), - new SetMap(), - new SetMap() - ) { } - - State( - IReadOnlyDictionary fills, - IReadOnlyDictionary orders, - IReadOnlyDictionary securities, - IncrementalUpdate last_update, - Dictionary state, - SetMap order_fill_tuples, - SetMap security_fill_tuples - ) { - _fills = fills; - _orders = orders; - _securities = securities; - _last_update = last_update; - _state = state; - _order_fills = order_fill_tuples; - _security_fills = security_fill_tuples; + public IDisposable Subscribe(IObserver> observer) { + using (_read()) { + observer.OnNext(new IncrementalUpdate(new Dictionary(_state), new Dictionary(0))); + IDisposable sub = _subject.Subscribe(observer); + return sub; } + } - public IncrementalUpdate last_update => _last_update; - public IReadOnlyDictionary full_state => _state; + public void Dispose() => Disposable.dispose(ref _dispose); - public void update_fills(TableUpdate fills) { - var removed = fills.removed.ToDictionary(f => f.FillKey, get_fill_tuple); - var added = fills.added.ToDictionary(f => f.FillKey, get_fill_tuple); - HashSet added_keys = new HashSet(fills.added.Select(f => f.FillKey)), - removed_keys = new HashSet(fills.removed.Select(f => f.FillKey)); - foreach (var kv in removed) { - if (!added_keys.Contains(kv.Key)) { - _order_fills.remove(kv.Value.fill.OrderKey, kv.Key); - _security_fills.remove(kv.Value.fill.SecurityKey, kv.Key); - } + void _update_fills(TableUpdate fills) { + using (_write()) { + Dictionary removed = new Dictionary(), added = new Dictionary(); + foreach (FillImmutable fill in fills.removed) { + FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security(fill.SecurityKey)); + removed.Add(old_ft.key, old_ft); + _security_fills.remove(fill.SecurityKey, fill.FillKey); + _order_fills.remove(fill.OrderKey, fill.FillKey); } - foreach (var kv in added) { - if (!removed_keys.Contains(kv.Key)) { - _order_fills.add(kv.Value.fill.OrderKey, kv.Key); - _security_fills.add(kv.Value.fill.SecurityKey, kv.Key); - } + foreach (FillImmutable fill in fills.added) { + FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security(fill.SecurityKey)); + added.Add(new_ft.key, new_ft); + _security_fills.add(fill.SecurityKey, fill.FillKey); + _order_fills.add(fill.OrderKey, fill.FillKey); } _fills = fills.state; _apply_changes(added, removed); - // - FillTuple get_fill_tuple(FillImmutable fill) => - new FillTuple(fill, _orders.TryGetValue(fill.OrderKey, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(), - _securities.TryGetValue(fill.SecurityKey, out SecurityMD s) ? s.security : new SecurityImmutable?()); } + } - public void update_orders(TableUpdate orders) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - Dictionary changed_orders = new Dictionary(); + void _update_orders(TableUpdate orders) { + using (_write()) { + Dictionary added = new Dictionary(), removed = new Dictionary(); + HashSet changed_keys = new HashSet(); foreach (var o in orders.removed) { - if (orders.try_get(o.Key, out OrderRecordImmutable newval)) - changed_orders.Add(o.Key, (o, newval)); - else if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { - foreach (FillTuple ft in get_fill_tuples(o, fill_keys)) - removed.Add(ft.key, ft); - foreach (FillTuple ft in get_fill_tuples(null, fill_keys)) - added.Add(ft.key, ft); + if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { + IEnumerable fills = fill_keys.Select(fk => _fills[fk]); + SecurityImmutable? sec = _get_security(o.security_key()); + foreach (FillImmutable fill in fills) { + FillTuple old_ft = new FillTuple(fill, o, sec); + removed.Add(old_ft.key, old_ft); + } + OrderRecordImmutable? curr_order = null; + if (orders.try_get(o.Key, out OrderRecordImmutable newval)) { + curr_order = newval; + changed_keys.Add(o.Key); + } + foreach (FillImmutable fill in fills) { + FillTuple new_ft = new FillTuple(fill, curr_order, sec); + added.Add(new_ft.key, new_ft); + } } } foreach (var o in orders.added) { - if (!changed_orders.ContainsKey(o.Key) && _order_fills.try_lookup(o.Key, out ISet fill_keys)) { - foreach (FillTuple ft in get_fill_tuples(null, fill_keys)) - removed.Add(ft.key, ft); - foreach (FillTuple ft in get_fill_tuples(o, fill_keys)) - added.Add(ft.key, ft); - } - } - foreach (var kv in changed_orders) { - if (_order_fills.try_lookup(kv.Key, out ISet fill_keys)) { - var fs_pairs = get_fill_security_pairs(fill_keys).ToList(); - foreach (FillTuple ft in fs_pairs.Select(fsp => new FillTuple(fsp.fill, kv.Value.oldval, fsp.security))) - removed.Add(ft.key, ft); - foreach (FillTuple ft in fs_pairs.Select(fsp => new FillTuple(fsp.fill, kv.Value.newval, fsp.security))) - added.Add(ft.key, ft); + if (!changed_keys.Contains(o.Key) && _order_fills.try_lookup(o.Key, out ISet fill_keys)) { + IEnumerable fills = fill_keys.Select(fk => _fills[fk]); + SecurityImmutable? sec = _get_security(o.security_key()); + foreach (FillImmutable fill in fills) { + FillTuple old_ft = new FillTuple(fill, null, sec); + removed.Add(old_ft.key, old_ft); + FillTuple new_ft = new FillTuple(fill, o, sec); + added.Add(new_ft.key, new_ft); + } } } _orders = orders.state; _apply_changes(added, removed); - // - IEnumerable get_fill_tuples(OrderRecordImmutable? o, ISet fill_keys) => - fill_keys.Select(k => { - var fill = _fills[k]; - return new FillTuple(fill, o, _securities.TryGetValue(fill.SecurityKey, out SecurityMD s) ? s.security : new SecurityImmutable?()); - }); - - IEnumerable<(FillImmutable fill, SecurityImmutable? security)> get_fill_security_pairs(ISet fill_keys) => - fill_keys.Select(k => { - var fill = _fills[k]; - return (fill, _securities.TryGetValue(fill.SecurityKey, out SecurityMD s) ? s.security : new SecurityImmutable?()); - }); } + } - public void update_securities(IReadOnlyDictionary securities) { - Dictionary added = new Dictionary(), removed = new Dictionary(); + void _update_securities(IReadOnlyDictionary securities) { + using (_write()) { + Dictionary added = new Dictionary(), removed = new Dictionary(); _securities.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); foreach (var k in removed_keys) { if (_security_fills.try_lookup(k, out ISet fill_keys)) { - foreach (FillTuple ft in get_fill_tuples(_securities[k].security, fill_keys)) - removed.Add(ft.key, ft); - foreach (FillTuple ft in get_fill_tuples(null, fill_keys)) - added.Add(ft.key, ft); + IEnumerable fills = fill_keys.Select(fk => _fills[fk]); + SecurityImmutable old_security = _securities[k].security; + foreach (FillImmutable fill in fills) { + OrderRecordImmutable? order = _get_order(fill.OrderKey); + FillTuple old_ft = new FillTuple(fill, order, old_security); + removed.Add(old_ft.key, old_ft); + FillTuple new_ft = new FillTuple(fill, order, null); + added.Add(new_ft.key, new_ft); + } } } foreach (var k in added_keys) { if (_security_fills.try_lookup(k, out ISet fill_keys)) { - foreach (FillTuple ft in get_fill_tuples(null, fill_keys)) - removed.Add(ft.key, ft); - foreach (FillTuple ft in get_fill_tuples(securities[k].security, fill_keys)) - added.Add(ft.key, ft); + IEnumerable fills = fill_keys.Select(fk => _fills[fk]); + SecurityImmutable new_security = securities[k].security; + foreach (FillImmutable fill in fills) { + OrderRecordImmutable? order = _get_order(fill.OrderKey); + FillTuple old_ft = new FillTuple(fill, order, null); + removed.Add(old_ft.key, old_ft); + FillTuple new_ft = new FillTuple(fill, order, new_security); + added.Add(new_ft.key, new_ft); + } } } _securities = securities; _apply_changes(added, removed); - // - IEnumerable get_fill_tuples(SecurityImmutable? s, ISet fill_keys) => - fill_keys.Select(k => { - var fill = _fills[k]; - return new FillTuple(fill, _orders.TryGetValue(fill.OrderKey, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(), s); - }); } + } - void _apply_changes(Dictionary added, Dictionary removed) { - foreach (var kv in removed) - _state.Remove(kv.Key); - foreach (var kv in added) - _state.Add(kv.Key, kv.Value); - _last_update = new IncrementalUpdate(added, removed); - } + SecurityImmutable? _get_security(uint key) => + _securities.TryGetValue(key, out SecurityMD s) ? s.security : new SecurityImmutable?(); + + OrderRecordImmutable? _get_order(uint key) => + _orders.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); + + void _apply_changes(Dictionary added, Dictionary removed) { + foreach (var kv in removed) + _state.Remove(kv.Key); + foreach (var kv in added) + _state.Add(kv.Key, kv.Value); + _last_update = new IncrementalUpdate(added, removed); + _subject.OnNext(_last_update); + } + + IDisposable _read() { + while (_n_write_ops > 0) + Thread.Yield(); + Interlocked.Increment(ref _n_read_ops); + return Disposable.Create(() => Interlocked.Decrement(ref _n_read_ops)); + } + + IDisposable _write() { + while (_n_read_ops > 0) + Thread.Yield(); + Interlocked.Increment(ref _n_write_ops); + return Disposable.Create(() => Interlocked.Decrement(ref _n_write_ops)); } } } \ No newline at end of file diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 5706ce854..74648a095 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -53,7 +53,7 @@ namespace mbts.spark { public SpreaderModel spreader => _spreader; public IObservable> on_orders { get; } public IObservable> on_fills { get; } - public IObservable> on_fill_tuples { get; } + public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } public IObservable> on_security_orders(uint security) => diff --git a/src/client/spark/core/Utility/AnonymousEqualityComparer.cs b/src/client/spark/core/Utility/AnonymousEqualityComparer.cs index 66115ca2f..5abc8f752 100644 --- a/src/client/spark/core/Utility/AnonymousEqualityComparer.cs +++ b/src/client/spark/core/Utility/AnonymousEqualityComparer.cs @@ -28,6 +28,20 @@ namespace mbts.utils { y != null && non_null_equals(x, y); + public static int aggregate_hash(int item1_hash, int item2_hash, int item3_hash) { + int rv = 63607; + unchecked { + _accumulate_hash(ref rv, ref item1_hash); + _accumulate_hash(ref rv, ref item2_hash); + _accumulate_hash(ref rv, ref item3_hash); + } + return rv; + } + + // WARNING: be sure to call this within in an unchecked scope + static void _accumulate_hash(ref int aggregate_hash, ref int item_hash) => + aggregate_hash = aggregate_hash * 6451 + item_hash; + private class AnonymousEqualityComparer : IEqualityComparer { readonly Func _equals; readonly Func _get_hash_code; diff --git a/src/client/spark/core/Utility/TableUpdateTransform.cs b/src/client/spark/core/Utility/TableUpdateTransform.cs index 65c2b612d..4fcc0c97a 100644 --- a/src/client/spark/core/Utility/TableUpdateTransform.cs +++ b/src/client/spark/core/Utility/TableUpdateTransform.cs @@ -19,6 +19,9 @@ namespace mbts.utils { } class IncrementalUpdate : IIncrementalUpdate where TKey : IEquatable { + public IncrementalUpdate() + : this(new Dictionary(0), new Dictionary(0)) { } + public IncrementalUpdate(IReadOnlyDictionary added, IReadOnlyDictionary removed) { this.added = added; this.removed = removed;