From 045418aceecaa4ded8008a1e1b3ef9ee479237e2 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 2 Aug 2018 15:36:28 -0400 Subject: [PATCH] OrdersModel - create OrderRecordTuples observable --- src/client/spark/Orders/OrderRecordTuple.cs | 37 +++- .../spark/Orders/OrderRecordTupleSource.cs | 171 ++++++++++++++++++ src/client/spark/Orders/OrdersModel.cs | 4 +- 3 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 src/client/spark/Orders/OrderRecordTupleSource.cs diff --git a/src/client/spark/Orders/OrderRecordTuple.cs b/src/client/spark/Orders/OrderRecordTuple.cs index da1c1e765..194da8563 100644 --- a/src/client/spark/Orders/OrderRecordTuple.cs +++ b/src/client/spark/Orders/OrderRecordTuple.cs @@ -1,26 +1,43 @@ using bts.api.orderman; +using mbts.utils; using System; namespace mbts.spark { + struct OrderRecordTupleKey : IEquatable { + public readonly uint order_key, security_key; + + public OrderRecordTupleKey(uint order_key, uint security_key) { + this.order_key = order_key; + this.security_key = security_key; + } + + public bool Equals(OrderRecordTupleKey other) => + other.order_key == order_key && + other.security_key == security_key; + + public override bool Equals(object obj) => + obj is OrderRecordTupleKey k && Equals(k); + + public override int GetHashCode() => + EqualityComparer.aggregate_hash( + (int)order_key, + (int)security_key); + } + class OrderRecordTuple : IEquatable { public readonly OrderRecordImmutable order; public readonly SecurityMD? security_md; - public readonly string key; + public readonly OrderRecordTupleKey key; - public OrderRecordTuple(OrderRecordImmutable order) { - this.order = order; - key = $"{order.Key}!s"; - } - - public OrderRecordTuple(OrderRecordImmutable order, SecurityMD security_md) { + public OrderRecordTuple(OrderRecordImmutable order, SecurityMD? security_md) { this.order = order; this.security_md = security_md; - key = order.Key.ToString(); + key = new OrderRecordTupleKey(order.Key, security_md.HasValue ? security_md.Value.security.Id : 0u); } public bool Equals(OrderRecordTuple other) => other != null - && other.security_md.HasValue == security_md.HasValue - && other.order.Equals(order); + && other.order.Equals(order) + && other.security_md.HasValue == security_md.HasValue; public override bool Equals(object obj) => Equals(obj as OrderRecordTuple); public override int GetHashCode() => key.GetHashCode(); diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs new file mode 100644 index 000000000..114859b1c --- /dev/null +++ b/src/client/spark/Orders/OrderRecordTupleSource.cs @@ -0,0 +1,171 @@ +using bts.api.core; +using bts.api.orderman; +using mbts.collections; +using mbts.utils; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading; + +namespace mbts.spark { + class OrderRecordTupleSource : IObservable>, IDisposable { + // returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] OrderRecordTupleSource. + // (initializes when the first subscription is made, and destroys when the last subscription is disposed). + public static IObservable> create_observable( + IObservable> securities, + IObservable> orders) { + OrderRecordTupleSource source = null; + int ref_count = 0; + return Observable.Create>(observer => { + if (Interlocked.Increment(ref ref_count) == 1) { + SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak.")); + source = new OrderRecordTupleSource(securities, orders); + } + IDisposable sub = source.Subscribe(observer); + return Disposable.Create(() => { + Disposable.dispose_once(ref sub); + if (Interlocked.Decrement(ref ref_count) == 0) + Disposable.dispose_once(ref source); + }); + }); + string fmt_assert_msg(string msg) => $"{nameof(OrderRecordTupleSource)}.{nameof(create_observable)}: {msg}"; + } + + // _n_write_ops should never be greater than 1, bc all 3 input observables fire 'OnDispatcher' + readonly ThreadYieldRWGate _rw_gate = new ThreadYieldRWGate(); + readonly Subject> _subject = new Subject>(); + readonly Dictionary _state = new Dictionary(); + readonly SetMap _security_orders = new SetMap(); + IReadOnlyDictionary _orders_cache = new Dictionary(0); + IReadOnlyDictionary _security_md_cache = new Dictionary(0); + IDisposable _dispose; + + OrderRecordTupleSource(IObservable> securities, IObservable> orders) => + _dispose = Disposable.Create( + securities.OnDispatcher().Subscribe(_update_securities), + orders.OnDispatcher().Subscribe(_update_orders)); + + public IDisposable Subscribe(IObserver> observer) { + using (_rw_gate.read()) { + observer.OnNext(new IncrementalUpdate(new Dictionary(_state), new Dictionary(0))); + return _subject.Subscribe(observer); + } + } + + public void Dispose() => Disposable.dispose(ref _dispose); + + void _update_orders(TableUpdate orders) { + using (_rw_gate.write()) { + Dictionary removed = new Dictionary(), added = new Dictionary(); + foreach (OrderRecordImmutable order in orders.removed) { + uint sec_key = order.security_key(); + OrderRecordTuple old_ft = new OrderRecordTuple(order, _get_security_md(sec_key)); + removed.Add(old_ft.order.Key, old_ft); + _security_orders.remove(sec_key, order.Key); + } + foreach (OrderRecordImmutable order in orders.added) { + uint sec_key = order.security_key(); + OrderRecordTuple new_ort = new OrderRecordTuple(order, _get_security_md(sec_key)); + added.Add(new_ort.order.Key, new_ort); + _security_orders.add(sec_key, order.Key); + } + _orders_cache = orders.state; + _apply_changes(added, removed); + } + } + + void _update_securities(IReadOnlyDictionary securities) { + using (_rw_gate.write()) { + Dictionary added = new Dictionary(), removed = new Dictionary(); + _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); + foreach (var k in removed_keys) { + if (_security_orders.try_lookup(k, out ISet order_keys)) { + IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); + SecurityMD old_sec_md = _security_md_cache[k]; + foreach (OrderRecordImmutable order in orders) { + OrderRecordTuple old_ft = new OrderRecordTuple(order, old_sec_md); + removed.Add(old_ft.order.Key, old_ft); + OrderRecordTuple new_ft = new OrderRecordTuple(order, null); + added.Add(new_ft.order.Key, new_ft); + } + } + } + foreach (var k in added_keys) { + if (_security_orders.try_lookup(k, out ISet order_keys)) { + IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); + SecurityMD new_sec_md = securities[k]; + foreach (OrderRecordImmutable order in orders) { + OrderRecordTuple old_ft = new OrderRecordTuple(order, null); + removed.Add(old_ft.order.Key, old_ft); + OrderRecordTuple new_ft = new OrderRecordTuple(order, new_sec_md); + added.Add(new_ft.order.Key, new_ft); + } + } + } + _security_md_cache = securities; + _apply_changes(added, removed); + } + } + + SecurityMD? _get_security_md(uint key) => + _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); + + OrderRecordImmutable? _get_order(uint key) => + _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); + + void _apply_changes(Dictionary added, Dictionary removed) { + foreach (var kv in removed) + _state.Remove(kv.Key); + foreach (var kv in added) + _state.Add(kv.Key, kv.Value); + _subject.OnNext(new IncrementalUpdate(added, removed)); + } + } + + /// + /// Provides mechanisms to ensure that only one write op may execute at a time, and calls to read or write while a write op is underway will Thread.yield until the write-op is completed. + /// If no write-op is underway, read-ops may run concurrently. + /// + class ThreadYieldRWGate { + int _n_write_ops, _n_read_ops; + + public IDisposable read() { + while (_n_write_ops > 0) + Thread.Yield(); + Interlocked.Increment(ref _n_read_ops); + if (_n_write_ops > 0) { + // A write-op was yielding at the same time and beat us to the punch! Try again + Interlocked.Decrement(ref _n_read_ops); + return read(); + } + return Disposable.Create(() => Interlocked.Decrement(ref _n_read_ops)); + } + + public IDisposable write() { + while (_n_read_ops > 0 || _n_write_ops > 0) + Thread.Yield(); + if (Interlocked.Increment(ref _n_write_ops) > 1 || _n_read_ops > 0) { + // Another write/read-op was yielding at the same time and beat us to the punch! Try again + Interlocked.Decrement(ref _n_write_ops); + return write(); + } + return Disposable.Create(() => Interlocked.Decrement(ref _n_write_ops)); + } + } + + class ThreadYieldGate { + int _n_ops; + + public IDisposable enter() { + while (_n_ops > 0) + Thread.Yield(); + if (Interlocked.Increment(ref _n_ops) > 1) { + Interlocked.Decrement(ref _n_ops); + return enter(); + } + return Disposable.Create(() => Interlocked.Decrement(ref _n_ops)); + } + } +} \ No newline at end of file diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index e3f8020bd..f5ab14855 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -35,6 +35,7 @@ namespace mbts.spark { IReadOnlyDictionary empty_orders = new Dictionary(); working_orders = on_orders.Scan(empty_orders, (worders, table) => _update_working_orders(worders, table)).Replay(1).RefCount(); + on_order_tuples = OrderRecordTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders); on_fills = _service.subscribei_fills().Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); @@ -42,7 +43,7 @@ namespace mbts.spark { _dispose = Disposable.Create( on_orders.Subscribe(_security_orders_demuxer), _bus.subscribe(SparkTags.CreateOrder).Subscribe(a => send_order(a.definition, a.price, a.size).ignore()), - _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach( x => send_order(x.definition, x.price, x.size).ignore())), + _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())), _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()), _bus.subscribe(SparkTags.CrackOrders).Subscribe(x => crack_orders(x).ignore()), _bus.subscribe(SparkTags.UpdateOrder).Subscribe(x => this.update_order(x.key, x.price, x.size).ignore()) @@ -53,6 +54,7 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public IObservable> on_orders { get; } + public IObservable> on_order_tuples { get; } public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; }