fix FillTupleSource

This commit is contained in:
John Colvin committed 2018-06-08 23:36:06 -04:00
1 parent 30c1dfd9a7
commit 4e68c98cfb
6 files changed
+171 -151

No files matched your search

@@ -24,7 +24,7 @@ namespace mbts.spark {
public FillsVM(IBus bus, OrdersModel model, StartupData state)
: this(bus, model, _get_observable(bus, model, state), state.sorting_criteria, state.filters) { }
FillsVM(IBus bus, OrdersModel model, IObservable<IIncrementalUpdate<FillTuple, string>> fills_source, IReadOnlyCollection<SortingCriteria> sorting_criteria, FilterStates filters)
FillsVM(IBus bus, OrdersModel model, IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> fills_source, IReadOnlyCollection<SortingCriteria> sorting_criteria, FilterStates filters)
: base(RXCollection.CreateIncrementalUpdate(fills_source), sorting_criteria) {
_model = model;
account_filter = DiscreteValuesFilterVM.Create<Row, string>(_model.factory.subscribe_accounts(), r => r.account, nameof(Row.account));
@@ -126,7 +126,7 @@ namespace mbts.spark {
protected override Row row_factory(FillTuple row_model) => new Row(row_model, _model);
protected override IEnumerable<SortingCriteria> get_default_sorting() { yield return new SortingCriteria(nameof(Row.fill_time), false); }
static IObservable<IIncrementalUpdate<FillTuple, string>> _get_observable(IBus bus, OrdersModel model, StartupData state) {
static IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> _get_observable(IBus bus, OrdersModel model, StartupData state) {
if (state.orders != null && state.orders.Length == 0) {
var orders_hash = new HashSet<uint>(state.orders);
return model.on_fill_tuples.filter(f => orders_hash.Contains(f.fill.OrderKey));
+28 -4
View File
@@ -1,5 +1,6 @@
using bts.api;
using bts.api.orderman;
using mbts.utils;
using System;
namespace mbts.spark
@@ -8,7 +9,7 @@ namespace mbts.spark
public readonly FillImmutable fill;
public readonly OrderRecordImmutable? order;
public readonly SecurityImmutable? security;
public readonly string key;
public readonly FillTupleKey key;
public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityImmutable? security) {
this.fill = fill;
@@ -16,9 +17,7 @@ namespace mbts.spark
this.security = security;
// a FillTuple that DOES NOT HAVE associated order/security needs to be distinguishable
// from a FillTuple (of the same underlying FillImmutable) that DOES HAVE associated order/security
key = fill.FillKey;
if (!order.HasValue) key += "!o";
if (!security.HasValue) key += "!s";
key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security.HasValue ? security.Value.Id : 0u);
}
public bool Equals(FillTuple other) => other != null
@@ -29,4 +28,29 @@ namespace mbts.spark
public override bool Equals(object obj) => Equals(obj as FillTuple);
public override int GetHashCode() => key.GetHashCode();
}
struct FillTupleKey : IEquatable<FillTupleKey> {
public readonly string fill_key;
public readonly uint order_key, security_key;
public FillTupleKey(string fill_key, uint order_key, uint security_key) {
this.fill_key = fill_key;
this.order_key = order_key;
this.security_key = security_key;
}
public bool Equals(FillTupleKey other) =>
other.order_key == order_key &&
other.security_key == security_key &&
other.fill_key == fill_key;
public override bool Equals(object obj) =>
obj is FillTupleKey k && Equals(k);
public override int GetHashCode() =>
EqualityComparer.aggregate_hash(
fill_key.GetHashCode(),
(int)order_key,
(int)security_key);
}
}
+123 -144
View File
@@ -11,20 +11,21 @@ using System.Reactive.Subjects;
using System.Threading;
namespace mbts.spark {
class FillTupleSource : IDisposable {
public static IObservable<IIncrementalUpdate<FillTuple, string>> create_observable(
class FillTupleSource : IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>>, IDisposable {
// returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] FillTupleSource.
// (initializes when the first subscription is made, and destroys when the last subscription is disposed).
public static IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> create_observable(
IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities,
IObservable<TableUpdate<OrderRecordImmutable, uint>> orders,
IObservable<TableUpdate<FillImmutable, string>> fills
) {
IObservable<TableUpdate<FillImmutable, string>> fills) {
FillTupleSource source = null;
int ref_count = 0;
return Observable.Create<IIncrementalUpdate<FillTuple, string>>(observer => {
return Observable.Create<IIncrementalUpdate<FillTuple, FillTupleKey>>(observer => {
if (Interlocked.Increment(ref ref_count) == 1) {
SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak."));
source = new FillTupleSource(securities, orders, fills);
}
IDisposable sub = source.fill_tuples.Subscribe(observer);
IDisposable sub = source.Subscribe(observer);
return Disposable.Create(() => {
Disposable.dispose_once(ref sub);
if (Interlocked.Decrement(ref ref_count) == 0)
@@ -34,186 +35,164 @@ namespace mbts.spark {
string fmt_assert_msg(string msg) => $"{nameof(FillTupleSource)}.{nameof(create_observable)}: {msg}";
}
readonly Subject<IIncrementalUpdate<FillTuple, FillTupleKey>> _subject = new Subject<IIncrementalUpdate<FillTuple, FillTupleKey>>();
readonly Dictionary<FillTupleKey, FillTuple> _state = new Dictionary<FillTupleKey, FillTuple>();
readonly SetMap<uint, string> _order_fills = new SetMap<uint, string>();
readonly SetMap<uint, string> _security_fills = new SetMap<uint, string>();
IReadOnlyDictionary<string, FillImmutable> _fills = new Dictionary<string, FillImmutable>(0);
IReadOnlyDictionary<uint, OrderRecordImmutable> _orders = new Dictionary<uint, OrderRecordImmutable>(0);
IReadOnlyDictionary<uint, SecurityMD> _securities = new Dictionary<uint, SecurityMD>(0);
IncrementalUpdate<FillTuple, FillTupleKey> _last_update = new IncrementalUpdate<FillTuple, FillTupleKey>();
// _n_write_ops should never be greater than 1, bc all 3 input observables fire 'OnDispatcher'
int _n_read_ops, _n_write_ops;
IDisposable _dispose;
FillTupleSource(
IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities,
IObservable<TableUpdate<OrderRecordImmutable, uint>> orders,
IObservable<TableUpdate<FillImmutable, string>> fills) {
State state = new State();
fill_tuples = Observable.Create<IIncrementalUpdate<FillTuple, string>>(observer => {
observer.OnNext(new IncrementalUpdate<FillTuple, string>(state.full_state.ToDictionary(), new Dictionary<string, FillTuple>(0)));
return _subject.Subscribe(observer);
});
//
_dispose = Disposable.Create(
securities.OnDispatcher().Subscribe(s => {
state.update_securities(s);
_subject.OnNext(state.last_update);
}),
orders.OnDispatcher().Subscribe(o => {
state.update_orders(o);
_subject.OnNext(state.last_update);
}),
fills.OnDispatcher().Subscribe(f => {
state.update_fills(f);
_subject.OnNext(state.last_update);
}));
securities.OnDispatcher().Subscribe(_update_securities),
orders.OnDispatcher().Subscribe(_update_orders),
fills.OnDispatcher().Subscribe(_update_fills));
}
readonly Subject<IIncrementalUpdate<FillTuple, string>> _subject = new Subject<IIncrementalUpdate<FillTuple, string>>();
public IncrementalUpdate<FillTuple, FillTupleKey> last_update => _last_update;
public IReadOnlyDictionary<FillTupleKey, FillTuple> full_state => _state;
IObservable<IIncrementalUpdate<FillTuple, string>> fill_tuples { get; }
public void Dispose() =>
Disposable.dispose(ref _dispose);
class State {
IReadOnlyDictionary<string, FillImmutable> _fills;
IReadOnlyDictionary<uint, OrderRecordImmutable> _orders;
IReadOnlyDictionary<uint, SecurityMD> _securities;
IncrementalUpdate<FillTuple, string> _last_update;
readonly Dictionary<string, FillTuple> _state;
readonly SetMap<uint, string> _order_fills;
readonly SetMap<uint, string> _security_fills;
public State() : this(
new Dictionary<string, FillImmutable>(),
new Dictionary<uint, OrderRecordImmutable>(),
new Dictionary<uint, SecurityMD>(),
new IncrementalUpdate<FillTuple, string>(new Dictionary<string, FillTuple>(), new Dictionary<string, FillTuple>()),
new Dictionary<string, FillTuple>(),
new SetMap<uint, string>(),
new SetMap<uint, string>()
) { }
State(
IReadOnlyDictionary<string, FillImmutable> fills,
IReadOnlyDictionary<uint, OrderRecordImmutable> orders,
IReadOnlyDictionary<uint, SecurityMD> securities,
IncrementalUpdate<FillTuple, string> last_update,
Dictionary<string, FillTuple> state,
SetMap<uint, string> order_fill_tuples,
SetMap<uint, string> security_fill_tuples
) {
_fills = fills;
_orders = orders;
_securities = securities;
_last_update = last_update;
_state = state;
_order_fills = order_fill_tuples;
_security_fills = security_fill_tuples;
public IDisposable Subscribe(IObserver<IIncrementalUpdate<FillTuple, FillTupleKey>> observer) {
using (_read()) {
observer.OnNext(new IncrementalUpdate<FillTuple, FillTupleKey>(new Dictionary<FillTupleKey, FillTuple>(_state), new Dictionary<FillTupleKey, FillTuple>(0)));
IDisposable sub = _subject.Subscribe(observer);
return sub;
}
}
public IncrementalUpdate<FillTuple, string> last_update => _last_update;
public IReadOnlyDictionary<string, FillTuple> full_state => _state;
public void Dispose() => Disposable.dispose(ref _dispose);
public void update_fills(TableUpdate<FillImmutable, string> fills) {
var removed = fills.removed.ToDictionary(f => f.FillKey, get_fill_tuple);
var added = fills.added.ToDictionary(f => f.FillKey, get_fill_tuple);
HashSet<string> added_keys = new HashSet<string>(fills.added.Select(f => f.FillKey)),
removed_keys = new HashSet<string>(fills.removed.Select(f => f.FillKey));
foreach (var kv in removed) {
if (!added_keys.Contains(kv.Key)) {
_order_fills.remove(kv.Value.fill.OrderKey, kv.Key);
_security_fills.remove(kv.Value.fill.SecurityKey, kv.Key);
}
void _update_fills(TableUpdate<FillImmutable, string> fills) {
using (_write()) {
Dictionary<FillTupleKey, FillTuple> removed = new Dictionary<FillTupleKey, FillTuple>(), added = new Dictionary<FillTupleKey, FillTuple>();
foreach (FillImmutable fill in fills.removed) {
FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security(fill.SecurityKey));
removed.Add(old_ft.key, old_ft);
_security_fills.remove(fill.SecurityKey, fill.FillKey);
_order_fills.remove(fill.OrderKey, fill.FillKey);
}
foreach (var kv in added) {
if (!removed_keys.Contains(kv.Key)) {
_order_fills.add(kv.Value.fill.OrderKey, kv.Key);
_security_fills.add(kv.Value.fill.SecurityKey, kv.Key);
}
foreach (FillImmutable fill in fills.added) {
FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security(fill.SecurityKey));
added.Add(new_ft.key, new_ft);
_security_fills.add(fill.SecurityKey, fill.FillKey);
_order_fills.add(fill.OrderKey, fill.FillKey);
}
_fills = fills.state;
_apply_changes(added, removed);
//
FillTuple get_fill_tuple(FillImmutable fill) =>
new FillTuple(fill, _orders.TryGetValue(fill.OrderKey, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(),
_securities.TryGetValue(fill.SecurityKey, out SecurityMD s) ? s.security : new SecurityImmutable?());
}
}
public void update_orders(TableUpdate<OrderRecordImmutable, uint> orders) {
Dictionary<string, FillTuple> added = new Dictionary<string, FillTuple>(), removed = new Dictionary<string, FillTuple>();
Dictionary<uint, (OrderRecordImmutable oldval, OrderRecordImmutable newval)> changed_orders = new Dictionary<uint, (OrderRecordImmutable oldval, OrderRecordImmutable newval)>();
void _update_orders(TableUpdate<OrderRecordImmutable, uint> orders) {
using (_write()) {
Dictionary<FillTupleKey, FillTuple> added = new Dictionary<FillTupleKey, FillTuple>(), removed = new Dictionary<FillTupleKey, FillTuple>();
HashSet<uint> changed_keys = new HashSet<uint>();
foreach (var o in orders.removed) {
if (orders.try_get(o.Key, out OrderRecordImmutable newval))
changed_orders.Add(o.Key, (o, newval));
else if (_order_fills.try_lookup(o.Key, out ISet<string> fill_keys)) {
foreach (FillTuple ft in get_fill_tuples(o, fill_keys))
removed.Add(ft.key, ft);
foreach (FillTuple ft in get_fill_tuples(null, fill_keys))
added.Add(ft.key, ft);
if (_order_fills.try_lookup(o.Key, out ISet<string> fill_keys)) {
IEnumerable<FillImmutable> fills = fill_keys.Select(fk => _fills[fk]);
SecurityImmutable? sec = _get_security(o.security_key());
foreach (FillImmutable fill in fills) {
FillTuple old_ft = new FillTuple(fill, o, sec);
removed.Add(old_ft.key, old_ft);
}
OrderRecordImmutable? curr_order = null;
if (orders.try_get(o.Key, out OrderRecordImmutable newval)) {
curr_order = newval;
changed_keys.Add(o.Key);
}
foreach (FillImmutable fill in fills) {
FillTuple new_ft = new FillTuple(fill, curr_order, sec);
added.Add(new_ft.key, new_ft);
}
}
}
foreach (var o in orders.added) {
if (!changed_orders.ContainsKey(o.Key) && _order_fills.try_lookup(o.Key, out ISet<string> fill_keys)) {
foreach (FillTuple ft in get_fill_tuples(null, fill_keys))
removed.Add(ft.key, ft);
foreach (FillTuple ft in get_fill_tuples(o, fill_keys))
added.Add(ft.key, ft);
}
}
foreach (var kv in changed_orders) {
if (_order_fills.try_lookup(kv.Key, out ISet<string> fill_keys)) {
var fs_pairs = get_fill_security_pairs(fill_keys).ToList();
foreach (FillTuple ft in fs_pairs.Select(fsp => new FillTuple(fsp.fill, kv.Value.oldval, fsp.security)))
removed.Add(ft.key, ft);
foreach (FillTuple ft in fs_pairs.Select(fsp => new FillTuple(fsp.fill, kv.Value.newval, fsp.security)))
added.Add(ft.key, ft);
if (!changed_keys.Contains(o.Key) && _order_fills.try_lookup(o.Key, out ISet<string> fill_keys)) {
IEnumerable<FillImmutable> fills = fill_keys.Select(fk => _fills[fk]);
SecurityImmutable? sec = _get_security(o.security_key());
foreach (FillImmutable fill in fills) {
FillTuple old_ft = new FillTuple(fill, null, sec);
removed.Add(old_ft.key, old_ft);
FillTuple new_ft = new FillTuple(fill, o, sec);
added.Add(new_ft.key, new_ft);
}
}
}
_orders = orders.state;
_apply_changes(added, removed);
//
IEnumerable<FillTuple> get_fill_tuples(OrderRecordImmutable? o, ISet<string> fill_keys) =>
fill_keys.Select(k => {
var fill = _fills[k];
return new FillTuple(fill, o, _securities.TryGetValue(fill.SecurityKey, out SecurityMD s) ? s.security : new SecurityImmutable?());
});
IEnumerable<(FillImmutable fill, SecurityImmutable? security)> get_fill_security_pairs(ISet<string> fill_keys) =>
fill_keys.Select(k => {
var fill = _fills[k];
return (fill, _securities.TryGetValue(fill.SecurityKey, out SecurityMD s) ? s.security : new SecurityImmutable?());
});
}
}
public void update_securities(IReadOnlyDictionary<uint, SecurityMD> securities) {
Dictionary<string, FillTuple> added = new Dictionary<string, FillTuple>(), removed = new Dictionary<string, FillTuple>();
void _update_securities(IReadOnlyDictionary<uint, SecurityMD> securities) {
using (_write()) {
Dictionary<FillTupleKey, FillTuple> added = new Dictionary<FillTupleKey, FillTuple>(), removed = new Dictionary<FillTupleKey, FillTuple>();
_securities.Keys.difference(securities.Keys, out IEnumerable<uint> added_keys, out IEnumerable<uint> removed_keys);
foreach (var k in removed_keys) {
if (_security_fills.try_lookup(k, out ISet<string> fill_keys)) {
foreach (FillTuple ft in get_fill_tuples(_securities[k].security, fill_keys))
removed.Add(ft.key, ft);
foreach (FillTuple ft in get_fill_tuples(null, fill_keys))
added.Add(ft.key, ft);
IEnumerable<FillImmutable> fills = fill_keys.Select(fk => _fills[fk]);
SecurityImmutable old_security = _securities[k].security;
foreach (FillImmutable fill in fills) {
OrderRecordImmutable? order = _get_order(fill.OrderKey);
FillTuple old_ft = new FillTuple(fill, order, old_security);
removed.Add(old_ft.key, old_ft);
FillTuple new_ft = new FillTuple(fill, order, null);
added.Add(new_ft.key, new_ft);
}
}
}
foreach (var k in added_keys) {
if (_security_fills.try_lookup(k, out ISet<string> fill_keys)) {
foreach (FillTuple ft in get_fill_tuples(null, fill_keys))
removed.Add(ft.key, ft);
foreach (FillTuple ft in get_fill_tuples(securities[k].security, fill_keys))
added.Add(ft.key, ft);
IEnumerable<FillImmutable> fills = fill_keys.Select(fk => _fills[fk]);
SecurityImmutable new_security = securities[k].security;
foreach (FillImmutable fill in fills) {
OrderRecordImmutable? order = _get_order(fill.OrderKey);
FillTuple old_ft = new FillTuple(fill, order, null);
removed.Add(old_ft.key, old_ft);
FillTuple new_ft = new FillTuple(fill, order, new_security);
added.Add(new_ft.key, new_ft);
}
}
}
_securities = securities;
_apply_changes(added, removed);
//
IEnumerable<FillTuple> get_fill_tuples(SecurityImmutable? s, ISet<string> fill_keys) =>
fill_keys.Select(k => {
var fill = _fills[k];
return new FillTuple(fill, _orders.TryGetValue(fill.OrderKey, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(), s);
});
}
}
void _apply_changes(Dictionary<string, FillTuple> added, Dictionary<string, FillTuple> removed) {
foreach (var kv in removed)
_state.Remove(kv.Key);
foreach (var kv in added)
_state.Add(kv.Key, kv.Value);
_last_update = new IncrementalUpdate<FillTuple, string>(added, removed);
}
SecurityImmutable? _get_security(uint key) =>
_securities.TryGetValue(key, out SecurityMD s) ? s.security : new SecurityImmutable?();
OrderRecordImmutable? _get_order(uint key) =>
_orders.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?();
void _apply_changes(Dictionary<FillTupleKey, FillTuple> added, Dictionary<FillTupleKey, FillTuple> removed) {
foreach (var kv in removed)
_state.Remove(kv.Key);
foreach (var kv in added)
_state.Add(kv.Key, kv.Value);
_last_update = new IncrementalUpdate<FillTuple, FillTupleKey>(added, removed);
_subject.OnNext(_last_update);
}
IDisposable _read() {
while (_n_write_ops > 0)
Thread.Yield();
Interlocked.Increment(ref _n_read_ops);
return Disposable.Create(() => Interlocked.Decrement(ref _n_read_ops));
}
IDisposable _write() {
while (_n_read_ops > 0)
Thread.Yield();
Interlocked.Increment(ref _n_write_ops);
return Disposable.Create(() => Interlocked.Decrement(ref _n_write_ops));
}
}
}
+1 -1
View File
@@ -53,7 +53,7 @@ namespace mbts.spark {
public SpreaderModel spreader => _spreader;
public IObservable<TableUpdate<OrderRecordImmutable, uint>> on_orders { get; }
public IObservable<TableUpdate<FillImmutable, string>> on_fills { get; }
public IObservable<IIncrementalUpdate<FillTuple, string>> on_fill_tuples { get; }
public IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> on_fill_tuples { get; }
public IObservable<IReadOnlyDictionary<uint,OrderRecordImmutable>> working_orders { get; }
public IObservable<ITableUpdateTransform<OrderRecordImmutable, uint>> on_security_orders(uint security) =>
@@ -28,6 +28,20 @@ namespace mbts.utils {
y != null &&
non_null_equals(x, y);
public static int aggregate_hash(int item1_hash, int item2_hash, int item3_hash) {
int rv = 63607;
unchecked {
_accumulate_hash(ref rv, ref item1_hash);
_accumulate_hash(ref rv, ref item2_hash);
_accumulate_hash(ref rv, ref item3_hash);
}
return rv;
}
// WARNING: be sure to call this within in an unchecked scope
static void _accumulate_hash(ref int aggregate_hash, ref int item_hash) =>
aggregate_hash = aggregate_hash * 6451 + item_hash;
private class AnonymousEqualityComparer<T> : IEqualityComparer<T> {
readonly Func<T, T, bool> _equals;
readonly Func<T, int> _get_hash_code;
@@ -19,6 +19,9 @@ namespace mbts.utils {
}
class IncrementalUpdate<T, TKey> : IIncrementalUpdate<T, TKey> where TKey : IEquatable<TKey> {
public IncrementalUpdate()
: this(new Dictionary<TKey, T>(0), new Dictionary<TKey, T>(0)) { }
public IncrementalUpdate(IReadOnlyDictionary<TKey, T> added, IReadOnlyDictionary<TKey, T> removed) {
this.added = added;
this.removed = removed;