From eaacafca86dfe5ae217865be2f46cad24ad6e369 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 5 Sep 2018 15:41:25 -0400 Subject: [PATCH] SPARK-1636 We can do better --- src/client/spark/Daemons/MDDaemon.cs | 41 ++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/client/spark/Daemons/MDDaemon.cs b/src/client/spark/Daemons/MDDaemon.cs index 32bc59eff..c3ae1779a 100644 --- a/src/client/spark/Daemons/MDDaemon.cs +++ b/src/client/spark/Daemons/MDDaemon.cs @@ -11,6 +11,8 @@ using System.Reactive.Linq; using System.Threading.Tasks; using System.Linq; using SecurityKey = System.UInt32; +using System.Collections.Concurrent; +using System.Threading; namespace bts { [Export(typeof(IInitializable))] @@ -19,9 +21,10 @@ namespace bts { readonly Lazy _ss; readonly MDConsumer _consumer = new MDConsumer(); readonly Co2RelayControl _relay; - HashSet _deleted_synthetics = new HashSet(); + readonly Dictionary _deleted_synthetics = new Dictionary(); + long _version = 0; + readonly ConcurrentDictionary _exchange_sec_map = new ConcurrentDictionary(); List _dispose = new List(); - IReadOnlyDictionary _exchange_sec_map = new Dictionary(); [ImportingConstructor] public MDSource(Lazy bus, Lazy ss) { @@ -36,7 +39,23 @@ namespace bts { _ss.Value.subscribei_metadata() .Select(upd => upd.all.ToDictionary(i => i.SecurityKey, i => i.Strategy)) .Subscribe(t => { - _deleted_synthetics = new HashSet(t.Where(kv => kv.Value.Deleted).Select(kv => kv.Key)); + var deleted_synthetic_keys = new HashSet(t.Where(kv => kv.Value.Deleted).Select(kv => kv.Key)); + // add back 'undeleted' synthetics + foreach (var kvp in _deleted_synthetics) { + if (!deleted_synthetic_keys.Contains(kvp.Key)) { + _exchange_sec_map.TryAdd(kvp.Key, kvp.Value); + _deleted_synthetics.Remove(kvp.Key); + _version += 1; + } + } + // Remove any newly deleted synthetics and store them for potential undelete + foreach (var key in deleted_synthetic_keys) { + if (_exchange_sec_map.TryRemove(key, out SecurityMD deleted)) { + _deleted_synthetics[key] = deleted; + _version += 1; + } + } + // Post full dictionary of syntheics here bus.post(SparkTags.SyntheticstrategiesTag, t); })); Task.Run(run); @@ -52,24 +71,22 @@ namespace bts { public void OnError(Exception error) => Log.Md.error(error.ToString()); public void OnNext(List values) { if (values.Count > 0) { - var temp = _exchange_sec_map.ToDictionary(k => k.Key, k => k.Value); foreach (var v in values) { - temp.Add(v.security.Id, new SecurityMD(v.security, v.handle)); + var smd = new SecurityMD(v.security, v.handle); + _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd); } - _exchange_sec_map = temp; + _version += 1; } } public void Dispose() => Disposable.dispose(_dispose); async Task run() { - int exch_sec_count = 0, deleted_synth_count = 0; + long run_version = 0; while (true) { var exch = _exchange_sec_map; - var deleted_synths = _deleted_synthetics; - if (exch.Count > exch_sec_count || deleted_synths.Count != deleted_synth_count) { - exch_sec_count = exch.Count; - deleted_synth_count = deleted_synths.Count; - exch = exch.Where(kv => !deleted_synths.Contains(kv.Key)).ToDictionary(); + long current_version = _version; + if (current_version > run_version) { + run_version = current_version; try { Log.Md.info($"MDSource publishing {exch.Count} securities"); bus.post(SparkTags.SecuritiesTag, exch);