using bts.api; using bts.api.core; using bts.co2; using mbts; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading.Tasks; using security_key = System.UInt32; namespace Spark.Api.Reactive { public class SecurityResolver : IDisposable { readonly MarketDataRelay _md; IDisposable _dispose; IReadOnlyDictionary _live_securities; readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); readonly Subject> _cache_subject = new Subject>(); public SecurityResolver( MarketDataRelay md, IObservable> securities) { _md = md; on_securities = securities; _dispose = securities.Subscribe(dict => _live_securities = dict); _cache_subject.OnNext(_cache); } IObservable> on_securities; SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live = null) { // Try live securities first live = live ?? _live_securities; if (live != null) { if (live.TryGetValue(key, out SecurityMD value)) { return value.security; } } // Then try cached securities if (_cache.TryGetValue(key, out SecurityImmutable security)) { return security; } return null; } async Task get_security(security_key key, IReadOnlyDictionary live) { var local_sec = try_get_local(key, live); if (local_sec.HasValue) { return local_sec; } // Query the server for the value var result = await _md.get_securities_async(new SecurityKeyList() { Keys = { key } }); if (result.success) { var security = result.value.Security.FirstOrDefault(); if (security != null) { return try_cache_security(new SecurityImmutable(security)); } } // Give up :( Log.Md.warn($"Failed to get security for key {key}"); return null; } async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { var result = new Dictionary(); var missing = new SecurityKeyList(); foreach (var key in keys) { var s = try_get_local(key); if (s.HasValue) { result.Add(key, s.Value); } else { missing.Keys.Add(key); } } if (missing.Keys.Count > 0) { var res = await _md.get_securities_async(missing); if (res.success) { foreach (var security in res.value.Security) { result.Add(security.Id, try_cache_security(new SecurityImmutable(security))); } } } return result; } // Request any missing keys from the server but do not wait for them to arrive IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { var result = new Dictionary(); var missing = new HashSet(); foreach (var key in keys) { var security = try_get_local(key); if (security.HasValue) { result.TryAdd(key, security.Value); } else { missing.Add(key); } } // Update cache in the background if (missing.Count > 0) { Task.Run(async () => { var skl = new SecurityKeyList(); skl.Keys.AddRange(missing); var r = await _md.get_securities_async(skl); if (r.success) { var retrieved = r.value.Security.Select(s => s.Id); Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]"); foreach (var security in r.value.Security) { try_cache_security(new SecurityImmutable(security)); } } }); } return result; } SecurityImmutable try_cache_security(SecurityImmutable s) { if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { _cache.TryAdd(s.Id, s); _cache_subject.OnNext(_cache); return s; } return live.security; } /////////////// // PUBLIC /////////////// #region public // Guaranteed to return a security if it has ever been seen by the system public Task get_security(security_key key) { return get_security(key, _live_securities); } // Guaranteed to return a security if it has ever been seen by the system public Task> get_securities(IEnumerable keys) { return get_securities(keys, _live_securities); } public void Dispose() { Disposable.dispose(ref _dispose); } public IObservable> join( IObservable> source, Func skeyfn) where TKey : IEquatable { return join(source, skeyfn, get_incremental_props, mk_update); (IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor) get_incremental_props(Update upd_in) => (upd_in.all, upd_in.added, upd_in.changes, upd_in.removed, upd_in.key_extractor); Update<(SecurityImmutable? sec, T val), TKey> mk_update( ImmutableDictionary state, IEnumerable<(SecurityImmutable? sec, T val)> added, IEnumerable> changes, IEnumerable<(SecurityImmutable? sec, T val)> removed, Func<(SecurityImmutable?, T), TKey> key_extractor) => // new Update<(SecurityImmutable? sec, T val), TKey>( state.Values, added, changes, removed, key_extractor); } public IObservable join( IObservable source, Func skeyfn, Func all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor)> get_incremental_props, Func, IEnumerable<(SecurityImmutable? sec, T val)>, IEnumerable>, IEnumerable<(SecurityImmutable? sec, T val)>, Func<(SecurityImmutable?, T), TKey>, TOut> create_result_update) where TKey : IEquatable { // return Observable.Create(observer => { var state = ImmutableDictionary.Empty; var gate = new object(); // We get thie function from the first source update Func<(SecurityImmutable?, T t), TKey> rowkeyfn = null; void on_securities_updated(object ambiguous) { if (rowkeyfn == null) return; lock (gate) { // If this is null we don't have any rows var changed = new List>(); foreach (var entry in state.Values) { if (entry.s == null) { var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); if (security.HasValue) { changed.Add( new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); } } } if (changed.Count > 0) { // update state var builder = state.ToBuilder(); foreach (var up in changed) { builder[rowkeyfn(up.New)] = up.New; } state = builder.ToImmutable(); // fire update observer.OnNext( create_result_update( state, Enumerable.Empty<(SecurityImmutable?, T)>(), changed, Enumerable.Empty<(SecurityImmutable?, T)>(), rowkeyfn)); } } } return Disposable.Create( // Fire update when source changes source.Subscribe((update_in) => { var update = get_incremental_props(update_in); lock (gate) { rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2)); var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changes.New()).Select(skeyfn)); var added = new List<(SecurityImmutable?, T)>(); var changed = new List>(); var removed = new List<(SecurityImmutable?, T)>(); foreach (var x in update.added) { if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) { added.Add((s, x)); } else { added.Add((null, x)); } } foreach (var x in update.removed) { if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) { removed.Add(value); } } foreach (var ch in update.changes) { SecurityImmutable? si = relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ? s : (SecurityImmutable?)null; changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); } var builder = state.ToBuilder(); builder.RemoveRange(update.removed.Select(update.key_extractor)); builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old))); foreach (var a in added) { builder.Add(rowkeyfn(a), a); } foreach (var ch in changed) { builder.Add(rowkeyfn(ch.New), ch.New); } state = builder.ToImmutable(); observer.OnNext(create_result_update(state, added, changed, removed, rowkeyfn)); } }), // Also fire update when securities change on_securities.Subscribe(on_securities_updated), // Also fire update when cache changes _cache_subject.Subscribe(on_securities_updated) ); }); } public IObservable> join_table( IObservable> source, Func get_sec_key) where TKey : IEquatable => join( source, get_sec_key, t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor), (state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>( state, added, changes, removed, key)); #endregion } public static class SecurityStoreExtensions { public static IObservable> join(this IObservable> source, SecurityResolver store, Func skeyfn) where TKey : IEquatable { return store.join(source, skeyfn); } public static IObservable> join_table(this IObservable> source, SecurityResolver store, Func skeyfn) where TKey : IEquatable { return store.join_table(source, skeyfn); } } }