Files
spark/src/client/spark_api/Reactive/SecurityResolver.cs
T

294 lines
13 KiB
C#

using bts.api;
using bts.api.core;
using bts.co2;
using mbts;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading.Tasks;
using security_key = System.UInt32;
namespace Spark.Api.Reactive {
public class SecurityResolver : IDisposable {
readonly MarketDataRelay _md;
IDisposable _dispose;
IReadOnlyDictionary<security_key, SecurityMD> _live_securities;
readonly ConcurrentDictionary<security_key, SecurityImmutable> _cache = new ConcurrentDictionary<security_key, SecurityImmutable>();
readonly Subject<IReadOnlyDictionary<security_key, SecurityImmutable>> _cache_subject = new Subject<IReadOnlyDictionary<security_key, SecurityImmutable>>();
public SecurityResolver(
MarketDataRelay md, IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities) {
_md = md;
on_securities = securities;
_dispose = securities.Subscribe(dict => _live_securities = dict);
_cache_subject.OnNext(_cache);
}
IObservable<IReadOnlyDictionary<security_key, SecurityMD>> on_securities;
SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary<security_key, SecurityMD> live = null) {
// Try live securities first
live = live ?? _live_securities;
if (live != null) {
if (live.TryGetValue(key, out SecurityMD value)) {
return value.security;
}
}
// Then try cached securities
if (_cache.TryGetValue(key, out SecurityImmutable security)) {
return security;
}
return null;
}
async Task<SecurityImmutable?> get_security(security_key key, IReadOnlyDictionary<security_key, SecurityMD> live) {
var local_sec = try_get_local(key, live);
if (local_sec.HasValue) {
return local_sec;
}
// Query the server for the value
var result = await _md.get_securities_async(new SecurityKeyList() {
Keys = { key }
});
if (result.success) {
var security = result.value.Security.FirstOrDefault();
if (security != null) {
return try_cache_security(new SecurityImmutable(security));
}
}
// Give up :(
Log.Md.warn($"Failed to get security for key {key}");
return null;
}
async Task<IReadOnlyDictionary<security_key, SecurityImmutable>> get_securities(IEnumerable<security_key> keys, IReadOnlyDictionary<security_key, SecurityMD> live) {
var result = new Dictionary<security_key, SecurityImmutable>();
var missing = new SecurityKeyList();
foreach (var key in keys) {
var s = try_get_local(key);
if (s.HasValue) {
result.Add(key, s.Value);
} else {
missing.Keys.Add(key);
}
}
if (missing.Keys.Count > 0) {
var res = await _md.get_securities_async(missing);
if (res.success) {
foreach (var security in res.value.Security) {
result.Add(security.Id, try_cache_security(new SecurityImmutable(security)));
}
}
}
return result;
}
// Request any missing keys from the server but do not wait for them to arrive
IReadOnlyDictionary<security_key, SecurityImmutable> get_and_ensure_securities(IEnumerable<security_key> keys) {
var result = new Dictionary<security_key, SecurityImmutable>();
var missing = new HashSet<security_key>();
foreach (var key in keys) {
var security = try_get_local(key);
if (security.HasValue) {
result.TryAdd(key, security.Value);
} else {
missing.Add(key);
}
}
// Update cache in the background
if (missing.Count > 0) {
Task.Run(async () => {
var skl = new SecurityKeyList();
skl.Keys.AddRange(missing);
var r = await _md.get_securities_async(skl);
if (r.success) {
var retrieved = r.value.Security.Select(s => s.Id);
Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]");
foreach (var security in r.value.Security) {
try_cache_security(new SecurityImmutable(security));
}
}
});
}
return result;
}
SecurityImmutable try_cache_security(SecurityImmutable s) {
if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) {
_cache.TryAdd(s.Id, s);
_cache_subject.OnNext(_cache);
return s;
}
return live.security;
}
///////////////
// PUBLIC
///////////////
#region public
// Guaranteed to return a security if it has ever been seen by the system
public Task<SecurityImmutable?> get_security(security_key key) {
return get_security(key, _live_securities);
}
// Guaranteed to return a security if it has ever been seen by the system
public Task<IReadOnlyDictionary<security_key, SecurityImmutable>> get_securities(IEnumerable<security_key> keys) {
return get_securities(keys, _live_securities);
}
public void Dispose() {
Disposable.dispose(ref _dispose);
}
public IObservable<Update<(SecurityImmutable? sec, T val), TKey>> join<T, TKey>(
IObservable<Update<T, TKey>> source, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
return join(source, skeyfn, get_incremental_props, mk_update);
(IEnumerable<T> all, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key_extractor) get_incremental_props(Update<T, TKey> upd_in) =>
(upd_in.all, upd_in.added, upd_in.changes, upd_in.removed, upd_in.key_extractor);
Update<(SecurityImmutable? sec, T val), TKey> mk_update(
ImmutableDictionary<TKey, (SecurityImmutable? sec, T val)> state,
IEnumerable<(SecurityImmutable? sec, T val)> added,
IEnumerable<Change<(SecurityImmutable? sec, T val)>> changes,
IEnumerable<(SecurityImmutable? sec, T val)> removed,
Func<(SecurityImmutable?, T), TKey> key_extractor) =>
//
new Update<(SecurityImmutable? sec, T val), TKey>(
state.Values, added, changes, removed, key_extractor);
}
public IObservable<TOut> join<TIn, T, TKey, TOut>(
IObservable<TIn> source,
Func<T, security_key> skeyfn,
Func<TIn, (IEnumerable<T> all, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key_extractor)> get_incremental_props,
Func<ImmutableDictionary<TKey, (SecurityImmutable? sec, T val)>, IEnumerable<(SecurityImmutable? sec, T val)>, IEnumerable<Change<(SecurityImmutable? sec, T val)>>, IEnumerable<(SecurityImmutable? sec, T val)>, Func<(SecurityImmutable?, T), TKey>, TOut> create_result_update)
where TKey : IEquatable<TKey> {
//
return Observable.Create<TOut>(observer => {
var state = ImmutableDictionary<TKey, (SecurityImmutable? s, T t)>.Empty;
var gate = new object();
// We get thie function from the first source update
Func<(SecurityImmutable?, T t), TKey> rowkeyfn = null;
void on_securities_updated(object ambiguous) {
if (rowkeyfn == null) return;
lock (gate) {
// If this is null we don't have any rows
var changed = new List<Change<(SecurityImmutable? s, T t)>>();
foreach (var entry in state.Values) {
if (entry.s == null) {
var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary<security_key, SecurityMD>);
if (security.HasValue) {
changed.Add(
new Change<(SecurityImmutable?, T)>(entry, (security, entry.t)));
}
}
}
if (changed.Count > 0) {
// update state
var builder = state.ToBuilder();
foreach (var up in changed) {
builder[rowkeyfn(up.New)] = up.New;
}
state = builder.ToImmutable();
// fire update
observer.OnNext(
create_result_update(
state,
Enumerable.Empty<(SecurityImmutable?, T)>(),
changed,
Enumerable.Empty<(SecurityImmutable?, T)>(),
rowkeyfn));
}
}
}
return Disposable.Create(
// Fire update when source changes
source.Subscribe((update_in) => {
var update = get_incremental_props(update_in);
lock (gate) {
rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2));
var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changes.New()).Select(skeyfn));
var added = new List<(SecurityImmutable?, T)>();
var changed = new List<Change<(SecurityImmutable? s, T t)>>();
var removed = new List<(SecurityImmutable?, T)>();
foreach (var x in update.added) {
if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) {
added.Add((s, x));
} else {
added.Add((null, x));
}
}
foreach (var x in update.removed) {
if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) {
removed.Add(value);
}
}
foreach (var ch in update.changes) {
SecurityImmutable? si =
relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ?
s : (SecurityImmutable?)null;
changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New)));
}
var builder = state.ToBuilder();
builder.RemoveRange(update.removed.Select(update.key_extractor));
builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old)));
foreach (var a in added) {
builder.Add(rowkeyfn(a), a);
}
foreach (var ch in changed) {
builder.Add(rowkeyfn(ch.New), ch.New);
}
state = builder.ToImmutable();
observer.OnNext(create_result_update(state, added, changed, removed, rowkeyfn));
}
}),
// Also fire update when securities change
on_securities.Subscribe(on_securities_updated),
// Also fire update when cache changes
_cache_subject.Subscribe(on_securities_updated)
);
});
}
public IObservable<TableUpdate<(SecurityImmutable? sec, T val), TKey>> join_table<T, TKey>(
IObservable<TableUpdate<T, TKey>> source, Func<T, security_key> get_sec_key) where TKey : IEquatable<TKey> =>
join(
source,
get_sec_key,
t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor),
(state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>(
state, added, changes, removed, key));
#endregion
}
public static class SecurityStoreExtensions {
public static IObservable<Update<(SecurityImmutable? sec, T val), TKey>>
join<T, TKey>(this IObservable<Update<T, TKey>> source, SecurityResolver store, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
return store.join(source, skeyfn);
}
public static IObservable<TableUpdate<(SecurityImmutable? sec, T val), TKey>>
join_table<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, SecurityResolver store, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
return store.join_table(source, skeyfn);
}
}
}