refactor: Move SecurityStore to spark_core, rename Resolver
This commit is contained in:
1 parent
203164f372
commit
c19d173983
10 files changed
+1061
-978
No files matched your search
@@ -0,0 +1,25 @@
|
||||
#nullable enable
|
||||
namespace Spark.Model {
|
||||
using bts;
|
||||
using bts.api;
|
||||
using bts.co2;
|
||||
|
||||
using mbts.spark;
|
||||
|
||||
using Spark.Api.Reactive;
|
||||
using Spark.Core;
|
||||
|
||||
using System.Composition;
|
||||
|
||||
class SecurityResolverExporter : VirtualSingleton<SecurityResolverExporter> {
|
||||
readonly LazyOnce<SecurityResolver>? _resolver;
|
||||
|
||||
[ImportingConstructor]
|
||||
public SecurityResolverExporter(IBus bus, MarketDataRelay md_relay) =>
|
||||
_resolver = new(() => new(md_relay, bus.subscribe(SparkTags.SecuritiesTag)));
|
||||
|
||||
[Export]
|
||||
public SecurityResolver resolver =>
|
||||
Log.notnull(_resolver);
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ namespace Spark.Model.Trading {
|
||||
|
||||
using mbts.spark;
|
||||
|
||||
using Spark.Api.Reactive;
|
||||
using Spark.Core.Reactive;
|
||||
|
||||
using System;
|
||||
@@ -25,7 +26,7 @@ namespace Spark.Model.Trading {
|
||||
public OrderDataSource(
|
||||
IBus bus,
|
||||
AdminModel admin_model,
|
||||
SecurityStore security_store,
|
||||
SecurityResolver sec_resolver,
|
||||
OrderService order_service,
|
||||
ISecurityAuxiliaryDataModel security_auxiliary_model,
|
||||
ErrorReporter error_reporter
|
||||
@@ -38,7 +39,7 @@ namespace Spark.Model.Trading {
|
||||
order_service.subscribei_orders()
|
||||
.Naggle(interval, scheduler),
|
||||
o => o.account_id())
|
||||
.join(security_store, o => o.security_key())
|
||||
.join(sec_resolver, o => o.security_key())
|
||||
.Table()
|
||||
.Cache();
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ namespace Spark.Trading {
|
||||
using mbts.spark;
|
||||
using mbts.ui;
|
||||
|
||||
using Spark.Api.Reactive;
|
||||
using Spark.Core;
|
||||
using Spark.Core.Reactive;
|
||||
using Spark.Model.Trading;
|
||||
@@ -38,7 +39,7 @@ namespace Spark.Trading {
|
||||
public PositionModel(
|
||||
IBus bus,
|
||||
AdminModel admin_model,
|
||||
SecurityStore security_store,
|
||||
SecurityResolver sec_resolver,
|
||||
PositionService position_service,
|
||||
OrderService order_service,
|
||||
OrderDataSource order_data,
|
||||
@@ -55,7 +56,7 @@ namespace Spark.Trading {
|
||||
on_exchange_positions = order_service.subscribei_exchange_positions()
|
||||
.Filter(ep => ep.SecurityKey != 0)
|
||||
.SelectKeyValue(ep => ep, ep => new AccountSecurityKey(ep.AccountId, ep.SecurityKey))
|
||||
.join(security_store, ep => ep.SecurityKey)
|
||||
.join(sec_resolver, ep => ep.SecurityKey)
|
||||
.Table()
|
||||
.Cache();
|
||||
|
||||
@@ -63,11 +64,11 @@ namespace Spark.Trading {
|
||||
.permission_filter(
|
||||
_position_service.subscribei_position_data().Naggle(interval, scheduler),
|
||||
p => p.AccountId)
|
||||
.join(security_store, pos => pos.SecurityId)
|
||||
.join(sec_resolver, pos => pos.SecurityId)
|
||||
.Table()
|
||||
.Cache();
|
||||
|
||||
day_fills = DayFillsDataSource.create(bus, security_store);
|
||||
day_fills = DayFillsDataSource.create(bus, sec_resolver);
|
||||
|
||||
pnl = bus.subscribe(SparkTags.ReportBlackboardTags.PNL.PNLData)
|
||||
.Select(d => d.position_data.ToDictionary(u => u.id(), PNLRowData.from_pnl_unit))
|
||||
|
||||
@@ -7,6 +7,8 @@ namespace Spark.Trading {
|
||||
using mbts.analytics;
|
||||
using mbts.spark;
|
||||
|
||||
using Spark.Api.Reactive;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -14,26 +16,26 @@ namespace Spark.Trading {
|
||||
|
||||
class DayFillsDataSource {
|
||||
public static IObservable<TableUpdate<DayFillsRowData, DayFillsRowKey>> create(
|
||||
IBus bus, SecurityStore sec_store) {
|
||||
IBus bus, SecurityResolver sec_resolver) {
|
||||
var users = bus.subscribe(SparkTags.Users);
|
||||
var inverse_users = users
|
||||
.Select(d => d.ToDictionary(kv => kv.Value.name, kv => kv.Value))
|
||||
.Table(u => u.name);
|
||||
var pnl_data_source = bus.subscribe(SparkTags.ReportBlackboardTags.PNL.PNLData);
|
||||
return create(pnl_data_source, inverse_users, sec_store);
|
||||
return create(pnl_data_source, inverse_users, sec_resolver);
|
||||
}
|
||||
|
||||
public static IObservable<TableUpdate<DayFillsRowData, DayFillsRowKey>> create(
|
||||
IObservable<PNLData> pnl_data_source,
|
||||
IObservable<TableUpdate<UserTuple, string>> inverse_users,
|
||||
SecurityStore sec_store) =>
|
||||
SecurityResolver sec_resolver) =>
|
||||
// join pnl data w/ security and user
|
||||
pnl_data_source
|
||||
.Select(d => d.day_trades
|
||||
.Select(DayFillsRowData.from_trade)
|
||||
.ToDictionary(DayFillsRowData.key_fn))
|
||||
.Table(DayFillsRowData.key_fn)
|
||||
.join_table(sec_store, rd => rd.security_key)
|
||||
.join_table(sec_resolver, rd => rd.security_key)
|
||||
.Select(tup => new DayFillsRowData(tup.val.trade, tup.sec), DayFillsRowData.key_fn)
|
||||
.LeftJoin(
|
||||
inverse_users,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
namespace bts.shell {
|
||||
namespace bts.shell {
|
||||
using bts.api;
|
||||
using bts.shell.fills;
|
||||
using bts.shell.manual_fills;
|
||||
@@ -9,6 +9,7 @@
|
||||
using mbts.ui;
|
||||
using mbts.ui.shell;
|
||||
|
||||
using Spark.Api.Reactive;
|
||||
using Spark.Core.Data;
|
||||
using Spark.Trading;
|
||||
|
||||
@@ -35,7 +36,7 @@
|
||||
INotificationServiceDataCache notification_service_data_cache,
|
||||
UserSettingsHub settings_hub,
|
||||
ISecurityMetadataStore security_metadata,
|
||||
SecurityStore security_store,
|
||||
SecurityResolver security_resolver,
|
||||
OrdersModel orders_model,
|
||||
md.MBOService mbo,
|
||||
IPositionModel position_model,
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -13,6 +13,7 @@
|
||||
using mbts.ui.settings;
|
||||
using mbts.ui.shell;
|
||||
|
||||
using Spark.Api.Reactive;
|
||||
using Spark.Core.Data;
|
||||
|
||||
using System;
|
||||
@@ -141,7 +142,7 @@
|
||||
typeof(OrderFactory),
|
||||
typeof(AlgoProviderFactory),
|
||||
typeof(DynamicOrderHelper),
|
||||
typeof(SecurityStore),
|
||||
typeof(SecurityResolver),
|
||||
typeof(AdminModel),
|
||||
typeof(AlgoStudioFlatDefinitionModel),
|
||||
typeof(AlgoTemplateMetadataService),
|
||||
|
||||
+293
-298
@@ -1,298 +1,293 @@
|
||||
using bts.api;
|
||||
using bts.api.core;
|
||||
using bts.co2;
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Composition;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
using System.Reactive.Subjects;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using security_key = System.UInt32;
|
||||
|
||||
namespace bts {
|
||||
|
||||
[Export]
|
||||
public class SecurityStore : IDisposable {
|
||||
readonly MarketDataRelay _md;
|
||||
readonly IBus _bus;
|
||||
IDisposable _dispose;
|
||||
IReadOnlyDictionary<security_key, SecurityMD> _live_securities;
|
||||
readonly ConcurrentDictionary<security_key, SecurityImmutable> _cache = new ConcurrentDictionary<security_key, SecurityImmutable>();
|
||||
readonly Subject<IReadOnlyDictionary<security_key, SecurityImmutable>> _cache_subject = new Subject<IReadOnlyDictionary<security_key, SecurityImmutable>>();
|
||||
|
||||
[ImportingConstructor]
|
||||
public SecurityStore(MarketDataRelay md, IBus bus) {
|
||||
_md = md;
|
||||
_bus = bus;
|
||||
_dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict);
|
||||
_cache_subject.OnNext(_cache);
|
||||
}
|
||||
|
||||
IObservable<IReadOnlyDictionary<security_key, SecurityMD>> on_securities => _bus.subscribe(SparkTags.SecuritiesTag);
|
||||
|
||||
SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary<security_key, SecurityMD> live = null) {
|
||||
// Try live securities first
|
||||
live = live ?? _live_securities;
|
||||
if (live != null) {
|
||||
if (live.TryGetValue(key, out SecurityMD value)) {
|
||||
return value.security;
|
||||
}
|
||||
}
|
||||
|
||||
// Then try cached securities
|
||||
if (_cache.TryGetValue(key, out SecurityImmutable security)) {
|
||||
return security;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async Task<SecurityImmutable?> get_security(security_key key, IReadOnlyDictionary<security_key, SecurityMD> live) {
|
||||
var local_sec = try_get_local(key, live);
|
||||
if (local_sec.HasValue) {
|
||||
return local_sec;
|
||||
}
|
||||
|
||||
// Query the server for the value
|
||||
var result = await _md.get_securities_async(new SecurityKeyList() {
|
||||
Keys = { key }
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
var security = result.value.Security.FirstOrDefault();
|
||||
if (security != null) {
|
||||
return try_cache_security(new SecurityImmutable(security));
|
||||
}
|
||||
}
|
||||
|
||||
// Give up :(
|
||||
Log.Md.warn($"Failed to get security for key {key}");
|
||||
return null;
|
||||
}
|
||||
|
||||
async Task<IReadOnlyDictionary<security_key, SecurityImmutable>> get_securities(IEnumerable<security_key> keys, IReadOnlyDictionary<security_key, SecurityMD> live) {
|
||||
var result = new Dictionary<security_key, SecurityImmutable>();
|
||||
var missing = new SecurityKeyList();
|
||||
foreach (var key in keys) {
|
||||
var s = try_get_local(key);
|
||||
if (s.HasValue) {
|
||||
result.Add(key, s.Value);
|
||||
} else {
|
||||
missing.Keys.Add(key);
|
||||
}
|
||||
}
|
||||
if (missing.Keys.Count > 0) {
|
||||
var res = await _md.get_securities_async(missing);
|
||||
if (res.success) {
|
||||
foreach (var security in res.value.Security) {
|
||||
result.Add(security.Id, try_cache_security(new SecurityImmutable(security)));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Request any missing keys from the server but do not wait for them to arrive
|
||||
IReadOnlyDictionary<security_key, SecurityImmutable> get_and_ensure_securities(IEnumerable<security_key> keys) {
|
||||
var result = new Dictionary<security_key, SecurityImmutable>();
|
||||
var missing = new HashSet<security_key>();
|
||||
foreach (var key in keys) {
|
||||
var security = try_get_local(key);
|
||||
if (security.HasValue) {
|
||||
result.TryAdd(key, security.Value);
|
||||
} else {
|
||||
missing.Add(key);
|
||||
}
|
||||
}
|
||||
// Update cache in the background
|
||||
if (missing.Count > 0) {
|
||||
Task.Run(async () => {
|
||||
var skl = new SecurityKeyList();
|
||||
skl.Keys.AddRange(missing);
|
||||
var r = await _md.get_securities_async(skl);
|
||||
if (r.success) {
|
||||
var retrieved = r.value.Security.Select(s => s.Id);
|
||||
Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]");
|
||||
foreach (var security in r.value.Security) {
|
||||
try_cache_security(new SecurityImmutable(security));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
SecurityImmutable try_cache_security(SecurityImmutable s) {
|
||||
if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) {
|
||||
_cache.TryAdd(s.Id, s);
|
||||
_cache_subject.OnNext(_cache);
|
||||
return s;
|
||||
}
|
||||
return live.security;
|
||||
}
|
||||
|
||||
///////////////
|
||||
// PUBLIC
|
||||
///////////////
|
||||
#region public
|
||||
|
||||
// Guaranteed to return a security if it has ever been seen by the system
|
||||
public Task<SecurityImmutable?> get_security(security_key key) {
|
||||
return get_security(key, _live_securities);
|
||||
}
|
||||
// Guaranteed to return a security if it has ever been seen by the system
|
||||
public Task<IReadOnlyDictionary<security_key, SecurityImmutable>> get_securities(IEnumerable<security_key> keys) {
|
||||
return get_securities(keys, _live_securities);
|
||||
}
|
||||
public void Dispose() {
|
||||
Disposable.dispose(ref _dispose);
|
||||
}
|
||||
|
||||
public IObservable<Update<(SecurityImmutable? sec, T val), TKey>> join<T, TKey>(
|
||||
IObservable<Update<T, TKey>> source, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
|
||||
return join(source, skeyfn, get_incremental_props, mk_update);
|
||||
|
||||
(IEnumerable<T> all, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key_extractor) get_incremental_props(Update<T, TKey> upd_in) =>
|
||||
(upd_in.all, upd_in.added, upd_in.changes, upd_in.removed, upd_in.key_extractor);
|
||||
|
||||
Update<(SecurityImmutable? sec, T val), TKey> mk_update(
|
||||
ImmutableDictionary<TKey, (SecurityImmutable? sec, T val)> state,
|
||||
IEnumerable<(SecurityImmutable? sec, T val)> added,
|
||||
IEnumerable<Change<(SecurityImmutable? sec, T val)>> changes,
|
||||
IEnumerable<(SecurityImmutable? sec, T val)> removed,
|
||||
Func<(SecurityImmutable?, T), TKey> key_extractor) =>
|
||||
//
|
||||
new Update<(SecurityImmutable? sec, T val), TKey>(
|
||||
state.Values, added, changes, removed, key_extractor);
|
||||
}
|
||||
|
||||
public IObservable<TOut> join<TIn, T, TKey, TOut>(
|
||||
IObservable<TIn> source,
|
||||
Func<T, security_key> skeyfn,
|
||||
Func<TIn, (IEnumerable<T> all, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key_extractor)> get_incremental_props,
|
||||
Func<ImmutableDictionary<TKey, (SecurityImmutable? sec, T val)>, IEnumerable<(SecurityImmutable? sec, T val)>, IEnumerable<Change<(SecurityImmutable? sec, T val)>>, IEnumerable<(SecurityImmutable? sec, T val)>, Func<(SecurityImmutable?, T), TKey>, TOut> create_result_update)
|
||||
where TKey : IEquatable<TKey> {
|
||||
//
|
||||
return Observable.Create<TOut>(observer => {
|
||||
var state = ImmutableDictionary<TKey, (SecurityImmutable? s, T t)>.Empty;
|
||||
var gate = new object();
|
||||
// We get thie function from the first source update
|
||||
Func<(SecurityImmutable?, T t), TKey> rowkeyfn = null;
|
||||
|
||||
void on_securities_updated(object ambiguous) {
|
||||
if (rowkeyfn == null) return;
|
||||
lock (gate) {
|
||||
// If this is null we don't have any rows
|
||||
var changed = new List<Change<(SecurityImmutable? s, T t)>>();
|
||||
foreach (var entry in state.Values) {
|
||||
if (entry.s == null) {
|
||||
var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary<security_key, SecurityMD>);
|
||||
if (security.HasValue) {
|
||||
changed.Add(
|
||||
new Change<(SecurityImmutable?, T)>(entry, (security, entry.t)));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (changed.Count > 0) {
|
||||
// update state
|
||||
var builder = state.ToBuilder();
|
||||
foreach (var up in changed) {
|
||||
builder[rowkeyfn(up.New)] = up.New;
|
||||
}
|
||||
state = builder.ToImmutable();
|
||||
// fire update
|
||||
observer.OnNext(
|
||||
create_result_update(
|
||||
state,
|
||||
Enumerable.Empty<(SecurityImmutable?, T)>(),
|
||||
changed,
|
||||
Enumerable.Empty<(SecurityImmutable?, T)>(),
|
||||
rowkeyfn));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Disposable.Create(
|
||||
// Fire update when source changes
|
||||
source.Subscribe((update_in) => {
|
||||
var update = get_incremental_props(update_in);
|
||||
lock (gate) {
|
||||
rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2));
|
||||
var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changes.New()).Select(skeyfn));
|
||||
|
||||
var added = new List<(SecurityImmutable?, T)>();
|
||||
var changed = new List<Change<(SecurityImmutable? s, T t)>>();
|
||||
var removed = new List<(SecurityImmutable?, T)>();
|
||||
|
||||
foreach (var x in update.added) {
|
||||
if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) {
|
||||
added.Add((s, x));
|
||||
} else {
|
||||
added.Add((null, x));
|
||||
}
|
||||
}
|
||||
foreach (var x in update.removed) {
|
||||
if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) {
|
||||
removed.Add(value);
|
||||
}
|
||||
}
|
||||
foreach (var ch in update.changes) {
|
||||
SecurityImmutable? si =
|
||||
relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ?
|
||||
s : (SecurityImmutable?)null;
|
||||
changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New)));
|
||||
}
|
||||
|
||||
var builder = state.ToBuilder();
|
||||
builder.RemoveRange(update.removed.Select(update.key_extractor));
|
||||
builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old)));
|
||||
foreach (var a in added) {
|
||||
builder.Add(rowkeyfn(a), a);
|
||||
}
|
||||
foreach (var ch in changed) {
|
||||
builder.Add(rowkeyfn(ch.New), ch.New);
|
||||
}
|
||||
state = builder.ToImmutable();
|
||||
|
||||
observer.OnNext(create_result_update(state, added, changed, removed, rowkeyfn));
|
||||
}
|
||||
}),
|
||||
// Also fire update when securities change
|
||||
on_securities.Subscribe(on_securities_updated),
|
||||
// Also fire update when cache changes
|
||||
_cache_subject.Subscribe(on_securities_updated)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public IObservable<TableUpdate<(SecurityImmutable? sec, T val), TKey>> join_table<T, TKey>(
|
||||
IObservable<TableUpdate<T, TKey>> source, Func<T, security_key> get_sec_key) where TKey : IEquatable<TKey> =>
|
||||
join(
|
||||
source,
|
||||
get_sec_key,
|
||||
t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor),
|
||||
(state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>(
|
||||
state, added, changes, removed, key));
|
||||
#endregion
|
||||
}
|
||||
|
||||
public static class SecurityStoreExtensions {
|
||||
public static IObservable<Update<(SecurityImmutable? sec, T val), TKey>>
|
||||
join<T, TKey>(this IObservable<Update<T, TKey>> source, SecurityStore store, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
|
||||
return store.join(source, skeyfn);
|
||||
}
|
||||
|
||||
public static IObservable<TableUpdate<(SecurityImmutable? sec, T val), TKey>>
|
||||
join_table<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, SecurityStore store, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
|
||||
return store.join_table(source, skeyfn);
|
||||
}
|
||||
}
|
||||
}
|
||||
using bts.api;
|
||||
using bts.api.core;
|
||||
using bts.co2;
|
||||
|
||||
using mbts;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
using System.Reactive.Subjects;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using security_key = System.UInt32;
|
||||
|
||||
namespace Spark.Api.Reactive {
|
||||
public class SecurityResolver : IDisposable {
|
||||
readonly MarketDataRelay _md;
|
||||
IDisposable _dispose;
|
||||
IReadOnlyDictionary<security_key, SecurityMD> _live_securities;
|
||||
readonly ConcurrentDictionary<security_key, SecurityImmutable> _cache = new ConcurrentDictionary<security_key, SecurityImmutable>();
|
||||
readonly Subject<IReadOnlyDictionary<security_key, SecurityImmutable>> _cache_subject = new Subject<IReadOnlyDictionary<security_key, SecurityImmutable>>();
|
||||
|
||||
public SecurityResolver(
|
||||
MarketDataRelay md, IObservable<IReadOnlyDictionary<uint, SecurityMD>> securities) {
|
||||
_md = md;
|
||||
on_securities = securities;
|
||||
_dispose = securities.Subscribe(dict => _live_securities = dict);
|
||||
_cache_subject.OnNext(_cache);
|
||||
}
|
||||
|
||||
IObservable<IReadOnlyDictionary<security_key, SecurityMD>> on_securities;
|
||||
|
||||
SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary<security_key, SecurityMD> live = null) {
|
||||
// Try live securities first
|
||||
live = live ?? _live_securities;
|
||||
if (live != null) {
|
||||
if (live.TryGetValue(key, out SecurityMD value)) {
|
||||
return value.security;
|
||||
}
|
||||
}
|
||||
|
||||
// Then try cached securities
|
||||
if (_cache.TryGetValue(key, out SecurityImmutable security)) {
|
||||
return security;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async Task<SecurityImmutable?> get_security(security_key key, IReadOnlyDictionary<security_key, SecurityMD> live) {
|
||||
var local_sec = try_get_local(key, live);
|
||||
if (local_sec.HasValue) {
|
||||
return local_sec;
|
||||
}
|
||||
|
||||
// Query the server for the value
|
||||
var result = await _md.get_securities_async(new SecurityKeyList() {
|
||||
Keys = { key }
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
var security = result.value.Security.FirstOrDefault();
|
||||
if (security != null) {
|
||||
return try_cache_security(new SecurityImmutable(security));
|
||||
}
|
||||
}
|
||||
|
||||
// Give up :(
|
||||
Log.Md.warn($"Failed to get security for key {key}");
|
||||
return null;
|
||||
}
|
||||
|
||||
async Task<IReadOnlyDictionary<security_key, SecurityImmutable>> get_securities(IEnumerable<security_key> keys, IReadOnlyDictionary<security_key, SecurityMD> live) {
|
||||
var result = new Dictionary<security_key, SecurityImmutable>();
|
||||
var missing = new SecurityKeyList();
|
||||
foreach (var key in keys) {
|
||||
var s = try_get_local(key);
|
||||
if (s.HasValue) {
|
||||
result.Add(key, s.Value);
|
||||
} else {
|
||||
missing.Keys.Add(key);
|
||||
}
|
||||
}
|
||||
if (missing.Keys.Count > 0) {
|
||||
var res = await _md.get_securities_async(missing);
|
||||
if (res.success) {
|
||||
foreach (var security in res.value.Security) {
|
||||
result.Add(security.Id, try_cache_security(new SecurityImmutable(security)));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Request any missing keys from the server but do not wait for them to arrive
|
||||
IReadOnlyDictionary<security_key, SecurityImmutable> get_and_ensure_securities(IEnumerable<security_key> keys) {
|
||||
var result = new Dictionary<security_key, SecurityImmutable>();
|
||||
var missing = new HashSet<security_key>();
|
||||
foreach (var key in keys) {
|
||||
var security = try_get_local(key);
|
||||
if (security.HasValue) {
|
||||
result.TryAdd(key, security.Value);
|
||||
} else {
|
||||
missing.Add(key);
|
||||
}
|
||||
}
|
||||
// Update cache in the background
|
||||
if (missing.Count > 0) {
|
||||
Task.Run(async () => {
|
||||
var skl = new SecurityKeyList();
|
||||
skl.Keys.AddRange(missing);
|
||||
var r = await _md.get_securities_async(skl);
|
||||
if (r.success) {
|
||||
var retrieved = r.value.Security.Select(s => s.Id);
|
||||
Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]");
|
||||
foreach (var security in r.value.Security) {
|
||||
try_cache_security(new SecurityImmutable(security));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
SecurityImmutable try_cache_security(SecurityImmutable s) {
|
||||
if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) {
|
||||
_cache.TryAdd(s.Id, s);
|
||||
_cache_subject.OnNext(_cache);
|
||||
return s;
|
||||
}
|
||||
return live.security;
|
||||
}
|
||||
|
||||
///////////////
|
||||
// PUBLIC
|
||||
///////////////
|
||||
#region public
|
||||
|
||||
// Guaranteed to return a security if it has ever been seen by the system
|
||||
public Task<SecurityImmutable?> get_security(security_key key) {
|
||||
return get_security(key, _live_securities);
|
||||
}
|
||||
// Guaranteed to return a security if it has ever been seen by the system
|
||||
public Task<IReadOnlyDictionary<security_key, SecurityImmutable>> get_securities(IEnumerable<security_key> keys) {
|
||||
return get_securities(keys, _live_securities);
|
||||
}
|
||||
public void Dispose() {
|
||||
Disposable.dispose(ref _dispose);
|
||||
}
|
||||
|
||||
public IObservable<Update<(SecurityImmutable? sec, T val), TKey>> join<T, TKey>(
|
||||
IObservable<Update<T, TKey>> source, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
|
||||
return join(source, skeyfn, get_incremental_props, mk_update);
|
||||
|
||||
(IEnumerable<T> all, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key_extractor) get_incremental_props(Update<T, TKey> upd_in) =>
|
||||
(upd_in.all, upd_in.added, upd_in.changes, upd_in.removed, upd_in.key_extractor);
|
||||
|
||||
Update<(SecurityImmutable? sec, T val), TKey> mk_update(
|
||||
ImmutableDictionary<TKey, (SecurityImmutable? sec, T val)> state,
|
||||
IEnumerable<(SecurityImmutable? sec, T val)> added,
|
||||
IEnumerable<Change<(SecurityImmutable? sec, T val)>> changes,
|
||||
IEnumerable<(SecurityImmutable? sec, T val)> removed,
|
||||
Func<(SecurityImmutable?, T), TKey> key_extractor) =>
|
||||
//
|
||||
new Update<(SecurityImmutable? sec, T val), TKey>(
|
||||
state.Values, added, changes, removed, key_extractor);
|
||||
}
|
||||
|
||||
public IObservable<TOut> join<TIn, T, TKey, TOut>(
|
||||
IObservable<TIn> source,
|
||||
Func<T, security_key> skeyfn,
|
||||
Func<TIn, (IEnumerable<T> all, IEnumerable<T> added, IEnumerable<Change<T>> changes, IEnumerable<T> removed, Func<T, TKey> key_extractor)> get_incremental_props,
|
||||
Func<ImmutableDictionary<TKey, (SecurityImmutable? sec, T val)>, IEnumerable<(SecurityImmutable? sec, T val)>, IEnumerable<Change<(SecurityImmutable? sec, T val)>>, IEnumerable<(SecurityImmutable? sec, T val)>, Func<(SecurityImmutable?, T), TKey>, TOut> create_result_update)
|
||||
where TKey : IEquatable<TKey> {
|
||||
//
|
||||
return Observable.Create<TOut>(observer => {
|
||||
var state = ImmutableDictionary<TKey, (SecurityImmutable? s, T t)>.Empty;
|
||||
var gate = new object();
|
||||
// We get thie function from the first source update
|
||||
Func<(SecurityImmutable?, T t), TKey> rowkeyfn = null;
|
||||
|
||||
void on_securities_updated(object ambiguous) {
|
||||
if (rowkeyfn == null) return;
|
||||
lock (gate) {
|
||||
// If this is null we don't have any rows
|
||||
var changed = new List<Change<(SecurityImmutable? s, T t)>>();
|
||||
foreach (var entry in state.Values) {
|
||||
if (entry.s == null) {
|
||||
var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary<security_key, SecurityMD>);
|
||||
if (security.HasValue) {
|
||||
changed.Add(
|
||||
new Change<(SecurityImmutable?, T)>(entry, (security, entry.t)));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (changed.Count > 0) {
|
||||
// update state
|
||||
var builder = state.ToBuilder();
|
||||
foreach (var up in changed) {
|
||||
builder[rowkeyfn(up.New)] = up.New;
|
||||
}
|
||||
state = builder.ToImmutable();
|
||||
// fire update
|
||||
observer.OnNext(
|
||||
create_result_update(
|
||||
state,
|
||||
Enumerable.Empty<(SecurityImmutable?, T)>(),
|
||||
changed,
|
||||
Enumerable.Empty<(SecurityImmutable?, T)>(),
|
||||
rowkeyfn));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Disposable.Create(
|
||||
// Fire update when source changes
|
||||
source.Subscribe((update_in) => {
|
||||
var update = get_incremental_props(update_in);
|
||||
lock (gate) {
|
||||
rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2));
|
||||
var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changes.New()).Select(skeyfn));
|
||||
|
||||
var added = new List<(SecurityImmutable?, T)>();
|
||||
var changed = new List<Change<(SecurityImmutable? s, T t)>>();
|
||||
var removed = new List<(SecurityImmutable?, T)>();
|
||||
|
||||
foreach (var x in update.added) {
|
||||
if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) {
|
||||
added.Add((s, x));
|
||||
} else {
|
||||
added.Add((null, x));
|
||||
}
|
||||
}
|
||||
foreach (var x in update.removed) {
|
||||
if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) {
|
||||
removed.Add(value);
|
||||
}
|
||||
}
|
||||
foreach (var ch in update.changes) {
|
||||
SecurityImmutable? si =
|
||||
relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ?
|
||||
s : (SecurityImmutable?)null;
|
||||
changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New)));
|
||||
}
|
||||
|
||||
var builder = state.ToBuilder();
|
||||
builder.RemoveRange(update.removed.Select(update.key_extractor));
|
||||
builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old)));
|
||||
foreach (var a in added) {
|
||||
builder.Add(rowkeyfn(a), a);
|
||||
}
|
||||
foreach (var ch in changed) {
|
||||
builder.Add(rowkeyfn(ch.New), ch.New);
|
||||
}
|
||||
state = builder.ToImmutable();
|
||||
|
||||
observer.OnNext(create_result_update(state, added, changed, removed, rowkeyfn));
|
||||
}
|
||||
}),
|
||||
// Also fire update when securities change
|
||||
on_securities.Subscribe(on_securities_updated),
|
||||
// Also fire update when cache changes
|
||||
_cache_subject.Subscribe(on_securities_updated)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public IObservable<TableUpdate<(SecurityImmutable? sec, T val), TKey>> join_table<T, TKey>(
|
||||
IObservable<TableUpdate<T, TKey>> source, Func<T, security_key> get_sec_key) where TKey : IEquatable<TKey> =>
|
||||
join(
|
||||
source,
|
||||
get_sec_key,
|
||||
t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor),
|
||||
(state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>(
|
||||
state, added, changes, removed, key));
|
||||
#endregion
|
||||
}
|
||||
|
||||
public static class SecurityStoreExtensions {
|
||||
public static IObservable<Update<(SecurityImmutable? sec, T val), TKey>>
|
||||
join<T, TKey>(this IObservable<Update<T, TKey>> source, SecurityResolver store, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
|
||||
return store.join(source, skeyfn);
|
||||
}
|
||||
|
||||
public static IObservable<TableUpdate<(SecurityImmutable? sec, T val), TKey>>
|
||||
join_table<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, SecurityResolver store, Func<T, security_key> skeyfn) where TKey : IEquatable<TKey> {
|
||||
return store.join_table(source, skeyfn);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
#nullable enable
|
||||
namespace Spark.Core {
|
||||
using System;
|
||||
|
||||
/// <summary>
|
||||
/// A thread-safe implementation of lazy initialization. Unlike <see cref="Lazy{T}"/>, it
|
||||
/// guarentees the creation delegate isn't invoked more than once.
|
||||
/// </summary>
|
||||
public class LazyOnce<T> {
|
||||
readonly Func<T> _create;
|
||||
Optional<T> _value;
|
||||
|
||||
public LazyOnce(Func<T> create) =>
|
||||
_create = create;
|
||||
|
||||
public bool IsValueCreated => _value.has_value;
|
||||
|
||||
public T Value {
|
||||
get {
|
||||
// check has_value outside of lock since, once set to true, it doesn't change
|
||||
if (!_value.has_value) {
|
||||
lock (_create) {
|
||||
// check has_value again (may have become true since first check)
|
||||
if (!_value.has_value) {
|
||||
_value = _create();
|
||||
}
|
||||
}
|
||||
}
|
||||
return _value;
|
||||
}
|
||||
}
|
||||
|
||||
public static implicit operator T(LazyOnce<T> lazy) =>
|
||||
lazy.Value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#nullable enable
|
||||
namespace Spark.Core {
|
||||
using bts;
|
||||
|
||||
using System.Threading;
|
||||
|
||||
/// <summary>
|
||||
/// A (base) class that asserts it isn't instantiated more than once
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// The underlying type, which is prevented from being instantiated more than once, as it
|
||||
/// inherits from <see cref="VirtualSingleton{T}"/> itself
|
||||
/// </typeparam>
|
||||
public abstract class VirtualSingleton<T> where T : VirtualSingleton<T> {
|
||||
static int _instances = 0;
|
||||
|
||||
protected VirtualSingleton() =>
|
||||
Log.assert(Interlocked.Increment(ref _instances) == 1);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user