improvement: Move system contracts sub off of dispatcher thread
Also add 200ms throttle. Contracts shouldn't change that often, so we don't need lightening speed here. Furthermore, such changes may trigger expensive cpu ops. Note: this change also replaces refs to ThreadedBlackboard w/ IBus (the former is somewhat tethered to the dispatcher thread, particularly by its timer_callback).
This commit is contained in:
1 parent
85529b2232
commit
6bcce832ac
3 files changed
+110
-85
No files matched your search
@@ -4,91 +4,79 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using bts.api;
|
||||
using System.Linq;
|
||||
using System.Reactive.Linq;
|
||||
using Spark.Core.Disposables;
|
||||
using Spark.Core.Reactive;
|
||||
|
||||
namespace mbts.analytics {
|
||||
|
||||
// Thin class that assembles all symbol based data published on different
|
||||
// tags, into IReadOnlyDictionary of symbol vs data publsihed under one tag
|
||||
public class SymbolAggregationPublisher<TData> : IDisposable {
|
||||
private readonly ThreadedBlackboard _bb;
|
||||
readonly IBus _bus;
|
||||
private readonly SecurityType _sec_type;
|
||||
Func<(string, Venue), ITag<TData>> _get_tag;
|
||||
// the tag on which the output will be published
|
||||
ITag<IReadOnlyDictionary<(string, Venue), TData>> _out_tag;
|
||||
private readonly ITag<SystemContractData> _sc_data_tag;
|
||||
private Dictionary<(string, Venue), ITag<TData>> _tag_map = new();
|
||||
private HashSet<object> _obj_tags = new HashSet<object>();
|
||||
Dictionary<(string, Venue), TData> _tag_map = new();
|
||||
private bool _dirty = true;
|
||||
readonly object _lock = new();
|
||||
IDisposable _dispose;
|
||||
|
||||
public SymbolAggregationPublisher(
|
||||
ThreadedBlackboard bb,
|
||||
IBus bus,
|
||||
SecurityType sec_type,
|
||||
Func<(string, Venue), ITag<TData>> get_tag,
|
||||
ITag<IReadOnlyDictionary<(string, Venue), TData>> out_tag) {
|
||||
_bb = bb;
|
||||
ITag<IReadOnlyDictionary<(string, Venue), TData>> out_tag,
|
||||
TimeSpan? contracts_throttle = null) {
|
||||
_bus = bus;
|
||||
var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
|
||||
if (contracts_throttle.HasValue) {
|
||||
contracts_source = contracts_source.Sample(contracts_throttle.Value);
|
||||
}
|
||||
var timer_source = Observable.Interval(TimeSpan.FromMilliseconds(133));
|
||||
_dispose = Disposable.Create(
|
||||
contracts_source.SubscribeEach(contracts => Disposable.Create(
|
||||
contracts.sectup_map.Where(st => st.security_type == sec_type).Select(subscribe_sec_data).ToList())),
|
||||
timer_source.Subscribe(_on_timer));
|
||||
_sec_type = sec_type;
|
||||
_get_tag = get_tag;
|
||||
_out_tag = out_tag;
|
||||
_sc_data_tag = SparkTags.SystemContractsTag;
|
||||
_bb.subscribe(SparkTags.SystemContractsTag);
|
||||
_bb.data_callback += _on_bb_data;
|
||||
_bb.timer_callback += _on_timer;
|
||||
_handle_sc_data();
|
||||
}
|
||||
|
||||
private void _on_bb_data(object tag, object data) {
|
||||
if (_sc_data_tag.Equals(tag)) {
|
||||
_handle_sc_data();
|
||||
} else {
|
||||
if (_obj_tags.Contains(tag)) {
|
||||
_dirty = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _handle_sc_data() {
|
||||
var sc_data = _bb.read_state(_sc_data_tag);
|
||||
if (sc_data == null) {
|
||||
return;
|
||||
}
|
||||
foreach (var st in sc_data.sectup_map) {
|
||||
if (st.security_type != _sec_type) {
|
||||
continue;
|
||||
}
|
||||
// this is not correct since it is assuming symbol/name is
|
||||
// unique across securty_tuples
|
||||
var symbol = (st.security_name, st.security_exchange);
|
||||
if (!_tag_map.ContainsKey(symbol)) {
|
||||
var tag = _get_tag(symbol);
|
||||
if (tag != null) {
|
||||
_tag_map.Add(symbol, tag);
|
||||
_obj_tags.Add(tag);
|
||||
_bb.subscribe(tag);
|
||||
_dirty = true;
|
||||
IDisposable subscribe_sec_data(SecurityTuple st) =>
|
||||
_bus.subscribe(_get_tag((st.security_name, st.security_exchange))).Subscribe(d => {
|
||||
lock (_lock) {
|
||||
_tag_map[(st.security_name, st.security_exchange)] = d;
|
||||
}
|
||||
}
|
||||
}
|
||||
_dirty = true;
|
||||
});
|
||||
}
|
||||
|
||||
private void _on_timer() {
|
||||
private void _on_timer(long _) {
|
||||
if (_dirty) {
|
||||
_dirty = false;
|
||||
var output = new Dictionary<(string, Venue), TData>();
|
||||
foreach (var kv in _tag_map) {
|
||||
var symbol = kv.Key;
|
||||
var symbol_tag = kv.Value;
|
||||
TData symbol_data = _bb.read_state(symbol_tag);
|
||||
output.Add(symbol, symbol_data);
|
||||
lock (_lock) {
|
||||
foreach (var kv in _tag_map) {
|
||||
var symbol = kv.Key;
|
||||
output.Add(symbol, kv.Value);
|
||||
}
|
||||
}
|
||||
IReadOnlyDictionary<(string, Venue), TData> ro_output = output;
|
||||
bts.Log.Gen.diag($"SymbolAggregationPublisher._on_timer publishing " +
|
||||
$"tag: {_out_tag} sz: {ro_output.Count}");
|
||||
_bb.post(_out_tag, ro_output);
|
||||
_bus.post(_out_tag, ro_output);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
_bb.data_callback -= _on_bb_data;
|
||||
_bb.timer_callback -= _on_timer;
|
||||
lock (_lock) {
|
||||
Disposable.dispose(ref _dispose);
|
||||
}
|
||||
}
|
||||
|
||||
class SymbolData {
|
||||
public TData data { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
@@ -31,12 +31,22 @@ namespace mbts.spark
|
||||
new Dictionary<SecurityTuple, SingleOptionDriverPricePublisher>();
|
||||
readonly IBus bus;
|
||||
ThreadedBlackboard _bb;
|
||||
public ResourceAllocator(IBus bus, ThreadedBlackboard bb)
|
||||
readonly object _lock = new object();
|
||||
|
||||
public ResourceAllocator(IBus bus, ThreadedBlackboard bb, TimeSpan? contracts_throttle = null)
|
||||
{
|
||||
bts.Log.Gen.info("ResourceAllocator started");
|
||||
this.bus = bus;
|
||||
_bb = bb;
|
||||
var s = bus.subscribe(SparkTags.SystemContractsTag).OnDispatcher().Subscribe(_on_data);
|
||||
var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
|
||||
if (contracts_throttle.HasValue) {
|
||||
contracts_source = contracts_source.Sample(contracts_throttle.Value);
|
||||
}
|
||||
var s = contracts_source.Subscribe(contracts => {
|
||||
lock (_lock) {
|
||||
_on_data(contracts);
|
||||
}
|
||||
});
|
||||
_disps.Add(s);
|
||||
}
|
||||
private void _on_data(SystemContractData d)
|
||||
@@ -53,9 +63,11 @@ namespace mbts.spark
|
||||
}
|
||||
public void Dispose()
|
||||
{
|
||||
_disps.ForEach(d => d.Dispose());
|
||||
_option_driver_map.Values.DisposeAll();
|
||||
_option_driver_map.Clear();
|
||||
lock (_lock) {
|
||||
_disps.ForEach(d => d.Dispose());
|
||||
_option_driver_map.Values.DisposeAll();
|
||||
_option_driver_map.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
// This is the anologous to the QuoteSetBlackboardManager. There
|
||||
@@ -106,6 +118,7 @@ namespace mbts.spark
|
||||
_bb.subscribe(_in_bb_req_vol);
|
||||
_bb.data_callback += _on_bb_update;
|
||||
_bb.timer_callback += _on_timer;
|
||||
var contracts_throttle = TimeSpan.FromMilliseconds(200);
|
||||
// sub to service
|
||||
var vs_sub = _analytics_service.subscribei_volsurfaces()
|
||||
.ObserveOn(SynchronizationContext.Current)
|
||||
@@ -116,18 +129,15 @@ namespace mbts.spark
|
||||
_subs.Add(vs_sub);
|
||||
_subs.Add(vs_usage_sub);
|
||||
_subs.Add(new AnalyticsParametersManager(bus, _analytics_service));
|
||||
_subs.Add(new ResourceAllocator(bus, _bb));
|
||||
_subs.Add(new ResourceAllocator(bus, _bb, contracts_throttle));
|
||||
_subs.Add(new SymbolAggregationPublisher<OptionDriverPriceMap>(
|
||||
_bb, SecurityType.Option, sym => SparkTags.OptionDriverBBOTag(sym),
|
||||
SparkTags.AllOptionDriverBBOTag));
|
||||
bus, SecurityType.Option, SparkTags.OptionDriverBBOTag, SparkTags.AllOptionDriverBBOTag, contracts_throttle));
|
||||
_subs.Add(new SymbolAggregationPublisher<OptionTheoInterpolationData>(
|
||||
_bb, SecurityType.Option, sym => SparkTags.SymbolInterpGreeksTag(sym),
|
||||
SparkTags.AllInterpGreeks));
|
||||
bus, SecurityType.Option, SparkTags.SymbolInterpGreeksTag, SparkTags.AllInterpGreeks, contracts_throttle));
|
||||
_subs.Add(new SymbolAggregationPublisher<ExpStrikeKeyedData<TheoPoint>>(
|
||||
_bb, SecurityType.Option, sym => SparkTags.TC2Tags.LiveTheosTag(sym),
|
||||
SparkTags.TC2Tags.AllLiveTheosTag));
|
||||
bus, SecurityType.Option, SparkTags.TC2Tags.LiveTheosTag, SparkTags.TC2Tags.AllLiveTheosTag, contracts_throttle));
|
||||
_subs.Add(new LiveTheoPublisher(_bb));
|
||||
_subs.Add(new InterpTheosServerPublisher(_bb, order_service));
|
||||
_subs.Add(new InterpTheosServerPublisher(_bb, bus, order_service, contracts_throttle));
|
||||
_subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
|
||||
// usage
|
||||
var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
|
||||
|
||||
@@ -8,6 +8,8 @@ using System.Linq;
|
||||
using bts.api.orderman;
|
||||
using bts.api.analytics;
|
||||
using bts;
|
||||
using System.Reactive.Linq;
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
namespace mbts.spark {
|
||||
// Listens to the interp theos, decorates the data
|
||||
@@ -15,23 +17,46 @@ namespace mbts.spark {
|
||||
public class InterpTheosServerPublisher : IDisposable {
|
||||
private readonly ThreadedBlackboard _bb;
|
||||
private readonly bts.api.orderman.OrderService _order_service;
|
||||
IReadOnlyDictionary<(string Symbol, Venue Venue), OptionTheoInterpolationData> _interp_greeks;
|
||||
IReadOnlyDictionary<(string Symbol, Venue Venue), OptionDriverPriceMap> _driver_bbos;
|
||||
VolSurfaceUsageListImmutable? _vs_usage;
|
||||
UserTuple _user;
|
||||
SystemContractData _system_contracts;
|
||||
IDisposable _dispose;
|
||||
|
||||
public InterpTheosServerPublisher(ThreadedBlackboard bb,
|
||||
bts.api.orderman.OrderService order_service) {
|
||||
public InterpTheosServerPublisher(
|
||||
ThreadedBlackboard bb, IBus bus, OrderService order_service, TimeSpan? contracts_throttle = null) {
|
||||
bts.Log.Ui.info(
|
||||
$"InterpTheosServerPublisher cons tid:{VolSurfaceTroubleshootingFunctions.tid()}");
|
||||
_bb = bb;
|
||||
_order_service = order_service;
|
||||
_bb.subscribe(SparkTags.AllInterpGreeks);
|
||||
_bb.subscribe(SparkTags.SystemContractsTag);
|
||||
_bb.subscribe(SparkTags.CurrentUser);
|
||||
_bb.subscribe(SparkTags.VolSurfaceUsageTag);
|
||||
_bb.data_callback += _on_data;
|
||||
}
|
||||
var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
|
||||
if (contracts_throttle.HasValue) {
|
||||
contracts_source = contracts_source.Sample(contracts_throttle.Value);
|
||||
}
|
||||
_dispose = Disposable.Create(
|
||||
bus.subscribe(SparkTags.CurrentUser).Subscribe(on_current_user),
|
||||
bus.subscribe(SparkTags.VolSurfaceUsageTag).Subscribe(on_vs_usage),
|
||||
bus.subscribe(SparkTags.AllOptionDriverBBOTag).Subscribe(on_driver_bbos),
|
||||
bus.subscribe(SparkTags.AllInterpGreeks).Subscribe(on_interp_greeks),
|
||||
contracts_source.Subscribe(on_system_contracts));
|
||||
|
||||
private void _on_data(object tag, object data) {
|
||||
if (tag.Equals(SparkTags.SystemContractsTag) ||
|
||||
tag.Equals(SparkTags.AllInterpGreeks)) {
|
||||
void on_current_user(UserTuple user) =>
|
||||
_user = user;
|
||||
|
||||
void on_vs_usage(VolSurfaceUsageListImmutable? vs_usage) =>
|
||||
_vs_usage = vs_usage;
|
||||
|
||||
void on_driver_bbos(IReadOnlyDictionary<(string Symbol, Venue Venue), OptionDriverPriceMap> driver_bbos) =>
|
||||
_driver_bbos = driver_bbos;
|
||||
|
||||
void on_interp_greeks(IReadOnlyDictionary<(string Symbol, Venue Venue), OptionTheoInterpolationData> interp_greeks) {
|
||||
_interp_greeks = interp_greeks;
|
||||
_publish_data();
|
||||
}
|
||||
|
||||
void on_system_contracts(SystemContractData system_contracts) {
|
||||
_system_contracts = system_contracts;
|
||||
_publish_data();
|
||||
}
|
||||
}
|
||||
@@ -63,8 +88,8 @@ namespace mbts.spark {
|
||||
// find the symbol vs vs_id, get the committed vs, see if it is
|
||||
// owned by the current user, and only then publish it
|
||||
var syms = new HashSet<(string, Venue)>();
|
||||
var usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
|
||||
string current_user = _bb.read_state(SparkTags.CurrentUser).name;
|
||||
var usage = _vs_usage;
|
||||
var current_user = _user.name;
|
||||
foreach (var u in (usage?.UsageList).empty_if_null()) {
|
||||
VolSurfaceInput vs_input =
|
||||
_bb.read_state(SparkTags.VolSurfaceCommittedInputTag(u.VsId));
|
||||
@@ -79,10 +104,12 @@ namespace mbts.spark {
|
||||
all_theos.Where(kv => syms.Contains(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value);
|
||||
}
|
||||
|
||||
private void _publish_data() {
|
||||
SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag, null);
|
||||
var all_theos = _bb.read_state(SparkTags.AllInterpGreeks, null);
|
||||
var all_drivers = _bb.read_state(SparkTags.AllOptionDriverBBOTag, null);
|
||||
void _publish_data() {
|
||||
// copy data to local vars because updates are multi-threaded,
|
||||
// and data must be consistent (during computation).
|
||||
var sc_data = _system_contracts;
|
||||
var all_theos = _interp_greeks;
|
||||
var all_drivers = _driver_bbos;
|
||||
if (sc_data == null || all_theos == null || all_drivers == null)
|
||||
return;
|
||||
var timer = Stopwatch.StartNew();
|
||||
@@ -130,7 +157,7 @@ namespace mbts.spark {
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
_bb.data_callback -= _on_data;
|
||||
Disposable.dispose(ref _dispose);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user