diff --git a/src/client/spark/Model/SecurityResolverExporter.cs b/src/client/spark/Model/SecurityResolverExporter.cs new file mode 100644 index 000000000..9bbf04c69 --- /dev/null +++ b/src/client/spark/Model/SecurityResolverExporter.cs @@ -0,0 +1,25 @@ +#nullable enable +namespace Spark.Model { + using bts; + using bts.api; + using bts.co2; + + using mbts.spark; + + using Spark.Api.Reactive; + using Spark.Core; + + using System.Composition; + + class SecurityResolverExporter : VirtualSingleton { + readonly LazyOnce? _resolver; + + [ImportingConstructor] + public SecurityResolverExporter(IBus bus, MarketDataRelay md_relay) => + _resolver = new(() => new(md_relay, bus.subscribe(SparkTags.SecuritiesTag))); + + [Export] + public SecurityResolver resolver => + Log.notnull(_resolver); + } +} diff --git a/src/client/spark/Model/Trading/OrderDataSource.cs b/src/client/spark/Model/Trading/OrderDataSource.cs index 6c5cb809a..f4a03530f 100644 --- a/src/client/spark/Model/Trading/OrderDataSource.cs +++ b/src/client/spark/Model/Trading/OrderDataSource.cs @@ -10,6 +10,7 @@ namespace Spark.Model.Trading { using mbts.spark; + using Spark.Api.Reactive; using Spark.Core.Reactive; using System; @@ -25,7 +26,7 @@ namespace Spark.Model.Trading { public OrderDataSource( IBus bus, AdminModel admin_model, - SecurityStore security_store, + SecurityResolver sec_resolver, OrderService order_service, ISecurityAuxiliaryDataModel security_auxiliary_model, ErrorReporter error_reporter @@ -38,7 +39,7 @@ namespace Spark.Model.Trading { order_service.subscribei_orders() .Naggle(interval, scheduler), o => o.account_id()) - .join(security_store, o => o.security_key()) + .join(sec_resolver, o => o.security_key()) .Table() .Cache(); diff --git a/src/client/spark/Model/Trading/PositionModel.cs b/src/client/spark/Model/Trading/PositionModel.cs index 669a35382..1e34fd101 100644 --- a/src/client/spark/Model/Trading/PositionModel.cs +++ b/src/client/spark/Model/Trading/PositionModel.cs @@ -14,6 +14,7 @@ namespace Spark.Trading { using mbts.spark; using mbts.ui; + using Spark.Api.Reactive; using Spark.Core; using Spark.Core.Reactive; using Spark.Model.Trading; @@ -38,7 +39,7 @@ namespace Spark.Trading { public PositionModel( IBus bus, AdminModel admin_model, - SecurityStore security_store, + SecurityResolver sec_resolver, PositionService position_service, OrderService order_service, OrderDataSource order_data, @@ -55,7 +56,7 @@ namespace Spark.Trading { on_exchange_positions = order_service.subscribei_exchange_positions() .Filter(ep => ep.SecurityKey != 0) .SelectKeyValue(ep => ep, ep => new AccountSecurityKey(ep.AccountId, ep.SecurityKey)) - .join(security_store, ep => ep.SecurityKey) + .join(sec_resolver, ep => ep.SecurityKey) .Table() .Cache(); @@ -63,11 +64,11 @@ namespace Spark.Trading { .permission_filter( _position_service.subscribei_position_data().Naggle(interval, scheduler), p => p.AccountId) - .join(security_store, pos => pos.SecurityId) + .join(sec_resolver, pos => pos.SecurityId) .Table() .Cache(); - day_fills = DayFillsDataSource.create(bus, security_store); + day_fills = DayFillsDataSource.create(bus, sec_resolver); pnl = bus.subscribe(SparkTags.ReportBlackboardTags.PNL.PNLData) .Select(d => d.position_data.ToDictionary(u => u.id(), PNLRowData.from_pnl_unit)) diff --git a/src/client/spark/Modules/Trading/DayFills/DayFillsDataSource.cs b/src/client/spark/Modules/Trading/DayFills/DayFillsDataSource.cs index d82f9b21e..ab71465f0 100644 --- a/src/client/spark/Modules/Trading/DayFills/DayFillsDataSource.cs +++ b/src/client/spark/Modules/Trading/DayFills/DayFillsDataSource.cs @@ -7,6 +7,8 @@ namespace Spark.Trading { using mbts.analytics; using mbts.spark; + using Spark.Api.Reactive; + using System; using System.Collections.Generic; using System.Linq; @@ -14,26 +16,26 @@ namespace Spark.Trading { class DayFillsDataSource { public static IObservable> create( - IBus bus, SecurityStore sec_store) { + IBus bus, SecurityResolver sec_resolver) { var users = bus.subscribe(SparkTags.Users); var inverse_users = users .Select(d => d.ToDictionary(kv => kv.Value.name, kv => kv.Value)) .Table(u => u.name); var pnl_data_source = bus.subscribe(SparkTags.ReportBlackboardTags.PNL.PNLData); - return create(pnl_data_source, inverse_users, sec_store); + return create(pnl_data_source, inverse_users, sec_resolver); } public static IObservable> create( IObservable pnl_data_source, IObservable> inverse_users, - SecurityStore sec_store) => + SecurityResolver sec_resolver) => // join pnl data w/ security and user pnl_data_source .Select(d => d.day_trades .Select(DayFillsRowData.from_trade) .ToDictionary(DayFillsRowData.key_fn)) .Table(DayFillsRowData.key_fn) - .join_table(sec_store, rd => rd.security_key) + .join_table(sec_resolver, rd => rd.security_key) .Select(tup => new DayFillsRowData(tup.val.trade, tup.sec), DayFillsRowData.key_fn) .LeftJoin( inverse_users, diff --git a/src/client/spark/Modules/Trading/TradingShellModule.cs b/src/client/spark/Modules/Trading/TradingShellModule.cs index 0cb041e4f..9b96f1225 100644 --- a/src/client/spark/Modules/Trading/TradingShellModule.cs +++ b/src/client/spark/Modules/Trading/TradingShellModule.cs @@ -1,4 +1,4 @@ -namespace bts.shell { +namespace bts.shell { using bts.api; using bts.shell.fills; using bts.shell.manual_fills; @@ -9,6 +9,7 @@ using mbts.ui; using mbts.ui.shell; + using Spark.Api.Reactive; using Spark.Core.Data; using Spark.Trading; @@ -35,7 +36,7 @@ INotificationServiceDataCache notification_service_data_cache, UserSettingsHub settings_hub, ISecurityMetadataStore security_metadata, - SecurityStore security_store, + SecurityResolver security_resolver, OrdersModel orders_model, md.MBOService mbo, IPositionModel position_model, diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index ad1a149a8..21dafde6f 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -1,667 +1,668 @@ -namespace mbts.spark { - using bts; - using bts.api; - using bts.api.core; - using bts.api.orderman; - using bts.api.position; - using bts.collections; - using bts.md; - using bts.shell.pnl; - using bts.spark; - using bts.utils; - - using mbts.config; - using mbts.spark.utilities; - using mbts.ui; - using mbts.utils; - - using Spark.Api.Position; - using Spark.Core.Reactive; - using Spark.Model.Trading; - using Spark.Trading; - - using System; - using System.Collections.Generic; - using System.Collections.Immutable; - using System.Collections.Specialized; - using System.Composition; - using System.Diagnostics.Contracts; - using System.Linq; - using System.Reactive.Concurrency; - using System.Reactive.Linq; - using System.Threading.Tasks; - - using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; - using OrderUpdatePiq = bts.api.core.TableUpdate; - - [Export] - class OrdersModel : IDisposable { - public static readonly bool piq_enabled = Configuration.get().enabled; - public readonly MarketDataAuxiliaryModel md_auxiliary_model; - public readonly ISecurityAuxiliaryDataModel security_auxiliary_model; - - readonly IBus _bus; - readonly IAlgoProvider _algo_provider, _attachable_algo_provider; - readonly OrderFactory _factory; - readonly OrderService _service; - readonly PositionService _position_service; - readonly MBOService _mbo_service; - readonly SpreaderModel _spreader; - readonly ErrorReporter _error; - readonly DynamicOrderHelper _dynamic_helper; - readonly OrderActivitySummarizer _activity; - readonly TableUpdateDemuxer _security_orders_demuxer; - readonly TableUpdateDemuxer _security_working_orders_with_piq_demuxer; - readonly OrderDataSource _data_source; - - /// is updated atomically from multiple threads (use a local copy to - /// perform lookups, etc.) - IReadOnlyDictionary _securities = new Dictionary(0); - IDisposable _dispose; - - [ImportingConstructor] - public OrdersModel(IBus bus, - OrderFactory factory, - OrderService service, - OrderDataSource data_source, - PositionService position_service, - MBOService mbo_service, - SpreaderModel spreader, - DynamicOrderHelper dynamic_helper, - AdminModel admin_model, - IAlgoStudioModel algo_model, - ISecurityAuxiliaryDataModel security_auxiliary_model, - MarketDataAuxiliaryModel md_auxiliary_model, - IAlgoProvider algo_provider, - IAttachableAlgoProvider attachable_algo_provider, - SecurityStore sstore, - ErrorReporter error) { - // - _bus = bus; - _factory = factory; - _service = service; - _data_source = data_source; - _position_service = position_service; - _mbo_service = mbo_service; - _spreader = spreader; - _error = error; - _algo_provider = algo_provider; - _attachable_algo_provider = attachable_algo_provider; - _dynamic_helper = dynamic_helper; - this.admin_model = admin_model; - this.algo_model = algo_model; - this.md_auxiliary_model = md_auxiliary_model; - this.security_auxiliary_model = security_auxiliary_model; - var scheduler = ThreadPoolScheduler.Instance; - var interval = TimeSpan.FromMilliseconds(100); - - on_order_securities = - on_orders - .as_iincremental() - .filter(t => t.sec.HasValue) - .Select(t => t.sec.Value); - - _activity = new OrderActivitySummarizer(bus, on_orders); - _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); - _security_working_orders_with_piq_demuxer = new TableUpdateDemuxer(o => o.order.Key, o => o.order.security_key()); - studio_algos_with_meta = algo_model.definitions.LeftJoin( - algo_model.metadata_service.subscribe_metadata().Table(m => m.id), - d => d.key, - m => m.id, - (algo, k, meta) => (algo, meta), - (algo, k) => (algo, null), - t => t.algo.key) - .Cache(); - // join orders with algo data/metadata - on_order_tuples = on_orders.LeftJoin( - studio_algos_with_meta, - t => t.order.Key, - am => am.algo.key, - t => AlgoKey.from_composite_key(t.order.algo_template_key()), - (os, okey, am, mkey) => new OrderSecurity(os.sec, os.order, null, am.algo, am.meta), - (os, okey, mkey) => new OrderSecurity(os.sec, os.order, null, null, null), - ot => ot.order.Key) - .Cache(); - - on_order_tuples_with_piq = on_order_tuples.LeftJoin< - OrderSecurity, - OrderPIQImmutable, - OrderSecurity, - uint>( - mbo_service.subscribei_piq().Table(), - OrderSecurityHelper.key_fn, - piq => piq.OrderId, - (osam, _, p) => new OrderSecurity(osam.security, osam.order, p.Position, osam.algo, osam.meta), - (osam, _) => new OrderSecurity(osam.security, osam.order, null, osam.algo, osam.meta), - osp => osp.order.Key - ).Cache(); - - // order tuples with root - on_order_tuples_with_root = on_order_tuples.RootDependent( - OrderSecurityHelper.key_fn, - OrderSecurityHelper.parent_key_fn, - OrderSecurityRootPair.from_order_security, - OrderSecurityRootPair.from_order_security, - OrderSecurityHelper.root_pair_key_fn - ).Cache(); - // order tuples with root and piq - on_order_tuples_with_root_and_piq = on_order_tuples_with_piq.RootDependent( - OrderSecurityHelper.key_fn, - OrderSecurityHelper.parent_key_fn, - OrderSecurityRootPair.from_order_security, - OrderSecurityRootPair.from_order_security, - OrderSecurityHelper.root_pair_key_fn - ).Cache(); - // fill tuples - var on_fills = admin_model - .permission_filter( - _service.subscribei_fills() - .Naggle(interval, scheduler), - f => f.Account) - .join(sstore, f => f.SecurityKey) - .Table().Cache(); - var fill_order_tuples_join = on_fills.LeftJoin( - on_order_tuples, - f => f.val.FillKey, - OrderSecurityHelper.key_fn, - f => f.val.OrderKey, - (f, fk, o, ok) => new FillTuple(f, o), - (f, fk, ok) => new FillTuple(f, null), - f => f.fill.FillKey); - on_fill_tuples = fill_order_tuples_join.Cache(); - // fill tuples with root (algo) - var fill_order_tuples_with_root_join = on_fills.LeftJoin( - on_order_tuples_with_root, - f => f.val.FillKey, - OrderSecurityHelper.root_pair_key_fn, - f => f.val.OrderKey, - (f, fk, o, ok) => new FillTuple(f.val, f.sec, o), - (f, fk, ok) => new FillTuple(f.val, f.sec, null), - FillTuple.key_fn); - on_fill_tuples_with_root = fill_order_tuples_with_root_join.Cache(); - // manual fills - on_manual_fills_securities = admin_model - .permission_filter( - _position_service.subscribei_manual_fills() - .Naggle(interval, scheduler), - f => f.AccountId) - .join(sstore, mf => mf.SecurityKey) - .Table().Cache(); - var manual_fill_tuples_account_join = on_manual_fills_securities.LeftJoin( - admin_model.on_accounts.Select(d => MappedDictionary.create(d, (k, v) => new AccountTuple(k, v))).Table(a => a.id), - mfs => mfs.Item2.RecordId, - AccountTuple.key_fn, - mfs => mfs.Item2.AccountId, - (mfs, mf_id, a, a_id) => new ManualFillTuple(mfs.Item2, mfs.Item1, a, null), - (mfs, mf_id, a_id) => new ManualFillTuple(mfs.Item2, mfs.Item1, null, null), - ManualFillTuple.key_fn); - on_manual_fill_tuples = manual_fill_tuples_account_join.LeftJoin( - admin_model.on_users.Select(d => MappedDictionary.create(d, (k, v) => new UserTuple(k, v))).Table(a => a.id), - ManualFillTuple.key_fn, - UserTuple.key_fn, - mft => mft.manual_fill.UserId, - (mft, mft_id, u, u_id) => new ManualFillTuple(mft.manual_fill, mft.security, mft.account, u), - (mft, mft_id, u_id) => mft, - ManualFillTuple.key_fn) - .Cache(); - // can attach - on_orders_can_attach = on_orders.ParentDependent( - t => t.order.Key, - t => t.order.ParentKey, - (child, parent) => child.order.is_non_terminal() && !parent.order.is_non_terminal(), - t => t.order.is_non_terminal()) - .Cache(); - // can detach - on_orders_can_detach = on_orders.ParentDependent( - t => t.order.Key, - t => t.order.ParentKey, - (child, parent) => child.order.is_non_terminal() && parent.order.is_non_terminal(), - t => false) - .Cache(); - _dispose = Disposable.Create( - on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), - on_order_tuples_with_piq - .Filter(os => os.order.is_active() && os.order.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible, key => key.order.Key) - .Subscribe(_security_working_orders_with_piq_demuxer), - _bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s), - _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())), - _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()), - _bus.subscribe(SparkTags.DetachOrders).Subscribe(x => detach_orders(x).ignore()), - _bus.subscribe(SparkTags.UpdateOrder).Subscribe(x => this.update_order(x.key, x.price, x.size).ignore()) - ); - } - - #region Output - public OrderFactory factory => _factory; - public SpreaderModel spreader => _spreader; - public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IObservable on_orders => _data_source.orders; - public IObservable on_order_tuples_with_piq { get; } - public IObservable> on_order_tuples { get; } - public IObservable> on_order_tuples_with_root { get; } - public IObservable> on_order_tuples_with_root_and_piq { get; } - public IObservable, uint>> on_orders_can_detach { get; } - public IObservable, uint>> on_orders_can_attach { get; } - public IObservable> on_order_securities { get; } - public IObservable> on_fill_tuples { get; } - public IObservable> on_fill_tuples_with_root { get; } - public IObservable working_orders => _data_source.working_orders; - public IObservable> studio_algos_with_meta { get; } - public IObservable> on_manual_fills_securities { get; } - public IObservable> on_manual_fill_tuples { get; } - public IObservable, uint>> security_orders => _data_source.security_orders; - - public AdminModel admin_model { get; } - public MBOService mbo_service => _mbo_service; - public IAlgoStudioModel algo_model { get; } - - public AlgoTypeSetImmutable get_permitted_algos(SecurityImmutable security) => - _algo_provider.get_permitted_algos(security); - - public IObservable> on_security_working_orders_with_piq(uint security) => - _security_working_orders_with_piq_demuxer.subscribe(security); - - public IObservable> on_security_orders(uint security) => - _security_orders_demuxer.subscribe(security); - - public IObservable> subscribe_algos(bool attachable_only = false) => - (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_algos(); - - public IObservable> subscribe_algos(SecurityImmutable security, bool attachable_only = false) => - (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_algos(security); - - public IObservable> subscribe_complete_algos(bool attachable_only = false, bool allow_undefined_child = false) => - (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_complete_algos(allow_undefined_child); - #endregion - - #region Input - // the security arg is needed for a temporary hack to prevent GlbxIceberg orders from going out in non-glbx exchanges (like Ice) - // eventually, we'll need an algo type model to handle this stuff. - // This hack leverages the fact that every place that calls 'send_order' already has the security handy (not just the sec id). - public Task send_order( - SecurityImmutable security, OrderDefinitionImmutable? order, double price, double size) { - if (order.HasValue) { -#if DEBUG - if (security.Exchange != Venue.Glbx && security.Exchange != Venue.Sim) { -#else - if (security.Exchange != Venue.Glbx) { -#endif - var algo_structure = OrderDataHelper.enumerate_algo_structure(order.Value.CustomDef); - if (algo_structure.Any(t => t.type == CustomOrderDefinition.CustomOneofCase.GlbxIceberg)) { - Log.Ui.info($"OrdersModel prevented a GlbxIceberg order from going out to a non-glbx exchange ({OrderUtils.short_description(security, order.Value.Side, price, size)})."); - return Task.FromResult(null); - } - } - return send_order(order.Value, price, size); - } - return Task.FromResult(null); - } - - /// Note: this method is private to force outside callers to use this overload . - /// This allows us to control which orders can be sent to which security/exchange (e.g., GlbxIceberg orders are only permitted for securities on the Glbx exchange). - async Task send_order(OrderDefinitionImmutable order, double price, double size) { - var result = (await send_order_get_error(order, price, size)); - _activity.on_send_order(order, price, size, result.key, result.error); - return result.key; - } - - async Task<(uint? key, string error)> send_order_get_error(OrderDefinitionImmutable order, double price, double size) { - var (iqty, fqty) = _break_quantity(order.SecurityKey, size); - var result = await _service.create_order_async(new Create() { - NewOrder = order.CloneToMutable(), - Price = price, - Quantity = iqty, - FractionalQuantity = fqty - }); - return _handle_error(result, "Error sending order: {0}"); - } - - Task<(OrderRecordImmutable, bool)[]> cancel_orders(IEnumerable orders) => - this.cancel_orders_where(o => orders.Contains(o.Key)); - - public async Task cancel_order(OrderRecordImmutable order) { - var result = await _service.cancel_order_async(new Cancel() { OrderKey = order.Key }); - (var id, var msg) = _handle_error(result, "Error canceling order: {0}"); - bool success = id.HasValue; - _activity.on_cancel_order(order, success, msg); - return success; - } - - public bool try_get_sec(uint sec_key, out SecurityMD result) => - _securities.TryGetValue(sec_key, out result); - - public SecurityMD? find_sec(uint sec_key) => - try_get_sec(sec_key, out var sec) ? sec : null; - - (uint iqty, FractionalQuantity fqty) _break_quantity( - uint sec_key, double qty) { - Contract.Requires(qty.is_real_non_negative()); - var display = try_get_sec(sec_key, out var sec) ? sec.quantity_display : - Security.Defaults.quantity_display_settingsi; - var rounded_qty = display.TickFactors.to_tick_factors().round(qty); - ExceptionAssert.InvalidOperation.That( - rounded_qty.is_real_non_negative(), - $"Expected rounded qty to be real, non-negative. " + - $"qty: {qty}; rounded_qty: {rounded_qty}; display: {display})"); - var iqty = Math.Floor(rounded_qty); - return ((uint)iqty, new() { Fraction = Math.Round(rounded_qty - iqty, 2) }); - } - - private CustomOrderDefinitionImmutable? _create_custom_def_for_stop_and_stop_limit(OrderRecordImmutable order, ref double stop_price) { - CustomOrderDefinition modified_custom_def = null; - var ord_type = order.order_type(); - if (ord_type == CustomOrderDefinition.CustomOneofCase.Stop) { - modified_custom_def = order.Def.CustomDef.CloneToMutable(); - modified_custom_def.Stop.StopPrice = stop_price; - } else if (ord_type == CustomOrderDefinition.CustomOneofCase.StopLimit) { - modified_custom_def = order.Def.CustomDef.CloneToMutable(); - double temp_price = order.price(); - modified_custom_def.StopLimit.StopPrice = stop_price; - stop_price = temp_price; - } - return modified_custom_def; - } - - public async Task update_order( - OrderRecordImmutable order, - double? new_price = null, - double? new_size = null, - CustomOrderDefinitionImmutable? new_algo = null) { - if (order.working_quantity() <= 0) { - // order already filled to completion - return false; - } - var size = new_size ?? order.quantity(); - var price = new_price ?? order.price(); - if (!(order.order_type() == CustomOrderDefinition.CustomOneofCase.Stop || order.order_type() == CustomOrderDefinition.CustomOneofCase.StopLimit) - && size == order.quantity() && price == order.price() && !new_algo.HasValue) { - // no parameters would be changed by this update - return false; - } - if ((order.order_type() == CustomOrderDefinition.CustomOneofCase.Stop || order.order_type() == CustomOrderDefinition.CustomOneofCase.StopLimit) && - !order.stop_or_stop_limit_has_been_triggered() && !new_algo.HasValue) { - // note the ref price parameter, if this is a stop limit then we swap the fields - new_algo = _create_custom_def_for_stop_and_stop_limit(order, ref price); - } - _activity.update_order(order, price, size, new_algo.HasValue); - var (iqty, fqty) = _break_quantity(order.security_key(), size); - var result = await _service.update_order_async(new Update() { - OrderKey = order.Key, - Price = price, - Quantity = iqty, - FractionalQuantity = fqty, - Custom = new_algo?.CloneToMutable() - }); - (var key, string msg) = _handle_error(result, "Failed to update order: {0}"); - bool success = key.HasValue; - if (!success) { - _activity.update_order_failed(order, price, size, new_algo.HasValue); - } - return success; - } - - public Task detach_orders(IEnumerable orders) => - Task.WhenAll(orders.Select(detach_order)); - - public async Task detach_order(uint order) { - var result = await _service.detach_order_async(new Detach() { OrderKey = order }); - return _handle_error(result, "Failed to break order: {0}").key.HasValue; - } - - (uint? key, string error) _handle_error(Result result, string template) => - _error.handle_error(result, template, r => r.ErrorCode, e => e == OrderError.Ok, r => new uint?(r.OrderKey)); - - #region Manual Fillls - public async Task create_manual_fills(IEnumerable fills) { - return await create_manual_fills(_create_fill_batch(fills)); - } - public async Task create_manual_fills(ManualFillBatchImmutable fills) { - var result = await _position_service.create_manual_fill_async(fills); - return _handle_error(result); - } - - public async Task update_manual_fills(IEnumerable fills) { - return await update_manual_fills(_create_fill_batch(fills)); - } - public async Task update_manual_fills(ManualFillBatchImmutable fills) { - var result = await _position_service.update_manual_fill_async(fills); - return _handle_error(result); - } - - public async Task delete_manual_fills(IEnumerable fills) { - return await delete_manual_fills(_create_fill_batch(fills)); - } - public async Task delete_manual_fills(ManualFillBatchImmutable fills) { - var result = await _position_service.remove_manual_fill_async(fills); - return _handle_error(result); - } - - ManualFillBatchImmutable _create_fill_batch(IEnumerable fills) { - var batch = new ManualFillBatch(); - batch.ManualFill.AddRange(fills); - return new ManualFillBatchImmutable(batch); - } - - bool _handle_error(Result result) { - if (!result.success) - _error.report_error(result.error_message); - return result.value.Response.All(r => r.Ok); - } - #endregion - - #endregion - - public void Dispose() => - Disposable.dispose(ref _dispose); - - static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdate table) { - var clone = target.ToBuilder(); - foreach (var o in table.removed) { - if (o.order.is_active()) { - clone.Remove(o.order.Key); - } - } - foreach (var o in table.changes) { - if (o.Old.order.is_active()) { - clone.Remove(o.Old.order.Key); - } - if (o.New.order.is_active()) { - clone.Add(o.New.order.Key, o.New); - } - } - foreach (var o in table.added) { - if (o.order.is_active()) { - clone.Add(o.order.Key, o); - } - } - return clone.ToImmutableDictionary(); - } - - static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { - var clone = target.ToBuilder(); - foreach (var o in table.removed) { - if (o.order.is_active()) { - clone.Remove(o.order.Key); - } - } - foreach (var o in table.changes) { - if (o.Old.order.is_active()) { - clone.Remove(o.Old.order.Key); - } - if (o.New.order.is_active()) { - clone.Add(o.New.order.Key, o.New); - } - } - foreach (var o in table.added) { - if (o.order.is_active()) { - clone.Add(o.order.Key, o); - } - } - return clone.ToImmutableDictionary(); - } - } - - // usinig new OrdersModel extensions to reduce the amount of code change needed to switch uis over to the new OrdersModel, - // without changing any code in the actual new OrdersModel class. - static class OrdersModelExtensions { - #region Cancel Orders - public static async Task cancel_order(this OrdersModel orders_model, uint order_id) { - var tu = await orders_model.on_orders.FirstAsync(); - if (tu.state.TryGetValue(order_id, out var o)) { - return await orders_model.cancel_order(o.order); - } - return false; - } - - public static async Task<(OrderRecordImmutable, bool)[]> cancel_orders_where(this OrdersModel model, Predicate pred) { - var tu = await model.on_orders.FirstAsync(); - var orders = - tu.state.Values - .Where(o => o.order.is_non_terminal() && pred(o.order)) - .Select(async o => { return (o.order, await model.cancel_order(o.order)); }); - return await Task.WhenAll(orders); - } - - public static Task<(OrderRecordImmutable, bool)[]> cancel_security_orders(this OrdersModel model, uint security) => - model.cancel_orders_where(o => o.Def.SecurityKey == security); - - public static Task<(OrderRecordImmutable, bool)[]> cancel_security_buy_orders(this OrdersModel model, uint security) => - model.cancel_orders_where(o => o.Def.SecurityKey == security && o.Def.Side == Side.Buy); - - public static Task<(OrderRecordImmutable, bool)[]> cancel_security_sell_orders(this OrdersModel model, uint security) => - model.cancel_orders_where(o => o.Def.SecurityKey == security && o.Def.Side == Side.Sell); - #endregion - - public static IObservable security_has_working_orders(this OrdersModel orders_model, uint security_key) => - orders_model - .on_security_orders(security_key) - .Select(t => t.state.Any(kv => kv.Value.is_active())) - .StartWith(false); - - public static IObservable> on_security_root_orders(this OrdersModel orders_model, uint security_key) => - orders_model.on_security_orders(security_key).Scan(new RootOrderData(), (rod, table) => { - // ** using HashSet/List pairs, rather than a Dictionary, because TableUpdateTransform calls for IReadOnlyLists, while Dictionary.Values only implements IReadOnlyCollection. - // (this way, we avoid copying both added/removed Dictionary.Values to new Lists at the end, see comment at bottom of method if this is unclear) ** - var added_root_keys = new HashSet(); - var added_roots = new List(); - var removed_root_keys = new HashSet(); - var removed_roots = new List(); - foreach (var o in table.removed.Concat(table.changes.Old())) { - if (rod.last_update.state.ContainsKey(o.Key)) { - // order was a root, needs to be removed - removed_roots.Add(o); - if (table.state.TryGetValue(o.Key, out OrderRecordImmutable newval)) { - // order was changed, if its an actual root, or its parent isn't present, the newval needs to be added - if (!o.has_parent() || !table.state.ContainsKey(o.ParentKey)) { - added_roots.Add(newval); - added_root_keys.Add(o.Key); - } - } else { - // order was indeed removed, not just changed - if it has a parent, remove its corresponding child-entry - if (o.has_parent()) { - rod.parent_to_children.remove(o.ParentKey, o.Key); - } - if (rod.parent_to_children.try_lookup(o.Key, out ISet children)) { - // order has left some orphans behind, any children that haven't been removed (i.e. still exist in table.state) are now roots - foreach (uint ck in children) { - if (table.state.TryGetValue(ck, out OrderRecordImmutable child)) { - added_roots.Add(child); - added_root_keys.Add(ck); - } - } - } - } - } else if (!table.state.ContainsKey(o.Key)) { - // order was indeed removed, not just changed - if it has a parent, remove its corresponding child-entry - rod.parent_to_children.remove(o.ParentKey, o.Key); - } - } - foreach (var o in table.added.Concat(table.changes.New())) { - if (added_root_keys.Contains(o.Key)) - continue; - if (rod.parent_to_children.try_lookup(o.Key, out ISet children)) { - // children were previously functioning as roots, need to remove them - foreach (uint ck in children) { - if (rod.last_update.state.TryGetValue(ck, out OrderRecordImmutable child)) - removed_roots.Add(child); - } - } - if (o.has_parent()) { - rod.parent_to_children.add(o.ParentKey, o.Key); - if (table.state.ContainsKey(o.ParentKey)) { - // not a root order - continue; - } - } - added_roots.Add(o); - } - // ** [continue comment at top of method] this call would otherwise be: - // rod.last_update = rod.last_update.update(removed_dict.Values.ToList(), added_dict.Values.ToList(), o => o.Key) ** - rod.last_update = rod.last_update.update(removed_roots, added_roots, o => o.Key); - return rod; - }).Select(r => r.last_update); - - public static IObservable> subscribe_all_algos(this OrdersModel orders_model) => - _subscribe_algo_meta_tuples(orders_model, orders_model.subscribe_algos()); - - public static IObservable> subscribe_algo_meta_tuples(this OrdersModel orders_model, SecurityImmutable security) => - _subscribe_algo_meta_tuples(orders_model, orders_model.subscribe_algos(security)); - - public static async Task order_from_key(this OrdersModel orders_model, uint order_key) { - (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3)); - if (success && tu.state.TryGetValue(order_key, out var rec)) { - return rec.order; - } - return null; - } - - public static async Task update_order( - this OrdersModel model, uint order, double price, double size) { - var result = await order_from_key(model, order); - if (result.HasValue) { - model.update_order(result.Value, price, size).ignore(); - } - } - - public static async Task update_order_price(this OrdersModel orders_model, uint order, double price) { - var result = await order_from_key(orders_model, order); - if (result.HasValue) { - var o = result.Value; - if (o.State.Price == price) return true; - var wq = o.working_quantity(); - if (wq > 0) { - return await orders_model.update_order(o, price, o.State.Quantity); - } - } - return false; - } - - public static async Task update_order_size( - this OrdersModel orders_model, uint order, double size) { - var result = await order_from_key(orders_model, order); - if (result.HasValue) { - var o = result.Value; - var wq = o.working_quantity(); - if (wq > 0) { - if (wq == size) return true; - return await orders_model.update_order(o, o.State.Price, size); - } - } - return false; - } - - static IObservable> _subscribe_algo_meta_tuples(OrdersModel orders_model, IObservable> algos) => - algos.LeftJoin( - orders_model.algo_model.metadata_service.subscribe_metadata().Table(AlgoDefinitionMetadata.key_fn), - AlgoData.key_fn, - AlgoDefinitionMetadata.key_fn, - (algo, key, meta) => (algo, (AlgoDefinitionMetadata?)meta), - (algo, key) => (algo, (AlgoDefinitionMetadata?)null), - tup => tup.algo.key); - - class RootOrderData { - public readonly SetMap parent_to_children = new SetMap(); - public TableUpdateTransform last_update { get; set; } = new TableUpdateTransform(); - } - } -} +namespace mbts.spark { + using bts; + using bts.api; + using bts.api.core; + using bts.api.orderman; + using bts.api.position; + using bts.collections; + using bts.md; + using bts.shell.pnl; + using bts.spark; + using bts.utils; + + using mbts.config; + using mbts.spark.utilities; + using mbts.ui; + using mbts.utils; + + using Spark.Api.Position; + using Spark.Api.Reactive; + using Spark.Core.Reactive; + using Spark.Model.Trading; + using Spark.Trading; + + using System; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Collections.Specialized; + using System.Composition; + using System.Diagnostics.Contracts; + using System.Linq; + using System.Reactive.Concurrency; + using System.Reactive.Linq; + using System.Threading.Tasks; + + using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; + using OrderUpdatePiq = bts.api.core.TableUpdate; + + [Export] + class OrdersModel : IDisposable { + public static readonly bool piq_enabled = Configuration.get().enabled; + public readonly MarketDataAuxiliaryModel md_auxiliary_model; + public readonly ISecurityAuxiliaryDataModel security_auxiliary_model; + + readonly IBus _bus; + readonly IAlgoProvider _algo_provider, _attachable_algo_provider; + readonly OrderFactory _factory; + readonly OrderService _service; + readonly PositionService _position_service; + readonly MBOService _mbo_service; + readonly SpreaderModel _spreader; + readonly ErrorReporter _error; + readonly DynamicOrderHelper _dynamic_helper; + readonly OrderActivitySummarizer _activity; + readonly TableUpdateDemuxer _security_orders_demuxer; + readonly TableUpdateDemuxer _security_working_orders_with_piq_demuxer; + readonly OrderDataSource _data_source; + + /// is updated atomically from multiple threads (use a local copy to + /// perform lookups, etc.) + IReadOnlyDictionary _securities = new Dictionary(0); + IDisposable _dispose; + + [ImportingConstructor] + public OrdersModel(IBus bus, + OrderFactory factory, + OrderService service, + OrderDataSource data_source, + PositionService position_service, + MBOService mbo_service, + SpreaderModel spreader, + DynamicOrderHelper dynamic_helper, + AdminModel admin_model, + IAlgoStudioModel algo_model, + ISecurityAuxiliaryDataModel security_auxiliary_model, + MarketDataAuxiliaryModel md_auxiliary_model, + IAlgoProvider algo_provider, + IAttachableAlgoProvider attachable_algo_provider, + SecurityResolver sec_resolver, + ErrorReporter error) { + // + _bus = bus; + _factory = factory; + _service = service; + _data_source = data_source; + _position_service = position_service; + _mbo_service = mbo_service; + _spreader = spreader; + _error = error; + _algo_provider = algo_provider; + _attachable_algo_provider = attachable_algo_provider; + _dynamic_helper = dynamic_helper; + this.admin_model = admin_model; + this.algo_model = algo_model; + this.md_auxiliary_model = md_auxiliary_model; + this.security_auxiliary_model = security_auxiliary_model; + var scheduler = ThreadPoolScheduler.Instance; + var interval = TimeSpan.FromMilliseconds(100); + + on_order_securities = + on_orders + .as_iincremental() + .filter(t => t.sec.HasValue) + .Select(t => t.sec.Value); + + _activity = new OrderActivitySummarizer(bus, on_orders); + _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); + _security_working_orders_with_piq_demuxer = new TableUpdateDemuxer(o => o.order.Key, o => o.order.security_key()); + studio_algos_with_meta = algo_model.definitions.LeftJoin( + algo_model.metadata_service.subscribe_metadata().Table(m => m.id), + d => d.key, + m => m.id, + (algo, k, meta) => (algo, meta), + (algo, k) => (algo, null), + t => t.algo.key) + .Cache(); + // join orders with algo data/metadata + on_order_tuples = on_orders.LeftJoin( + studio_algos_with_meta, + t => t.order.Key, + am => am.algo.key, + t => AlgoKey.from_composite_key(t.order.algo_template_key()), + (os, okey, am, mkey) => new OrderSecurity(os.sec, os.order, null, am.algo, am.meta), + (os, okey, mkey) => new OrderSecurity(os.sec, os.order, null, null, null), + ot => ot.order.Key) + .Cache(); + + on_order_tuples_with_piq = on_order_tuples.LeftJoin< + OrderSecurity, + OrderPIQImmutable, + OrderSecurity, + uint>( + mbo_service.subscribei_piq().Table(), + OrderSecurityHelper.key_fn, + piq => piq.OrderId, + (osam, _, p) => new OrderSecurity(osam.security, osam.order, p.Position, osam.algo, osam.meta), + (osam, _) => new OrderSecurity(osam.security, osam.order, null, osam.algo, osam.meta), + osp => osp.order.Key + ).Cache(); + + // order tuples with root + on_order_tuples_with_root = on_order_tuples.RootDependent( + OrderSecurityHelper.key_fn, + OrderSecurityHelper.parent_key_fn, + OrderSecurityRootPair.from_order_security, + OrderSecurityRootPair.from_order_security, + OrderSecurityHelper.root_pair_key_fn + ).Cache(); + // order tuples with root and piq + on_order_tuples_with_root_and_piq = on_order_tuples_with_piq.RootDependent( + OrderSecurityHelper.key_fn, + OrderSecurityHelper.parent_key_fn, + OrderSecurityRootPair.from_order_security, + OrderSecurityRootPair.from_order_security, + OrderSecurityHelper.root_pair_key_fn + ).Cache(); + // fill tuples + var on_fills = admin_model + .permission_filter( + _service.subscribei_fills() + .Naggle(interval, scheduler), + f => f.Account) + .join(sec_resolver, f => f.SecurityKey) + .Table().Cache(); + var fill_order_tuples_join = on_fills.LeftJoin( + on_order_tuples, + f => f.val.FillKey, + OrderSecurityHelper.key_fn, + f => f.val.OrderKey, + (f, fk, o, ok) => new FillTuple(f, o), + (f, fk, ok) => new FillTuple(f, null), + f => f.fill.FillKey); + on_fill_tuples = fill_order_tuples_join.Cache(); + // fill tuples with root (algo) + var fill_order_tuples_with_root_join = on_fills.LeftJoin( + on_order_tuples_with_root, + f => f.val.FillKey, + OrderSecurityHelper.root_pair_key_fn, + f => f.val.OrderKey, + (f, fk, o, ok) => new FillTuple(f.val, f.sec, o), + (f, fk, ok) => new FillTuple(f.val, f.sec, null), + FillTuple.key_fn); + on_fill_tuples_with_root = fill_order_tuples_with_root_join.Cache(); + // manual fills + on_manual_fills_securities = admin_model + .permission_filter( + _position_service.subscribei_manual_fills() + .Naggle(interval, scheduler), + f => f.AccountId) + .join(sec_resolver, mf => mf.SecurityKey) + .Table().Cache(); + var manual_fill_tuples_account_join = on_manual_fills_securities.LeftJoin( + admin_model.on_accounts.Select(d => MappedDictionary.create(d, (k, v) => new AccountTuple(k, v))).Table(a => a.id), + mfs => mfs.Item2.RecordId, + AccountTuple.key_fn, + mfs => mfs.Item2.AccountId, + (mfs, mf_id, a, a_id) => new ManualFillTuple(mfs.Item2, mfs.Item1, a, null), + (mfs, mf_id, a_id) => new ManualFillTuple(mfs.Item2, mfs.Item1, null, null), + ManualFillTuple.key_fn); + on_manual_fill_tuples = manual_fill_tuples_account_join.LeftJoin( + admin_model.on_users.Select(d => MappedDictionary.create(d, (k, v) => new UserTuple(k, v))).Table(a => a.id), + ManualFillTuple.key_fn, + UserTuple.key_fn, + mft => mft.manual_fill.UserId, + (mft, mft_id, u, u_id) => new ManualFillTuple(mft.manual_fill, mft.security, mft.account, u), + (mft, mft_id, u_id) => mft, + ManualFillTuple.key_fn) + .Cache(); + // can attach + on_orders_can_attach = on_orders.ParentDependent( + t => t.order.Key, + t => t.order.ParentKey, + (child, parent) => child.order.is_non_terminal() && !parent.order.is_non_terminal(), + t => t.order.is_non_terminal()) + .Cache(); + // can detach + on_orders_can_detach = on_orders.ParentDependent( + t => t.order.Key, + t => t.order.ParentKey, + (child, parent) => child.order.is_non_terminal() && parent.order.is_non_terminal(), + t => false) + .Cache(); + _dispose = Disposable.Create( + on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_order_tuples_with_piq + .Filter(os => os.order.is_active() && os.order.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible, key => key.order.Key) + .Subscribe(_security_working_orders_with_piq_demuxer), + _bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s), + _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())), + _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()), + _bus.subscribe(SparkTags.DetachOrders).Subscribe(x => detach_orders(x).ignore()), + _bus.subscribe(SparkTags.UpdateOrder).Subscribe(x => this.update_order(x.key, x.price, x.size).ignore()) + ); + } + + #region Output + public OrderFactory factory => _factory; + public SpreaderModel spreader => _spreader; + public DynamicOrderHelper dynamic_helper => _dynamic_helper; + public IObservable on_orders => _data_source.orders; + public IObservable on_order_tuples_with_piq { get; } + public IObservable> on_order_tuples { get; } + public IObservable> on_order_tuples_with_root { get; } + public IObservable> on_order_tuples_with_root_and_piq { get; } + public IObservable, uint>> on_orders_can_detach { get; } + public IObservable, uint>> on_orders_can_attach { get; } + public IObservable> on_order_securities { get; } + public IObservable> on_fill_tuples { get; } + public IObservable> on_fill_tuples_with_root { get; } + public IObservable working_orders => _data_source.working_orders; + public IObservable> studio_algos_with_meta { get; } + public IObservable> on_manual_fills_securities { get; } + public IObservable> on_manual_fill_tuples { get; } + public IObservable, uint>> security_orders => _data_source.security_orders; + + public AdminModel admin_model { get; } + public MBOService mbo_service => _mbo_service; + public IAlgoStudioModel algo_model { get; } + + public AlgoTypeSetImmutable get_permitted_algos(SecurityImmutable security) => + _algo_provider.get_permitted_algos(security); + + public IObservable> on_security_working_orders_with_piq(uint security) => + _security_working_orders_with_piq_demuxer.subscribe(security); + + public IObservable> on_security_orders(uint security) => + _security_orders_demuxer.subscribe(security); + + public IObservable> subscribe_algos(bool attachable_only = false) => + (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_algos(); + + public IObservable> subscribe_algos(SecurityImmutable security, bool attachable_only = false) => + (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_algos(security); + + public IObservable> subscribe_complete_algos(bool attachable_only = false, bool allow_undefined_child = false) => + (attachable_only ? _attachable_algo_provider : _algo_provider).subscribe_complete_algos(allow_undefined_child); + #endregion + + #region Input + // the security arg is needed for a temporary hack to prevent GlbxIceberg orders from going out in non-glbx exchanges (like Ice) + // eventually, we'll need an algo type model to handle this stuff. + // This hack leverages the fact that every place that calls 'send_order' already has the security handy (not just the sec id). + public Task send_order( + SecurityImmutable security, OrderDefinitionImmutable? order, double price, double size) { + if (order.HasValue) { +#if DEBUG + if (security.Exchange != Venue.Glbx && security.Exchange != Venue.Sim) { +#else + if (security.Exchange != Venue.Glbx) { +#endif + var algo_structure = OrderDataHelper.enumerate_algo_structure(order.Value.CustomDef); + if (algo_structure.Any(t => t.type == CustomOrderDefinition.CustomOneofCase.GlbxIceberg)) { + Log.Ui.info($"OrdersModel prevented a GlbxIceberg order from going out to a non-glbx exchange ({OrderUtils.short_description(security, order.Value.Side, price, size)})."); + return Task.FromResult(null); + } + } + return send_order(order.Value, price, size); + } + return Task.FromResult(null); + } + + /// Note: this method is private to force outside callers to use this overload . + /// This allows us to control which orders can be sent to which security/exchange (e.g., GlbxIceberg orders are only permitted for securities on the Glbx exchange). + async Task send_order(OrderDefinitionImmutable order, double price, double size) { + var result = (await send_order_get_error(order, price, size)); + _activity.on_send_order(order, price, size, result.key, result.error); + return result.key; + } + + async Task<(uint? key, string error)> send_order_get_error(OrderDefinitionImmutable order, double price, double size) { + var (iqty, fqty) = _break_quantity(order.SecurityKey, size); + var result = await _service.create_order_async(new Create() { + NewOrder = order.CloneToMutable(), + Price = price, + Quantity = iqty, + FractionalQuantity = fqty + }); + return _handle_error(result, "Error sending order: {0}"); + } + + Task<(OrderRecordImmutable, bool)[]> cancel_orders(IEnumerable orders) => + this.cancel_orders_where(o => orders.Contains(o.Key)); + + public async Task cancel_order(OrderRecordImmutable order) { + var result = await _service.cancel_order_async(new Cancel() { OrderKey = order.Key }); + (var id, var msg) = _handle_error(result, "Error canceling order: {0}"); + bool success = id.HasValue; + _activity.on_cancel_order(order, success, msg); + return success; + } + + public bool try_get_sec(uint sec_key, out SecurityMD result) => + _securities.TryGetValue(sec_key, out result); + + public SecurityMD? find_sec(uint sec_key) => + try_get_sec(sec_key, out var sec) ? sec : null; + + (uint iqty, FractionalQuantity fqty) _break_quantity( + uint sec_key, double qty) { + Contract.Requires(qty.is_real_non_negative()); + var display = try_get_sec(sec_key, out var sec) ? sec.quantity_display : + Security.Defaults.quantity_display_settingsi; + var rounded_qty = display.TickFactors.to_tick_factors().round(qty); + ExceptionAssert.InvalidOperation.That( + rounded_qty.is_real_non_negative(), + $"Expected rounded qty to be real, non-negative. " + + $"qty: {qty}; rounded_qty: {rounded_qty}; display: {display})"); + var iqty = Math.Floor(rounded_qty); + return ((uint)iqty, new() { Fraction = Math.Round(rounded_qty - iqty, 2) }); + } + + private CustomOrderDefinitionImmutable? _create_custom_def_for_stop_and_stop_limit(OrderRecordImmutable order, ref double stop_price) { + CustomOrderDefinition modified_custom_def = null; + var ord_type = order.order_type(); + if (ord_type == CustomOrderDefinition.CustomOneofCase.Stop) { + modified_custom_def = order.Def.CustomDef.CloneToMutable(); + modified_custom_def.Stop.StopPrice = stop_price; + } else if (ord_type == CustomOrderDefinition.CustomOneofCase.StopLimit) { + modified_custom_def = order.Def.CustomDef.CloneToMutable(); + double temp_price = order.price(); + modified_custom_def.StopLimit.StopPrice = stop_price; + stop_price = temp_price; + } + return modified_custom_def; + } + + public async Task update_order( + OrderRecordImmutable order, + double? new_price = null, + double? new_size = null, + CustomOrderDefinitionImmutable? new_algo = null) { + if (order.working_quantity() <= 0) { + // order already filled to completion + return false; + } + var size = new_size ?? order.quantity(); + var price = new_price ?? order.price(); + if (!(order.order_type() == CustomOrderDefinition.CustomOneofCase.Stop || order.order_type() == CustomOrderDefinition.CustomOneofCase.StopLimit) + && size == order.quantity() && price == order.price() && !new_algo.HasValue) { + // no parameters would be changed by this update + return false; + } + if ((order.order_type() == CustomOrderDefinition.CustomOneofCase.Stop || order.order_type() == CustomOrderDefinition.CustomOneofCase.StopLimit) && + !order.stop_or_stop_limit_has_been_triggered() && !new_algo.HasValue) { + // note the ref price parameter, if this is a stop limit then we swap the fields + new_algo = _create_custom_def_for_stop_and_stop_limit(order, ref price); + } + _activity.update_order(order, price, size, new_algo.HasValue); + var (iqty, fqty) = _break_quantity(order.security_key(), size); + var result = await _service.update_order_async(new Update() { + OrderKey = order.Key, + Price = price, + Quantity = iqty, + FractionalQuantity = fqty, + Custom = new_algo?.CloneToMutable() + }); + (var key, string msg) = _handle_error(result, "Failed to update order: {0}"); + bool success = key.HasValue; + if (!success) { + _activity.update_order_failed(order, price, size, new_algo.HasValue); + } + return success; + } + + public Task detach_orders(IEnumerable orders) => + Task.WhenAll(orders.Select(detach_order)); + + public async Task detach_order(uint order) { + var result = await _service.detach_order_async(new Detach() { OrderKey = order }); + return _handle_error(result, "Failed to break order: {0}").key.HasValue; + } + + (uint? key, string error) _handle_error(Result result, string template) => + _error.handle_error(result, template, r => r.ErrorCode, e => e == OrderError.Ok, r => new uint?(r.OrderKey)); + + #region Manual Fillls + public async Task create_manual_fills(IEnumerable fills) { + return await create_manual_fills(_create_fill_batch(fills)); + } + public async Task create_manual_fills(ManualFillBatchImmutable fills) { + var result = await _position_service.create_manual_fill_async(fills); + return _handle_error(result); + } + + public async Task update_manual_fills(IEnumerable fills) { + return await update_manual_fills(_create_fill_batch(fills)); + } + public async Task update_manual_fills(ManualFillBatchImmutable fills) { + var result = await _position_service.update_manual_fill_async(fills); + return _handle_error(result); + } + + public async Task delete_manual_fills(IEnumerable fills) { + return await delete_manual_fills(_create_fill_batch(fills)); + } + public async Task delete_manual_fills(ManualFillBatchImmutable fills) { + var result = await _position_service.remove_manual_fill_async(fills); + return _handle_error(result); + } + + ManualFillBatchImmutable _create_fill_batch(IEnumerable fills) { + var batch = new ManualFillBatch(); + batch.ManualFill.AddRange(fills); + return new ManualFillBatchImmutable(batch); + } + + bool _handle_error(Result result) { + if (!result.success) + _error.report_error(result.error_message); + return result.value.Response.All(r => r.Ok); + } + #endregion + + #endregion + + public void Dispose() => + Disposable.dispose(ref _dispose); + + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + var clone = target.ToBuilder(); + foreach (var o in table.removed) { + if (o.order.is_active()) { + clone.Remove(o.order.Key); + } + } + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); + } + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); + } + } + foreach (var o in table.added) { + if (o.order.is_active()) { + clone.Add(o.order.Key, o); + } + } + return clone.ToImmutableDictionary(); + } + + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { + var clone = target.ToBuilder(); + foreach (var o in table.removed) { + if (o.order.is_active()) { + clone.Remove(o.order.Key); + } + } + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); + } + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); + } + } + foreach (var o in table.added) { + if (o.order.is_active()) { + clone.Add(o.order.Key, o); + } + } + return clone.ToImmutableDictionary(); + } + } + + // usinig new OrdersModel extensions to reduce the amount of code change needed to switch uis over to the new OrdersModel, + // without changing any code in the actual new OrdersModel class. + static class OrdersModelExtensions { + #region Cancel Orders + public static async Task cancel_order(this OrdersModel orders_model, uint order_id) { + var tu = await orders_model.on_orders.FirstAsync(); + if (tu.state.TryGetValue(order_id, out var o)) { + return await orders_model.cancel_order(o.order); + } + return false; + } + + public static async Task<(OrderRecordImmutable, bool)[]> cancel_orders_where(this OrdersModel model, Predicate pred) { + var tu = await model.on_orders.FirstAsync(); + var orders = + tu.state.Values + .Where(o => o.order.is_non_terminal() && pred(o.order)) + .Select(async o => { return (o.order, await model.cancel_order(o.order)); }); + return await Task.WhenAll(orders); + } + + public static Task<(OrderRecordImmutable, bool)[]> cancel_security_orders(this OrdersModel model, uint security) => + model.cancel_orders_where(o => o.Def.SecurityKey == security); + + public static Task<(OrderRecordImmutable, bool)[]> cancel_security_buy_orders(this OrdersModel model, uint security) => + model.cancel_orders_where(o => o.Def.SecurityKey == security && o.Def.Side == Side.Buy); + + public static Task<(OrderRecordImmutable, bool)[]> cancel_security_sell_orders(this OrdersModel model, uint security) => + model.cancel_orders_where(o => o.Def.SecurityKey == security && o.Def.Side == Side.Sell); + #endregion + + public static IObservable security_has_working_orders(this OrdersModel orders_model, uint security_key) => + orders_model + .on_security_orders(security_key) + .Select(t => t.state.Any(kv => kv.Value.is_active())) + .StartWith(false); + + public static IObservable> on_security_root_orders(this OrdersModel orders_model, uint security_key) => + orders_model.on_security_orders(security_key).Scan(new RootOrderData(), (rod, table) => { + // ** using HashSet/List pairs, rather than a Dictionary, because TableUpdateTransform calls for IReadOnlyLists, while Dictionary.Values only implements IReadOnlyCollection. + // (this way, we avoid copying both added/removed Dictionary.Values to new Lists at the end, see comment at bottom of method if this is unclear) ** + var added_root_keys = new HashSet(); + var added_roots = new List(); + var removed_root_keys = new HashSet(); + var removed_roots = new List(); + foreach (var o in table.removed.Concat(table.changes.Old())) { + if (rod.last_update.state.ContainsKey(o.Key)) { + // order was a root, needs to be removed + removed_roots.Add(o); + if (table.state.TryGetValue(o.Key, out OrderRecordImmutable newval)) { + // order was changed, if its an actual root, or its parent isn't present, the newval needs to be added + if (!o.has_parent() || !table.state.ContainsKey(o.ParentKey)) { + added_roots.Add(newval); + added_root_keys.Add(o.Key); + } + } else { + // order was indeed removed, not just changed - if it has a parent, remove its corresponding child-entry + if (o.has_parent()) { + rod.parent_to_children.remove(o.ParentKey, o.Key); + } + if (rod.parent_to_children.try_lookup(o.Key, out ISet children)) { + // order has left some orphans behind, any children that haven't been removed (i.e. still exist in table.state) are now roots + foreach (uint ck in children) { + if (table.state.TryGetValue(ck, out OrderRecordImmutable child)) { + added_roots.Add(child); + added_root_keys.Add(ck); + } + } + } + } + } else if (!table.state.ContainsKey(o.Key)) { + // order was indeed removed, not just changed - if it has a parent, remove its corresponding child-entry + rod.parent_to_children.remove(o.ParentKey, o.Key); + } + } + foreach (var o in table.added.Concat(table.changes.New())) { + if (added_root_keys.Contains(o.Key)) + continue; + if (rod.parent_to_children.try_lookup(o.Key, out ISet children)) { + // children were previously functioning as roots, need to remove them + foreach (uint ck in children) { + if (rod.last_update.state.TryGetValue(ck, out OrderRecordImmutable child)) + removed_roots.Add(child); + } + } + if (o.has_parent()) { + rod.parent_to_children.add(o.ParentKey, o.Key); + if (table.state.ContainsKey(o.ParentKey)) { + // not a root order + continue; + } + } + added_roots.Add(o); + } + // ** [continue comment at top of method] this call would otherwise be: + // rod.last_update = rod.last_update.update(removed_dict.Values.ToList(), added_dict.Values.ToList(), o => o.Key) ** + rod.last_update = rod.last_update.update(removed_roots, added_roots, o => o.Key); + return rod; + }).Select(r => r.last_update); + + public static IObservable> subscribe_all_algos(this OrdersModel orders_model) => + _subscribe_algo_meta_tuples(orders_model, orders_model.subscribe_algos()); + + public static IObservable> subscribe_algo_meta_tuples(this OrdersModel orders_model, SecurityImmutable security) => + _subscribe_algo_meta_tuples(orders_model, orders_model.subscribe_algos(security)); + + public static async Task order_from_key(this OrdersModel orders_model, uint order_key) { + (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3)); + if (success && tu.state.TryGetValue(order_key, out var rec)) { + return rec.order; + } + return null; + } + + public static async Task update_order( + this OrdersModel model, uint order, double price, double size) { + var result = await order_from_key(model, order); + if (result.HasValue) { + model.update_order(result.Value, price, size).ignore(); + } + } + + public static async Task update_order_price(this OrdersModel orders_model, uint order, double price) { + var result = await order_from_key(orders_model, order); + if (result.HasValue) { + var o = result.Value; + if (o.State.Price == price) return true; + var wq = o.working_quantity(); + if (wq > 0) { + return await orders_model.update_order(o, price, o.State.Quantity); + } + } + return false; + } + + public static async Task update_order_size( + this OrdersModel orders_model, uint order, double size) { + var result = await order_from_key(orders_model, order); + if (result.HasValue) { + var o = result.Value; + var wq = o.working_quantity(); + if (wq > 0) { + if (wq == size) return true; + return await orders_model.update_order(o, o.State.Price, size); + } + } + return false; + } + + static IObservable> _subscribe_algo_meta_tuples(OrdersModel orders_model, IObservable> algos) => + algos.LeftJoin( + orders_model.algo_model.metadata_service.subscribe_metadata().Table(AlgoDefinitionMetadata.key_fn), + AlgoData.key_fn, + AlgoDefinitionMetadata.key_fn, + (algo, key, meta) => (algo, (AlgoDefinitionMetadata?)meta), + (algo, key) => (algo, (AlgoDefinitionMetadata?)null), + tup => tup.algo.key); + + class RootOrderData { + public readonly SetMap parent_to_children = new SetMap(); + public TableUpdateTransform last_update { get; set; } = new TableUpdateTransform(); + } + } +} diff --git a/src/client/spark/view/Shell/ShellModel.cs b/src/client/spark/view/Shell/ShellModel.cs index e27a2b047..30d6c4de9 100644 --- a/src/client/spark/view/Shell/ShellModel.cs +++ b/src/client/spark/view/Shell/ShellModel.cs @@ -13,6 +13,7 @@ using mbts.ui.settings; using mbts.ui.shell; + using Spark.Api.Reactive; using Spark.Core.Data; using System; @@ -141,7 +142,7 @@ typeof(OrderFactory), typeof(AlgoProviderFactory), typeof(DynamicOrderHelper), - typeof(SecurityStore), + typeof(SecurityResolver), typeof(AdminModel), typeof(AlgoStudioFlatDefinitionModel), typeof(AlgoTemplateMetadataService), diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark_api/Reactive/SecurityResolver.cs similarity index 93% rename from src/client/spark/Security/SecurityStore.cs rename to src/client/spark_api/Reactive/SecurityResolver.cs index 9c7362591..349ab78f2 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark_api/Reactive/SecurityResolver.cs @@ -1,298 +1,293 @@ -using bts.api; -using bts.api.core; -using bts.co2; - -using mbts; -using mbts.spark; - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Composition; -using System.Linq; -using System.Reactive.Linq; -using System.Reactive.Subjects; -using System.Threading.Tasks; - -using security_key = System.UInt32; - -namespace bts { - - [Export] - public class SecurityStore : IDisposable { - readonly MarketDataRelay _md; - readonly IBus _bus; - IDisposable _dispose; - IReadOnlyDictionary _live_securities; - readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); - readonly Subject> _cache_subject = new Subject>(); - - [ImportingConstructor] - public SecurityStore(MarketDataRelay md, IBus bus) { - _md = md; - _bus = bus; - _dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict); - _cache_subject.OnNext(_cache); - } - - IObservable> on_securities => _bus.subscribe(SparkTags.SecuritiesTag); - - SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live = null) { - // Try live securities first - live = live ?? _live_securities; - if (live != null) { - if (live.TryGetValue(key, out SecurityMD value)) { - return value.security; - } - } - - // Then try cached securities - if (_cache.TryGetValue(key, out SecurityImmutable security)) { - return security; - } - - return null; - } - - async Task get_security(security_key key, IReadOnlyDictionary live) { - var local_sec = try_get_local(key, live); - if (local_sec.HasValue) { - return local_sec; - } - - // Query the server for the value - var result = await _md.get_securities_async(new SecurityKeyList() { - Keys = { key } - }); - - if (result.success) { - var security = result.value.Security.FirstOrDefault(); - if (security != null) { - return try_cache_security(new SecurityImmutable(security)); - } - } - - // Give up :( - Log.Md.warn($"Failed to get security for key {key}"); - return null; - } - - async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { - var result = new Dictionary(); - var missing = new SecurityKeyList(); - foreach (var key in keys) { - var s = try_get_local(key); - if (s.HasValue) { - result.Add(key, s.Value); - } else { - missing.Keys.Add(key); - } - } - if (missing.Keys.Count > 0) { - var res = await _md.get_securities_async(missing); - if (res.success) { - foreach (var security in res.value.Security) { - result.Add(security.Id, try_cache_security(new SecurityImmutable(security))); - } - } - } - return result; - } - - // Request any missing keys from the server but do not wait for them to arrive - IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { - var result = new Dictionary(); - var missing = new HashSet(); - foreach (var key in keys) { - var security = try_get_local(key); - if (security.HasValue) { - result.TryAdd(key, security.Value); - } else { - missing.Add(key); - } - } - // Update cache in the background - if (missing.Count > 0) { - Task.Run(async () => { - var skl = new SecurityKeyList(); - skl.Keys.AddRange(missing); - var r = await _md.get_securities_async(skl); - if (r.success) { - var retrieved = r.value.Security.Select(s => s.Id); - Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]"); - foreach (var security in r.value.Security) { - try_cache_security(new SecurityImmutable(security)); - } - } - }); - } - return result; - } - - SecurityImmutable try_cache_security(SecurityImmutable s) { - if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { - _cache.TryAdd(s.Id, s); - _cache_subject.OnNext(_cache); - return s; - } - return live.security; - } - - /////////////// - // PUBLIC - /////////////// - #region public - - // Guaranteed to return a security if it has ever been seen by the system - public Task get_security(security_key key) { - return get_security(key, _live_securities); - } - // Guaranteed to return a security if it has ever been seen by the system - public Task> get_securities(IEnumerable keys) { - return get_securities(keys, _live_securities); - } - public void Dispose() { - Disposable.dispose(ref _dispose); - } - - public IObservable> join( - IObservable> source, Func skeyfn) where TKey : IEquatable { - return join(source, skeyfn, get_incremental_props, mk_update); - - (IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor) get_incremental_props(Update upd_in) => - (upd_in.all, upd_in.added, upd_in.changes, upd_in.removed, upd_in.key_extractor); - - Update<(SecurityImmutable? sec, T val), TKey> mk_update( - ImmutableDictionary state, - IEnumerable<(SecurityImmutable? sec, T val)> added, - IEnumerable> changes, - IEnumerable<(SecurityImmutable? sec, T val)> removed, - Func<(SecurityImmutable?, T), TKey> key_extractor) => - // - new Update<(SecurityImmutable? sec, T val), TKey>( - state.Values, added, changes, removed, key_extractor); - } - - public IObservable join( - IObservable source, - Func skeyfn, - Func all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor)> get_incremental_props, - Func, IEnumerable<(SecurityImmutable? sec, T val)>, IEnumerable>, IEnumerable<(SecurityImmutable? sec, T val)>, Func<(SecurityImmutable?, T), TKey>, TOut> create_result_update) - where TKey : IEquatable { - // - return Observable.Create(observer => { - var state = ImmutableDictionary.Empty; - var gate = new object(); - // We get thie function from the first source update - Func<(SecurityImmutable?, T t), TKey> rowkeyfn = null; - - void on_securities_updated(object ambiguous) { - if (rowkeyfn == null) return; - lock (gate) { - // If this is null we don't have any rows - var changed = new List>(); - foreach (var entry in state.Values) { - if (entry.s == null) { - var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); - if (security.HasValue) { - changed.Add( - new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); - } - } - } - if (changed.Count > 0) { - // update state - var builder = state.ToBuilder(); - foreach (var up in changed) { - builder[rowkeyfn(up.New)] = up.New; - } - state = builder.ToImmutable(); - // fire update - observer.OnNext( - create_result_update( - state, - Enumerable.Empty<(SecurityImmutable?, T)>(), - changed, - Enumerable.Empty<(SecurityImmutable?, T)>(), - rowkeyfn)); - } - } - } - - return Disposable.Create( - // Fire update when source changes - source.Subscribe((update_in) => { - var update = get_incremental_props(update_in); - lock (gate) { - rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2)); - var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changes.New()).Select(skeyfn)); - - var added = new List<(SecurityImmutable?, T)>(); - var changed = new List>(); - var removed = new List<(SecurityImmutable?, T)>(); - - foreach (var x in update.added) { - if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) { - added.Add((s, x)); - } else { - added.Add((null, x)); - } - } - foreach (var x in update.removed) { - if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) { - removed.Add(value); - } - } - foreach (var ch in update.changes) { - SecurityImmutable? si = - relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ? - s : (SecurityImmutable?)null; - changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); - } - - var builder = state.ToBuilder(); - builder.RemoveRange(update.removed.Select(update.key_extractor)); - builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old))); - foreach (var a in added) { - builder.Add(rowkeyfn(a), a); - } - foreach (var ch in changed) { - builder.Add(rowkeyfn(ch.New), ch.New); - } - state = builder.ToImmutable(); - - observer.OnNext(create_result_update(state, added, changed, removed, rowkeyfn)); - } - }), - // Also fire update when securities change - on_securities.Subscribe(on_securities_updated), - // Also fire update when cache changes - _cache_subject.Subscribe(on_securities_updated) - ); - }); - } - - public IObservable> join_table( - IObservable> source, Func get_sec_key) where TKey : IEquatable => - join( - source, - get_sec_key, - t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor), - (state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>( - state, added, changes, removed, key)); - #endregion - } - - public static class SecurityStoreExtensions { - public static IObservable> - join(this IObservable> source, SecurityStore store, Func skeyfn) where TKey : IEquatable { - return store.join(source, skeyfn); - } - - public static IObservable> - join_table(this IObservable> source, SecurityStore store, Func skeyfn) where TKey : IEquatable { - return store.join_table(source, skeyfn); - } - } -} +using bts.api; +using bts.api.core; +using bts.co2; + +using mbts; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading.Tasks; + +using security_key = System.UInt32; + +namespace Spark.Api.Reactive { + public class SecurityResolver : IDisposable { + readonly MarketDataRelay _md; + IDisposable _dispose; + IReadOnlyDictionary _live_securities; + readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + readonly Subject> _cache_subject = new Subject>(); + + public SecurityResolver( + MarketDataRelay md, IObservable> securities) { + _md = md; + on_securities = securities; + _dispose = securities.Subscribe(dict => _live_securities = dict); + _cache_subject.OnNext(_cache); + } + + IObservable> on_securities; + + SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live = null) { + // Try live securities first + live = live ?? _live_securities; + if (live != null) { + if (live.TryGetValue(key, out SecurityMD value)) { + return value.security; + } + } + + // Then try cached securities + if (_cache.TryGetValue(key, out SecurityImmutable security)) { + return security; + } + + return null; + } + + async Task get_security(security_key key, IReadOnlyDictionary live) { + var local_sec = try_get_local(key, live); + if (local_sec.HasValue) { + return local_sec; + } + + // Query the server for the value + var result = await _md.get_securities_async(new SecurityKeyList() { + Keys = { key } + }); + + if (result.success) { + var security = result.value.Security.FirstOrDefault(); + if (security != null) { + return try_cache_security(new SecurityImmutable(security)); + } + } + + // Give up :( + Log.Md.warn($"Failed to get security for key {key}"); + return null; + } + + async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { + var result = new Dictionary(); + var missing = new SecurityKeyList(); + foreach (var key in keys) { + var s = try_get_local(key); + if (s.HasValue) { + result.Add(key, s.Value); + } else { + missing.Keys.Add(key); + } + } + if (missing.Keys.Count > 0) { + var res = await _md.get_securities_async(missing); + if (res.success) { + foreach (var security in res.value.Security) { + result.Add(security.Id, try_cache_security(new SecurityImmutable(security))); + } + } + } + return result; + } + + // Request any missing keys from the server but do not wait for them to arrive + IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { + var result = new Dictionary(); + var missing = new HashSet(); + foreach (var key in keys) { + var security = try_get_local(key); + if (security.HasValue) { + result.TryAdd(key, security.Value); + } else { + missing.Add(key); + } + } + // Update cache in the background + if (missing.Count > 0) { + Task.Run(async () => { + var skl = new SecurityKeyList(); + skl.Keys.AddRange(missing); + var r = await _md.get_securities_async(skl); + if (r.success) { + var retrieved = r.value.Security.Select(s => s.Id); + Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]"); + foreach (var security in r.value.Security) { + try_cache_security(new SecurityImmutable(security)); + } + } + }); + } + return result; + } + + SecurityImmutable try_cache_security(SecurityImmutable s) { + if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { + _cache.TryAdd(s.Id, s); + _cache_subject.OnNext(_cache); + return s; + } + return live.security; + } + + /////////////// + // PUBLIC + /////////////// + #region public + + // Guaranteed to return a security if it has ever been seen by the system + public Task get_security(security_key key) { + return get_security(key, _live_securities); + } + // Guaranteed to return a security if it has ever been seen by the system + public Task> get_securities(IEnumerable keys) { + return get_securities(keys, _live_securities); + } + public void Dispose() { + Disposable.dispose(ref _dispose); + } + + public IObservable> join( + IObservable> source, Func skeyfn) where TKey : IEquatable { + return join(source, skeyfn, get_incremental_props, mk_update); + + (IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor) get_incremental_props(Update upd_in) => + (upd_in.all, upd_in.added, upd_in.changes, upd_in.removed, upd_in.key_extractor); + + Update<(SecurityImmutable? sec, T val), TKey> mk_update( + ImmutableDictionary state, + IEnumerable<(SecurityImmutable? sec, T val)> added, + IEnumerable> changes, + IEnumerable<(SecurityImmutable? sec, T val)> removed, + Func<(SecurityImmutable?, T), TKey> key_extractor) => + // + new Update<(SecurityImmutable? sec, T val), TKey>( + state.Values, added, changes, removed, key_extractor); + } + + public IObservable join( + IObservable source, + Func skeyfn, + Func all, IEnumerable added, IEnumerable> changes, IEnumerable removed, Func key_extractor)> get_incremental_props, + Func, IEnumerable<(SecurityImmutable? sec, T val)>, IEnumerable>, IEnumerable<(SecurityImmutable? sec, T val)>, Func<(SecurityImmutable?, T), TKey>, TOut> create_result_update) + where TKey : IEquatable { + // + return Observable.Create(observer => { + var state = ImmutableDictionary.Empty; + var gate = new object(); + // We get thie function from the first source update + Func<(SecurityImmutable?, T t), TKey> rowkeyfn = null; + + void on_securities_updated(object ambiguous) { + if (rowkeyfn == null) return; + lock (gate) { + // If this is null we don't have any rows + var changed = new List>(); + foreach (var entry in state.Values) { + if (entry.s == null) { + var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); + if (security.HasValue) { + changed.Add( + new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); + } + } + } + if (changed.Count > 0) { + // update state + var builder = state.ToBuilder(); + foreach (var up in changed) { + builder[rowkeyfn(up.New)] = up.New; + } + state = builder.ToImmutable(); + // fire update + observer.OnNext( + create_result_update( + state, + Enumerable.Empty<(SecurityImmutable?, T)>(), + changed, + Enumerable.Empty<(SecurityImmutable?, T)>(), + rowkeyfn)); + } + } + } + + return Disposable.Create( + // Fire update when source changes + source.Subscribe((update_in) => { + var update = get_incremental_props(update_in); + lock (gate) { + rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2)); + var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changes.New()).Select(skeyfn)); + + var added = new List<(SecurityImmutable?, T)>(); + var changed = new List>(); + var removed = new List<(SecurityImmutable?, T)>(); + + foreach (var x in update.added) { + if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) { + added.Add((s, x)); + } else { + added.Add((null, x)); + } + } + foreach (var x in update.removed) { + if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) { + removed.Add(value); + } + } + foreach (var ch in update.changes) { + SecurityImmutable? si = + relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ? + s : (SecurityImmutable?)null; + changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); + } + + var builder = state.ToBuilder(); + builder.RemoveRange(update.removed.Select(update.key_extractor)); + builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old))); + foreach (var a in added) { + builder.Add(rowkeyfn(a), a); + } + foreach (var ch in changed) { + builder.Add(rowkeyfn(ch.New), ch.New); + } + state = builder.ToImmutable(); + + observer.OnNext(create_result_update(state, added, changed, removed, rowkeyfn)); + } + }), + // Also fire update when securities change + on_securities.Subscribe(on_securities_updated), + // Also fire update when cache changes + _cache_subject.Subscribe(on_securities_updated) + ); + }); + } + + public IObservable> join_table( + IObservable> source, Func get_sec_key) where TKey : IEquatable => + join( + source, + get_sec_key, + t => (t.state.Values, t.added, t.changes, t.removed, t.key_extractor), + (state, added, changes, removed, key) => new TableUpdate<(SecurityImmutable?, T), TKey>( + state, added, changes, removed, key)); + #endregion + } + + public static class SecurityStoreExtensions { + public static IObservable> + join(this IObservable> source, SecurityResolver store, Func skeyfn) where TKey : IEquatable { + return store.join(source, skeyfn); + } + + public static IObservable> + join_table(this IObservable> source, SecurityResolver store, Func skeyfn) where TKey : IEquatable { + return store.join_table(source, skeyfn); + } + } +} diff --git a/src/client/spark_core/LazyOnce.cs b/src/client/spark_core/LazyOnce.cs new file mode 100644 index 000000000..4babc3ab3 --- /dev/null +++ b/src/client/spark_core/LazyOnce.cs @@ -0,0 +1,36 @@ +#nullable enable +namespace Spark.Core { + using System; + + /// + /// A thread-safe implementation of lazy initialization. Unlike , it + /// guarentees the creation delegate isn't invoked more than once. + /// + public class LazyOnce { + readonly Func _create; + Optional _value; + + public LazyOnce(Func create) => + _create = create; + + public bool IsValueCreated => _value.has_value; + + public T Value { + get { + // check has_value outside of lock since, once set to true, it doesn't change + if (!_value.has_value) { + lock (_create) { + // check has_value again (may have become true since first check) + if (!_value.has_value) { + _value = _create(); + } + } + } + return _value; + } + } + + public static implicit operator T(LazyOnce lazy) => + lazy.Value; + } +} diff --git a/src/client/spark_core/VirtualSingleton.cs b/src/client/spark_core/VirtualSingleton.cs new file mode 100644 index 000000000..30ad78097 --- /dev/null +++ b/src/client/spark_core/VirtualSingleton.cs @@ -0,0 +1,20 @@ +#nullable enable +namespace Spark.Core { + using bts; + + using System.Threading; + + /// + /// A (base) class that asserts it isn't instantiated more than once + /// + /// + /// The underlying type, which is prevented from being instantiated more than once, as it + /// inherits from itself + /// + public abstract class VirtualSingleton where T : VirtualSingleton { + static int _instances = 0; + + protected VirtualSingleton() => + Log.assert(Interlocked.Increment(ref _instances) == 1); + } +}