diff --git a/.gitignore b/.gitignore index d87a92c1b..a8d93ca09 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ src/server/sparkd/orderman/perf_analysis/logs/ src/server/sparkd/orderman/perf_analysis/trace_data/ .fake *.fsx.lock +*.code-workspace src/lib/orderman/perf_analysis/logs/latency_data/ trace_data/ /src/lib/orderman/perf_analysis/last_pretty_log.csv diff --git a/.vscode/launch.json b/.vscode/launch.json index 7b1fe24a8..0cd375b54 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test","AuthToken"], + "args": ["test", "[md]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index 06358a7a0..27775d2d9 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -5,9 +5,11 @@ #### User Interface Improvements - Removed security key field from most grids - Removed trade key field from time and sales grid - - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM + - Fixed the floating order book so that rows are no longer highlighted after an order is modified +#### Bug Fixes + - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM #### Algos - Added OCO Algo - An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. Composing this order with Cover simulates scalping functionality @@ -18,7 +20,7 @@ - Improved robustness of limit system test - Extensive market data refactoring allow for security definition updates - + ## 1.4.11 ### New Features - Floating Order Book diff --git a/remote-sparkvm.code-workspace b/remote-sparkvm.code-workspace deleted file mode 100644 index 03c875df7..000000000 --- a/remote-sparkvm.code-workspace +++ /dev/null @@ -1,27 +0,0 @@ -{ - "folders": [ - { - "path": "." - }, - { - "uri": "sftp://bts:BT5u53r@sparkvm:22/home/bts/", - "name": "sparkvm" - } - ], - "settings": { - "files.associations": { - "utility": "cpp", - "initializer_list": "cpp", - "list": "cpp", - "type_traits": "cpp", - "vector": "cpp", - "xhash": "cpp", - "xstring": "cpp", - "system_error": "cpp", - "array": "cpp", - "string_view": "cpp", - "xutility": "cpp", - "*.ipp": "cpp" - } - } -} \ No newline at end of file diff --git a/src/client/spark/Daemons/DeleteSyntheticDaemon.cs b/src/client/spark/Daemons/DeleteSyntheticDaemon.cs index 15bc8d00c..52819f4e3 100644 --- a/src/client/spark/Daemons/DeleteSyntheticDaemon.cs +++ b/src/client/spark/Daemons/DeleteSyntheticDaemon.cs @@ -32,7 +32,7 @@ namespace bts { if (!orders_success) { _error_reporter.report_error($"Unable to delete synthetic security ({id}): unable to determine whether or not security ({id}) has working orders (deletion is not permitted while there are working orders)."); Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because the orders lookup timed out (must confirm no working orders)."); - } else if (orders.state.Values.Any(o => o.Def.SecurityKey == id && o.is_non_terminal())) { + } else if (orders.state.Values.Any(o => o.order.Def.SecurityKey == id && o.order.is_non_terminal())) { Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because non-terminal-orders were detected."); } else { ExecutionStrategyImmutable? strategy = await _orders_model.spreader.lookup_strategy_async(id); diff --git a/src/client/spark/Daemons/MDDaemon.cs b/src/client/spark/Daemons/MDDaemon.cs index 816faac4a..70361557b 100644 --- a/src/client/spark/Daemons/MDDaemon.cs +++ b/src/client/spark/Daemons/MDDaemon.cs @@ -14,19 +14,21 @@ using SecurityKey = System.UInt32; using System.Collections.Concurrent; using System.Threading; using bts.api.user; +using bts.md; namespace bts { [Export(typeof(IInitializable))] - class MDDaemon : bts.md.MDSource, IObserver>, IInitializable, IDisposable { + class MDDaemon : md.MDSource, IObserver>, IInitializable, IDisposable { readonly Lazy _bus; readonly Lazy _token; readonly Lazy _ss; readonly Dictionary _deleted_synthetics = new Dictionary(); - static uint get_naggle() => mbts.config.Configuration.get().naggle_ms; + static uint get_naggle() => mbts.config.Configuration.get().naggle_ms; [ImportingConstructor] - public MDDaemon(Lazy bus, Lazy token, Lazy ss) : base(get_naggle(), true) { + public MDDaemon(Lazy bus, Lazy token, Lazy ss) : + base(get_naggle(), MDSourceOptions.DeferInit) { _bus = bus; _token = token; _ss = ss; @@ -44,14 +46,14 @@ namespace bts { if (!deleted_synthetic_keys.Contains(kvp.Key)) { _exchange_sec_map.TryAdd(kvp.Key, kvp.Value); _deleted_synthetics.Remove(kvp.Key); - _version += 1; + _version = DateTime.UtcNow; } } // Remove any newly deleted synthetics and store them for potential undelete foreach (var key in deleted_synthetic_keys) { if (_exchange_sec_map.TryRemove(key, out SecurityMD deleted)) { _deleted_synthetics[key] = deleted; - _version += 1; + _version = DateTime.UtcNow; } } // Post full dictionary of syntheics here diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index f0dfc0583..5b8ac95c3 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; IDisposable _dispose; [ImportingConstructor] @@ -57,14 +57,16 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out OrderRecordImmutable ori) && _md_map.TryGetValue(ori.Def.SecurityKey, out SecurityMD smd) - && (ori.HasDef && ori.Def.HasCustomDef && !(ori.Def.CustomDef.HasStop || ori.Def.CustomDef.HasStopLimit))) - _show_order_window(new OrderModificationDialog { - DataContext = new ExistingOrderTicketVM(_model, - Observable.Return(new OrderRecordTuple(ori, smd)), - Observable.Return(_model.factory.default_ladder_settings(smd.security).calculator_buttons) - ) - }); + if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue + && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { + var md = _md_map.TryGetValue(ori.order.security_key()); + _show_order_window(new OrderModificationDialog { + DataContext = new ExistingOrderTicketVM(_model, + Observable.Return((md,ori.order)), + Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) + ) + }); + } } } } \ No newline at end of file diff --git a/src/client/spark/Daemons/ShellServices.cs b/src/client/spark/Daemons/ShellServices.cs index eb9deee90..c51a6b0da 100644 --- a/src/client/spark/Daemons/ShellServices.cs +++ b/src/client/spark/Daemons/ShellServices.cs @@ -5,7 +5,7 @@ using bts.api.orderman; using bts.api.persistence; using bts.api.position; using bts.api.synthetic; -using bts.api.user; +using bts.co2; using bts.metrics; using bts.support; using System.ComponentModel.Composition; @@ -41,5 +41,8 @@ namespace Spark { [Export] public MarketDataAuxiliaryService md_auxiliary_service => context.locate(); + + [Export] + public MarketDataRelay md_replay => context.locate(); } } \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index a7abb9199..043e88512 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -39,7 +39,7 @@ namespace mbts.spark { user_filter = DiscreteValuesFilterVM.Create(_model.factory.subscribe_users().Select(d => d.Values), r => r.user, nameof(Row.user)); security_filter = DiscreteValuesFilterVM.Create( fills_source.with_respect_to( - ft => ft.security_md.HasValue ? ft.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable, + ft => ft.security ?? SecurityDataHelper.Defaults.SecurityImmutable, s => s.Id, ft => ft.key ).Select(t => (t.added.Values, t.removed.Values)), r => r.security, @@ -186,7 +186,8 @@ namespace mbts.spark { readonly FillImmutable _fill; readonly OrderRecordImmutable _order; - public Row(FillTuple fill_tuple, OrdersModel model) : base(fill_tuple.security_md.HasValue ? fill_tuple.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable) { + public Row(FillTuple fill_tuple, OrdersModel model) : + base(fill_tuple.security ?? SecurityDataHelper.Defaults.SecurityImmutable) { _fill = fill_tuple.fill; _order = fill_tuple.order ?? OrderDataHelper.Defaults.OrderRecordImmutable; _model = model; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index 682075084..e0bebe3f6 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -30,7 +30,7 @@ namespace mbts.spark { interface ICVOrderBookColumn : IKeyedColumn, IGridColumn { string sort_member_path { get; } ListSortDirection? SortDirection { get; set; } - IDataGridColumnFilter filter { get; } + IDataGridColumnFilter filter { get; } IComparer comparer { get; } } @@ -65,9 +65,9 @@ namespace mbts.spark { public OrderBookColumnCollection(IBus bus, OrdersModel model, IObservable<(IEnumerable added, IEnumerable removed)> order_securities) { // security Add(security_column = new SecurityColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( order_securities, - ort => ort.security_md.HasValue ? ort.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable, + ort => ort.security ?? SecurityDataHelper.Defaults.SecurityImmutable, SecurityCriteria.from_securities, SecurityCriteria.get_to_securities(bus), nameof(OrderBookRowVM.security), @@ -81,9 +81,9 @@ namespace mbts.spark { nameof(OrderBookRowVM.symbol), r => r.symbol, DefaultColumnWidths.Symbol, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( mbts.co2.UserSymbol.get_all().Select(u => u.symbol).Distinct().ToList(), - ort => ort.security_md.HasValue ? ort.security_md.Value.security.Symbol : string.Empty, + ort => ort.security?.Symbol ?? string.Empty, nameof(OrderBookRowVM.symbol) ), DataGridFilterKnownTemplates.DiscreteValues, @@ -95,7 +95,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.time_created), r => r.time_created_text, DefaultColumnWidths.Time, - new TimeFilterVM( + new TimeFilterVM( ort => ort.order.created(), nameof(OrderBookRowVM.time_created)), DataGridFilterKnownTemplates.TimeFilter, @@ -107,7 +107,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.last_change), r => r.last_change_text, DefaultColumnWidths.Time, - new TimeFilterVM( + new TimeFilterVM( ort => ort.order.last_change(), nameof(OrderBookRowVM.last_change)), DataGridFilterKnownTemplates.TimeFilter, @@ -119,7 +119,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.user), r => r.user, DefaultColumnWidths.User, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( model.factory.subscribe_users().Select(d => d.Values), ort => ort.order.user(), nameof(OrderBookRowVM.user)), @@ -132,7 +132,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.account), r => r.account, DefaultColumnWidths.Account, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( model.factory.subscribe_accounts().Select(d => d.Values), ort => ort.order.account(), vals => vals.ToArray(), @@ -147,9 +147,9 @@ namespace mbts.spark { nameof(OrderBookRowVM.exchange), r => r.exchange.ToString(), DefaultColumnWidths.Exchange, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values_except(Venue.Unset).ToList(), - ort => ort.security_md.HasValue ? ort.security_md.Value.security.Exchange : Venue.Unset, + ort => ort.security.exchange_or_default(), nameof(OrderBookRowVM.exchange)), DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.exchange))); @@ -160,7 +160,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.time_in_force), r => r.time_in_force.ToString(), DefaultColumnWidths.TimeInForce, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.time_in_force(), nameof(OrderBookRowVM.time_in_force)), @@ -168,7 +168,7 @@ namespace mbts.spark { _wrap_comparer(r => r.time_in_force))); // order type Add(order_type_column = new OrderBookOrderTypeColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( utilities.OrderFactory.all_order_types, ort => ort.order.order_type(), nameof(OrderBookRowVM.order_type)))); @@ -179,7 +179,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.order_sub_type), r => r.order_sub_type_text, 80, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.sub_type(), nameof(OrderBookRowVM.order_sub_type)), @@ -187,20 +187,20 @@ namespace mbts.spark { _wrap_comparer(r => r.order_sub_type))); // side Add(side_column = new SideColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( new[] { Side.Buy, Side.Sell }, ort=> ort.order.side(), nameof(OrderBookRowVM.side)))); // quantity Add(qty_column = new SizeColumn( - new UIntMinMax(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity)))); + new UIntMinMax(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity)))); // edit quantity column // not usually part of the main order book, but it's useful to add it // rather than debugging the popup window //Add(edit_qty_column = new EditableQtyColumn()); // filled quantity Add(filled_qty_column = new FilledColumn( - new UIntMinMax(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity)))); + new UIntMinMax(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity)))); // price Add(price_column = new SimpleTextColumn( Columns.Price, @@ -208,7 +208,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.price), r => r.price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.price(), nameof(OrderBookRowVM.price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -224,7 +224,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.avg_price), r => r.avg_price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.avg_price(), nameof(OrderBookRowVM.avg_price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -236,7 +236,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.last_price), r => r.last_price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.last_price(), nameof(OrderBookRowVM.last_price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -248,7 +248,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.status), r => r.status.ToString(), 80, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.status(), nameof(OrderBookRowVM.status)), @@ -256,7 +256,7 @@ namespace mbts.spark { _wrap_comparer(r => r.status))); // last message Add(last_message_column = new OrderBookLastMessageColumn( - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.last_msg(), nameof(OrderBookRowVM.last_msg)))); // security key @@ -266,7 +266,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.security_key), r => r.security_key.ToString(), DefaultColumnWidths.SecurityKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.security_key().ToString(), nameof(OrderBookRowVM.security_key)), DataGridFilterKnownTemplates.Regex, @@ -278,7 +278,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.order_key), r => r.order_key.ToString(), DefaultColumnWidths.OrderKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.Key.ToString(), nameof(OrderBookRowVM.order_key)), DataGridFilterKnownTemplates.Regex, @@ -290,7 +290,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.parent_order_key), r => r.parent_order_key.ToString(), DefaultColumnWidths.ParentKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.ParentKey.ToString(), nameof(OrderBookRowVM.parent_order_key)), DataGridFilterKnownTemplates.Regex, @@ -389,9 +389,9 @@ namespace mbts.spark { } } - public Func get_filter() { + public Func get_filter() { var active_filters = this.OfType().Where(c => c.filter.active.GetValueOrDefault()).Select(c => c.filter).ToArray(); - return active_filters.Length == 0 ? null : new Func(r => active_filters.All(f => f.filter(r))); + return active_filters.Length == 0 ? null : new Func(r => active_filters.All(f => f.filter(r))); } public OrderBookSortingData get_sorting_data() { @@ -481,7 +481,7 @@ namespace mbts.spark { GlyphContext _root_glyph_context, _child_glyph_context; protected OrderbookColumnBase(Columns col_key, string header, Func get_text, string sort_prop, double width, - IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(col_key) { + IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(col_key) { id = key.ToString(); Width = width; Header = header; @@ -499,10 +499,10 @@ namespace mbts.spark { } } - public IDataGridColumnPersistentFilter filter { get; } + public IDataGridColumnPersistentFilter filter { get; } public IComparer comparer { get; } - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; IDataGridColumnFilter IFilterableColumn.filter => filter; string IFilterableColumn.filter_template => _filter_template; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } @@ -516,7 +516,7 @@ namespace mbts.spark { public class SimpleTextColumn : OrderbookColumnBase, IRenderColumn { public SimpleTextColumn(Columns key, string header, string sort_prop, Func get_text, double width, - IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) + IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(key, header, get_text, sort_prop, width, filter, filter_template, comparer) { } public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => @@ -526,7 +526,7 @@ namespace mbts.spark { #region Explicit Column Defs public class SideColumn : OrderbookColumnBase, IRenderColumn { - public SideColumn(IDataGridColumnPersistentFilter filter) + public SideColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Side, "B/S", r => r.side.ToString(), nameof(OrderBookRowVM.side), DefaultColumnWidths.Side, filter, DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.side)) { } public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => @@ -535,7 +535,7 @@ namespace mbts.spark { } public class SizeColumn : OrderbookColumnBase, IRenderColumn { - public SizeColumn(IDataGridColumnPersistentFilter filter) + public SizeColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Size, "Qty", r => r.quantity.ToString(), nameof(OrderBookRowVM.quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.quantity)) { sort_member_path = nameof(OrderBookRowVM.quantity); } @@ -546,7 +546,7 @@ namespace mbts.spark { } public class FilledColumn : OrderbookColumnBase, IRenderColumn { - public FilledColumn(IDataGridColumnPersistentFilter filter) + public FilledColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Filled, "Filled Qty", r => r.filled_quantity.ToString(), nameof(OrderBookRowVM.filled_quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.filled_quantity)) { sort_member_path = nameof(OrderBookRowVM.filled_quantity); } @@ -558,7 +558,7 @@ namespace mbts.spark { public class OrderBookOrderTypeColumn : OrderTypeColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; - public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) + public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) : base(Columns.OrderType, new Binding(nameof(OrderBookRowVM.order_type)), new Binding(nameof(OrderBookRowVM.order_metadata)), new Binding(nameof(OrderBookRowVM.dynamic_order_type_display_name)), filter) { id = key.ToString(); this.filter = filter; @@ -567,10 +567,10 @@ namespace mbts.spark { _cached_sort_direction = SortDirection; } - public new IDataGridColumnPersistentFilter filter { get; } + public new IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.order_type); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } protected override void OnSortDirectionChanged(ListSortDirection? new_value) { @@ -582,7 +582,7 @@ namespace mbts.spark { public class OrderBookLastMessageColumn : TextColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; - public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter filter) + public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter filter) : base(Columns.LastMessage, new Binding(nameof(OrderBookRowVM.last_msg)), filter, DataGridFilterKnownTemplates.Regex) { id = key.ToString(); this.filter = filter; @@ -592,10 +592,10 @@ namespace mbts.spark { tooltip_binding = new Binding(nameof(OrderBookRowVM.last_msg)); } - public new IDataGridColumnPersistentFilter filter { get; } + public new IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.last_msg); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } protected override void OnSortDirectionChanged(ListSortDirection? new_value) { @@ -611,7 +611,7 @@ namespace mbts.spark { ListSortDirection? _cached_sort_direction; - public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { + public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { id = key.ToString(); this.filter = filter; Header = DefaultColumnHeaders.Security; @@ -620,10 +620,10 @@ namespace mbts.spark { _cached_sort_direction = SortDirection; } - public IDataGridColumnPersistentFilter filter { get; } + public IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.security, SecurityHelper.Comparer.InstanceImmutable.to_nullable()); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; IDataGridColumnFilter IFilterableColumn.filter => filter; string IFilterableColumn.filter_template => DataGridFilterKnownTemplates.DiscreteValues; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs index b572530fc..f65aeb4be 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs @@ -1,5 +1,7 @@ using bts; +using bts.api; using bts.api.core; +using bts.api.orderman; using bts.collections; using bts.utils; using mbts.utils; @@ -13,26 +15,26 @@ using System.Reactive.Linq; namespace mbts.spark { class OrderBookRowCollection : ObservableCollection { static readonly IEqualityComparer> _node_equality_comparer = EqualityComparer.Create, uint>(n => n.value.order_key); - readonly Func _row_factory; + readonly Func _row_factory; /// Critical: Any and all row mutations must be executed through . /// Mutations can change the sort-position of a row, and will be broken if its elements aren't properly sorted (it relies heavily on binary-search logic). SortedTree _tree; IComparer _comparer; - Func _filter; + Func _filter; bool _is_refresh_pending; int _deferment_count = 0; /// When processing changed-items, the method is used to update the changed row. It is not expected to fail (return false), /// but if it does, we don't want to overwhelm the logs with a bunch of useless noise in case it starts happening repeatedly, hence the int fields below. - /// , , . + /// , , . const int _max_replace_misses_logged = 10; int _n_replace_misses_logged; - public OrderBookRowCollection(Func row_factory, IComparer comparer) { + public OrderBookRowCollection(Func row_factory, IComparer comparer) { _row_factory = row_factory; _tree = _mk_tree(_comparer = comparer); } - public Func filter { + public Func filter { get => _filter; set { _filter = value; @@ -144,24 +146,24 @@ namespace mbts.spark { }); } - public void update(IIncrementalTableUpdateTransform table) { + public void update(TableUpdate<(SecurityImmutable? sec,OrderRecordImmutable order),uint> table) { IEnumerable> impacted_branch_roots; int n_removed = table.removed.Count(); if (n_removed == total_row_count) { // full refresh (most likely this is the first update or a server restart) - _tree = _mk_tree(_comparer, table.added.Select(_row_factory)); + _tree = _mk_tree(_comparer, table.added.Select(x => _row_factory(x))); impacted_branch_roots = _tree.RootNodes; } else { - foreach (OrderRecordTuple ort in table.removed) + foreach (OrderSecurity ort in table.removed) _tree.Remove(ort.order.Key); - foreach (Change change in table.changes) { + foreach (var change in table.changes) { if (!_tree.TryReplace(change.New.order.Key, r => { r.update(change.New); return r; })) { // uh oh, replace failed! Let's log this, but not a million times (this seems like the kind of error that, if it happens once, could happen a lot). if (_n_replace_misses_logged++ < _max_replace_misses_logged) - Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderRecordTuple)} in update (old:{change.Old.key}, new:{change.New.key}). This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook."); + Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderSecurity)} in update (old:{change.Old.order.Key}, new:{change.New.order.Key}). This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook."); } } - foreach (OrderRecordTuple ort in table.added) + foreach (OrderSecurity ort in table.added) _tree.Insert(_row_factory(ort)); impacted_branch_roots = table.added.Concat(table.changed).Select(ort => ort.order.Key).Select(_tree.GetRootNode) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 25e44a88b..03f3343de 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -19,14 +19,15 @@ namespace mbts.spark { IRaiseCanExecuteChangedCommand _show_order_modify_cmd, _toggle_expanded_cmd, _cancel_cmd, _modify_order_quantity, _modify_order_price; bool _is_parent, _is_expanded, _is_selected; int _depth_level; - SecurityMD? _security_md => order_tuple.security_md; + SecurityImmutable? _security => order_tuple.security; + // for mini-ob uint _prospective_quantity; double _prospective_price; - public OrderBookRowVM(OrderBookVM owner, OrderRecordTuple order_tuple, bool is_expanded) { + public OrderBookRowVM(OrderBookVM owner, OrderSecurity order_tuple, bool is_expanded) { _owner = owner; this.order_tuple = order_tuple; is_visible = !is_child; @@ -49,17 +50,17 @@ namespace mbts.spark { public event Action invalidated; // Core - public OrderRecordTuple order_tuple { get; private set; } + public OrderSecurity order_tuple { get; private set; } public OrderRecordImmutable order => order_tuple.order; public OrderMetadata order_metadata => _owner.get_metadata(this); public uint order_key => order.Key; public uint parent_order_key => order.ParentKey; public uint security_key => order.security_key(); - public SecurityImmutable? security => _security_md.security(); + public SecurityImmutable? security => _security; public ContractSpecificationImmutable? contract_spec => security.spec_or_default(); public string security_text => security.display_name(); - public Venue exchange => _security_md.exchange(); - public string symbol => _security_md.symbol(); + public Venue exchange => _security.exchange_or_default(); + public string symbol => _security.symbol_or_default(); // Def public CustomOrderDefinition.CustomOneofCase order_type => order.order_type(); public string dynamic_order_type_display_name => order.dynamic_order_type_display_name(); @@ -163,9 +164,9 @@ namespace mbts.spark { internal void expand_internal() => Set(ref _is_expanded, true); internal void collapse_internal() => Set(ref _is_expanded, false); - public void update(OrderRecordTuple order_tuple) { + public void update(OrderSecurity order_tuple) { Log.assert(order_tuple.order.Key == this.order.Key, "OrderBookRowVM: update_order was called with an order with the wrong key."); - bool had_security = _security_md.HasValue; + bool had_security = _security.HasValue; // _show_order_modify_cmd is lazy (instantiated in the default_command getter); it might be null, don't not force instantiation bool? could_modify_execute = _show_order_modify_cmd?.CanExecute(null); OrderRecordImmutable old_order = order; @@ -190,7 +191,7 @@ namespace mbts.spark { RaisePropertyChanged(nameof(price)); RaisePropertyChanged(nameof(is_price_dirty)); } - if (!had_security && _security_md.HasValue) { + if (!had_security && _security.HasValue) { RaisePropertyChanged(nameof(security), nameof(security_text)); // need to raise invalidated to refresh the price cells (the security.ContractSpec determines how they are formatted - Ticks!) invalidate = true; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 5b2e9ec19..452842c64 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -3,7 +3,6 @@ using bts.api; using bts.api.orderman; using bts.shell; using bts.spark; -using bts.utils; using mbts.persist; using mbts.ui; using mbts.ui.controls; @@ -21,9 +20,15 @@ using System.Windows.Controls; using System.Windows.Input; using static bts.shell.OrderBookWindowFactory; using static mbts.spark.OrderBookView; -using _GridSettingsManager = mbts.ui.GridSettingsManager; +using _GridSettingsManager = + mbts.ui.GridSettingsManager< + mbts.spark.OrderBookRowVM, + bts.api.orderman.OrderSecurity, + bts.shell.INotificationServiceDataCache, + mbts.OrderRecordTuplePredicate, + bts.shell.OrderBookWindowFactory.OrderBookGridSettings>; -namespace mbts.spark { +namespace mbts.spark { partial class OrderBookVM : ViewModelBase, IDisposable, IHandleRowSelection, IHandleGridSorting, IWillLetYouKnowWhenIDispose { readonly IBus _bus; readonly OrdersModel _model; @@ -40,7 +45,7 @@ namespace mbts.spark { _bus = bus; _model = model; _settings_manager = settings_manager; - _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(u => (u.added.Values.Select(smd => smd.security), u.removed.Values.Select(smd => smd.security)))); + _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(x => (x.added.Values, x.removed.Values))); // we use an ObservableAppendCollection b/c the mini-ob needs a cancel column at the front columns = new ObservableAppendCollection(_columns); _row_collection = new OrderBookRowCollection(ort => new OrderBookRowVM(this, ort, rows_are_initially_expanded), _columns.get_sorting_data().comparer) { filter = _columns.get_filter() }; @@ -51,7 +56,7 @@ namespace mbts.spark { expand_all_rows_command = new DelegateCommand(_row_collection.expand_all); grid_settings_manager = _create_grid_settings_manager(owner_title, bus, notification_service, model, rows, core_columns); _resources = new List { - model.on_order_tuples_table.OnDispatcher().Subscribe(_row_collection.update), + model.on_orders.OnDispatcher().Subscribe(_row_collection.update), grid_settings_manager, Disposable.Create(() => { _columns.filter_changed -= on_filter_changed; @@ -225,13 +230,13 @@ namespace mbts.spark { return new _GridSettingsManager( owner_title, bus, notification_service, model, rows, columns, () => { - var pvm = PredicateBuilderVMWrapper.create( + var pvm = PredicateBuilderVMWrapper.create( OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); return pvm; }, col_id => { - var pvm = PredicateBuilderVMWrapper.create( + var pvm = PredicateBuilderVMWrapper.create( OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); return pvm; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs index 3b62863a4..d0a878a08 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs @@ -16,7 +16,7 @@ using System.Windows.Data; using System.Windows.Media; //using IOrderBookWSM = mbts.persist.IWindowSettingsManager>; using OrderBookGridSettings = bts.shell.OrderBookWindowFactory.OrderBookGridSettings; -using OrderBookGridSettingsVM = mbts.spark.GridSettingsVM; +using OrderBookGridSettingsVM = mbts.spark.GridSettingsVM; namespace mbts.spark { internal class OrderBookWindow : SparkWindow, ISharedWindow { @@ -51,7 +51,7 @@ namespace mbts.spark { _settings_manager = settings_manager; } - public override OrderBookWindowFactory.OrderBookStartupData SaveState => _state = _vm.create(); + public override OrderBookWindowFactory.OrderBookStartupData SaveState => _state = _vm?.create(); public override string window_type => _window_type; public override UIElement Content => _view; @@ -86,12 +86,12 @@ namespace mbts.spark { ); _cancel_buy_orders_command = SparkCommands.CancelBuyOrders.complete( - _model.working_orders.Select(os => os.Values.Where(o => o.side() == Side.Buy).Select(o => o.Key)), + _model.working_orders.Select(os => os.Values.Where(o => o.order.side() == Side.Buy).Select(o => o.order.Key)), wbids => wbids.Any() ); _cancel_sell_orders_command = SparkCommands.CancelSellOrders.complete( - _model.working_orders.Select(os => os.Values.Where(o => o.side() == Side.Sell).Select(o => o.Key)), + _model.working_orders.Select(os => os.Values.Where(o => o.order.side() == Side.Sell).Select(o => o.order.Key)), wasks => wasks.Any() ); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs index 0733cb42b..5101a4149 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs @@ -233,7 +233,7 @@ namespace bts.shell { } [ProtoContract] - public class OrderBookGridSettings : IGridSettingsWithNotifications, IEquatable { + public class OrderBookGridSettings : IGridSettingsWithNotifications, IEquatable { [ProtoMember(1)] public GridSettings general { get; set; } [ProtoMember(2)] @@ -248,7 +248,7 @@ namespace bts.shell { } public IReadOnlyList predicate_animations => _predicate_animations ?? (_predicate_animations = new List()); - IEnumerable> IGridSettingsWithNotifications.predicate_animations => predicate_animations; + IEnumerable> IGridSettingsWithNotifications.predicate_animations => predicate_animations; public OrderBookGridSettings with(GridSettings general) => new OrderBookGridSettings(general ?? this.general, predicate_animations); diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index a01dd2506..eb3823cc7 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -1,4 +1,6 @@ using bts.api; +using bts.api.core; +using bts.api.orderman; using bts.spark; using mbts; using mbts.collections; @@ -9,6 +11,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Linq; +using System.Reactive.Linq; +using OrderTuple = System.ValueTuple; namespace bts.shell { interface INotificationService { @@ -16,13 +20,13 @@ namespace bts.shell { INotificationServiceDataCache data_cache { get; } IDisposable link_to_activity(IPredicate predicate, SoundNotification sound); IDisposable link_to_activity(INotificationPredicate toast_notification_predicate); - IDisposable link_to_order(IPredicate predicate, SoundNotification sound); - IDisposable link_to_order(INotificationPredicate toast_notification_predicate); + IDisposable link_to_order(IPredicate predicate, SoundNotification sound); + IDisposable link_to_order(INotificationPredicate toast_notification_predicate); } static class NotificationServiceExtensions { public static IDisposable link(this INotificationService notification_service, IPredicate predicate, SoundNotification sound) { - if (predicate is IPredicate orp) + if (predicate is IPredicate orp) return notification_service.link_to_order(orp, sound); if (predicate is IPredicate ap) return notification_service.link_to_activity(ap, sound); @@ -30,7 +34,7 @@ namespace bts.shell { } public static IDisposable link(this INotificationService notification_service, INotificationPredicate toast_notification_predicate) { - if (toast_notification_predicate is INotificationPredicate orp) + if (toast_notification_predicate is INotificationPredicate orp) return notification_service.link_to_order(orp); if (toast_notification_predicate is INotificationPredicate ap) return notification_service.link_to_activity(ap); @@ -41,7 +45,7 @@ namespace bts.shell { [Export(typeof(INotificationService))] partial class NotificationService : INotificationService, IDisposable { readonly NotificationProcessor _activity_notifications; - readonly NotificationProcessor _order_notifications; + readonly NotificationProcessor _order_notifications; readonly DataCache _data_cache = new DataCache(); IDisposable _dispose; @@ -50,10 +54,10 @@ namespace bts.shell { public NotificationService(IBus bus, INotifier notifier, OrdersModel orders_model) { this.notifier = notifier; _activity_notifications = new NotificationProcessor(notifier); - _order_notifications = new NotificationProcessor(notifier); + _order_notifications = new NotificationProcessor(notifier); _dispose = Disposable.Create( bus.subscribe(SparkTags.LogActivity).Subscribe(_on_activity), - orders_model.on_order_tuples.Subscribe(_on_orders) + orders_model.on_orders.Subscribe(_on_orders) ); } @@ -63,17 +67,17 @@ namespace bts.shell { public IDisposable link_to_activity(IPredicate predicate, SoundNotification sound) => _activity_notifications.link(predicate, sound); public IDisposable link_to_activity(INotificationPredicate toast_notification_predicate) => _activity_notifications.link(toast_notification_predicate); - public IDisposable link_to_order(IPredicate predicate, SoundNotification sound) => _order_notifications.link(predicate, sound); - public IDisposable link_to_order(INotificationPredicate toast_notification_predicate) => _order_notifications.link(toast_notification_predicate); + public IDisposable link_to_order(IPredicate predicate, SoundNotification sound) => _order_notifications.link(predicate, sound); + public IDisposable link_to_order(INotificationPredicate toast_notification_predicate) => _order_notifications.link(toast_notification_predicate); public void Dispose() => Disposable.dispose(ref _dispose); void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(IIncrementalUpdate update) { + void _on_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable),uint> update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history - _data_cache.update(update); - foreach (var o in update.added.Values) + _data_cache.update(update.as_iincremental()); + foreach (var o in update.added) _order_notifications.process_item(o); } diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index aed90fbdb..fac9deeed 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -1,4 +1,6 @@ -using mbts.spark; +using bts.api; +using bts.api.orderman; +using mbts.spark; using mbts.utils; using System; using System.Collections.Generic; @@ -13,15 +15,21 @@ namespace bts.shell { interface INotificationServiceDataCache { DateTime last_order_reset_time { get; } bool is_first_order_update { get; } - IReadOnlyDictionary>> order_history { get; } + IReadOnlyDictionary>> order_history { get; } } static class NotificationServiceDataCacheExtensions { - public static OrderRecordTuple get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) => - notification_service_data_cache.order_history.TryGetValue(key, out List> l) && l.Count > 1 ? l[l.Count - 2].data : null; + public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) { + if (notification_service_data_cache.order_history.TryGetValue(key, out List> l)) { + if (l.Count > 1) { + return l[l.Count - 2].data; + } + } + return null; + } } - class TimeStampedData { + struct TimeStampedData { public readonly T data; public readonly DateTime time_stamp; @@ -34,14 +42,15 @@ namespace bts.shell { partial class NotificationService : INotificationService, IDisposable { class DataCache : INotificationServiceDataCache { const int _buffer_size = 10; - readonly Dictionary _order_cache = new Dictionary(); - readonly Dictionary>> _order_history = new Dictionary>>(); + readonly Dictionary _order_cache = new Dictionary(); + readonly Dictionary>> _order_history = new Dictionary>>(); + + public IReadOnlyDictionary>> order_history => _order_history; - public IReadOnlyDictionary>> order_history => _order_history; public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - public void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; @@ -51,7 +60,7 @@ namespace bts.shell { foreach (var kv in data.removed) { if (data.added.ContainsKey(kv.Key)) { _order_cache[kv.Key] = kv.Value; - _order_history[kv.Key].Add(new TimeStampedData(kv.Value, now)); + _order_history[kv.Key].Add(new TimeStampedData(kv.Value, now)); } else { _order_cache.Remove(kv.Key); // TODO: maybe these shouldn't be removed? @@ -65,9 +74,9 @@ namespace bts.shell { continue; _order_cache.Add(kv.Key, kv.Value); _order_history.AddOrUpdate(kv.Key, - k => new List> { new TimeStampedData(kv.Value, now) }, + k => new List> { new TimeStampedData(kv.Value, now) }, (k, l) => { - l.Add(new TimeStampedData(kv.Value, now)); + l.Add(new TimeStampedData(kv.Value, now)); if (l.Count >= 10) l.RemoveAt(0); }); diff --git a/src/client/spark/Notifications/ToastNotification.cs b/src/client/spark/Notifications/ToastNotification.cs index f2590e5db..892a42cb4 100644 --- a/src/client/spark/Notifications/ToastNotification.cs +++ b/src/client/spark/Notifications/ToastNotification.cs @@ -60,14 +60,14 @@ namespace bts.shell { public static IToastNotification order_filled(OrderRecordImmutable oldval, OrderRecordImmutable newval, SecurityImmutable? security) => new OrderFilledToastNotification(oldval, newval, security); - public static IToastNotification order_filled(OrderRecordTuple oldval, OrderRecordTuple newval) => - new OrderFilledToastNotification(oldval.order, newval.order, newval.security_md?.security); + public static IToastNotification order_filled(OrderSecurity oldval, OrderSecurity newval) => + new OrderFilledToastNotification(oldval.order, newval.order, newval.security); public static IToastNotification order_completed(OrderRecordImmutable order, SecurityImmutable? security) => new OrderCompletedToastNotification(order, security); - public static IToastNotification order_completed(OrderRecordTuple order) => - new OrderCompletedToastNotification(order.order, order.security_md?.security); + public static IToastNotification order_completed(OrderSecurity order) => + new OrderCompletedToastNotification(order.order, order.security); public static IToastNotification order_status_changed(uint key, OrderStatus? old_status, OrderStatus new_status, SecurityImmutable? security) => new OrderStatusChangeToastNotification(key, old_status, new_status, security); diff --git a/src/client/spark/Orders/ExistingOrderTicketVM.cs b/src/client/spark/Orders/ExistingOrderTicketVM.cs index 18583a019..ddbf8f6f4 100644 --- a/src/client/spark/Orders/ExistingOrderTicketVM.cs +++ b/src/client/spark/Orders/ExistingOrderTicketVM.cs @@ -19,7 +19,7 @@ namespace mbts.spark { readonly Property _current_order_type; readonly Property _price_dirty, _size_dirty, _account_dirty, _time_in_force_dirty, _order_type_dirty; - public ExistingOrderTicketVM(OrdersModel model, IObservable order_source, IObservable calculator_buttons) { + public ExistingOrderTicketVM(OrdersModel model, IObservable<(SecurityMD? security_md, OrderRecordImmutable order)> order_source, IObservable calculator_buttons) { uint? key = null; var security_md_source = order_source.Where(ot => ot.security_md.HasValue).Select(ot => ot.security_md.Value); // diff --git a/src/client/spark/Orders/FillSecurity.cs b/src/client/spark/Orders/FillSecurity.cs new file mode 100644 index 000000000..01d8d0a20 --- /dev/null +++ b/src/client/spark/Orders/FillSecurity.cs @@ -0,0 +1,25 @@ +using System; + +namespace bts.api.orderman { + public readonly struct FillSecurity : IEquatable { + public FillSecurity(SecurityImmutable? sec, FillImmutable fill) { + this.sec = sec; + this.fill = fill; + } + public FillSecurity((SecurityImmutable? sec, FillImmutable fill) t) { + sec = t.sec; + fill = t.fill; + } + public static implicit operator FillSecurity(ValueTuple x) => new FillSecurity(x); + public SecurityImmutable? sec { get; } + public FillImmutable fill { get; } + + public override bool Equals(object obj) => (obj is FillSecurity os) && Equals(os); + public bool Equals(FillSecurity other) => fill.FillKey == other.fill.FillKey && sec == other.sec; + public override int GetHashCode() => (sec, fill).GetHashCode(); + + public static bool operator ==(FillSecurity left, FillSecurity right) => left.Equals(right); + public static bool operator !=(FillSecurity left, FillSecurity right) => !left.Equals(right); + + } +} \ No newline at end of file diff --git a/src/client/spark/Orders/FillTuple.cs b/src/client/spark/Orders/FillTuple.cs index 4481d2ffa..80f7eb946 100644 --- a/src/client/spark/Orders/FillTuple.cs +++ b/src/client/spark/Orders/FillTuple.cs @@ -10,14 +10,14 @@ namespace mbts.spark { class FillTuple : IEquatable { public readonly FillImmutable fill; public readonly OrderRecordImmutable? order; - public readonly SecurityMD? security_md; + public readonly SecurityImmutable? security; public readonly FillTupleKey key; - public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityMD? security_md) { + public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityImmutable? security) { this.fill = fill; this.order = order; - this.security_md = security_md; - key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security_md.HasValue ? security_md.Value.security.Id : 0u); + this.security = security; + key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security?.Id ?? 0); } public string account { @@ -31,7 +31,7 @@ namespace mbts.spark { public bool Equals(FillTuple other) => other != null && other.order.HasValue == order.HasValue - && other.security_md.HasValue == security_md.HasValue + && other.security.HasValue == security.HasValue && other.fill.Equals(fill); public override bool Equals(object obj) => Equals(obj as FillTuple); diff --git a/src/client/spark/Orders/FillTupleSource.cs b/src/client/spark/Orders/FillTupleSource.cs index 18629764e..679ca6a1f 100644 --- a/src/client/spark/Orders/FillTupleSource.cs +++ b/src/client/spark/Orders/FillTupleSource.cs @@ -1,4 +1,5 @@ -using bts.api.core; +using bts.api; +using bts.api.core; using bts.api.orderman; using mbts.collections; using mbts.utils; @@ -14,15 +15,14 @@ namespace mbts.spark { // returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] FillTupleSource. // (initializes when the first subscription is made, and destroys when the last subscription is disposed). public static IObservable> create_observable( - IObservable> securities, - IObservable> orders, - IObservable> fills) { + IObservable> orders, + IObservable> fills) { FillTupleSource source = null; int ref_count = 0; return Observable.Create>(observer => { if (Interlocked.Increment(ref ref_count) == 1) { SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak.")); - source = new FillTupleSource(securities, orders, fills); + source = new FillTupleSource(orders, fills); } IDisposable sub = source.Subscribe(observer); return Disposable.Create(() => { @@ -38,19 +38,16 @@ namespace mbts.spark { readonly Dictionary _state = new Dictionary(); readonly SetMap _order_fills = new SetMap(); readonly SetMap _security_fills = new SetMap(); - IReadOnlyDictionary _fills_cache = new Dictionary(0); - IReadOnlyDictionary _orders_cache = new Dictionary(0); - IReadOnlyDictionary _security_md_cache = new Dictionary(0); + IReadOnlyDictionary _fills_cache = new Dictionary(); + IReadOnlyDictionary _orders_cache = new Dictionary(); IDisposable _dispose; object _lock = new object(); FillTupleSource( - IObservable> securities, - IObservable> orders, - IObservable> fills) => + IObservable> orders, + IObservable> fills) => // _dispose = Disposable.Create( - securities.Subscribe(_update_securities), orders.Subscribe(_update_orders), fills.AddRemoveOnly().Subscribe(_update_fills)); @@ -63,19 +60,19 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - void _update_fills(LegacyTableUpdate fills) { + void _update_fills(LegacyTableUpdate<(SecurityImmutable?,FillImmutable), string> fills) { lock (_lock) { var removed = new Dictionary(); var added = new Dictionary(); - foreach (FillImmutable fill in fills.removed) { - FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security_md(fill.SecurityKey)); + foreach (var (security,fill) in fills.removed) { + FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), security); removed.Add(old_ft.key, old_ft); _security_fills.remove(fill.SecurityKey, fill.FillKey); _order_fills.remove(fill.OrderKey, fill.FillKey); } - foreach (FillImmutable fill in fills.added) { - FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security_md(fill.SecurityKey)); + foreach (var (security,fill) in fills.added) { + FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), security); added.Add(new_ft.key, new_ft); _security_fills.add(fill.SecurityKey, fill.FillKey); _order_fills.add(fill.OrderKey, fill.FillKey); @@ -85,35 +82,33 @@ namespace mbts.spark { } } - void _update_orders(TableUpdate orders) { + void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable order), uint> orders) { lock (_lock) { Dictionary added = new Dictionary(), removed = new Dictionary(); HashSet changed_keys = new HashSet(); - foreach (var o in orders.removed) { + foreach (var (_,o) /*winking owl*/ in orders.removed) { if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { OrderRecordImmutable? curr_order = null; - if (orders.try_get(o.Key, out OrderRecordImmutable newval)) { - curr_order = newval; + if (orders.try_get(o.Key, out var newval)) { + curr_order = newval.order; changed_keys.Add(o.Key); } - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - foreach (FillImmutable fill in fills) { - SecurityMD? sec_md = _get_security_md(fill.SecurityKey); - FillTuple old_ft = new FillTuple(fill, o, sec_md); + var fills = fill_keys.Select(fk => _fills_cache[fk]); + foreach (var (security, fill) in fills) { + FillTuple old_ft = new FillTuple(fill, o, security); removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, curr_order, sec_md); + FillTuple new_ft = new FillTuple(fill, curr_order, security); added.Add(new_ft.key, new_ft); } } } - foreach (var o in orders.added) { + foreach (var (_,o) in orders.added) { if (!changed_keys.Contains(o.Key) && _order_fills.try_lookup(o.Key, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - foreach (FillImmutable fill in fills) { - SecurityMD? sec_md = _get_security_md(fill.SecurityKey); - FillTuple old_ft = new FillTuple(fill, null, sec_md); + var fills = fill_keys.Select(fk => _fills_cache[fk]); + foreach (var (security,fill) in fills) { + FillTuple old_ft = new FillTuple(fill, null, security); removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, o, sec_md); + FillTuple new_ft = new FillTuple(fill, o, security); added.Add(new_ft.key, new_ft); } } @@ -123,46 +118,8 @@ namespace mbts.spark { } } - void _update_securities(IReadOnlyDictionary securities) { - lock (_lock) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); - foreach (var k in removed_keys) { - if (_security_fills.try_lookup(k, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - SecurityMD old_sec_md = _security_md_cache[k]; - foreach (FillImmutable fill in fills) { - OrderRecordImmutable? order = _get_order(fill.OrderKey); - FillTuple old_ft = new FillTuple(fill, order, old_sec_md); - removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, order, null); - added.Add(new_ft.key, new_ft); - } - } - } - foreach (var k in added_keys) { - if (_security_fills.try_lookup(k, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - SecurityMD new_sec_md = securities[k]; - foreach (FillImmutable fill in fills) { - OrderRecordImmutable? order = _get_order(fill.OrderKey); - FillTuple old_ft = new FillTuple(fill, order, null); - removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, order, new_sec_md); - added.Add(new_ft.key, new_ft); - } - } - } - _security_md_cache = securities; - _apply_changes(added, removed); - } - } - - SecurityMD? _get_security_md(uint key) => - _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); - OrderRecordImmutable? _get_order(uint key) => - _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); + _orders_cache.TryGetValue(key, out var o) ? o.Item2 : new OrderRecordImmutable?(); void _apply_changes(Dictionary added, Dictionary removed) { foreach (var kv in removed) diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index dd30e5962..3bcb6de13 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Reactive.Linq; using System.Text; using System.Threading.Tasks; +using OrderPair = System.ValueTuple; namespace bts.spark { internal class Activity { @@ -26,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -38,11 +39,11 @@ namespace bts.spark { #region formatting - (SecurityImmutable?, ContractSpecificationImmutable?) get_security(uint security_key) { + SecurityImmutable? try_get_security(uint security_key) { if (_security.TryGetValue(security_key, out SecurityMD md)) { - return (md.security, md.security.ContractSpec); + return md.security; } - return (null, null); + return null; } public static string format_order_success(uint key, double price, uint size, Side side, ContractSpecificationImmutable? spec) { return $"({key}) {side.ToString().ToLower()} {size} @ {spec.format(price)}"; @@ -123,24 +124,23 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { - foreach (var order in update.added) { - if (!string.IsNullOrEmpty(order.State.LastMsg) && order.State.Created.ToDateTime().Date == DateTime.Today) { - (var security, var spec) = get_security(order.Def.SecurityKey); - if (security.HasValue) { - log_status(ActivityLogCategory.warn, security, order.State.LastMsg); + foreach (var o in update.added) { + if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { + if (o.sec.HasValue) { + log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); } } } foreach (var order in update.changes) { - var neworder = order.New; - var oldorder = order.Old; - (var security, var spec) = get_security(neworder.Def.SecurityKey); + var neworder = order.New.order; + var oldorder = order.Old.order; + var security = order.New.sec ?? order.Old.sec; // // description - var mod_desc = modified_description(oldorder, neworder, spec); + var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); if (!string.IsNullOrEmpty(mod_desc)) { log_status(ActivityLogCategory.modify, security, mod_desc); } @@ -171,28 +171,30 @@ namespace bts.spark { } internal void on_send_order(OrderDefinitionImmutable order, double price, uint size, uint? key, string error) { - (var security, var spec) = get_security(order.SecurityKey); + var security = try_get_security(order.SecurityKey); log_status(ActivityLogCategory.order, security, key.HasValue ? - format_order_success(key.Value, price, size, order.Side, spec) : - format_order_fail(price, size, order.Side, spec, error)); + format_order_success(key.Value, price, size, order.Side, security?.ContractSpec) : + format_order_fail(price, size, order.Side, security?.ContractSpec, error)); } internal void on_cancel_order(OrderRecordImmutable order, bool success, string error) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); log_status(ActivityLogCategory.cancel, security, success ? - format_cancel_success(order, spec): - format_cancel_error(order, spec, error)); + format_cancel_success(order, security?.ContractSpec): + format_cancel_error(order, security?.ContractSpec, error)); } internal void update_order(OrderRecordImmutable order, double price, uint size) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); + var spec = security?.ContractSpec; log_status(ActivityLogCategory.modify, security, $"({order.Key}) change from {order.State.Quantity}@{spec.format(order.State.Price)} to {size}@{spec.format(price)}"); } internal void update_order_failed(OrderRecordImmutable order, double price, uint size) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); + var spec = security?.ContractSpec; log_status(ActivityLogCategory.modify, security, $"({order.Key}) failed to change from {order.State.Quantity}@{spec.format(order.State.Price)} to {size}@{spec.format(price)}"); diff --git a/src/client/spark/Orders/OrderRecordTuple.cs b/src/client/spark/Orders/OrderRecordTuple.cs deleted file mode 100644 index 4e6255364..000000000 --- a/src/client/spark/Orders/OrderRecordTuple.cs +++ /dev/null @@ -1,47 +0,0 @@ -using bts.api.orderman; -using bts.utils; -using mbts.utils; -using System; - -namespace mbts.spark { - struct OrderRecordTupleKey : IEquatable { - public readonly uint order_key, security_key; - - public OrderRecordTupleKey(uint order_key, uint security_key) { - this.order_key = order_key; - this.security_key = security_key; - } - - public bool Equals(OrderRecordTupleKey other) => - other.order_key == order_key && - other.security_key == security_key; - - public override bool Equals(object obj) => - obj is OrderRecordTupleKey k && Equals(k); - - public override int GetHashCode() => - EqualityComparer.aggregate_hash( - (int)order_key, - (int)security_key); - } - - class OrderRecordTuple : IEquatable { - public readonly OrderRecordImmutable order; - public readonly SecurityMD? security_md; - public readonly OrderRecordTupleKey key; - - public OrderRecordTuple(OrderRecordImmutable order, SecurityMD? security_md) { - this.order = order; - this.security_md = security_md; - key = new OrderRecordTupleKey(order.Key, security_md.HasValue ? security_md.Value.security.Id : 0u); - } - - public bool Equals(OrderRecordTuple other) => other != null - && other.order.Equals(order) - && other.security_md.HasValue == security_md.HasValue; - - public override bool Equals(object obj) => Equals(obj as OrderRecordTuple); - public override int GetHashCode() => key.GetHashCode(); - public override string ToString() => $"order:{order.Key}, sec:{security_md.display_name()}"; - } -} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs deleted file mode 100644 index 0c8aeaccf..000000000 --- a/src/client/spark/Orders/OrderRecordTupleSource.cs +++ /dev/null @@ -1,147 +0,0 @@ -using bts.api.core; -using bts.api.orderman; -using mbts.collections; -using mbts.utils; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reactive.Linq; -using System.Reactive.Subjects; - -namespace mbts.spark { - class OrderRecordTupleSource : IDisposable { - readonly object _lock = new object(); - // subjects - readonly Subject> _subject = new Subject>(); - readonly Subject> _table_subject = new Subject>(); - readonly Subject> _order_securities_subject = new Subject>(); - // data cache - readonly Dictionary _state = new Dictionary(); - readonly SetMap _security_orders = new SetMap(); - readonly IDictionary _security_md_cache = new Dictionary(0); - IReadOnlyDictionary _orders_cache = new Dictionary(0); - IDisposable _dispose; - - public OrderRecordTupleSource(IObservable> securities, IObservable> orders) => - _dispose = Disposable.Create( - securities.Subscribe(_update_securities), - orders.Subscribe(_update_orders)); - - public IObservable> subscribe_incremental_update() => - Observable.Create>(observer => { - lock (_lock) { - observer.OnNext(new IncrementalUpdate(new Dictionary(_state), new Dictionary(0))); - return _subject.Subscribe(observer); - } - }); - - public IObservable> subscribe_incremental_table_update() => - Observable.Create>(observer => { - lock (_lock) { - observer.OnNext(IncrementalTableUpdateTranform.init(_state.Values)); - return _table_subject.Subscribe(observer); - } - }); - - public IObservable> subscribe_order_securities() => - Observable.Create>(observer => { - lock (_lock) { - var init = new Dictionary(); - foreach (uint sec_key in _security_orders.Keys) { - if (_security_md_cache.TryGetValue(sec_key, out SecurityMD smd)) - init.Add(sec_key, smd); - } - observer.OnNext(new IncrementalUpdate(init, new Dictionary(0))); - return _order_securities_subject.Subscribe(observer); - } - }); - - public void Dispose() => Disposable.dispose(ref _dispose); - - void _update_orders(TableUpdate orders) { - lock (_lock) { - Dictionary removed = new Dictionary(), added = new Dictionary(); - Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); - foreach (OrderRecordImmutable order in orders.removed) { - uint sec_key = order.security_key(); - OrderRecordTuple old_ft = new OrderRecordTuple(order, _get_security_md(sec_key)); - removed.Add(old_ft.order.Key, old_ft); - if (_security_orders.removes_key(sec_key, order.Key) && old_ft.security_md.HasValue) - removed_securities.Add(sec_key, old_ft.security_md.Value); - } - foreach (Change ch in orders.changes) { - // orders cannot change securities; ch.New can use the same security_md as ch.Old - var sec_md = _get_security_md(ch.Old.security_key()); - removed.Add(ch.Old.Key, new OrderRecordTuple(ch.Old, sec_md)); - added.Add(ch.New.Key, new OrderRecordTuple(ch.New, sec_md)); - } - foreach (OrderRecordImmutable order in orders.added) { - uint sec_key = order.security_key(); - OrderRecordTuple new_ort = new OrderRecordTuple(order, _get_security_md(sec_key)); - added.Add(new_ort.order.Key, new_ort); - if (_security_orders.adds_key(sec_key, order.Key) && new_ort.security_md.HasValue) - added_securities.Add(sec_key, new_ort.security_md.Value); - } - _orders_cache = orders.state; - _apply_changes(added, removed, added_securities, removed_securities); - } - } - - void _update_securities(IReadOnlyDictionary securities) { - lock (_lock) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); - _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); - /// Warning: must NOT be reassigned with ; the latter is always the same object. - /// This would mean that the - /// call above would literally be calculating the difference between the keys of the same dictionary! - foreach (var k in removed_keys.ToList()) { - if (_security_orders.try_lookup(k, out ISet order_keys)) { - SecurityMD old_sec_md = _security_md_cache[k]; - removed_securities.Add(k, old_sec_md); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - foreach (OrderRecordImmutable order in orders) { - OrderRecordTuple old_ft = new OrderRecordTuple(order, old_sec_md); - removed.Add(old_ft.order.Key, old_ft); - OrderRecordTuple new_ft = new OrderRecordTuple(order, null); - added.Add(new_ft.order.Key, new_ft); - } - } - _security_md_cache.Remove(k); - } - foreach (var k in added_keys.ToList()) { - SecurityMD new_sec_md = securities[k]; - if (_security_orders.try_lookup(k, out ISet order_keys)) { - added_securities.Add(k, new_sec_md); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - foreach (OrderRecordImmutable order in orders) { - OrderRecordTuple old_ft = new OrderRecordTuple(order, null); - removed.Add(old_ft.order.Key, old_ft); - OrderRecordTuple new_ft = new OrderRecordTuple(order, new_sec_md); - added.Add(new_ft.order.Key, new_ft); - } - } - _security_md_cache.Add(k, new_sec_md); - } - _apply_changes(added, removed, added_securities, removed_securities); - } - } - - SecurityMD? _get_security_md(uint key) => - _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); - - OrderRecordImmutable? _get_order(uint key) => - _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); - - void _apply_changes(Dictionary added, Dictionary removed, Dictionary added_securities, Dictionary removed_securities) { - foreach (var kv in removed) - _state.Remove(kv.Key); - foreach (var kv in added) - _state.Add(kv.Key, kv.Value); - var update = new IncrementalUpdate(added, removed); - _subject.OnNext(update); - _table_subject.OnNext(update.as_table()); - _order_securities_subject.OnNext(new IncrementalUpdate(added_securities, removed_securities)); - } - } -} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs new file mode 100644 index 000000000..a703e4c2b --- /dev/null +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -0,0 +1,25 @@ +using System; + +namespace bts.api.orderman { + public readonly struct OrderSecurity: IEquatable { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + this.security = sec; + this.order = order; + } + public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + security = t.sec; + order = t.order; + } + public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); + public SecurityImmutable? security { get; } + public OrderRecordImmutable order { get; } + + public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); + public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security; + public override int GetHashCode() => (security, order).GetHashCode(); + + public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); + public static bool operator != (OrderSecurity left, OrderSecurity right) => !left.Equals(right); + + } +} diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 461e2c015..89bce07da 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -16,8 +16,10 @@ using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Threading.Tasks; +using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; namespace mbts.spark { + [Export] class OrdersModel : IDisposable { readonly IBus _bus; @@ -34,7 +36,8 @@ namespace mbts.spark { [ImportingConstructor] private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, - SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model) { + SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model, + SecurityStore sstore) { _bus = bus; _factory = factory; _service = service; @@ -44,26 +47,39 @@ namespace mbts.spark { _dynamic_helper = dynamic_helper; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); + on_orders = + _service + .subscribei_orders() + .Naggle(interval, scheduler) + .PermissionFilter(admin_model.on_admin_data) + .join(sstore, order => order.security_key()) + .Table().Cache(); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( - ImmutableDictionary.Empty, + ImmutableDictionary.Empty, (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); - var order_record_tuple_source = new OrderRecordTupleSource(bus.subscribe(SparkTags.SecuritiesTag), on_orders); - on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); - on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); - on_order_securities = order_record_tuple_source.subscribe_order_securities(); - on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); + + on_order_securities = + on_orders + .as_iincremental() + .filter(t => t.sec.HasValue) + .Select(t => t.sec.Value); + + on_fills = + _service.subscribei_fills() + .Naggle(interval, scheduler) + .PermissionFilter(admin_model.on_admin_data) + .join(sstore, f => f.SecurityKey) + .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); - on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); + on_fill_tuples = FillTupleSource.create_observable(on_orders, on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); _dispose = Disposable.Create( - on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer), + //on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer), on_positions.AddRemoveOnly().Subscribe(_security_positions_demuxer), - order_record_tuple_source, _bus.subscribe(SparkTags.CreateOrder).Subscribe(a => send_order(a.definition, a.price, a.size).ignore()), _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())), _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()), @@ -76,13 +92,11 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IObservable> on_orders { get; } - public IObservable> on_order_tuples { get; } - public IObservable> on_order_tuples_table { get; } - public IObservable> on_order_securities { get; } - public IObservable> on_fills { get; } + public IObservable on_orders { get; } + public IObservable> on_order_securities { get; } + public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } - public IObservable> working_orders { get; } + public IObservable> working_orders { get; } public IObservable> on_positions { get; } public IObservable> on_security_orders(uint security) => @@ -153,24 +167,25 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary _update_working_orders(ImmutableDictionary target, TableUpdate table) { + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { - if (o.is_active()) { - clone.Remove(o.Key); + if (o.order.is_active()) { + clone.Remove(o.order.Key); } } - foreach (var o in table.changed) { - if (o.is_active()) { - clone[o.Key] = o; + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); } - else { - clone.Remove(o.Key); + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); } } foreach (var o in table.added) { - if (o.is_active()) { - clone.Add(o.Key, o); + if (o.order.is_active()) { + clone.Add(o.order.Key, o); } } return clone.ToImmutableDictionary(); @@ -185,8 +200,8 @@ namespace mbts.spark { public static async Task cancel_order(this OrdersModel orders_model, uint order_id) { var tu = await orders_model.on_orders.FirstAsync(); - if (tu.state.TryGetValue(order_id, out OrderRecordImmutable order)) { - return await orders_model.cancel_order(order); + if (tu.state.TryGetValue(order_id, out var o)) { + return await orders_model.cancel_order(o.order); } return false; } @@ -195,8 +210,8 @@ namespace mbts.spark { var tu = await model.on_orders.FirstAsync(); var orders = tu.state.Values - .Where(o => o.is_non_terminal() && pred(o)) - .Select(async o => { return (o, await model.cancel_order(o)); } ); + .Where(o => o.order.is_non_terminal() && pred(o.order)) + .Select(async o => { return (o.order, await model.cancel_order(o.order)); } ); return await Task.WhenAll(orders); } @@ -304,8 +319,8 @@ namespace mbts.spark { public static async Task order_from_key(this OrdersModel orders_model, uint order_key) { (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3)); - if (success && tu.state.TryGetValue(order_key, out OrderRecordImmutable rec)) { - return rec; + if (success && tu.state.TryGetValue(order_key, out var rec)) { + return rec.order; } return null; } diff --git a/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs index 4546dfd6d..9ed71bb6e 100644 --- a/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs @@ -26,7 +26,7 @@ namespace mbts { } [ProtoContract] - class OrderCanceledEventPredicate : IPredicateBuilder, IEquatable { + class OrderCanceledEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [Description("(any order)")] AnyOrder, @@ -45,13 +45,13 @@ namespace mbts { } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache data_cache) => OrderEventPredicateHelper.create_status_transition_predicate(OrderStatus.Canceled, security, data_cache); - public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => new MyToaster(security?.create(data_cache), data_cache); public bool Equals(OrderCanceledEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -61,7 +61,7 @@ namespace mbts { static bool _is_canceled(OrderRecordImmutable o) => o.HasState && o.State.Status == OrderStatus.Canceled; - class MyToaster : INotificationPredicate, IEquatable { + class MyToaster : INotificationPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; @@ -70,12 +70,13 @@ namespace mbts { _data_cache = data_cache; } - public IToastNotification trigger(OrderRecordTuple order) => - OrderEventPredicateHelper.check_order_transition_trigger(order, _is_canceled, _security_predicate, _data_cache, out OrderRecordTuple oldval) ? - ToastNotifications.order_status_changed(order.order.Key, OrderEventPredicateHelper.get_nullable_status(oldval), OrderStatus.Canceled, order.security_md.HasValue ? order.security_md.Value.security : new SecurityImmutable?()) : + public IToastNotification trigger(OrderSecurity order) => + OrderEventPredicateHelper.check_order_transition_trigger(order, _is_canceled, _security_predicate, _data_cache, out var oldval) ? + ToastNotifications.order_status_changed(order.order.Key, + OrderEventPredicateHelper.get_nullable_status(oldval), OrderStatus.Canceled, order.security) : ToastNotifications.empty; - bool IPredicate.trigger(OrderRecordTuple order) => + bool IPredicate.trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_canceled, _security_predicate, _data_cache); public bool Equals(MyToaster other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs index 881201de5..f6942d908 100644 --- a/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs @@ -26,7 +26,7 @@ namespace mbts { } [ProtoContract] - class OrderCompletedEventPredicate : IPredicateBuilder, IEquatable { + class OrderCompletedEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [Description("(any order)")] AnyOrder, @@ -45,13 +45,13 @@ namespace mbts { } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache data_cache) => OrderEventPredicateHelper.create_status_transition_predicate(OrderStatus.Completed, security, data_cache); - public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => new MyToaster(security?.create(data_cache), data_cache); public bool Equals(OrderCompletedEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -61,7 +61,7 @@ namespace mbts { static bool _is_completed(OrderRecordImmutable o) => o.HasState && o.State.Status == OrderStatus.Completed; - class MyToaster : INotificationPredicate, IEquatable { + class MyToaster : INotificationPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; @@ -70,11 +70,11 @@ namespace mbts { _data_cache = data_cache; } - public IToastNotification trigger(OrderRecordTuple order) => + public IToastNotification trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_completed, _security_predicate, _data_cache) ? ToastNotifications.order_completed(order) : ToastNotifications.empty; - bool IPredicate.trigger(OrderRecordTuple order) => + bool IPredicate.trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_completed, _security_predicate, _data_cache); public bool Equals(MyToaster other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs b/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs index 25c019787..f9e27e62f 100644 --- a/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs +++ b/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs @@ -11,16 +11,16 @@ namespace mbts { /// /// Checks if the order has transitioned from 'not passing the predicate, or not existing' to 'passing the predicate.' /// - public static bool check_order_transition_predicate(OrderRecordTuple order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache) => - check_order_transition_trigger(order, order_predicate_core, security_predicate, data_cache, out OrderRecordTuple _); + public static bool check_order_transition_predicate(OrderSecurity order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache) => + check_order_transition_trigger(order, order_predicate_core, security_predicate, data_cache, out var _); /// /// Checks if the order has transitioned from 'not passing the predicate, or not existing' to 'passing the predicate.' Oldval will be null if the order did not previously exist, or does not currently pass the predicate. /// - public static bool check_order_transition_trigger(OrderRecordTuple order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache, out OrderRecordTuple oldval) { + public static bool check_order_transition_trigger(OrderSecurity order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache, out OrderSecurity? oldval) { if (check_security_predicate(order, security_predicate) && order_predicate_core(order.order)) { oldval = data_cache.get_previous_order_state(order.order.Key); - return oldval == null ? !data_cache.is_first_order_update : !order_predicate_core(oldval.order); + return oldval == null ? !data_cache.is_first_order_update : !order_predicate_core(oldval.Value.order); } oldval = null; return false; @@ -29,19 +29,19 @@ namespace mbts { /// /// Checks if the 'security predicate is null,' or 'it is not null, and the order has a non-null security which passes.' /// - public static bool check_security_predicate(OrderRecordTuple order, IPredicate security_predicate) => - security_predicate == null || (order.security_md.HasValue && security_predicate.trigger(order.security_md.Value.security)); + public static bool check_security_predicate(OrderSecurity order, IPredicate security_predicate) => + security_predicate == null || (order.security.HasValue && security_predicate.trigger(order.security.Value)); - public static OrderStatus? get_nullable_status(OrderRecordTuple order) { - if (order == null && order.order.HasState) - return order.order.State.Status; + public static OrderStatus? get_nullable_status(OrderSecurity? order) { + if (order?.order.HasState == true) + return order?.order.State.Status; return null; } - public static IPredicate create_status_transition_predicate(OrderStatus status, SecurityPredicate security_predicate, INotificationServiceDataCache data_cache) => + public static IPredicate create_status_transition_predicate(OrderStatus status, SecurityPredicate security_predicate, INotificationServiceDataCache data_cache) => new OrderStatusTransitionPredicate(status, security_predicate?.create(data_cache), data_cache); - class OrderStatusTransitionPredicate : IPredicate, IEquatable { + class OrderStatusTransitionPredicate : IPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; readonly OrderStatus _status; @@ -52,7 +52,7 @@ namespace mbts { _data_cache = data_cache; } - public bool trigger(OrderRecordTuple value) => + public bool trigger(OrderSecurity value) => check_order_transition_predicate(value, o => o.HasState && o.State.Status == _status, _security_predicate, _data_cache); public bool Equals(OrderStatusTransitionPredicate other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index 9b67bd56f..80767eec6 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -1,4 +1,5 @@ using bts.api; +using bts.api.orderman; using bts.shell; using mbts.spark; using mbts.ui.settings; @@ -13,8 +14,9 @@ namespace mbts { { OrderFilledEventPredicate.PredicateTypeOneofCase.AnyOrder, PredicateBuilderVMWrapper.create(p => p.security = null) }, { OrderFilledEventPredicate.PredicateTypeOneofCase.SecurityOrder, SecurityCriteriaVMFactory.create(bus, o => o.security?.is_value ?? new SecurityCriteria(), (p, s) => { - if (p.security == null) + if (p.security == null) { p.security = new SecurityPredicate(); + } p.security.is_value = s; }) } @@ -22,7 +24,7 @@ namespace mbts { } [ProtoContract] - class OrderFilledEventPredicate : IPredicateBuilder, IEquatable { + class OrderFilledEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [PredicateCase("(any order)")] AnyOrder, @@ -35,19 +37,21 @@ namespace mbts { public PredicateTypeOneofCase predicate_type_case { get { - if (security != null) + if (security != null) { return PredicateTypeOneofCase.SecurityOrder; + } + return PredicateTypeOneofCase.AnyOrder; } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache notification_service_data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache notification_service_data_cache) => new FillTrigger(_create_model(notification_service_data_cache)); - public INotificationPredicate create_toast(INotificationServiceDataCache notification_service_data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache notification_service_data_cache) => new FillToastNotificationTrigger(_create_model(notification_service_data_cache)); public bool Equals(OrderFilledEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -66,35 +70,43 @@ namespace mbts { _security_predicate = security_predicate; } - public (bool trigger, OrderRecordTuple oldval, OrderRecordTuple newval) get_filled_order_args(OrderRecordTuple ort) { + // Please review carfully + public (OrderSecurity oldval, OrderSecurity newval)? get_filled_order_args(OrderSecurity ort) { if (ort.order.is_exchange_order() && OrderEventPredicateHelper.check_security_predicate(ort, _security_predicate)) { - if (_data_cache.get_previous_order_state(ort.order.Key) is OrderRecordTuple oldval) - return (ort.order.filled() > oldval.order.filled(), oldval, ort); + if (_data_cache.get_previous_order_state(ort.order.Key) is var oldval && oldval.HasValue) { + if (ort.order.filled() > oldval?.order.filled()) { + return (oldval.Value, ort); + } + } else if (!_data_cache.is_first_order_update && ort.order.filled() > 0) { // old order will be a copy of new order, except with filledQuantity set to 0 var old_state = ort.order.State.CloneToMutable(); old_state.FilledQuantity = 0; - return (true, new OrderRecordTuple(ort.order.with(state: old_state), ort.security_md), ort); + return (new OrderSecurity(ort.security, ort.order.with(state: old_state)), ort); } } - return (false, null, null); + return null; } } - class FillTrigger : IPredicate { + class FillTrigger : IPredicate { protected readonly FillTriggerModel _model; public FillTrigger(FillTriggerModel model) => _model = model; - public bool trigger(OrderRecordTuple order) => _model.get_filled_order_args(order).trigger; + public bool trigger(OrderSecurity order) => _model.get_filled_order_args(order).HasValue; } - class FillToastNotificationTrigger : FillTrigger, INotificationPredicate { + class FillToastNotificationTrigger : FillTrigger, INotificationPredicate { public FillToastNotificationTrigger(FillTriggerModel model) : base(model) { } - public new IToastNotification trigger(OrderRecordTuple value) { - (var trigger, var oldval, var newval) = _model.get_filled_order_args(value); - return trigger ? ToastNotifications.order_filled(oldval, newval) : ToastNotifications.empty; + public new IToastNotification trigger(OrderSecurity value) { + var args = _model.get_filled_order_args(value); + if (args.HasValue) { + return ToastNotifications.order_filled(args.Value.oldval, args.Value.newval); + } + return ToastNotifications.empty; + //return args ? ToastNotifications.order_filled(oldval, newval) : ToastNotifications.empty; } } } diff --git a/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs b/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs index 1a938f50a..e12c4d480 100644 --- a/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs +++ b/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs @@ -61,7 +61,7 @@ namespace mbts { } [ProtoContract] - class OrderRecordTuplePredicate : INotificationPredicateBuilder, IEquatable { + class OrderRecordTuplePredicate : INotificationPredicateBuilder, IEquatable { SecurityPredicate _security; SidePredicate _side; DoublePredicate _price, _last_price, _avg_price; @@ -206,14 +206,14 @@ namespace mbts { } } - public IPredicate create_predicate(INotificationServiceDataCache data_cache) { + public IPredicate create_predicate(INotificationServiceDataCache data_cache) { switch (predicate_type_case) { - case PTOC.Security: return wrap(_security, r => r.security_md.HasValue ? r.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable); + case PTOC.Security: return wrap(_security, r => r.security ?? SecurityDataHelper.Defaults.SecurityImmutable); case PTOC.Time: return wrap(_time_created, r => r.order.created()); case PTOC.LastChange: return wrap(_last_change, r => r.order.last_change()); case PTOC.User: return wrap(_user, r => r.order.user_id()); case PTOC.Account: return wrap(_account, r => r.order.account_id()); - case PTOC.Exchange: return wrap(_exchange, r => r.security_md.HasValue ? r.security_md.Value.security.Exchange : Venue.Unset); + case PTOC.Exchange: return wrap(_exchange, r => r.security.exchange_or_default()); case PTOC.TimeInForce: return wrap(_time_in_force, r => r.order.time_in_force()); case PTOC.OrderType: return wrap(_order_type, r => r.order.order_type()); case PTOC.SubType: return wrap(_order_sub_type, r => r.order.sub_type()); @@ -234,14 +234,14 @@ namespace mbts { default: return null; } // - IPredicate wrap(IPredicateBuilder trigger, Func select) => - trigger.create(data_cache).wrap(select); + IPredicate wrap(IPredicateBuilder trigger, Func select) => + trigger.create(data_cache).wrap(select); } - IPredicate IPredicateBuilder.create(object obj) => + IPredicate IPredicateBuilder.create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public INotificationPredicate create_notification_predicate(INotificationServiceDataCache notification_service_data_cache) { + public INotificationPredicate create_notification_predicate(INotificationServiceDataCache notification_service_data_cache) { if (!is_toast_enabled) return null; switch (predicate_type_case) { @@ -252,7 +252,7 @@ namespace mbts { } } - INotificationPredicate INotificationPredicateBuilder.create_notification_predicate(object obj) => + INotificationPredicate INotificationPredicateBuilder.create_notification_predicate(object obj) => create_notification_predicate((INotificationServiceDataCache)obj); public bool Equals(OrderRecordTuplePredicate other) { @@ -325,8 +325,8 @@ namespace mbts { return EqualityComparer.phase_hash(raw_hash, (int)ptoc); } - static string _get_order_placed_toast_msg(OrderRecordTuple order) => - $"Order Sent: {order.security_md.display_name()} {order.order.short_description(order.security_md.spec_or_default())}"; + static string _get_order_placed_toast_msg(OrderSecurity order) => + $"Order Sent: {order.security.display_name()} {order.order.short_description(order.security.spec_or_default())}"; void _set(ref T field, T value) { if (value != null) { @@ -359,7 +359,7 @@ namespace mbts { } [ProtoContract] - class OrderRecordTuplePredicateAnimations : IPredicateAnimationsData { + class OrderRecordTuplePredicateAnimations : IPredicateAnimationsData { [ProtoMember(1)] OrderRecordTuplePredicate _predicate; [ProtoMember(2)] diff --git a/src/client/spark/Security/MDLookup.cs b/src/client/spark/Security/MDLookup.cs new file mode 100644 index 000000000..e7949dda7 --- /dev/null +++ b/src/client/spark/Security/MDLookup.cs @@ -0,0 +1,30 @@ +using mbts; +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.Reactive.Linq; + +namespace bts { + class MDLookup : IDisposable { + IReadOnlyDictionary _state; + IDisposable _dispose; + + [ImportingConstructor] + public MDLookup(IObservable> src) { + _dispose = src.Subscribe(s => _state = s); + } + + public void Dispose() { Disposable.dispose(ref _dispose); } + + public SecurityMD? get_md(uint key) { + if (_state?.TryGetValue(key, out var md) == true) { + return md; + } + return null; + } + public SecurityMD? get_md(uint? key) { + if (key.HasValue) return get_md(key.Value); + return null; + } + } +} diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs new file mode 100644 index 000000000..ed848ef2b --- /dev/null +++ b/src/client/spark/Security/SecurityStore.cs @@ -0,0 +1,261 @@ +using bts.api; +using bts.api.core; +using bts.co2; +using mbts; +using mbts.spark; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.ComponentModel.Composition; +using System.Linq; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading.Tasks; +using security_key = System.UInt32; + +namespace bts { + + [Export] + public class SecurityStore : IDisposable { + readonly MarketDataRelay _md; + readonly IBus _bus; + IDisposable _dispose; + IReadOnlyDictionary _live_securities; + readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + readonly Subject> _cache_subject = new Subject>(); + + [ImportingConstructor] + public SecurityStore(MarketDataRelay md, IBus bus) { + _md = md; + _bus = bus; + _dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict); + _cache_subject.OnNext(_cache); + } + + IObservable> on_securities => _bus.subscribe(SparkTags.SecuritiesTag); + + SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live=null) { + // Try live securities first + live = live ?? _live_securities; + if (live != null) { + if (live.TryGetValue(key, out SecurityMD value)) { + return value.security; + } + } + + // Then try cached securities + if (_cache.TryGetValue(key, out SecurityImmutable security)) { + return security; + } + + return null; + } + + async Task get_security(security_key key, IReadOnlyDictionary live) { + var local_sec = try_get_local(key, live); + if (local_sec.HasValue) { + return local_sec; + } + + // Query the server for the value + var result = await _md.get_securities_async(new SecurityKeyList() { + Keys = { key } + }); + + if (result.success) { + var security = result.value.Security.FirstOrDefault(); + if (security != null) { + return try_cache_security(new SecurityImmutable(security)); + } + } + + // Give up :( + Log.Md.warn($"Failed to get security for key {key}"); + return null; + } + + async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { + var result = new Dictionary(); + var missing = new SecurityKeyList(); + foreach (var key in keys) { + var s = try_get_local(key); + if (s.HasValue) { + result.Add(key, s.Value); + } + else { + missing.Keys.Add(key); + } + } + if (missing.Keys.Count > 0) { + var res = await _md.get_securities_async(missing); + if (res.success) { + foreach (var security in res.value.Security) { + result.Add(security.Id, try_cache_security(new SecurityImmutable(security))); + } + } + } + return result; + } + + // Request any missing keys from the server but do not wait for them to arrive + IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { + var result = new Dictionary(); + var missing = new HashSet(); + foreach (var key in keys) { + var security = try_get_local(key); + if (security.HasValue) { + result.TryAdd(key, security.Value); + } + else { + missing.Add(key); + } + } + // Update cache in the background + if (missing.Count > 0) { + Task.Run(async () => { + var skl = new SecurityKeyList(); + skl.Keys.AddRange(missing); + var r = await _md.get_securities_async(skl); + if (r.success) { + var retrieved = r.value.Security.Select(s => s.Id); + Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]"); + foreach (var security in r.value.Security) { + try_cache_security(new SecurityImmutable(security)); + } + } + }); + } + return result; + } + + SecurityImmutable try_cache_security(SecurityImmutable s) { + if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { + _cache.TryAdd(s.Id, s); + _cache_subject.OnNext(_cache); + return s; + } + return live.security; + } + + /////////////// + // PUBLIC + /////////////// + #region public + + // Guaranteed to return a security if it has ever been seen by the system + public Task get_security(security_key key) { + return get_security(key, _live_securities); + } + // Guaranteed to return a security if it has ever been seen by the system + public Task> get_securities(IEnumerable keys) { + return get_securities(keys, _live_securities); + } + public void Dispose() { + Disposable.dispose(ref _dispose); + } + + public IObservable> join( + IObservable> source, Func skeyfn) where TKey : IEquatable { + return Observable.Create>(observer => { + var state = ImmutableDictionary.Empty; + var gate = new object(); + // We get thie function from the first source update + Func<(SecurityImmutable?,T t), TKey> rowkeyfn = null; + + void on_securities_updated(object ambiguous) { + if (rowkeyfn == null) return; + lock (gate) { + // If this is null we don't have any rows + var changed = new List>(); + foreach (var entry in state.Values) { + if (entry.s == null) { + var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); + if (security.HasValue) { + changed.Add( + new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); + } + } + } + if (changed.Count > 0) { + // update state + var builder = state.ToBuilder(); + foreach (var up in changed) { + builder[rowkeyfn(up.New)] = up.New; + } + state = builder.ToImmutable(); + // fire update + observer.OnNext( + new Update<(SecurityImmutable?, T), TKey>( + state.Values, + Enumerable.Empty<(SecurityImmutable?,T)>(), + changed, + Enumerable.Empty<(SecurityImmutable?,T)>(), + rowkeyfn)); + } + } + } + + return Disposable.Create( + // Fire update when source changes + source.Subscribe((update) => { + lock (gate) { + rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2)); + var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(skeyfn)); + + var added = new List<(SecurityImmutable?, T)>(); + var changed = new List>(); + var removed = new List<(SecurityImmutable?, T)>(); + + foreach (var x in update.added) { + if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) { + added.Add((s, x)); + } + else { + added.Add((null, x)); + } + } + foreach (var x in update.removed) { + if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) { + removed.Add(value); + } + } + foreach (var ch in update.changes) { + SecurityImmutable? si = + relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ? + s : (SecurityImmutable?)null; + changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); + } + + var builder = state.ToBuilder(); + builder.RemoveRange(update.removed.Select(update.key_extractor)); + builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old))); + foreach (var a in added) { + builder.Add(rowkeyfn(a), a); + } + foreach (var ch in changed) { + builder.Add(rowkeyfn(ch.New), ch.New); + } + state = builder.ToImmutable(); + + observer.OnNext(new Update<(SecurityImmutable?, T t), TKey>( + state.Values, added, changed, removed, rowkeyfn)); + } + }), + // Also fire update when securities change + on_securities.Subscribe(on_securities_updated), + // Also fire update when cache changes + _cache_subject.Subscribe(on_securities_updated) + ); + }); + } + #endregion + } + + public static class SecurityStoreExtensions { + public static IObservable> + join(this IObservable> source, SecurityStore store, Func skeyfn) where TKey : IEquatable { + return store.join(source, skeyfn); + } + } +} diff --git a/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs b/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs index 93ba83022..36c545789 100644 --- a/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs +++ b/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs @@ -233,6 +233,7 @@ namespace mbts return has_value; } + } } diff --git a/src/client/spark/core/Utility/TableUpdateTransform.cs b/src/client/spark/core/Utility/TableUpdateTransform.cs index 6dee3f511..c733d1202 100644 --- a/src/client/spark/core/Utility/TableUpdateTransform.cs +++ b/src/client/spark/core/Utility/TableUpdateTransform.cs @@ -1,4 +1,5 @@ using bts.api.core; +using bts.collections; using mbts.collections; using System; using System.Collections.Generic; @@ -26,9 +27,6 @@ namespace mbts.utils { } class IncrementalUpdate : IIncrementalUpdate where TKey : IEquatable { - public IncrementalUpdate() - : this(new Dictionary(0), new Dictionary(0)) { } - public IncrementalUpdate(IReadOnlyDictionary added, IReadOnlyDictionary removed) { this.added = added; this.removed = removed; @@ -305,6 +303,40 @@ namespace mbts.utils { state_clone.Add(kv.Key, kv.Value); return new PivotTableUpdateTransform(state_clone, added.Values.ToList(), removed.Values.ToList(), _key_map, _selector, _result_key, _key); } - } + } + + public static IObservable> as_iincremental(this IObservable> src) where TKey : IEquatable { + return src.Select(as_iincremental); + } + + public static IIncrementalUpdate as_iincremental(this TableUpdate update) where TKey : IEquatable { + var key = update.key_extractor; + var added = update.added.ToDictionary(key); + var removed = update.removed.ToDictionary(key); + foreach (var ch in update.changes) { + added[key(ch.New)] = ch.New; + removed[key(ch.Old)] = ch.Old; + } + return new IncrementalUpdate(added, removed); + } + + public static IObservable> + Select(this IObservable> src, Func map) where TKey : IEquatable { + return src.Select(update => { + return new IncrementalUpdate( + new MappedDictionary(update.added, map), + new MappedDictionary(update.removed, map)); + }); + } + + public static IObservable> + Where(this IObservable> src, Func where) where TKey : IEquatable { + return src.Select(update => { + return new IncrementalUpdate( + new FilteredDictionary(update.added, where), + new FilteredDictionary(update.removed, where)); + }); + } + } } \ No newline at end of file diff --git a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs index 8c3a7f20c..b3c153195 100644 --- a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs +++ b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs @@ -17,7 +17,8 @@ using System.Windows.Media; using static mbts.ui.controls.DIYGrid; namespace mbts.ui { - class GridSettingsManager : ViewModelBase, IValueFactory, IObserver, IDisposable + class GridSettingsManager : + ViewModelBase, IValueFactory, IObserver, IDisposable where TRow : IRowVM where TPredicateBuilder : IPredicateBuilder, new() where TSettingsData : class, IGridSettingsWithNotifications { diff --git a/src/client/spark/view/Spark/Core/SparkUtils.cs b/src/client/spark/view/Spark/Core/SparkUtils.cs index fb620cb3a..3474ea79f 100644 --- a/src/client/spark/view/Spark/Core/SparkUtils.cs +++ b/src/client/spark/view/Spark/Core/SparkUtils.cs @@ -48,9 +48,6 @@ namespace mbts.spark { } #endregion - public static string display_name(this SecurityMD? security_md) => - (security_md?.security).display_name(); - public static string display_name(this SecurityImmutable? security) => security.HasValue ? security.Value.ToString() : "unknown security"; @@ -59,11 +56,11 @@ namespace mbts.spark { public static SecurityImmutable? security(this SecurityMD? security_md) => security_md.HasValue ? new SecurityImmutable?(security_md.Value.security) : new SecurityImmutable?(); - public static Venue exchange(this SecurityMD? security_md) => - security_md.HasValue ? security_md.Value.security.Exchange : Venue.Unset; + public static Venue exchange_or_default(this SecurityImmutable? security) => + security?.Exchange ?? Venue.Unset; - public static string symbol(this SecurityMD? security_md) => - security_md.HasValue ? security_md.Value.security.Symbol : "unknown"; + public static string symbol_or_default(this SecurityImmutable? security) => + security?.Symbol ?? "unknown"; public static async Task get_security_async(IBus bus, uint security_key) => await bus.subscribe(SparkTags.SecuritiesTag).Where(map => map.ContainsKey(security_key)).Select(map => map[security_key].security).FirstAsync(); diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 4b7528808..034725b4a 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -134,8 +134,14 @@ namespace mbts.spark { // Here we manage N orders to 1 price slot in the ladder Dictionary> orders_used_in_this_update = new Dictionary>(_equality); - var ladders_orders = working.Values.Where(o => o.Def.SecurityKey == security.Id && (o.HasState && o.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible)); - foreach (var wo in ladders_orders) { + var ladders_orders = + working.Values + .Where(o => + o.order.Def.SecurityKey == security.Id && + (o.order.HasState && o.order.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible)); + + foreach (var osec in ladders_orders) { + var wo = osec.order; var slot_price_target = wo.price(); var order_type = wo.order_type(); var filled = wo.filled(); @@ -186,10 +192,9 @@ namespace mbts.spark { var traversal_candidate = only_order; bool exit_early = false; while (!exit_early && traversal_candidate.ParentKey != 0) { - var found_parent = working.Where(o => o.Key == traversal_candidate.ParentKey); - if (found_parent.Count() == 1) - traversal_candidate = found_parent.First().Value; - else + if (working.TryGetFirst(o => o.Key == traversal_candidate.ParentKey, out var first)) + traversal_candidate = first.Value.order; + else exit_early = true; } if (!exit_early) diff --git a/src/client/spark/view/Spark/PAndL/PAndLModel.cs b/src/client/spark/view/Spark/PAndL/PAndLModel.cs index 558f2305a..9ffa2d95b 100644 --- a/src/client/spark/view/Spark/PAndL/PAndLModel.cs +++ b/src/client/spark/view/Spark/PAndL/PAndLModel.cs @@ -1,4 +1,5 @@ -using bts.api; +using bts; +using bts.api; using mbts.utils; using System; using System.ComponentModel.Composition; @@ -8,18 +9,18 @@ namespace mbts.spark { class PAndLModel : IDisposable { readonly IBus _bus; readonly OrdersModel _order_model; - //readonly AccountService _account_service; + readonly MDLookup _mdlookup; IDisposable _dispose; [ImportingConstructor] private PAndLModel(IBus bus, OrdersModel order_model/*, AccountService account_service*/) { _bus = bus; _order_model = order_model; - //_account_service = account_service; + _mdlookup = new MDLookup(bus.subscribe(SparkTags.SecuritiesTag)); } public IObservable> on_pnl_data => - _order_model.on_fill_tuples.convert(ft => new PnLData(ft)).table(pd => pd.key); + _order_model.on_fill_tuples.Select(ft => new PnLData(ft, _mdlookup.get_md(ft.security?.Id))).table(pd => pd.key); //public async Task get_accounts() { // var request = await _account_service.get_accounts_async(new GetAccountsRequest()); diff --git a/src/client/spark/view/Spark/PAndL/PnLData.cs b/src/client/spark/view/Spark/PAndL/PnLData.cs index 99c441c16..5b44015a5 100644 --- a/src/client/spark/view/Spark/PAndL/PnLData.cs +++ b/src/client/spark/view/Spark/PAndL/PnLData.cs @@ -28,19 +28,23 @@ namespace mbts.spark { class PnLData { readonly FillTuple _fill_tuple; + readonly SecurityMD? _md; - public PnLData(FillTuple fill_tuple) => + public PnLData(FillTuple fill_tuple, SecurityMD? md) { + // It is entirely possible for the FillTuple to have a security but for no SecrityMD to be available. For example: expired security + _md = md; _fill_tuple = fill_tuple; + } public FillTupleKey key => _fill_tuple.key; public FillImmutable fill => _fill_tuple.fill; - public SecurityMD? sec_md => _fill_tuple.security_md; + public SecurityMD? sec_md => _md; + public SecurityImmutable? security => _fill_tuple.security; public OrderRecordImmutable? order => _fill_tuple.order; public string account => _fill_tuple.account; - public Venue? exchange => sec_md?.security.Exchange; - public string symbol => sec_md?.security.Symbol; - public SecurityImmutable? security => sec_md?.security; + public Venue? exchange => security?.Exchange; + public string symbol => security?.Symbol; public double buy_quantity => (fill.Side == Side.Buy) ? fill.LastShares : 0; public double sell_quantity => (fill.Side == Side.Sell) ? fill.LastShares : 0; public SecurityType? security_type => security?.security_type(); diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index 6f9604a01..e97fc196b 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -64,15 +64,19 @@ namespace bts.api.core { public Change merge(Change newer) { return new Change(Old, newer.New); } + public void Deconstruct(out T Old, out T New) { + Old = this.Old; + New = this.New; + } public override string ToString() { return $"{{{Old}=>{New}}}"; } } public class Update where TKey : IEquatable { - private Func _key; + readonly Func _key; internal Update( - IReadOnlyCollection all, + IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, @@ -122,7 +126,7 @@ namespace bts.api.core { ); } - public IReadOnlyCollection all { get; } + public IEnumerable all { get; } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); @@ -208,11 +212,13 @@ namespace bts.api.core { ImmutableDictionary init, IEnumerable added, IEnumerable> changed, - IEnumerable removed) { + IEnumerable removed, + Func maybe_key_extractor) { _state = init; this.added = added; this.changes = changed; this.removed = removed; + this.key_extractor = maybe_key_extractor; } public IReadOnlyDictionary state { get { return _state; } } @@ -229,12 +235,13 @@ namespace bts.api.core { foreach (var i in update.changed) { builder[update.key(i)] = i; } - return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed); + return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed, update.key_extractor); } internal TableUpdate as_snapshot() { - return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0]); + return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0], key_extractor); } + internal Func key_extractor { get; private set; } } internal class LegacyTableUpdate where TKey : IEquatable { @@ -283,7 +290,7 @@ namespace bts.api.core { public static IObservable> Table(this IObservable> source) where TKey : IEquatable => source.Scan( - new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0]), + new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => TableUpdate.update(table,update)); // Fold updates and emit at max 1/interval @@ -415,5 +422,40 @@ namespace bts.api.core { changed = diffs; removed = rems; } + + // This method does everything lazily so it's cheap until the collections are enumerated + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + return src.Select(update => { + return new Update( + update.all.Select(map), + update.added.Select(map), + update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), + update.removed.Select(map), + key); + }); + } + + // Constructs a new dictionary so not as cheap as Select on the Update<> type + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + var cache = ImmutableDictionary.Empty; + return src.Select(update => { + var added = update.added.Select(map); + var removed = update.removed.Select(map); + var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); + var builder = cache.ToBuilder(); + builder.RemoveRange(removed.Select(key)); + builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); + foreach (var (o,n) in changes) { + Debug.Assert(key(o).Equals(key(n))); + builder[key(o)] = n; + } + cache = builder.ToImmutable(); + + return new TableUpdate(cache, added, changes, removed, key); + }); + } + } } \ No newline at end of file diff --git a/src/client/spark_api/md/MDSource.cs b/src/client/spark_api/md/MDSource.cs index 9fdebb63b..392fc6844 100644 --- a/src/client/spark_api/md/MDSource.cs +++ b/src/client/spark_api/md/MDSource.cs @@ -15,20 +15,26 @@ using System.Reactive.Subjects; namespace bts.md { + [Flags] + enum MDSourceOptions { + DeferInit = 1, + } class MDSource : IObserver>, IDisposable { protected readonly MDConsumer _consumer = new MDConsumer(); protected readonly Co2RelayControl _relay; - protected long _version = 0; + protected DateTime _version = DateTime.MinValue; protected readonly ConcurrentDictionary _exchange_sec_map = new ConcurrentDictionary(); protected List _dispose = new List(); + protected readonly MDSourceOptions _options; - public MDSource(uint naggle_ms, bool defer_init = false) - { + public MDSource(uint naggle_ms, MDSourceOptions options = 0) + { + _options = options; _relay = new Co2RelayControl(_consumer, naggle_ms); _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore(); _dispose.Add(_consumer.on_security.Subscribe(this)); - if(!defer_init) + if(!options.HasFlag(MDSourceOptions.DeferInit)) initialize(); } @@ -57,18 +63,18 @@ namespace bts.md var smd = new SecurityMD(new SecurityImmutable(v.security), v.handle); _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd); } - _version += 1; + _version = DateTime.UtcNow; } } public void Dispose() => Disposable.dispose(_dispose); async protected Task run() { - long run_version = 0; + DateTime run_version = DateTime.MinValue; while (true) { var exch = _exchange_sec_map; - long current_version = _version; + var current_version = _version; if (current_version > run_version) { run_version = current_version; diff --git a/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs b/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs index 5f7576ad8..415cb0ef3 100644 --- a/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs @@ -9,6 +9,14 @@ namespace System.Collections.Generic { else update_value(key, value); } + public static bool TryAdd(this IDictionary dict, TKey key, TValue value) { + if (dict.ContainsKey(key)) return false; + else { + dict[key] = value; + return true; + } + } + public static void AddOrUpdate(this IDictionary dictionary, TKey key, Func add_value_factory, Func update_value) { if (!dictionary.TryGetValue(key, out TValue value)) dictionary.Add(key, add_value_factory(key)); @@ -77,5 +85,20 @@ namespace System.Collections.Generic { /// true if the dictionary contains an element with the specified key, AND returns true; otherwise, false. public static bool TryPopValueIf(this IDictionary dictionary, TKey key, Func should_pop) => dictionary.TryGetValue(key, out TValue result) && should_pop(result) && dictionary.Remove(key); + + + public static T? TryGetValue(this IDictionary src, TKey key) where T : struct { + if (src.TryGetValue(key, out T value)) { + return value; + } + return null; + } + public static T? TryGetValue(this IReadOnlyDictionary src, TKey key) where T : struct { + if (src.TryGetValue(key, out T value)) { + return value; + } + return null; + } + } } \ No newline at end of file diff --git a/src/client/spark_core/TestRunner/TestAssert.cs b/src/client/spark_core/TestRunner/TestAssert.cs index 5bc09ae02..0306073cc 100644 --- a/src/client/spark_core/TestRunner/TestAssert.cs +++ b/src/client/spark_core/TestRunner/TestAssert.cs @@ -65,5 +65,25 @@ namespace bts.test { $"Expected: {item} in [{aa}]. {msg}", file, line); } } + + public static void throws(Action runme, string msg="", [CallerFilePath] string file="", [CallerLineNumber] int line=0) where T : Exception { + bool threw = false; + try { + runme(); + } + catch (T) { + threw = true; + } + catch (Exception e) { + throw new AssertFailture( + $"Expected '{typeof(T).Name}' but threw '{e.GetType().Name}'", file, line); + } + + if (!threw) { + throw new AssertFailture( + msg ?? $"Expected '{typeof(T).Name}' exception but ran successfully.", file, line); + } + + } } } \ No newline at end of file diff --git a/src/client/spark_core/TestRunner/ConsoleEx.cs b/src/client/spark_core/utils/ConsoleEx.cs similarity index 100% rename from src/client/spark_core/TestRunner/ConsoleEx.cs rename to src/client/spark_core/utils/ConsoleEx.cs diff --git a/src/client/spark_core/utils/FilteredDictionary.cs b/src/client/spark_core/utils/FilteredDictionary.cs new file mode 100644 index 000000000..cb0ca57f1 --- /dev/null +++ b/src/client/spark_core/utils/FilteredDictionary.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace bts.collections { + public class FilteredDictionary : IReadOnlyDictionary { + readonly IReadOnlyDictionary _dict; + readonly Func _filter; + int? _count; + + public FilteredDictionary(IReadOnlyDictionary src, Func filter) { + _dict = src; + _filter = filter; + _count = null; + } + + public T this[TKey key] { + get { + var result = _dict[key]; + if (_filter(result)) { + return result; + } + throw new KeyNotFoundException(key.ToString()); + } + } + + public IEnumerable Keys => _dict.Where(kvp => _filter(kvp.Value)).Select(kvp => kvp.Key); + + public IEnumerable Values => _dict.Values.Where(_filter); + + public int Count { + get { + if (_count.HasValue) return _count.Value; + _count = Values.Count(); + return _count.Value; + } + } + + public bool ContainsKey(TKey key) => Keys.Contains(key); + + public IEnumerator> GetEnumerator() { + return new FilteredEnumerator>(_dict.GetEnumerator(), kvp => _filter(kvp.Value)); + } + + public bool TryGetValue(TKey key, out T value) { + if (_dict.TryGetValue(key, out var val)) { + if (_filter(val)) { + value = val; + return true; + } + } + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class FilteredEnumerator : IEnumerator { + readonly Func _filter; + readonly IEnumerator _src; + + public FilteredEnumerator(IEnumerator src, Func filter) { + _filter = filter; + _src = src; + } + public T Current => _src.Current; + + object IEnumerator.Current => Current; + + public void Dispose() => _src.Dispose(); + + public bool MoveNext() { + do { + var ok = _src.MoveNext(); + if (!ok) return false; + } while (!_filter(_src.Current)); + return true; + } + + public void Reset() { _src.MoveNext(); } + } + + +} \ No newline at end of file diff --git a/src/client/spark_core/utils/MappedDictionary.cs b/src/client/spark_core/utils/MappedDictionary.cs new file mode 100644 index 000000000..d66ce775f --- /dev/null +++ b/src/client/spark_core/utils/MappedDictionary.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace bts.collections { + internal class MappedDictionary : IReadOnlyDictionary { + readonly IReadOnlyDictionary _dict; + readonly Func _map; + + public MappedDictionary(IReadOnlyDictionary src, Func map) { + _dict = src; + _map = map; + } + + public TRes this[TKey key] => _map(_dict[key]); + + public IEnumerable Keys => _dict.Keys; + + public IEnumerable Values => _dict.Values.Select(_map); + + public int Count => _dict.Count; + + public bool ContainsKey(TKey key) => _dict.ContainsKey(key); + + public IEnumerator> GetEnumerator() { + return new MappedEnumerator, KeyValuePair>( + _dict.GetEnumerator(), + kvp => new KeyValuePair(kvp.Key, _map(kvp.Value))); + } + + public bool TryGetValue(TKey key, out TRes value) { + if (_dict.TryGetValue(key, out TSrc src)) { + value = _map(src); + return true; + } + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class MappedEnumerator : IEnumerator { + readonly IEnumerator _src; + readonly Func _map; + + public MappedEnumerator(IEnumerator src, Func map) { + _src = src; + _map = map; + } + + public TRes Current => _map(_src.Current); + + object IEnumerator.Current => Current; + + public void Dispose() { _src.Dispose(); } + + public bool MoveNext() => _src.MoveNext(); + + public void Reset() { _src.Reset(); } + } + +} \ No newline at end of file diff --git a/src/client/spark_test/Utils/FilteredDictionaryTests.cs b/src/client/spark_test/Utils/FilteredDictionaryTests.cs new file mode 100644 index 000000000..8a99f1c7a --- /dev/null +++ b/src/client/spark_test/Utils/FilteredDictionaryTests.cs @@ -0,0 +1,85 @@ +using bts.collections; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace bts.test { + + [TestGroup] + public class FilteredDictionaryTests { + IEnumerable range = Enumerable.Range(5,6); + Dictionary source; + IReadOnlyDictionary filtered; + + public FilteredDictionaryTests() { + // Create a source dictionary + source = Enumerable.Range(1, 10).ToDictionary(x => x); + // create a filtered dictionary + filtered = new FilteredDictionary(source, x => x >= 5); + } + + void ignore(T t) { } + + [Test] + public void index() { + foreach (var i in Enumerable.Range(-10,20)) { + if (range.Contains(i)) { + Assert.AreEqual(i, filtered[i]); + } + else { + Assert.throws(() => ignore(filtered[i])); + } + } + } + [Test] + public void keys() { + Assert.SequenceEqual(range, filtered.Keys); + } + [Test] + public void values() { + Assert.SequenceEqual(range, filtered.Values); + } + [Test] + public void count() { + Assert.AreEqual(range.Count(), filtered.Count); + Assert.AreEqual(range.Count(), filtered.Count()); + } + [Test] + public void contains_key() { + foreach (var i in Enumerable.Range(-10, 20)) { + if (range.Contains(i)) { + Assert.IsTrue(filtered.ContainsKey(i)); + } + else { + Assert.IsFalse(filtered.ContainsKey(i)); + } + } + } + [Test] + public void enumerator() { + Assert.SequenceEqual( + range.Select(i => new KeyValuePair(i, i)), filtered); + } + [Test] + public void try_get_value() { + foreach (var i in Enumerable.Range(-10, 20)) { + if (range.Contains(i)) { + Assert.IsTrue(filtered.TryGetValue(i, out var _)); + } + else { + Assert.IsFalse(filtered.TryGetValue(i, out var _)); + } + } + } + + [Test] + public void FilterAndMap() { + string transform(int i) => (i * i).ToString(); + var mapped = new MappedDictionary(filtered, transform); + Assert.SequenceEqual( + range.Select(transform), + mapped.Values); + + } + } +} diff --git a/src/client/spark_test/Utils/MappedDictionaryTests.cs b/src/client/spark_test/Utils/MappedDictionaryTests.cs new file mode 100644 index 000000000..e426de542 --- /dev/null +++ b/src/client/spark_test/Utils/MappedDictionaryTests.cs @@ -0,0 +1,71 @@ +using bts.collections; +using System.Collections.Generic; +using System.Linq; + +namespace bts.test { + + [TestGroup] + public class MappedDictionaryTests { + IEnumerable range = Enumerable.Range(1, 10); + Dictionary source; + IReadOnlyDictionary mapped; + + public MappedDictionaryTests() { + // Create a source dictionary + source = range.ToDictionary(x => x); + // create a filtered dictionary + mapped = new MappedDictionary(source, x => x.ToString()); + } + + [Test] + public void index() { + foreach(var i in range) { + Assert.AreEqual(i.ToString(), mapped[i]); + } + } + [Test] + public void keys() { + Assert.SequenceEqual(range, mapped.Keys); + } + [Test] + public void values() { + Assert.SequenceEqual(range.Select(x => x.ToString()), mapped.Values); + } + [Test] + public void count() { + Assert.AreEqual(range.Count(), mapped.Count); + } + [Test] + public void contains_key() { + foreach (var key in range) { + Assert.IsTrue(mapped.ContainsKey(key)); + } + Assert.IsFalse(mapped.ContainsKey(range.Max() + 1)); + Assert.IsFalse(mapped.ContainsKey(range.Min() - 1)); + } + [Test] + public void enumerator() { + Assert.SequenceEqual(range, mapped.Select(kvp => kvp.Key)); + Assert.SequenceEqual( + range.Select(x => x.ToString()), + mapped.Select(kvp => kvp.Value)); + } + [Test] + public void try_get_value() { + foreach (var i in Enumerable.Range(-20, 20)) { + Assert.AreEqual( + range.Contains(i), + mapped.TryGetValue(i, out string _)); + } + } + [Test] + public void MapAndFilter() { + var set = new HashSet(new[] { "4", "5" }); + var filtered = new FilteredDictionary(mapped, set.Contains); + Assert.AreEqual(2, filtered.Count); + Assert.SequenceEqual( + Enumerable.Range(4, 2).Select(i => (i, i.ToString())), + filtered.Select(kvp => (kvp.Key, kvp.Value))); + } + } +} diff --git a/src/client/spark_test/api/ApiUpdateTests.cs b/src/client/spark_test/api/ApiUpdateTests.cs index 791d9a773..e1f6f08b7 100644 --- a/src/client/spark_test/api/ApiUpdateTests.cs +++ b/src/client/spark_test/api/ApiUpdateTests.cs @@ -1,12 +1,12 @@ using bts.api.core; -using System.Collections.Generic; using bts.test; using System; -using System.Reactive.Linq; -using System.Threading.Tasks; +using System.Collections.Generic; using System.Linq; using System.Reactive.Concurrency; +using System.Reactive.Linq; using System.Reactive.Subjects; +using System.Threading.Tasks; namespace mbts.api.Test { [TestGroup] @@ -44,7 +44,7 @@ namespace mbts.api.Test { static int id(Item i) => i.id; Item mkitem() { - return new Item(++_id, Guid.NewGuid().ToString().Substring(0,8)); + return new Item(++_id, Guid.NewGuid().ToString().Substring(0, 8)); } IObservable> validate_incremental(IObservable> source) { @@ -68,7 +68,7 @@ namespace mbts.api.Test { // This will mostly be collapsing to add() for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -80,7 +80,7 @@ namespace mbts.api.Test { // Do it again with TableUpdate for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -94,35 +94,32 @@ namespace mbts.api.Test { // collapse rolling window var update_list = raw.reconstitute(id).ToEnumerable().ToList(); var window = 10; - for (int x = 0; x+window < total; x += 5) { + for (int x = 0; x + window < total; x += 5) { var start = update_list[x]; var end = update_list[x + window]; - var super_update = update_list.Skip(x).Take(window+1).Aggregate((u1, u2) => u1.merge(u2)); + var super_update = update_list.Skip(x).Take(window + 1).Aggregate((u1, u2) => u1.merge(u2)); start.merge(super_update); Assert.SequenceEqual(end.all, super_update.all); } } - void validate_filter(IObservable> raw, IObservable> filtered, Func pred) - { + void validate_filter(IObservable> raw, IObservable> filtered, Func pred) { var total = raw.ToEnumerable().Count(); // Validate that update is right at each point - for (var count = 1; count < total; ++count) - { - var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); + for (var count = 1; count < total; ++count) { + var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.all.All(pred)); Assert.That(next.added.All(pred)); Assert.That(next.removed.All(pred)); - Assert.That(next.changed.All(pred)); + Assert.That(next.changed.All(pred)); } // Do it again with TableUpdate - for (var count = 1; count < total; ++count) - { + for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.added.All(pred)); @@ -132,36 +129,32 @@ namespace mbts.api.Test { } - void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) - { - raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => - { + void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) { + raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => { Assert.That(tup.Item2.all.All(pred.Value)); Assert.That(tup.Item2.added.All(pred.Value)); - Assert.That(!tup.Item2.removed.Any(pred.Value)); + Assert.That(!tup.Item2.removed.Any(pred.Value)); Assert.That(tup.Item2.changed.All(pred.Value)); Dictionary rawd = tup.Item1.added.ToDictionary(x => x.id); Dictionary fd = tup.Item2.added.ToDictionary(x => x.id); - foreach (var kv in rawd) - { - if(pred.Value(kv.Value)) + foreach (var kv in rawd) { + if (pred.Value(kv.Value)) { Assert.That(fd.ContainsKey(kv.Key)); - else + } + else { Assert.That(!fd.ContainsKey(kv.Key)); + } } }); } [Test] - public void test_filter() - { - var raw = Observable.Create>(observer => - { + public void test_filter() { + var raw = Observable.Create>(observer => { RawUpdate rup = new RawUpdate(); - for (int x = 0; x < 50; ++x) - { + for (int x = 0; x < 50; ++x) { rup.Clear(); var item = _item_pool[x]; rup.added_items.Add(item); @@ -169,39 +162,35 @@ namespace mbts.api.Test { observer.OnNext(rup); } var rand = new Random(); - for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) - { + for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) { rup.Clear(); rup.removed_keys.Add(_item_pool[r].id); observer.OnNext(rup); } rup.Clear(); - for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) - { + for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) { var ch = new Item(_item_pool[c].id, _item_pool[c].name + "_change"); rup.changed_items.Add(ch); - rup.changed_keys.Add(ch.id); + rup.changed_keys.Add(ch.id); } observer.OnNext(rup); - observer.OnCompleted(); + observer.OnCompleted(); return Disposable.empty; }); Func evens = (x => x.id % 2 == 0); Func odds = (x => x.id % 2 == 1); - var predeven = Observable.Create>(observer => - { - observer.OnNext(evens); + var predeven = Observable.Create>(observer => { + observer.OnNext(evens); observer.OnCompleted(); return Disposable.empty; }); - var filtered = raw.reconstitute(x=> x.id).Filter(predeven); + var filtered = raw.reconstitute(x => x.id).Filter(predeven); validate_filter(raw, filtered, evens); - var predodd = Observable.Create>(observer => - { + var predodd = Observable.Create>(observer => { observer.OnNext(odds); observer.OnCompleted(); return Disposable.empty; @@ -212,13 +201,13 @@ namespace mbts.api.Test { - var sub= new BehaviorSubject>(evens); + var sub = new BehaviorSubject>(evens); filtered = raw.reconstitute(x => x.id).Filter(sub); validate_filter_sub(raw, filtered, sub); sub.OnNext(odds); sub.OnCompleted(); - } + } [Test] public async Task test_add() { @@ -249,7 +238,7 @@ namespace mbts.api.Test { rup.added_keys.AddRange(_item_pool.Take(20).Select(id)); observer.OnNext(rup); - for (int x = 0; x < 50; x+=2) { + for (int x = 0; x < 50; x += 2) { rup.Clear(); var item = _item_pool[x]; rup.removed_keys.Add(item.id); @@ -347,11 +336,121 @@ namespace mbts.api.Test { public void test_merge_begin_end_state() { int key(string s) => s[0]; var empty = new string[0]; - var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); + var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); var a3 = new Update(new[] { "a3" }, empty, new[] { new Change("a2", "a3") }, empty, key); var merged = a2.merge(a3); Assert.SequenceEqual(new[] { new Change("a1", "a3") }, merged.changes); } + T id(T t) => t; + + [Test,Ignore("SPARK-1988")] + public void test_reconstitute_twice() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + var input = + Enumerable.Range(1, 10) + .Select(create) + .ToObservable() + .reconstitute(id); + + var expected = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i)).SelectMany(id); + var a = input.ToEnumerable().SelectMany(u => u.all).ToList(); + var b = input.ToEnumerable().SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected, a); + Assert.SequenceEqual(expected, b); + } + + [Test] + public void test_select_update() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + } + + [Test] + public void test_select_tableupdate() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Table() + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.state.Values).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + + } + + //public class Box { + // public int value; + // public override string ToString() { + // return value.ToString(); + // } + //} + + //[Test] + //public void test_scan() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan( + // new Box { value = 0 }, + // (box, i) => { + // box.value += i; + // return box; + // }); + // var a = obs.Select(box => box.value).ToEnumerable(); + // var b = obs.Select(box => box.value).ToEnumerable(); + // Assert.SequenceEqual(a, b); + //} + + //[Test] + //public void test_scan2() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan>( + // () => new Box { value = 0 }, + // (getbox, i) => { + // var box = getbox(); + // box.value += i; + // return () => box; + // }); + // var a = obs.Select(box => box().value).ToEnumerable().ToList(); + // var b = obs.Select(box => box().value).ToEnumerable().ToList(); + // Assert.SequenceEqual(a, b); + //} } } diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index abff1711f..c520a5933 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/IcebergOrder2.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/IcebergOrder2.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index afd768425..f3e9584f4 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d93256c7c..d83daf33b 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -14,14 +14,20 @@ message ExpirationNameList { bts.api.Product product = 1; map names = 2; } - message ExpirationNameListReply { repeated ExpirationNameList securities = 2; } +message SecurityKeyList { + repeated uint32 keys = 1; +} + service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); + rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); + + rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); } diff --git a/src/lib/config/ConfigDownloader.cpp b/src/lib/config/ConfigDownloader.cpp index 2c6e94af8..da23232eb 100755 --- a/src/lib/config/ConfigDownloader.cpp +++ b/src/lib/config/ConfigDownloader.cpp @@ -53,7 +53,7 @@ ConfigDownloader::Result ConfigDownloader::get(const string& url, stringstream& curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream); curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 30); + curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 6); curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger); CURLcode res = curl_easy_perform(_curl); diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 17d800d6b..08603497a 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -35,3 +35,23 @@ CMEOptions { } } +MDBootDelayWindow { + rule: "DeploymentType.type == 'dev'" + value { + delay_seconds: 0 + } +} + +MDBootDelayWindow { + value { + delay_seconds: 120 + standard { + start: "16:30" + end: "16:40" + } + sunday { + start: "14:00" + end: "14:10" + } + } +} \ No newline at end of file diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 4dde48bc2..aba6237c6 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -4,8 +4,9 @@ package bts.md; message MarketDataConfig { // Allows you to turn off in-proc market data entirely optional bool use_md = 1 [default = true]; - optional bool use_replay = 2 [default=false]; - optional bool use_sim = 3 [default=true]; + optional bool use_onixs = 2 [default=true]; + optional bool use_replay = 3 [default=false]; + optional bool use_sim = 4 [default=true]; } message PlatformMap @@ -14,6 +15,17 @@ message PlatformMap optional string glbx_platform = 1 [default = "PLATFORM_SIM"]; } +// LOCAL TIME +message MDBootDelayWindow { + message TimeSpan { + optional string start = 1 [default = "16:30"]; + optional string end = 2 [default = "16:40"]; + } + optional uint32 delay_seconds = 1 [default = 120]; + optional TimeSpan standard = 2; + optional TimeSpan sunday = 3; +} + // // Options that apply to market data server diff --git a/src/lib/domain_model/Security.proto b/src/lib/domain_model/Security.proto index d8c870b98..2450074aa 100644 --- a/src/lib/domain_model/Security.proto +++ b/src/lib/domain_model/Security.proto @@ -1,6 +1,7 @@ syntax="proto3"; package bts.api; option csharp_namespace="bts.api"; +option cc_enable_arenas=true; import "lib/api/apioptions.proto"; enum Venue { @@ -171,6 +172,9 @@ message Security { ICEExchangeData ice = 230; } } +message SecurityList { + repeated Security security = 1; +} message Product { bts.api.Venue exchange = 1; diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index f751f51df..46e5bfbcd 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -1,5 +1,6 @@ #include "MDPeriodicDumper.h" #include "lib/md/MDConsumer.h" +#include "lib/md/SecurityCache.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/datetime.h" @@ -25,7 +26,7 @@ namespace md { struct Entry { uint32_t seq; MDConsumerHandle handle; - std::unique_ptr security; + SecurityPtr security; }; std::string print_book( @@ -158,12 +159,13 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, auto thread_name = "tail-" + qualifier; threading::configure_current_thread(thread_name); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols); // read all securities into a vector std::vector entries; - consumer.walk_security_list([&](const api::Security& sec, MDConsumerHandle handle) { - entries.push_back({ 0, handle, std::make_unique(sec) }); + consumer.walk_security_list([&](const SecurityPtr& sec, MDConsumerHandle handle) { + entries.push_back({ 0, handle, sec }); }); if (!monitor_all) { diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp new file mode 100644 index 000000000..1cb454971 --- /dev/null +++ b/src/lib/md/MDClient.cpp @@ -0,0 +1,96 @@ +#include "MDClient.h" +#include "get_symbols.h" +#include "SecurityCache.h" +#include "SecurityOracle.h" +#include "lib/utils/TimeStamp.h" +#include + +namespace bts { +namespace md { + +struct MDClientImpl{ + utils::Dispatcher* _dispatcher; + api::ISyntheticSecurityService* _ss; + SecurityCache _cache; + std::unique_ptr _security_timer; + std::unique_ptr _recovery_timer; + std::set _symbols; + MDConsumer _consumer; + utils::Event _event; + + MDClientImpl( + utils::Dispatcher* d, + api::ISyntheticSecurityService* s, + const std::set* symbols): + _dispatcher(d), + _ss(s), + _security_timer(), + _recovery_timer(), + _symbols(symbols ? *symbols : get_system_md_symbols()), + _consumer(_cache, _ss, &_symbols), + _event() + { + if (d) { + _security_timer = + std::make_unique(*d, config::get().security_scan_interval(),[this]() { scan_securities(); }), + _recovery_timer = + std::make_unique(*d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), + _security_timer->start(); + _recovery_timer->start(); + } + } + + void check_heartbeats() { + if (!_consumer.all_heartbeats_ok()) { + MDConsumer temp(_cache,_ss); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); + // Subscribers are responsible for keeping the old consumer reference if they need it + _event.publish(_consumer); + } + } + } + + size_t scan_securities() { + if (_consumer.all_heartbeats_ok()) { + TimeStamp start = TimeStamp::now(); + auto count = _consumer.scan_securities(); + auto delta = (TimeStamp::now() - start); + if (delta.milliseconds() > 10) { + LOG(md,Diag,"Scanned handles: {}ms found: {} securities. " , + delta.milliseconds(),count); + } + return count; + } + return 0; + } +}; + + +MDClient::MDClient( + utils::Dispatcher* d, + api::ISyntheticSecurityService* s, + const std::set* symbols) : + _self(new MDClientImpl(d,s, symbols)) + { } +MDClient::~MDClient() { +} + +size_t MDClient::scan_securities() { return _self->scan_securities(); } +void MDClient::check_heartbeats() { return _self->check_heartbeats(); } + +utils::EventSubscriber& MDClient::on_consumer() { + return _self->_event; +} +MDConsumer& MDClient::current_consumer() const { + return _self->_consumer; +} +SecurityOracle MDClient::oracle() const { + return SecurityOracle(*this); +} +std::set MDClient::symbols() const { + return _self->_symbols; +} + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h new file mode 100644 index 000000000..47925bc88 --- /dev/null +++ b/src/lib/md/MDClient.h @@ -0,0 +1,37 @@ +#pragma once +#include "MDConsumer.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/Event.h" +#include "ISyntheticSecurityService.h" +#include "UserSymbol.h" +#include +#include + +namespace bts { +namespace md { + +class SecurityOracle; +struct MDClientImpl; +class MDClient { + std::unique_ptr _self; + +public: + // Will check for recovery + MDClient( + utils::Dispatcher* d=nullptr, + api::ISyntheticSecurityService* ss=nullptr, + const std::set* s=nullptr); + ~MDClient(); + + // Check heartbeats and attempt recovery if they are bad + void check_heartbeats(); + size_t scan_securities(); + + utils::EventSubscriber& on_consumer(); + MDConsumer& current_consumer() const; + SecurityOracle oracle() const; + std::set symbols() const; +}; + + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index dba9eda76..e296fc1d6 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -2,6 +2,7 @@ #include "MDConsumer.h" #include "MemorySegmentUtils.h" #include "PlatformLoader.h" +#include "SecurityCache.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -39,15 +40,15 @@ bool match_symbol(const string& val, const string& b) { return b.empty() || (val == b); } -typedef shared_ptr PSecurity; +typedef shared_ptr MutSecurityPtr; struct SecurityLess { - bool operator()(const PSecurity& left, const PSecurity& right) { + bool operator()(const SecurityPtr& left, const SecurityPtr& right) { return left->id() < right->id(); } }; -PSecurity convert_psecurity(const FixedSecurity& security) { - PSecurity psec = make_shared(); +MutSecurityPtr convert_psecurity(const FixedSecurity& security) { + auto psec = make_shared(); convert_security(security, psec.get()); return psec; } @@ -306,7 +307,7 @@ class SyntheticConsumerInst { // LOCK only public methods Mutex _workmutex; - shared_ptr _ss; + api::ISyntheticSecurityService* _ss; map _data; vector> _pending; set _seen; @@ -314,9 +315,10 @@ class SyntheticConsumerInst { bool _initialized; int _security_sub; int _meta_sub; + std::vector> _old_strategies; public: - SyntheticConsumerInst(std::shared_ptr& ss, const IGetHandle& gh) : + SyntheticConsumerInst(api::ISyntheticSecurityService* ss, const IGetHandle& gh) : _workmutex(), _ss(ss), _data(), @@ -325,7 +327,8 @@ class SyntheticConsumerInst { _get_handle(gh), _initialized(false), _security_sub(0), - _meta_sub(0) + _meta_sub(0), + _old_strategies() { } @@ -418,8 +421,13 @@ private: // update security metadata auto iter = _data.find(m.security_key()); if (iter != _data.end()) { - iter->second->strategy.CopyFrom(m.strategy()); + auto updated = std::make_unique(); + updated->CopyFrom(m.strategy()); + iter->second->strategy.swap(updated); iter->second->updated = TimeStamp::now(); + // Keep the old strategy alive because it could be in use. + // metadata is changed so seldomly this is not a significant use of memory + if (updated != nullptr) _old_strategies.push_back(std::move(updated)); } } @@ -475,7 +483,7 @@ private: if (legs.size() == (size_t)strategy.security->synthetic().legs_size()) { auto s = make_shared(); s->security.CopyFrom(*strategy.security); - s->strategy.CopyFrom(strategy.strategy); + s->strategy = std::make_unique(strategy.strategy); s->legs = legs; s->updated = TimeStamp::now(); result.insert( @@ -523,8 +531,28 @@ private: // Pimple for MDConsumer /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { + typedef tbb::queuing_mutex Mutex; + Mutex _workmutex; + vector> _instances; + std::shared_ptr _sci; + map> _deferred_handles; + vector _callbacks; + mutable SecurityCache _cache; + // co2::Platform _platform; + public: - MDConsumerImpl(const set& symbols, std::shared_ptr& ss, bool shared, co2::Platform platform = co2::PLATFORM_UNSET ) { + MDConsumerImpl( + const set& symbols, + SecurityCache& cache, + api::ISyntheticSecurityService* ss, + bool shared, co2::Platform platform) : + _workmutex(), + _instances(), + _sci(), + _deferred_handles(), + _callbacks(), + _cache(cache) + { std::ostringstream msg; for (const auto& symbol : symbols) { if (msg.tellp() > 0) { msg << ", "; } @@ -556,16 +584,21 @@ public: if (ss != nullptr) { _sci = std::make_shared(ss, [=](const set& keys) { return get_handles(keys); }); } + + // Add the SecurityCache to on_security so it can cache discovered securities + _callbacks.push_back([this](const SecurityPtr& security, MDConsumerHandle) { + _cache.add_live_security(security); + }); } ~MDConsumerImpl() { } // This is a terrible name size_t walk_sec_list(SecurityCallback cb, const string& symbol, SecurityType type) { size_t result = 0; - vector securities; + vector securities; for (auto& inst : _instances) { if (match_symbol(inst->symbol(), symbol) && match_type(inst->type(), type)) { - result += inst->walk_sec_list([&](PSecurity security) { securities.push_back(security); }, symbol, type); + result += inst->walk_sec_list([&](MutSecurityPtr security) { securities.push_back(security); }, symbol, type); } } auto complete_securities = fill_leg_details(securities); @@ -573,7 +606,7 @@ public: for (const auto& security : complete_securities) { auto find_result = handles.find(security->id()); if (find_result != handles.end()) { - cb(*security, find_result->second); + cb(security, find_result->second); } } return result; @@ -602,7 +635,8 @@ public: return empty_handle; } - map get_handles(const vector& securities) { + template //maybe const + map get_handles(const vector& securities) { set keys; for (const auto& sec : securities) { keys.insert(sec->id()); @@ -641,8 +675,8 @@ public: return handle; } - vector fill_leg_details(const vector& securities, int depth=0) { - vector result; + vector fill_leg_details(const vector& securities, int depth=0) { + vector result; if (securities.empty() || depth > 64) return result; // // Gather ids for legs @@ -655,14 +689,14 @@ public: } } // If there are no spreads everything is (by definition) resolved already - if (keys.empty()) return securities; + if (keys.empty()) return vector{securities.begin(), securities.end()}; // // gather handles for direct legs auto handles = get_handles(keys); // // Some of the legs will themselves be spreads - vector legs; + vector legs; legs.reserve(handles.size()); for (const auto& pair : handles) { if (pair.second.is_initialized()) { @@ -670,9 +704,8 @@ public: } } // recurse to find details - legs = fill_leg_details(legs, depth+1); - map leg_map; - for (auto& leg : legs) { + map leg_map; + for (auto& leg : fill_leg_details(legs, depth+1)) { leg_map.insert(make_pair(leg->id(), move(leg))); } @@ -705,7 +738,7 @@ public: if (_callbacks.empty() && _deferred_handles.empty()) return 0; // Call each _instance and pass it a callback for (auto& inst : _instances) { - vector found_securities; + vector found_securities; inst->scan_new_securities([&](const ColdEntry& entry) { // Check if any deferred handles need this security (could be sped up with get_handles() but it isn't hot auto iter = _deferred_handles.find(entry.security.key); @@ -719,10 +752,11 @@ public: if (resolve_legs && !found_securities.empty()) { sort(found_securities.begin(), found_securities.end(), SecurityLess()); - vector resolved_securities = fill_leg_details(found_securities); + vector resolved_securities = fill_leg_details(found_securities); sort(resolved_securities.begin(), resolved_securities.end(), SecurityLess()); + // Extract the list of unresolved securities - vector unresolved_securities; + vector unresolved_securities; set_difference(found_securities.begin(), found_securities.end(), resolved_securities.begin(), resolved_securities.end(), inserter(unresolved_securities, unresolved_securities.begin()), SecurityLess()); @@ -748,7 +782,7 @@ public: if (find_result != handles.end()) { // Pass new securities to each callback for (auto& cb : _callbacks) { - cb(*security, find_result->second); + cb(security, find_result->second); } } else { @@ -758,13 +792,13 @@ public: } count += resolved_securities.size(); } - else /*!resolve_legs*/ { + else /* do NOT resolve_legs*/ { auto handles = get_handles(found_securities); for (const auto& security : found_securities) { auto find_result = handles.find(security->id()); if (find_result != handles.end()) { for (auto& cb : _callbacks) { - cb(*security, find_result->second); + cb(security, find_result->second); } } else { @@ -788,11 +822,12 @@ public: } // call all subscribers for (auto& cb : _callbacks) { - cb(security, handle); + cb(std::make_shared(security), handle); } } } + _cache.flush_to_disk(); return count; } @@ -803,17 +838,20 @@ public: // Note: We have to lock over the initial walk because if scan_securities() finds a new security concurrently with this walk // it might mark it seen without this callback getting called. for (auto& inst : _instances) { - vector securities; + vector securities; inst->walk_old_securities([&](const ColdEntry& entry) { securities.push_back(convert_psecurity(entry.security)); }); - auto filled_securities = resolve_legs ? fill_leg_details(securities) : securities; + auto filled_securities = + resolve_legs ? + fill_leg_details(securities) : + vector{securities.begin(), securities.end()}; auto handles = get_handles(filled_securities); for (const auto& security : filled_securities) { auto find_result = handles.find(security->id()); bts_assert_d(find_result != handles.end()); if (find_result != handles.end()) { - cb(*security, find_result->second); + cb(security, find_result->second); } } } @@ -822,7 +860,7 @@ public: for (const auto& item : _sci->get_old_synthetics()) { const auto& security = item->security; auto handle = MDConsumerHandle(item); - cb(security, handle); + cb(std::make_shared(security), handle); } } // store the callback @@ -832,6 +870,10 @@ public: scan_securities(resolve_legs); } + SecurityPtr get_security(SecurityKey key) const { + return _cache.get_security(key); + } + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { for (auto& inst : _instances) { if ((inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) || @@ -858,14 +900,6 @@ public: return result; } -private: - typedef tbb::queuing_mutex Mutex; - Mutex _workmutex; - vector> _instances; - std::shared_ptr _sci; - map> _deferred_handles; - vector _callbacks; - // co2::Platform _platform; }; @@ -873,27 +907,20 @@ private: // MDConsumer public interface, forwards to pimple /////////////////////////////////////////////////////////////////////////////////////////////////// -MDConsumer::MDConsumer(std::shared_ptr& synt, bool shared) : _pimpl() { - set all_symbols; - get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); +MDConsumer::MDConsumer( + SecurityCache& cache, + api::ISyntheticSecurityService* ss, + const std::set* symbols, + bool shared, + co2::Platform platform) : _pimpl() { - _pimpl.reset(new MDConsumerImpl(all_symbols, synt, shared, co2::PLATFORM_UNSET)); -} -MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, bool shared) : - _pimpl() -{ - set symbols; - symbols.insert(s); - _pimpl.reset(new MDConsumerImpl(symbols, synt, shared, co2::PLATFORM_UNSET)); -} -MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, bool shared, co2::Platform platform) : - _pimpl(new MDConsumerImpl(symbols, s, shared, platform)) {} + set all_symbols; + if (symbols == nullptr || symbols->empty()) { + all_symbols = get_system_md_symbols(); + symbols = &all_symbols; + } -MDConsumer::MDConsumer(api::Venue ven, co2::Platform platform) : _pimpl() { - set all_symbols; - get_system_md_symbols(ven, SECURITYTYPE_UNSET, all_symbols); - std::shared_ptr null = nullptr; - _pimpl.reset(new MDConsumerImpl(all_symbols, null, true, platform)); + _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, shared, platform)); } MDConsumer::~MDConsumer() {} @@ -911,6 +938,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } +const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 643ef2635..02c762a8f 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -7,6 +7,7 @@ #include "MDConsumerHandle.h" #include "UserSymbol.h" #include "ISyntheticSecurityService.h" +#include "SecurityPtr.h" #include #include @@ -18,22 +19,23 @@ namespace bts { namespace md { -/* typedef void (security_callback)(const Security& security); */ -typedef std::function SecurityCallback; +typedef std::function SecurityCallback; using std::size_t; class MDConsumerImpl; class MDConsumerInst; class MDConsumerChannel; +class SecurityCache; + class MDConsumer { public: - // Consumer for all symbols - MDConsumer(std::shared_ptr&, bool shared=true); - MDConsumer(const UserSymbol&, std::shared_ptr, bool shared=true); - // Consumer for a given set of symbols - MDConsumer(std::set const& symbols, std::shared_ptr, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); - MDConsumer(api::Venue, co2::Platform); + MDConsumer( + SecurityCache&, + api::ISyntheticSecurityService* ss, + const std::set* sym = nullptr, + bool shared=true, + co2::Platform platform=co2::PLATFORM_UNSET); ~MDConsumer(); bool all_heartbeats_ok() const; @@ -51,6 +53,10 @@ public: size_t scan_securities(bool resolve_legs = true); void on_security(SecurityCallback cb, bool resolve_legs = true); + // Retrieve any security that has been seen in the past + // You still need to call scan_securities() periodically to discover new securities + const api::Security* get_security(SecurityKey key) const; + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 4062f1353..891f2ee90 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -16,7 +16,7 @@ MDConsumerHandle::MDConsumerHandle() { _type = HANDLETYPE_VOID; static_assert(sizeof(MDConsumerHandle) <= 32, ""); } -MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { +MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { memset(this, 0, sizeof(MDConsumerHandle)); synthetic.def = def; _type = HANDLETYPE_SYNTHETIC; @@ -223,6 +223,17 @@ void copy_synthetic_side(size_t count, api::Side side, const api::ContractSpecif } } +const spark::LegSpreaderParams& get_spreader_params(const SyntheticStrategy& synth, security_key key) { + auto strategy = synth.strategy.get(); + if (strategy) { + auto iter = strategy->leg_params_map().find(key); + if (iter != strategy->leg_params_map().end()) { + return iter->second; + } + } + return spark::LegSpreaderParams::default_instance(); +} + TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) { small_vector bid_leg_travs; small_vector ask_leg_travs; @@ -232,13 +243,7 @@ TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, Pri for (size_t x = 0; x < synth.legs.size(); ++x) { std::array bids, asks; TimeStamp t; - auto key = synth.legs[x].key(); - auto iter = synth.strategy.leg_params_map().find(key); - const spark::LegSpreaderParams& leg_params = - iter == synth.strategy.leg_params_map().end() ? - spark::LegSpreaderParams::default_instance() : - iter->second; - + const spark::LegSpreaderParams& leg_params = get_spreader_params(synth, synth.legs[x].key()); auto mdtype = leg_params.market_data_type(); HandleReadResult r; if (mdtype == spark::LegSpreaderParams::DIRECT) { diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 2000ab7cd..770b8ee59 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -14,7 +14,7 @@ namespace md { class MDConsumerHandle; struct SyntheticStrategy { - spark::ExecutionStrategy strategy; + std::unique_ptr strategy; api::Security security; std::vector legs; TimeStamp updated; @@ -147,7 +147,7 @@ class BTS_EXPORT MDConsumerHandle { public: MDConsumerHandle(); - explicit MDConsumerHandle(const std::shared_ptr&); + explicit MDConsumerHandle(const std::shared_ptr&); ~MDConsumerHandle(); MDConsumerHandle(const MDConsumerHandle& other); @@ -335,7 +335,7 @@ private: const ColdHeader* cheader; }; struct SyntheticHandle { - std::shared_ptr def; + std::shared_ptr def; }; HandleType _type; union { diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 01b0616bb..5027f3d75 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -2,6 +2,7 @@ #include "common.h" #include "UserSymbol.h" #include "MDConsumer.h" +#include "SecurityCache.h" #include "get_symbols.h" #include "relay/RelayClient.h" #include "lib/utils/Dispatcher.h" @@ -26,17 +27,12 @@ struct BlitUserSymbol { }; CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, bool shared) { - std::shared_ptr pnull = nullptr; - if (usersymbols == nullptr || count == 0) { - return new MDConsumer(pnull, shared); - } - else { - std::set symbols; - for (size_t x = 0; x < count; ++x) { - symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); - } - return new MDConsumer(symbols, pnull, shared); + SecurityCache cache; + std::set symbols; + for (size_t x = 0; x < count; ++x) { + symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } + return new MDConsumer(cache, nullptr/*synthetics*/, &symbols, shared); } CAPI void destroy_md_consumer(MDConsumer* consumer) { @@ -64,10 +60,10 @@ typedef void(*security_cb)(security_key, const char*, size_t, MDConsumerHandle& CAPI size_t md_consumer_walk_security_list(MDConsumer* consumer, security_cb cb) { bts_assert(consumer); std::string buffer; - return consumer->walk_security_list([&](const Security& security, MDConsumerHandle handle) { + return consumer->walk_security_list([&](const SecurityPtr& security, MDConsumerHandle handle) { buffer.clear(); - security.SerializePartialToString(&buffer); - cb(security.id(), buffer.c_str(), buffer.size(), handle); + security->SerializePartialToString(&buffer); + cb(security->id(), buffer.c_str(), buffer.size(), handle); }); } @@ -94,10 +90,10 @@ void safe_call_clr_callback(security_cb cb, security_key key, const char* buffer CAPI void md_consumer_set_on_security(MDConsumer* consumer, security_cb cb) { bts_assert(consumer); - consumer->on_security([=](const Security& security, MDConsumerHandle handle) { + consumer->on_security([=](const SecurityPtr& security, MDConsumerHandle handle) { std::string buffer; - security.SerializePartialToString(&buffer); - safe_call_clr_callback(cb, security.id(), buffer.c_str(), buffer.size(), handle); + security->SerializePartialToString(&buffer); + safe_call_clr_callback(cb, security->id(), buffer.c_str(), buffer.size(), handle); }); } #endif diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp deleted file mode 100644 index 31a554ee5..000000000 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef NO_CPP0X -#include "MDRecoverableConsumer.h" -#include "PlatformLoader.h" -#include "lib/logging/logging.h" -#include - - -using namespace std; -namespace bts { -namespace md { - -set get_symbols(Venue v=api::VENUE_UNSET); - -MDRecoverableConsumer::MDRecoverableConsumer(std::shared_ptr ss) : - _symbols(get_symbols()), - _ss(ss), - _current(init(_symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{} - -MDRecoverableConsumer::MDRecoverableConsumer(Venue exchange, std::shared_ptr ss) : // (visual studio 2012) MDRecoverableConsumer(get_symbols(exchange)) - _symbols(get_symbols(exchange)), - _ss(ss), - _current(init(_symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{} - -MDRecoverableConsumer::MDRecoverableConsumer(std::set symbols, std::shared_ptr ss) : - _symbols(symbols), - _ss(ss), - _current(init(symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{ -} - -MDRecoverableConsumer::~MDRecoverableConsumer() {} - -unique_ptr MDRecoverableConsumer::init(std::set& symbols, std::shared_ptr& ss) { - return std::make_unique(symbols, ss); -} - -size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { - return _current->consumer.walk_security_list(callback); -} - -void MDRecoverableConsumer::on_security(SecurityCallback cb) { - // Add a filter to the callback to avoid calling the on_security callback with duplicates - auto filtered_cb = [this,cb](const Security& security, MDConsumerHandle handle) { - auto key = security.id(); - if (_seen.find(key) == _seen.end()) { - _seen.insert(key); - cb(security, handle); - } - }; - _current->consumer.on_security(filtered_cb); - Mutex::scoped_lock l (_lock); - _callbacks.push_back(filtered_cb); -} - -size_t MDRecoverableConsumer::scan_securities() { - return _current->consumer.scan_securities(); -} - -std::shared_ptr MDRecoverableConsumer::get_managed_handle(security_key key) { - Mutex::scoped_lock l (_lock); - auto handle = _current->consumer.get_managed_handle(key); - _current->handles.push_back(handle); - return handle; -} - -bool MDRecoverableConsumer::all_heartbeats_ok() const { - return _current->consumer.all_heartbeats_ok(); -} - -bool MDRecoverableConsumer::try_recover() { - Mutex::scoped_lock l (_lock); - // Replace current with a new one (if the new one is better) - auto temp = init(_symbols, _ss); - if (!temp->consumer.all_heartbeats_ok()) return false; - - std::swap(temp, _current); - // push current to old - _old.push_back(std::move(temp)); - // copy old callbacks so we can reinstate them - auto old_callbacks = _callbacks; - - // Add an on_security callback to current to check the old handles - l.release(); - - // re-instante old callbacks - for (auto &cb : old_callbacks) { - _current->consumer.on_security(cb); - } - - _current->consumer.on_security([&](const Security& security, MDConsumerHandle newhandle) { - Mutex::scoped_lock l (_lock); - for (auto& group : _old) { - for (auto& handle : group->handles) { - if (handle->key() == security.id()) { - // update the handle! (if this is called guarunteed success) - *handle = newhandle; - bts_assert(handle->is_initialized()); - _current->handles.push_back(move(handle)); - } - } - } - cleanup(); - }); - - return true; -} - -// cleanup should only ever be called while _lock is locked -void MDRecoverableConsumer::cleanup() { - for (auto& group : _old) { - auto& handles = group->handles; - handles.erase( - remove_if( - handles.begin(), - handles.end(), - [](shared_ptr& handle) { return handle == nullptr; }), - handles.end()); - } - auto old_sz = _old.size (); - _old.erase( - remove_if( - _old.begin(), - _old.end(), - [](unique_ptr& pgroup) -> bool { return pgroup->handles.empty(); }), - _old.end()); - if (old_sz > _old.size ()) { - LOG (md,Info,"Cleaned up {} MDConsumer instances" ,old_sz - _old.size () ); - } -} - - -// -// Utility functions -set get_symbols(Venue exchange) { - set all_symbols; - get_system_md_symbols(exchange, api::SECURITYTYPE_UNSET, all_symbols); - return all_symbols; -} - - -}}//namespace -#endif - diff --git a/src/lib/md/MDRecoverableConsumer.h b/src/lib/md/MDRecoverableConsumer.h deleted file mode 100644 index 95068587d..000000000 --- a/src/lib/md/MDRecoverableConsumer.h +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once - -#include "lib/utils/bts_no_copy.h" -#include "common.h" -#include "MDConsumer.h" -#include "MDConsumerHandle.h" -#include "UserSymbol.h" -#include -#include -#include - -namespace bts { -namespace md { - -class BTS_EXPORT MDRecoverableConsumer -{ -public: - MDRecoverableConsumer(std::shared_ptr ss); - // Map only a certain exchange for more granular recovery - MDRecoverableConsumer(Venue, std::shared_ptr ss); - MDRecoverableConsumer(std::set symbols, std::shared_ptr ss); - ~MDRecoverableConsumer(); - - // - // Security functions - size_t walk_security_list(SecurityCallback callback); - void on_security(SecurityCallback cb); - // Scans for added securities - updates managed handles - size_t scan_securities(); - - // - // Data Access - std::shared_ptr get_managed_handle(security_key key); - - // - // Recover from publisher failure - bool all_heartbeats_ok() const; - bool try_recover(); - -private: - BTS_NO_COPY(MDRecoverableConsumer); - - void cleanup(); - - struct MDConsumerGroup { - MDConsumerGroup( - std::set& symbols, - std::shared_ptr& ss) : consumer(symbols, ss), handles() {} - MDConsumer consumer; - std::vector> handles; - }; - std::unique_ptr init(std::set& symbols, std::shared_ptr& ss); - - std::set _symbols; - std::shared_ptr _ss; - std::unique_ptr _current; - std::vector> _old; - - std::vector _callbacks; - std::set _seen; - typedef tbb::mutex Mutex; - Mutex _lock; -}; - -}}//namespace - diff --git a/src/lib/md/PlatformLoader.cpp b/src/lib/md/PlatformLoader.cpp index acd1ace54..717e8e88c 100644 --- a/src/lib/md/PlatformLoader.cpp +++ b/src/lib/md/PlatformLoader.cpp @@ -172,6 +172,12 @@ size_t get_system_md_symbols(const md::PlatformConfig &config, return get_system_md_symbols(v, st, out_result); } +std::set get_system_md_symbols() { + std::set all_symbols; + get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); + return all_symbols; +} + extern "C" BTS_EXPORT Platform platform_for_exchange(Venue v) { const auto &pmap = config::get(); co2::Platform platform; diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp new file mode 100644 index 000000000..160633145 --- /dev/null +++ b/src/lib/md/SecurityCache.cpp @@ -0,0 +1,91 @@ +#include "SecurityCache.h" +#include "SecurityDB.h" +#include "lib/utils/Dispatcher.h" + +#include +#include +#include + +namespace bts { +namespace md { + +struct SecurityCacheImpl { + std::mutex cache_mutex; + std::unordered_map cache; + tbb::concurrent_queue update_queue; + void enqueue_update_db(const SecurityPtr&); + void flush(); +}; + +void SecurityCacheImpl::enqueue_update_db(const SecurityPtr& security) { + update_queue.push(security); +} + +void SecurityCacheImpl::flush() { + if (!update_queue.empty()) { + std::vector work; + SecurityPtr temp; + while(update_queue.try_pop(temp)) { work.push_back(temp); } + if (!work.empty()) { + if (db::insert_or_replace(work) == Result::error) { + // If we fail to insert then retry later + for (auto& s : work) update_queue.push(std::move(s)); + } + } + } +} + +////////////////// + +SecurityCache::SecurityCache() : _me(std::make_shared()) {} +SecurityCache::~SecurityCache() {} + +SecurityPtr SecurityCache::try_get_security(SecurityKey key) const { + std::lock_guard lock(_me->cache_mutex); + auto iter = _me->cache.find(key); + if (iter != _me->cache.end()) { + return iter->second; + } + return nullptr; +} + +SecurityPtr SecurityCache::get_security(SecurityKey key) { + std::unique_lock lock(_me->cache_mutex); + auto iter = _me->cache.find(key); + if (iter != _me->cache.end()) { + return iter->second; + } + else { + // unlock for db read + lock.unlock(); + auto security = db::find_security(key); + if (security) { + LOG(md,Info, "Loaded security from database: {} {}", security->symbol(), security->id()); + return add_db_security(security); + } + return security; + } +} + +SecurityPtr SecurityCache::add_db_security(const SecurityPtr& security) { + bts_assert(security); + std::lock_guard lock(_me->cache_mutex); + SecurityKey key(security->id()); + auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); + // If result was inserted result.first->second is equal to security + // If the value was already present it must be a recently inserted 'live' security and we should use it + return result.first->second; +} +bool SecurityCache::add_live_security(const SecurityPtr& security) { + bts_assert(security); + std::lock_guard lock(_me->cache_mutex); + SecurityKey key(security->id()); + // Insert but don't overwrite existing + auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); + _me->enqueue_update_db(security); + return result.second; +} + +void SecurityCache::flush_to_disk() { _me->flush(); } + +}}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h new file mode 100644 index 000000000..e4c2ef2c4 --- /dev/null +++ b/src/lib/md/SecurityCache.h @@ -0,0 +1,36 @@ +#pragma once + +#include "lib/domain_model/Security.pb.h" +#include "MDConsumerHandle.h" +#include "lib/domain_model/SecurityKey.h" +#include "SecurityPtr.h" +#include + +namespace bts { +namespace md { + +// Get security on demand +// Store security+handle +struct SecurityCacheImpl; +class SecurityCache { + std::shared_ptr _me; + +public: + SecurityCache(); + ~SecurityCache(); + + //Load security from cache with fallback to database + SecurityPtr get_security(SecurityKey key); + + // Get the security if it's in the cache otherwise return nullptr. Does not query security database. + SecurityPtr try_get_security(SecurityKey key) const; + + // returns true if the cache was updated + bool add_live_security(const SecurityPtr& security); + SecurityPtr add_db_security(const SecurityPtr& security); + + void flush_to_disk(); + +}; + +}}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 03ac7d67e..e16bb7584 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -1,7 +1,9 @@ #include "SecurityDB.h" #ifndef WIN32 -#include "lib/utils/Sqlite_utils.h" +#include +#include + #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" #include "lib/md/Shm/FixedSecurity.h" @@ -15,6 +17,7 @@ using std::string; using namespace bts::utils; +using namespace SQLite; namespace bts { namespace md { @@ -25,7 +28,7 @@ namespace db { * key, created_date, expiration_date */ -sqlite3_ptr open_db() { +std::unique_ptr open_db() { // Directory location on disk auto db_dir = config::get().data_path() + "databases/md/"; auto db_path = db_dir + "/securities.sqlite"; @@ -34,20 +37,14 @@ sqlite3_ptr open_db() { } // Create table if it does not exist - auto dbptr = sqlite::open_db(db_path.c_str()); - bts_assert(dbptr); - auto db = dbptr.get(); - sqlite3_busy_timeout(db, 9999); - sqlite::exec_nofail(db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(db, "PRAGMA syncronous=0"); + auto db = std::make_unique(db_path, OPEN_READWRITE | OPEN_CREATE); + db->setBusyTimeout(9999); + db->exec("PRAGMA journal_mode=WAL"); + db->exec("PRAGMA syncronous=0"); + db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); + db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); - sqlite::exec_nofail(db, - "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); - - sqlite::exec_nofail(db, - "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); - - return dbptr; + return db; } datetime::Date sec_expiration_cut_off() @@ -59,44 +56,45 @@ datetime::Date sec_expiration_cut_off() return datetime::Date(pt.date().year(),pt.date().month(),pt.date().day()); } +void apply_compatibilit_mutations(Security& security) { + // Migrate from old tick stuff + if (security.has_glbx()) { + if (security.has_glbx()) { + if (security.contract_spec().main_fraction() == 0) { + security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); + } + if (security.contract_spec().sub_fraction() == 0) { + security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); + } + if (security.contract_spec().price_display_format() == 0) { + security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); + } + } + } +} + size_t load_securities( const std::set& symbols, const std::function& callback) { size_t count = 0; - auto dbptr = open_db(); + auto db = open_db(); // Load all existing securities and call the callback - auto db = dbptr.get(); - auto select_stmt = sqlite::prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); - auto select = select_stmt.get(); - sqlite3_bind_int(select, 1, sec_expiration_cut_off().to_short()); - auto result = sqlite3_step(select); + SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp"); + select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); + Security security; - while (result == SQLITE_ROW) { - security_key security_key = sqlite3_column_int(select, 0); - auto symbol = std::string( - (char*)sqlite3_column_text(select, 1), - sqlite3_column_bytes(select, 1)); - auto venue = (api::Venue)sqlite3_column_int(select, 2); - auto type = (api::SecurityType)sqlite3_column_int(select, 3); + while (select_stmt.tryExecuteStep() == SQLITE_ROW) { + security_key security_key = select_stmt.getColumn("key"); + std::string symbol = select_stmt.getColumn("symbol"); + api::Venue venue = (api::Venue)(int) select_stmt.getColumn("venue"); + auto type = (api::SecurityType)(int) select_stmt.getColumn("type"); UserSymbol us(symbol, venue, type); if (symbols.find(us) != symbols.end()) { - auto data = sqlite3_column_blob(select, 4); - auto len = sqlite3_column_bytes(select, 4); - // Migrate from old tick stuff - if (security.ParseFromArray(data, len)) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); - } - } + auto blob = select_stmt.getColumn("security_blob"); + if (security.ParseFromArray(blob, blob.getBytes())) { + apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -105,64 +103,85 @@ size_t load_securities( LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); } } - result = sqlite3_step(select); } return count; } -void store_if_new(const std::vector& securities) { - auto dbptr = open_db(); - auto insert = sqlite::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - sqlite::exec_nofail(dbptr, "BEGIN TRANSACTION"); - std::string buffer; - for (const api::Security* psecurity : securities) { - if (psecurity && psecurity->SerializeToString(&buffer)) { - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, psecurity->id()), dbptr); - SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, psecurity->symbol().c_str(), psecurity->symbol().size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, psecurity->exchange()), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, expiration(*psecurity).to_short()), dbptr); - sqlite3_step(insert.get()); - sqlite3_reset(insert.get()); +SecurityPtr find_security(SecurityKey key) { + try { + auto db = open_db(); + SQLite::Statement select(*db, "select security_blob from securities where key = :key"); + select.bind(":key", key.value()); + if(select.tryExecuteStep() == SQLITE_ROW) { + auto result = std::make_shared(); + auto blob = select.getColumn(0); + if(result->ParseFromArray(blob, blob.getBytes())) { + apply_compatibilit_mutations(*result); + return result; + } } } - sqlite::exec_nofail(dbptr, "COMMIT TRANSACTION"); + catch(const SQLite::Exception& e) { + LOG(md,Error, "Error reading security database. {}", std::string(e.what())); + } + return nullptr; +} + +Result insert_or_replace(const std::vector& securities) { + try { + auto db = open_db(); + SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + SQLite::Transaction transaction(*db); + std::string buffer; + for (const auto& psecurity : securities) { + if (psecurity && psecurity->SerializeToString(&buffer)) { + insert.bind(1, psecurity->id()); + insert.bind(2, psecurity->symbol()); + insert.bind(3, psecurity->exchange()); + insert.bind(4, securitytype_of(*psecurity)); + insert.bind(5, buffer); + insert.bind(6, expiration(*psecurity).to_short()); + insert.exec(); + insert.reset(); + } + } + transaction.commit(); + } + catch(const SQLite::Exception& e) { + LOG(md,Warn, "Error writing to security db (retry pending): {}", std::string(e.what())); + return Result::error; + } + return Result::ok; } std::vector load_metadata() { std::vector values; - auto dbptr = open_db(); - auto db = dbptr.get(); - auto select_stmt = sqlite::prepare_query(db, "select key,data from metadata"); - auto select = select_stmt.get(); - auto result = sqlite3_step(select); - while(result == SQLITE_ROW) { - uint32_t key = sqlite3_column_int(select, 0); - auto blob = sqlite3_column_blob(select, 1); - auto len = sqlite3_column_bytes(select, 1); - api::synthetic::Metadata data; - if (data.ParseFromArray(blob, len)) { - bts_assert(key == data.security_key()); - values.push_back(data); + auto db = open_db(); + SQLite::Statement select(*db, "select key,data from metadata"); + while(select.tryExecuteStep() == SQLITE_ROW) { + uint32_t key = select.getColumn(0); + auto blob = select.getColumn(1); + api::synthetic::Metadata meta; + if (meta.ParseFromArray(blob, blob.getBytes())) { + bts_assert(key == meta.security_key()); + values.push_back(meta); } else { LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); } - result = sqlite3_step(select); } return values; } bool store_metadata(const api::synthetic::Metadata& data) { - auto dbptr = open_db(); - auto insert = sqlite::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); + auto db = open_db(); + SQLite::Statement insert(*db, "insert or replace into metadata (key, data) values (?,?)"); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); + insert.bind(1, data.security_key()); std::string buffer = data.SerializeAsString(); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 2, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - auto result = sqlite3_step(insert.get()); + insert.bind(2, buffer); + auto result = insert.tryExecuteStep(); if (result != SQLITE_DONE) { LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); return false; @@ -180,13 +199,8 @@ size_t load_securities( const std::function& callback) { return 0; } -void store_if_new(const std::vector& handles) {} -// std::vector load_metadata() { -// std::vector result; -// return result; -// } -// void store_metadata(const api::synthetic::Metadata&) { -// } +Result insert_or_replace(const std::vector& handles) { return Result::error; } +SecurityPtr find_security(SecurityKey) { return nullptr; } #endif diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 3e7b9e2ec..b04f355da 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -3,10 +3,13 @@ #include "lib/domain_model/common.h" #include "lib/domain_model/Security.pb.h" #include "lib/utils/datetime.h" +#include "lib/utils/ResultEnum.h" #include "lib/md/MDProducerHandle.h" #include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" #include "UserSymbol.h" +#include "lib/domain_model/SecurityKey.h" +#include "SecurityPtr.h" #include #include @@ -25,7 +28,9 @@ size_t load_securities( const std::set& symbols, const std::function& callback); -void store_if_new(const std::vector& handles); +SecurityPtr find_security(SecurityKey); + +Result insert_or_replace(const std::vector&); // metadata std::vector load_metadata(); diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index f8cc32613..afeea6f3b 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -5,7 +5,6 @@ #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" -#include "lib/md/SecurityDB.h" #include #include @@ -279,19 +278,6 @@ void SecurityIDRecorderState::work_db() { sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } - // record the securities - if (!new_entries.empty() || !existing_securities.empty()) { - std::vector save_these; - save_these.reserve(new_entries.size() + existing_securities.size()); - for (const auto& e : new_entries) { - save_these.push_back(&e->security); - } - for (const auto& e : existing_securities) { - save_these.push_back(&e->security); - } - db::store_if_new(save_these); - } - if (!new_entries.empty()) { for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp new file mode 100644 index 000000000..7691f6e4c --- /dev/null +++ b/src/lib/md/SecurityOracle.cpp @@ -0,0 +1,13 @@ +#include "SecurityOracle.h" +#include "MDClient.h" + +namespace bts { +namespace md { + +SecurityOracle::SecurityOracle(const MDClient& client): _client(client) {} +SecurityOracle::~SecurityOracle() { } +const api::Security* SecurityOracle::get_security(SecurityKey key) const { + return _client.current_consumer().get_security(key); +} + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h new file mode 100644 index 000000000..82163a1e1 --- /dev/null +++ b/src/lib/md/SecurityOracle.h @@ -0,0 +1,17 @@ +#pragma once +#include "lib/domain_model/SecurityKey.h" +#include + +namespace bts { +namespace api {class Security;} +namespace md { +class MDClient; +class SecurityOracle { + const MDClient& _client; +public: + SecurityOracle(const MDClient&); + ~SecurityOracle(); + const api::Security* get_security(SecurityKey key) const; +}; + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityPtr.h b/src/lib/md/SecurityPtr.h new file mode 100644 index 000000000..228c9a198 --- /dev/null +++ b/src/lib/md/SecurityPtr.h @@ -0,0 +1,12 @@ +#pragma once + +#include "lib/domain_model/SecurityKey.h" +#include + +namespace bts { +namespace api {class Security;} +namespace md { + + typedef std::shared_ptr SecurityPtr; + +}} \ No newline at end of file diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h index 0f07c0080..32b169192 100644 --- a/src/lib/md/SyntheticSecurityService.h +++ b/src/lib/md/SyntheticSecurityService.h @@ -20,9 +20,9 @@ class SyntheticSecurityService : struct Pimpl; std::unique_ptr _me; - public: - SyntheticSecurityService(api::ServiceContext& context); - ~SyntheticSecurityService(); +public: + SyntheticSecurityService(api::ServiceContext& context); + ~SyntheticSecurityService(); const std::unordered_map securities() override; utils::EventSubscriber& on_securities() override; diff --git a/src/lib/md/coconut2.h b/src/lib/md/coconut2.h index 79230eb4c..4195a0a40 100644 --- a/src/lib/md/coconut2.h +++ b/src/lib/md/coconut2.h @@ -9,5 +9,4 @@ #include "SettlementRecorder.h" // TODO: separate coconut2.h into consumer and producer sides #include "SecurityIndex.h" -#include "MDRecoverableConsumer.h" diff --git a/src/lib/md/get_symbols.h b/src/lib/md/get_symbols.h index 3088e1b77..d0226432d 100644 --- a/src/lib/md/get_symbols.h +++ b/src/lib/md/get_symbols.h @@ -8,6 +8,7 @@ namespace bts { namespace md { +std::set get_system_md_symbols(); size_t get_system_md_symbols(api::Venue, api::SecurityType, std::vector* result); size_t get_system_md_symbols(api::Venue, api::SecurityType, std::set& out_result); size_t get_system_md_symbols(const md::PlatformConfig& config, std::set& out_result); diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index dea02db98..e8331308b 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -7,12 +7,14 @@ #include "lib/utils/AuthToken.h" #include "../MDConsumer.h" +#include "../MDClient.h" #include "../PlatformLoader.h" #include "lib/logging/logging.h" #include "lib/config/protos/system.pb.h" #include "lib/md/protos/co2.pb.h" +#include "lib/md/SecurityCache.h" #include "lib/api/apicore.pb.h" #include "lib/api/services/MarketDataRelay.server.pb.h" #include "lib/api/public/ServiceContext.h" @@ -162,29 +164,34 @@ void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) { // Handles market data queries and responds class QueryServerImpl : public IQueryServer { + MDClient& _client; MDConsumer _consumer; - const std::set _symbols; - const std::shared_ptr _ss; + int _sub; boost::container::flat_map> _groups; public: - QueryServerImpl(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) : - _consumer(symbols, ss), - _symbols(symbols), - _ss(ss), + QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : + _client(client), + _consumer(client.current_consumer()), + _sub(0), _groups() { - for (const auto& sym : symbols) { + for (const auto& sym : client.symbols()) { _groups.emplace(std::make_pair(sym, std::make_unique(sym, _consumer, stats))); } _groups.shrink_to_fit(); + _sub = client.on_consumer().subscribe([=](const MDConsumer& c) { handle_recovery(c); }); subscribe_securities(); } + QueryServerImpl(const QueryServerImpl&)=delete; + ~QueryServerImpl() { + _client.on_consumer().unsubscribe(_sub); + } void subscribe_securities() { auto id = _consumer.id(); - _consumer.on_security([this,id](const Security&, MDConsumerHandle handle) { + _consumer.on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { if (_consumer.id() == id) { auto iter = _groups.find(handle.get_usersymbol()); bts_assert(iter != _groups.end()); @@ -196,31 +203,16 @@ public: } // not-reentrant - void scan_securities() { - if (_consumer.all_heartbeats_ok()) { - TimeStamp start = TimeStamp::now(); - size_t count = _consumer.scan_securities(false); - auto delta = (TimeStamp::now() - start); - if (delta.milliseconds() > 10) { - LOG(md,Diag,"Scanned handles: {}ms found: {} securities. " , - delta.milliseconds(),count); - } - } - } - - // not-reentrant - void check_heartbeats(utils::Dispatcher& dispatcher) { - if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_symbols, _ss); - if (temp.all_heartbeats_ok()) { - _consumer.swap(temp); - LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); - for (auto& pair : _groups) { - pair.second->set_consumer(_consumer); - } - subscribe_securities(); - dispatcher.post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); + void handle_recovery(const MDConsumer& consumer) { + MDConsumer temp(consumer); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); + for (auto& pair : _groups) { + pair.second->set_consumer(_consumer); } + subscribe_securities(); + utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } } @@ -388,6 +380,18 @@ public: } } + void get_securities(google::protobuf::RepeatedField keys, api::SecurityList* result) const { + bts_assert(result); + // Ensure keys are unique + std::set unique_keys(keys.begin(), keys.end()); + for (auto k : unique_keys) { + auto ptr = _consumer.get_security(SecurityKey(k)); + if (ptr) { + result->add_security()->CopyFrom(*ptr); + } + } + } + }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -474,56 +478,37 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } + + virtual void get_securities(const api::AuthToken&, const SecurityKeyList& keys, const std::function& reply) override { + _threadpool.post([=]() { + google::protobuf::Arena arena; + api::SecurityList* preply = google::protobuf::Arena::CreateMessage(&arena); + _db.get_securities(keys.keys(), preply); + reply(*preply); + }); + } }; struct QueryServer { - QueryServer(std::set symbols, - utils::Dispatcher& threadpool, + QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, - std::shared_ptr& ss, + MDClient& client, metrics::MetricsContainer& stats): - db(symbols, ss, stats), + db(client, stats), relay(db, threadpool, ctx, stats), - admin_client(ctx.locate()), - security_timer(), - health_timer() + admin_client(ctx.locate()) { } QueryServerImpl db; MDRelayServer relay; api::admin::AdminService admin_client; - std::unique_ptr security_timer; - std::unique_ptr health_timer; rxcpp::composite_subscription entitlement_sub; }; -QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr& ss, metrics::MetricsContainer& stats) { +QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, MDClient& client, metrics::MetricsContainer& stats) { auto& threadpool = utils::Dispatcher::get_named("MDQuery"); - std::set all_symbols; - Venue venue; - bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); - SecurityType type; - bts_assert(api::SecurityType_Parse(config::get().type(), &type)); - get_system_md_symbols(venue, type, all_symbols); + QueryServerPtr container = std::make_shared(threadpool, ctx, client, stats); - // auto admin = ctx.locate(); - // admin.subscribe_md_entitlement_data().subscribe( - - QueryServerPtr container = std::make_shared(all_symbols, threadpool, ctx, ss, stats); - auto& db = container->db; - - // Scan for new securities every half second - container->security_timer = std::make_unique(threadpool, config::get().security_scan_interval(), [&db]() { - db.scan_securities(); - }); - container->security_timer->start(); - - // Scan for dead producers every 30 seconds - container->health_timer = std::make_unique(threadpool, config::get().recovery_interval(), [&db, &threadpool]() { - db.check_heartbeats(threadpool); - }); - container->health_timer->start(); - // Subscribe to entitlement data container->entitlement_sub = container->admin_client.subscribe_md_entitlement_data().subscribe([=](const api::admin::Md_entitlement_dataUpdate& update) { @@ -564,8 +549,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared return container; } -std::unique_ptr create_query_server(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) { - return std::make_unique(symbols, ss, stats); +std::unique_ptr create_query_server(MDClient& mdc, metrics::MetricsContainer& mets) { + return std::make_unique(mdc, mets); } }}//namespace diff --git a/src/lib/md/relay/QueryServer.h b/src/lib/md/relay/QueryServer.h index 13a8d056e..186089658 100644 --- a/src/lib/md/relay/QueryServer.h +++ b/src/lib/md/relay/QueryServer.h @@ -14,18 +14,18 @@ namespace bts { namespace api { class ServiceContext; } namespace metrics { class MetricsContainer; } namespace md { +class MDClient; typedef boost::container::flat_set Entitlements; struct IQueryServer { virtual ~IQueryServer() {} - virtual void scan_securities() = 0; virtual void query(const co2::MDRequest& request, co2::MDReply* reply, const Entitlements*, bool) const = 0; }; -std::unique_ptr create_query_server(const std::set&, std::shared_ptr&, metrics::MetricsContainer&); +std::unique_ptr create_query_server(MDClient&, metrics::MetricsContainer&); struct QueryServer; typedef std::shared_ptr QueryServerPtr; -QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr&, metrics::MetricsContainer&); +QueryServerPtr run_query_server(const bts::api::ServiceContext&, MDClient&, metrics::MetricsContainer&); }}//namespace diff --git a/src/lib/md/relay/RelayRecorder.cpp b/src/lib/md/relay/RelayRecorder.cpp index 124cdf920..d2a094b52 100644 --- a/src/lib/md/relay/RelayRecorder.cpp +++ b/src/lib/md/relay/RelayRecorder.cpp @@ -26,6 +26,7 @@ namespace bts { namespace md { const char* magic = "Co2"; +const uint32_t recorder_version = 3; template void writebuf(ofstream& out, const T& pb) { @@ -61,7 +62,7 @@ RelayRecorder::RelayRecorder(const std::string& base_path, const std::string& fi CoconutReplayHeader header; header.set_magic(magic); - header.set_version(3); + header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().config_host()); header.set_naggle_ms(naggle); @@ -132,7 +133,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { #ifndef _WIN32 md::SecurityIDRecorder idrec {{}}; #endif - if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) { + if (readbuf(in, &header) && header.magic() == magic && header.version() == recorder_version) { co2::MDReply message; while (readbuf(in, &message)) { bool has_securities = false; @@ -249,7 +250,7 @@ void unprint_recording(const std::string& src, const std::string& dest) { // Write out the header CoconutReplayHeader header; header.set_magic(magic); - header.set_version(2); + header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().host_name()); writebuf(out, header); diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 0ec5f24a2..9f0bf2aca 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1,8 +1,9 @@ - #include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" #include "../relay/QueryServer.h" +#include "../SecurityCache.h" +#include "../MDClient.h" #include "Co2TestInstrument.h" #include "lib/config/config.h" #include "lib/os/os.h" @@ -368,18 +369,19 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); vector consumer_securities; // Poll for security list - none visible until they have keys - size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); // Assign a unique ID to each security (from the producer) assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client - count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); @@ -438,14 +440,15 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback vector securities; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() != invalid_security_key); - REQUIRE(security.id() == handle.key()); - securities.push_back(security); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() != invalid_security_key); + REQUIRE(security->id() == handle.key()); + securities.push_back(*security); }); // No securities have ids yet REQUIRE(securities.size() == 0); @@ -477,7 +480,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -534,7 +538,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -593,15 +598,16 @@ public: assign_ids(producers.begin(), producers.end()); // Symbol list for the consumer - auto syms = get_test_symbols(); - MDConsumer consumer = MDConsumer(syms, nullptr); + auto symbols = get_test_symbols(); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback - callback fires for existing securities vector securities; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() != invalid_security_key); - REQUIRE(security.id() == handle.key()); - securities.push_back(security); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() != invalid_security_key); + REQUIRE(security->id() == handle.key()); + securities.push_back(*security); }); // Found all securities REQUIRE(securities.size() == producers.size()); @@ -620,20 +626,23 @@ public: for (auto& pro : producers) { pro.update_bbo(TimeStamp::now(), PriceSize({1, 0}, 1), PriceSize({2, 0}, 2)); } // Symbol list for the consumer - MDRecoverableConsumer consumer(get_test_symbols(), nullptr); - map> products; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - products.insert(make_pair(security.id(), consumer.get_managed_handle(security.id()))); + SecurityCache cache; + auto symbols = get_test_symbols(); + MDClient client(nullptr, nullptr, &symbols); + MDConsumer consumer = client.current_consumer(); + map products; + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + products.insert(make_pair(security->id(), handle)); }); - consumer.scan_securities(); + client.scan_securities(); REQUIRE(products.size() == producers.size()); - // Choose a handle + // Check handles for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating()); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating()); } // Delete the producer @@ -655,15 +664,15 @@ public: try { for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating() == false); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating() == false); } } catch(std::exception&) { LOG(md,Error,"Failed to validate heartbeat while producer is down."); REQUIRE(false); } - // Create a new publisher and attempt to recover + // Create a new producer and attempt to recover shared_ptr producer2; try { producer2 = make_shared(TEST_PLATFORM); @@ -683,16 +692,30 @@ public: } // Attempt to recover the managed handles - REQUIRE(consumer.try_recover()); + int on_consumer_count = 0; + auto handle = client.on_consumer().subscribe([&](const MDConsumer& c) { + ++on_consumer_count; + consumer = c; + products.clear(); + }); + client.check_heartbeats(); + REQUIRE(on_consumer_count == 1); + REQUIRE(products.empty()); + // Consumer has been updated + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + products.insert(make_pair(security->id(), handle)); + }); + client.on_consumer().unsubscribe(handle); - // Since we know that both securities exist they should have been updated to the new section + // Check that the new product handles have been updated for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating()); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating()); TimeStamp tzero; PriceSize bid, ask; - REQUIRE(handle->get_direct_book(tzero, 1, &bid, &ask) == HandleReadResult::SUCCESS); + REQUIRE(handle.get_direct_book(tzero, 1, &bid, &ask) == HandleReadResult::SUCCESS); REQUIRE(bid.price() == 4.0); REQUIRE(bid.size() == 5); REQUIRE(ask.price() == 6.0); @@ -737,10 +760,11 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); set keys; - consumer.walk_security_list([&](const Security& security, MDConsumerHandle) { keys.insert(security.id()); }); + consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); REQUIRE(keys.size() == producers.size()); auto map = consumer.get_handles(keys); REQUIRE(map.size() == keys.size()); @@ -781,18 +805,19 @@ public: csymbols.insert(UserSymbol("S2", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(csymbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); - auto check_sec = [&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); + auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security->id(); }); REQUIRE(iter != end(all_secs_array)); - REQUIRE(iter->symbol() == (security.symbol())); - if (security.Contract_case() == Security::kSpread) { - REQUIRE(security.spread().normalized() == false); - REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); - for (int x = 0; x < security.spread().legs_size(); ++x) { - REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); + REQUIRE(iter->symbol() == (security->symbol())); + if (security->Contract_case() == Security::kSpread) { + REQUIRE(security->spread().normalized() == false); + REQUIRE(security->spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security->spread().legs_size(); ++x) { + REQUIRE((security->spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; @@ -827,18 +852,19 @@ public: csymbols.insert(UserSymbol("FF", VENUE_BTS, SECURITYTYPE_FUTURE)); csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(csymbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); - auto check_sec = [&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); + auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security->id(); }); REQUIRE(iter != end(all_secs_array)); - REQUIRE((iter->id()) == (security.id())); - if (security.Contract_case() == Security::kSpread) { - REQUIRE(security.spread().normalized() == false); - REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); - for (int x = 0; x < security.spread().legs_size(); ++x) { - REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); + REQUIRE((iter->id()) == (security->id())); + if (security->Contract_case() == Security::kSpread) { + REQUIRE(security->spread().normalized() == false); + REQUIRE(security->spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security->spread().legs_size(); ++x) { + REQUIRE((security->spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; @@ -878,7 +904,8 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(consumer_symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); size_t security_count = 0; // Indirection for check function @@ -886,7 +913,7 @@ public: check_func = [&](const Security&) { REQUIRE(false); }; - consumer.on_security([&](const Security& sec, MDConsumerHandle) { ++security_count; check_func(sec); }); + consumer.on_security([&](const SecurityPtr& sec, MDConsumerHandle) { ++security_count; check_func(*sec); }); consumer.scan_securities(); REQUIRE(security_count == 0); @@ -946,7 +973,8 @@ public: auto ph2 = *add_security(o2); set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); - MDConsumer consumer(consumer_symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1212,6 +1240,7 @@ struct Co2IndexTests { } ; struct Co2RelayTests { + SecurityCache cache; Co2RelayTests() : _stats() {} void setup() { config::internal::initialize_for_test( @@ -1230,7 +1259,8 @@ struct Co2RelayTests { symbols.insert(future_symbol); UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols); return consumer; } @@ -1245,7 +1275,21 @@ struct Co2RelayTests { } metrics::MetricsContainer _stats; - std::unique_ptr setup_query_server(MDProducer& producer, Entitlements* out_entitlements, vector* out_producers) { + + const std::set setup_symbols() { + std::set result; + for (const auto& symbol : { "FOO", "FOOF" }) { + result.insert({symbol, VENUE_BTS, SECURITYTYPE_FUTURE}); + } + return result; + } + + std::unique_ptr setup_query_server( + MDClient& client, + MDProducer& producer, + Entitlements* out_entitlements, + vector* out_producers) { + // Create a bunch of securities for (size_t x = 0; x < 100; ++x) { // That's right. Expirations daily. @@ -1265,23 +1309,26 @@ struct Co2RelayTests { for (const auto& s : symbols) { out_entitlements->insert(s.symbol()); } - shared_ptr empty; - auto server = create_query_server(symbols, empty, _stats); + auto server = create_query_server(client, _stats); REQUIRE(server); - server->scan_securities(); + client.scan_securities(); return server; } void relay_nans() { MDProducer producer(TEST_PLATFORM); - UserSymbol symbol("FOO", VENUE_BTS, SECURITYTYPE_FUTURE); Entitlements entitlements; // initialize data vector producers; - auto server = setup_query_server(producer, &entitlements, &producers); + // + // todo: setup_query_server should use client.symbols() instead of it's own list + // + const auto symbols = setup_symbols(); + MDClient client(nullptr, nullptr, &symbols); + auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices PriceSize price35 = PriceSize_from_double<2>(3.5, 5); @@ -1292,14 +1339,15 @@ struct Co2RelayTests { // Request via query server co2::MDRequest request; - for (const auto& symbol : { "FOO", "FOOF" }) { + for (const auto& symbol : symbols) { auto* chan = request.add_channels(); - chan->set_type(SECURITYTYPE_FUTURE); - chan->set_symbol(symbol); - chan->set_exchange(VENUE_BTS); + chan->set_type(symbol.type()); + chan->set_symbol(symbol.symbol()); + chan->set_exchange(symbol.exchange()); chan->set_flags(co2::MDRequest::BBO | co2::MDRequest::SECURITY); chan->set_entitled(true); } + // This request should include everything co2::MDReply reply; @@ -1411,7 +1459,9 @@ struct Co2RelayTests { // initialize data vector producers; Entitlements entitlements; - auto server = setup_query_server(producer, &entitlements, &producers); + const auto symbols = setup_symbols(); + MDClient client(nullptr, nullptr, &symbols); + auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels auto set_direct = [&](PriceSize bid, PriceSize ask) { @@ -1587,16 +1637,17 @@ namespace ImpliedTests { symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols); vector consumer_securities; // Poll for security list - none visible until they have keys - size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client - count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); vector consumers; @@ -2360,4 +2411,3 @@ TEST_CASE("IntPrice", "[md]") { }}} - diff --git a/src/lib/md/test/synthetic_book_merge_tests.cpp b/src/lib/md/test/synthetic_book_merge_tests.cpp index 4d174207a..87e8ce79a 100644 --- a/src/lib/md/test/synthetic_book_merge_tests.cpp +++ b/src/lib/md/test/synthetic_book_merge_tests.cpp @@ -135,7 +135,10 @@ tuple,vector> make_securities(c void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) { - auto* map = strat.strategy.mutable_leg_params_map(); + if (!strat.strategy) { + strat.strategy = std::make_unique(); + } + auto* map = strat.strategy->mutable_leg_params_map(); map->insert({ security_key, params }); } diff --git a/src/lib/orderman/actors/AsyncFactory.cpp b/src/lib/orderman/actors/AsyncFactory.cpp index 91b76045e..f6543a5b2 100644 --- a/src/lib/orderman/actors/AsyncFactory.cpp +++ b/src/lib/orderman/actors/AsyncFactory.cpp @@ -7,7 +7,6 @@ #include "lib/orderman/events/CreateOrderReplyEvent.h" #include "lib/orderman/events/CreateOrderRequestEvent.h" #include "lib/orderman/events/NewAdminDataEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/order_factory/OrderFactory.h" #include "lib/orderman/order_router/OrderRouter.h" #include "lib/orderman/securities/MapSecurityDB.h" @@ -16,8 +15,8 @@ namespace bts { namespace orderman { struct AsyncFactory::PImpl { - PImpl(DynamicOrderRegistry const &b) - : orders_stream(), fix_buffer_stream(), db(), order_router(), + PImpl(DynamicOrderRegistry const &b, md::SecurityOracle o) + : orders_stream(), fix_buffer_stream(), db(o), order_router(), order_factory(db, order_router, b) {} mercury::Stream orders_stream; @@ -27,7 +26,7 @@ struct AsyncFactory::PImpl { OrderFactory order_factory; }; -AsyncFactory::AsyncFactory(DynamicOrderRegistry const &b) : me(new PImpl(b)) {} +AsyncFactory::AsyncFactory(DynamicOrderRegistry const &b, md::SecurityOracle o) : me(new PImpl(b, o)) {} AsyncFactory::~AsyncFactory() {} void AsyncFactory::init() { @@ -40,12 +39,6 @@ void AsyncFactory::init() { me->order_router.update(*ev.admin_data); }); - subscribe([this](auto const &u) { - if (u.events) - for (auto const &ev : *(u.events)) - me->db.securities[SecurityKey(ev.security->id())] = ev.security; - }); - subscribe([this](const auto &u) { if (u.events) for (auto const &ev : *u.events) { diff --git a/src/lib/orderman/actors/AsyncFactory.h b/src/lib/orderman/actors/AsyncFactory.h index 76214701c..d66aebab8 100644 --- a/src/lib/orderman/actors/AsyncFactory.h +++ b/src/lib/orderman/actors/AsyncFactory.h @@ -4,6 +4,7 @@ #include "lib/orderman/orderman_utils/orderman_common.h" #include "lib/mercury/actors/Actor.h" #include "lib/config/protos/jetfix.pb.h" +#include "lib/md/SecurityOracle.h" namespace bts { namespace orderman { @@ -13,7 +14,7 @@ class DynamicOrderRegistry; class AsyncFactory : public mercury::Actor { public: - AsyncFactory(DynamicOrderRegistry const&); + AsyncFactory(DynamicOrderRegistry const&, md::SecurityOracle); ~AsyncFactory(); void init() override; diff --git a/src/lib/orderman/actors/FillStore.cpp b/src/lib/orderman/actors/FillStore.cpp index 1e1f7bf85..3d6652726 100644 --- a/src/lib/orderman/actors/FillStore.cpp +++ b/src/lib/orderman/actors/FillStore.cpp @@ -2,8 +2,8 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/orderman/actors/SODPosition.h" #include "lib/orderman/events/NewExecutionReportEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderDefinitionUpdateEvent.h" +#include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/persistence/SqliteStore.h" #include "lib/utils/PBUtils.h" @@ -15,104 +15,69 @@ namespace orderman { auto const fills_ns = "fills"; -enum class FillStore::State { WAIT_SEC, READY }; - struct FillStore::UnresolvedFills { std::vector fills; api::Security const *security; }; struct FillStore::PImpl { - PImpl() - : state(State::WAIT_SEC), store(new persistence::SqliteStore("fills")), - known_orders(), fills_with_unresolved_accounts(), sod_pos_map() {} + PImpl(md::SecurityOracle &oracle) + : store(new persistence::SqliteStore("fills")), known_orders(), + fills_with_unresolved_accounts(), sod_pos_map(), oracle(oracle) {} - State state; std::unique_ptr store; std::unordered_set known_fills; std::unordered_map known_orders; std::unordered_map fills_with_unresolved_accounts; mercury::Stream fill_stream; - mercury::Actor::SubHandle stored_sec_sub; SODPosition sod_pos_map; + md::SecurityOracle oracle; }; -FillStore::FillStore() : me(new PImpl()) {} +FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {} void FillStore::init() { - me->state = State::WAIT_SEC; - me->stored_sec_sub = - subscribe([this](auto const &update) { - if (update.events) { - if (me->state == State::WAIT_SEC) { + // Subscribe to NewExecutionReportEvent for new fills + subscribe( + [this](auto const &ev) { this->on_execution_report(ev); }); - LOG(OrderMan, Info, - "Fill store received initial list of {} securities", - update.events->size()); + // Load stored fills, resolve their securities and publish the lot + api::orderman::Fill fill; + std::vector stored_batch; + LOG(persist, Info, "Loading load all fills"); + auto all = me->store->retrieve_all()[fills_ns]; + LOG(persist, Info, "Finished loading all {} fills", all.size()); + for (auto const &kvp : all) + if (utils::pb::from_string(kvp.second, &fill)) { - // don't need securities anymore, NewExecutionReportEvent - // events will carry the security with them - unsubscribe(me->stored_sec_sub); + auto sec = me->oracle.get_security(SecurityKey(fill.security_key())); + if (sec) { + auto fk = kvp.first; + if (!me->sod_pos_map.add_fill_to_sod_pos(fill)) + stored_batch.emplace_back(fill, *sec); + me->known_fills.insert(fk); + } else + LOG(persist, Error, + "Stored fill {} contains unresolved security {}. ", kvp.first, + fill.security_key()); + } else + LOG(persist, Error, + "Could not recover fill {} from storage. " + "This should be investigated", + kvp.first); + me->fill_stream = make_stream(); + LOG(OrderMan, Info, + "Fill Store published {} fills recoved from disk out of {} " + "total fills", + stored_batch.size(), all.size()); + me->sod_pos_map.add_sod_pos_as_fill_events(stored_batch, me->oracle); + publish_batch(me->fill_stream, stored_batch); - // Subscribe to NewExecutionReportEvent for new fills - subscribe( - [this](auto const &ev) { this->on_execution_report(ev); }); - - // index securities - std::map sec_idx; - for (auto const &ev : *(update.events)) { - auto sec_ptr = ev.security; - sec_idx[SecurityKey(sec_ptr->id())] = sec_ptr; - } - - // Load stored fills, resolve their securities and publish the - // lot - api::orderman::Fill fill; - std::vector stored_batch; - LOG(persist, Info, "Loading load all fills"); - auto all = me->store->retrieve_all()[fills_ns]; - LOG(persist, Info, "Finished loading all {} fills", all.size()); - for (auto const &kvp : all) { - auto fk = kvp.first; - me->known_fills.insert(fk); - - if (utils::pb::from_string(kvp.second, - &fill)) { - - auto sec = sec_idx[SecurityKey(fill.security_key())]; - if (sec) { - if (!me->sod_pos_map.add_fill_to_sod_pos(fill)) - stored_batch.emplace_back(fill, *sec); - } else - LOG(persist, Diag, - "Stored fill {} contains unresolved security {}. ", - kvp.first, fill.security_key()); - } else - LOG(persist, Error, - "Could not recover fill {} from storage. " - "This should be investigated", - kvp.first); - } - me->fill_stream = make_stream(); - LOG(OrderMan, Info, - "Fill Store published {} fills recoved from disk out of {} " - "total fills", - stored_batch.size(), all.size()); - me->sod_pos_map.add_sod_pos_as_fill_events(stored_batch, - sec_idx); - publish_batch(me->fill_stream, stored_batch); - - // subscribe to order updates so we can resolve - // account ids for future execution reports - subscribe([this](auto const &ev) { - this->on_order_definition_update(ev); - }); - - me->state = State::READY; - } - } - }); + // subscribe to order definition updates so we can resolve + // account ids for future execution reports + subscribe( + [this](auto const &ev) { this->on_order_definition_update(ev); }); } FillStore::~FillStore() {} diff --git a/src/lib/orderman/actors/FillStore.h b/src/lib/orderman/actors/FillStore.h index 732559cda..f42a1ac7a 100644 --- a/src/lib/orderman/actors/FillStore.h +++ b/src/lib/orderman/actors/FillStore.h @@ -2,6 +2,7 @@ #include #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" namespace bts { namespace orderman { @@ -14,12 +15,11 @@ class OrderDefinitionUpdateEvent; class FillStore : public mercury::Actor { public: - FillStore(); + FillStore(md::SecurityOracle); ~FillStore(); void init() override; private: - enum class State; struct PImpl; struct UnresolvedFills; diff --git a/src/lib/orderman/actors/OrderStore.cpp b/src/lib/orderman/actors/OrderStore.cpp index ca139a1ad..292272751 100644 --- a/src/lib/orderman/actors/OrderStore.cpp +++ b/src/lib/orderman/actors/OrderStore.cpp @@ -5,9 +5,8 @@ #include "lib/config/protos/orders_fills_cleanup.pb.h" #include "lib/config/protos/toggles.pb.h" #include "lib/orderman/actors/SODPosition.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" -#include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/OrderDefinitionUpdateEvent.h" +#include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" #include "lib/persistence/SqliteStore.h" #include "lib/utils/PBUtils.h" @@ -19,16 +18,14 @@ namespace orderman { static auto const orders_ns = "orders"; -enum class OrderStore::State { WAIT_SEC, READY }; - struct OrderStore::PImpl { - PImpl() - : state(State::WAIT_SEC), known_orders(), - store(new persistence::SqliteStore("orders")), order_stream(), + PImpl(md::SecurityOracle o) + : known_orders(), store(new persistence::SqliteStore("orders")), + order_stream(), soft_cleanup_enabled( - config::get().enabled()) {} + config::get().enabled()), + oracle(o) {} - State state; const GatewayName gw_name; struct ORInfo { api::orderman::OrderRecord rec; @@ -37,88 +34,63 @@ struct OrderStore::PImpl { std::unordered_map known_orders; std::unique_ptr store; mercury::Stream order_stream; - mercury::Actor::SubHandle stored_sec_sub; bool soft_cleanup_enabled; + md::SecurityOracle oracle; timeval cutoff_time; }; -OrderStore::OrderStore() : me(new PImpl()) {} +OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {} OrderStore::~OrderStore() {} void OrderStore::init() { - me->state = State::WAIT_SEC; - me->stored_sec_sub = subscribe([this]( - auto const &update) { - if (update.events) { - if (me->state == State::WAIT_SEC) { - LOG(OrderMan, Info, - "Order store received initial list of {} securities", - update.events->size()); + // subscribe to order updates so we can store them + subscribe( + [this](auto const &ev) { this->on_order_state_update(ev); }); - // don't need securities anymore, we just need the initial - // batch so we can filter out orders on expired securities - unsubscribe(me->stored_sec_sub); + subscribe( + [this](auto const &ev) { this->on_order_definition_update(ev); }); - // subscribe to order state and order definition updates so - // we publish newly created order records - subscribe( - [this](auto const &ev) { this->on_order_state_update(ev); }); - subscribe( - [this](auto const &ev) { this->on_order_definition_update(ev); }); + // now load orders from disk, for each order check if + // it reffers to a valid security. if so publish it. + api::orderman::OrderRecord rec; + std::vector stored_batch; - // index securities - std::map sec_idx; - for (auto const &ev : *(update.events)) { - auto sec_ptr = ev.security; - sec_idx[SecurityKey(sec_ptr->id())] = sec_ptr; - } + LOG(persist, Info, "Loading all orders"); + auto all = me->store->retrieve_all()[orders_ns]; + LOG(persist, Info, "Finished loading all {} orders", all.size()); - // now load orders from disk, for each order check if - // it reffers to a valid security. if so publish it. - api::orderman::OrderRecord rec; - std::vector stored_batch; + me->cutoff_time = SODPosition::get_order_cutoff_time(all); - LOG(persist, Info, "Loading all orders"); - auto all = me->store->retrieve_all()[orders_ns]; - LOG(persist, Info, "Finished loading all {} orders", all.size()); + for (auto const &kvp : all) + if (utils::pb::from_string(kvp.second, + &rec)) { + auto ok = OrderKey(std::stoi(kvp.first)); + auto sk = SecurityKey(rec.def().security_key()); + auto sec = me->oracle.get_security(sk); - me->cutoff_time = SODPosition::get_order_cutoff_time(all); - - for (auto const &kvp : all) - if (utils::pb::from_string( - kvp.second, &rec)) { - auto ok = OrderKey(std::stoi(kvp.first)); - auto sk = SecurityKey(rec.def().security_key()); - - if (config::get< - config::toggles::FilterOrdersWithDanglingSecurities>() - .enabled() && - sec_idx.find(sk) == sec_idx.end()) { - LOG(OrderMan, Diag, - "Order {} refers to unresolved security {}", - ok.value(), sk.value()); - } else { - if (!_order_cutoff(rec)) - stored_batch.emplace_back(rec); - me->known_orders[ok] = PImpl::ORInfo {rec,true}; - } - } else - LOG(OrderMan, Error, - "Could not recover order {} from storage. " - "This should be investigated", - kvp.first); - - me->order_stream = make_stream(); - LOG(OrderMan, Info, - "Order Store published {} orders recoved from disk out of {} " - "total orders", - stored_batch.size(), all.size()); - publish_batch(me->order_stream, stored_batch); - me->state = State::READY; + if (config::get() + .enabled() && + sec == nullptr) { + LOG(OrderMan, Diag, "Order {} refers to unresolved security {}", + ok.value(), sk.value()); + } else { + if (!_order_cutoff(rec)) + stored_batch.emplace_back(rec); + me->known_orders[ok] = PImpl::ORInfo{rec, true}; } - } - }); + } else + LOG(OrderMan, Error, + "Could not recover order {} from storage. " + "This should be investigated", + kvp.first); + + me->order_stream = make_stream(); + LOG(OrderMan, Info, + "Order Store published {} orders recoved from disk out of {} " + "total orders", + stored_batch.size(), all.size()); + publish_batch(me->order_stream, stored_batch); } void OrderStore::on_order_state_update( @@ -134,7 +106,7 @@ void OrderStore::on_order_state_update( rec.set_key(ev.key.value()); rec.set_parent_key(ev.parent_key.value()); modified_orders.insert(ev.key); - if (rec.has_state()) + if (rec.has_state()) rec.mutable_state()->MergeFrom(ev.rebuild_state()); else { *rec.mutable_state() = ev.rebuild_state(); @@ -161,14 +133,14 @@ void OrderStore::on_order_definition_update( store_and_publish_if_complete(modified_orders); } -void OrderStore::store_and_publish_if_complete(std::set const&changed) -{ +void OrderStore::store_and_publish_if_complete( + std::set const &changed) { for (auto k : changed) { auto &rec_info = me->known_orders[k]; auto &rec = rec_info.rec; if (rec.has_def() && rec.has_state()) { - publish(me->order_stream,rec); - if (rec_info.is_saved) + publish(me->order_stream, rec); + if (rec_info.is_saved) me->store->update(orders_ns, std::to_string(k.value()), utils::pb::to_string(rec)); else { diff --git a/src/lib/orderman/actors/OrderStore.h b/src/lib/orderman/actors/OrderStore.h index 0e766f056..14a7e459e 100644 --- a/src/lib/orderman/actors/OrderStore.h +++ b/src/lib/orderman/actors/OrderStore.h @@ -2,6 +2,7 @@ #include #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/orderman_utils/orderman_common.h" namespace bts { @@ -16,12 +17,11 @@ class OrderDefinitionUpdateEvent; class OrderStore : public mercury::Actor { public: - OrderStore(); + OrderStore(md::SecurityOracle); ~OrderStore(); void init() override; private: - enum class State; struct PImpl; std::unique_ptr me; diff --git a/src/lib/orderman/actors/SODPosition.cpp b/src/lib/orderman/actors/SODPosition.cpp index ee9515be7..18f6d83c4 100644 --- a/src/lib/orderman/actors/SODPosition.cpp +++ b/src/lib/orderman/actors/SODPosition.cpp @@ -171,22 +171,22 @@ bool SODPosition::add_fill_to_sod_pos(const Fill &fill) { void SODPosition::add_sod_pos_as_fill_events( std::vector &fe, - std::map sec_map) { + md::SecurityOracle& oracle) { if (_enabled) for (auto &acct_p : _sod_map) { for (auto &sec_p : acct_p.second) { Fill &long_pos = sec_p.second.first; Fill &short_pos = sec_p.second.second; bts_assert(long_pos.security_key() == short_pos.security_key()); - auto sec_it = sec_map.find(SecurityKey(long_pos.security_key())); - if (sec_it == sec_map.end()) + auto sec = oracle.get_security(SecurityKey(long_pos.security_key())); + if (!sec) LOG(persist, Error, "Could not recover fill for key:{} from storage. " "This should be investigated", long_pos.security_key()); else { - fe.emplace_back(long_pos, *sec_it->second); - fe.emplace_back(short_pos, *sec_it->second); + fe.emplace_back(long_pos, *sec); + fe.emplace_back(short_pos, *sec); } } } diff --git a/src/lib/orderman/actors/SODPosition.h b/src/lib/orderman/actors/SODPosition.h index d31e39446..860e0cf2d 100644 --- a/src/lib/orderman/actors/SODPosition.h +++ b/src/lib/orderman/actors/SODPosition.h @@ -4,6 +4,7 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/mercury/actors/Actor.h" #include "lib/orderman/events/StoredFillEvent.h" +#include "lib/md/SecurityOracle.h" using namespace bts::api::orderman; @@ -34,7 +35,7 @@ public: static timeval get_order_cutoff_time(const std::map& ordermap); bool add_fill_to_sod_pos(const Fill& fill); void add_sod_pos_as_fill_events(std::vector& fill_events, - std::map sec_map); + md::SecurityOracle&); }; }} diff --git a/src/lib/orderman/actors/SecuritySrc.cpp b/src/lib/orderman/actors/SecuritySrc.cpp index 8067ef85f..eacf58745 100644 --- a/src/lib/orderman/actors/SecuritySrc.cpp +++ b/src/lib/orderman/actors/SecuritySrc.cpp @@ -15,7 +15,7 @@ struct SecuritySrc::PImpl { mercury::Stream new_securities_stream; std::vector to_publish; md::MDConsumer md_consumer; - std::vector> securities_keep_alive; + std::vector securities_keep_alive; }; SecuritySrc::SecuritySrc(md::MDConsumer &md) : me(new PImpl(md)) {} @@ -24,11 +24,9 @@ SecuritySrc::~SecuritySrc() {} void SecuritySrc::init() { me->new_securities_stream = make_stream(); me->md_consumer.on_security( - [this](api::Security const &s, md::MDConsumerHandle) { - auto sec = std::make_unique(s); - auto sec_ptr = sec.get(); - me->securities_keep_alive.push_back(std::move(sec)); - me->to_publish.push_back(sec_ptr); + [this](md::SecurityPtr const &sec, md::MDConsumerHandle) { + me->securities_keep_alive.push_back(sec); + me->to_publish.push_back(sec.get()); }); subscribe([this](auto) { this->on_scan_securities(); }); diff --git a/src/lib/orderman/actors/ServiceBridge.cpp b/src/lib/orderman/actors/ServiceBridge.cpp index 46a63e9b2..8e6296843 100644 --- a/src/lib/orderman/actors/ServiceBridge.cpp +++ b/src/lib/orderman/actors/ServiceBridge.cpp @@ -6,7 +6,6 @@ #include "lib/orderman/events/CancelOrderEvent.h" #include "lib/orderman/events/DetachOrderEvent.h" #include "lib/orderman/events/NewAdminDataEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderManagerReadyEvent.h" #include "lib/orderman/events/OrderOpResultEvent.h" #include "lib/orderman/events/StoredFillEvent.h" @@ -30,11 +29,10 @@ struct ServiceBridge::PImpl { PImpl(OnReady const &on_ready, OnOrders const &on_orders, OnFills const &on_fills, OnOrderOpResult const &on_order_op_result, DynamicOrderRegistry const &dyn_order_registry, - mercury::BrokerProxy &service_call_marshaller) + mercury::BrokerProxy &service_call_marshaller, md::SecurityOracle o) : on_ready(on_ready), on_orders(on_orders), on_fills(on_fills), on_order_op_result(on_order_op_result), have_orders(false), - have_securities(false), order_manager_ready(false), sec_db(), - order_router(), + order_manager_ready(false), sec_db(o), order_router(), order_factory(sec_db, order_router, dyn_order_registry), known_orders(), service_call_marshaller(service_call_marshaller) {} @@ -43,7 +41,6 @@ struct ServiceBridge::PImpl { OnFills on_fills; OnOrderOpResult on_order_op_result; bool have_orders; - bool have_securities; bool order_manager_ready; securities::MapSecurityDB sec_db; OrderRouter order_router; @@ -63,9 +60,10 @@ ServiceBridge::ServiceBridge(OnReady const &on_ready, OnOrders const &on_orders, OnFills const &on_fills, OnOrderOpResult const &on_order_op_result, DynamicOrderRegistry const &dyn_order_registry, - mercury::BrokerProxy &service_call_marshaller) + mercury::BrokerProxy &service_call_marshaller, + md::SecurityOracle o) : me(new PImpl(on_ready, on_orders, on_fills, on_order_op_result, - dyn_order_registry, service_call_marshaller)) {} + dyn_order_registry, service_call_marshaller, o)) {} ServiceBridge::~ServiceBridge() {} @@ -73,7 +71,6 @@ void ServiceBridge::block_until_ready() { me->ready.get_future().wait(); } void ServiceBridge::init() { subscribe([this](auto const &u) { this->on(u); }); - subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); @@ -113,20 +110,6 @@ void ServiceBridge::on(mercury::Update const &update) { } } -void ServiceBridge::on(mercury::Update const &update) { - if (update.events) { - - // update our security db - for (auto ev : *(update.events)) { - auto sec_ptr = ev.security; - me->sec_db.securities[SecurityKey(sec_ptr->id())] = sec_ptr; - } - bool was_ready = is_ready(); - me->have_securities = true; - maybe_signal_ready(was_ready); - } -} - void ServiceBridge::on(mercury::Update const &update) { if (update.events) { bool was_ready = is_ready(); @@ -214,11 +197,13 @@ void ServiceBridge::detach_order(uint32_t rid, api::orderman::Detach const &c) { if (me->known_orders.count(ok)) publish(me->detach_order_pub, rid, ok); else - me->on_order_op_result(OrderOpResultEvent(ok, - rid, OrderOpResult(api::orderman::ORDER_ERROR_UNKNOWN_ORDER))); + me->on_order_op_result(OrderOpResultEvent( + ok, rid, + OrderOpResult(api::orderman::ORDER_ERROR_UNKNOWN_ORDER))); } else - me->on_order_op_result(OrderOpResultEvent(ok, - rid, OrderOpResult(api::orderman::ORDER_ERROR_SERVER_NOT_READY))); + me->on_order_op_result(OrderOpResultEvent( + ok, rid, + OrderOpResult(api::orderman::ORDER_ERROR_SERVER_NOT_READY))); }); } @@ -231,7 +216,7 @@ void ServiceBridge::update_admin_data( } bool ServiceBridge::is_ready() const { - return me->have_orders && me->have_securities && me->order_manager_ready; + return me->have_orders && me->order_manager_ready; } void ServiceBridge::maybe_signal_ready(bool old_ready) { diff --git a/src/lib/orderman/actors/ServiceBridge.h b/src/lib/orderman/actors/ServiceBridge.h index 97136b982..bcf225db1 100644 --- a/src/lib/orderman/actors/ServiceBridge.h +++ b/src/lib/orderman/actors/ServiceBridge.h @@ -2,6 +2,7 @@ #include "lib/config/protos/jetfix.pb.h" #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" #include // fwds @@ -25,7 +26,6 @@ class Detach; namespace orderman { class StoredFillEvent; class StoredOrderEvent; -class NewSecuritiesEvent; class OrderManagerReadyEvent; class NewAdminDataEvent; class OrderOpResultEvent; @@ -55,7 +55,8 @@ class ServiceBridge : public mercury::Actor { ServiceBridge(OnReady const &, OnOrders const &, OnFills const &, OnOrderOpResult const&, DynamicOrderRegistry const &, - mercury::BrokerProxy &service_call_marshaller); + mercury::BrokerProxy &service_call_marshaller, + md::SecurityOracle o); void init() override; ~ServiceBridge(); @@ -79,7 +80,6 @@ class ServiceBridge : public mercury::Actor { void on(mercury::Update const &); void on(mercury::Update const &); - void on(mercury::Update const &); void on(mercury::Update const &); void on(mercury::Update const &); void on(mercury::Update const &); diff --git a/src/lib/orderman/actors/order_manager/OrderManager.cpp b/src/lib/orderman/actors/order_manager/OrderManager.cpp index d2fbd3168..0349ed84c 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.cpp +++ b/src/lib/orderman/actors/order_manager/OrderManager.cpp @@ -8,6 +8,7 @@ #include "lib/orderman/events/OrderManagerReadyEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" +#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/order_factory/OrderFactory.h" #include "lib/orderman/orders/Order.h" #include "lib/orderman/securities/ISecurityDB.h" @@ -31,8 +32,8 @@ enum class OrderManager::State { }; struct OrderManager::PImpl { - PImpl(OrderManager &owner, DynamicOrderRegistry const &b, trace::PerfLog &l) - : state(State::WAIT_ADMIN), services_data(owner, b, l) {} + PImpl(OrderManager &owner, DynamicOrderRegistry const &b, trace::PerfLog &l, md::SecurityOracle o) + : state(State::WAIT_ADMIN), services_data(owner, b, l, o) {} State state; std::unordered_map orders; @@ -42,8 +43,8 @@ struct OrderManager::PImpl { mercury::Stream pub_ready; }; -OrderManager::OrderManager(DynamicOrderRegistry const &b, trace::PerfLog &l) - : me(new PImpl(*this, b, l)) {} +OrderManager::OrderManager(DynamicOrderRegistry const &b, trace::PerfLog &l, md::SecurityOracle o) + : me(new PImpl(*this, b, l, o)) {} OrderManager::~OrderManager() {} void OrderManager::init() { @@ -85,8 +86,6 @@ void OrderManager::on_securities( for (auto const &ev : *(update.events)) { auto sec_ptr = ev.security; secs.push_back(sec_ptr); - me->services_data.sec_db.securities[SecurityKey(sec_ptr->id())] = - sec_ptr; } me->services_data.limiter.add_securities(secs); diff --git a/src/lib/orderman/actors/order_manager/OrderManager.h b/src/lib/orderman/actors/order_manager/OrderManager.h index 461705c1f..0e54a2e65 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.h +++ b/src/lib/orderman/actors/order_manager/OrderManager.h @@ -4,6 +4,7 @@ #include "lib/jetfix/common/OutFIXPtr.h" #include "lib/mercury/actors/Actor.h" #include "lib/mercury/actors/IPollableActor.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/events/ActivateOrderEvent.h" #include "lib/orderman/events/CancelOrderEvent.h" #include "lib/orderman/events/DetachOrderEvent.h" @@ -62,8 +63,8 @@ namespace orderman { class OrderManager : public mercury::Actor, public mercury::IPollableActor { public: - OrderManager(DynamicOrderRegistry const &dyn_order_builder, - trace::PerfLog &); + OrderManager(DynamicOrderRegistry const &dyn_order_builder, trace::PerfLog &, md::SecurityOracle); + ~OrderManager(); void init() override; diff --git a/src/lib/orderman/actors/order_manager/OrderServicesData.cpp b/src/lib/orderman/actors/order_manager/OrderServicesData.cpp index 5b19f6e44..253f56317 100644 --- a/src/lib/orderman/actors/order_manager/OrderServicesData.cpp +++ b/src/lib/orderman/actors/order_manager/OrderServicesData.cpp @@ -7,8 +7,8 @@ namespace orderman { OrderServicesData::OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b, - trace::PerfLog &l) - : owner(owner), log(l), order_router(), + trace::PerfLog &l, md::SecurityOracle o) + : owner(owner), log(l), sec_db(o), order_router(), order_factory(sec_db, order_router, b), last_timer_ev( Timer::now(), diff --git a/src/lib/orderman/actors/order_manager/OrderServicesData.h b/src/lib/orderman/actors/order_manager/OrderServicesData.h index 97c47acb8..259f02397 100644 --- a/src/lib/orderman/actors/order_manager/OrderServicesData.h +++ b/src/lib/orderman/actors/order_manager/OrderServicesData.h @@ -8,6 +8,7 @@ #include "lib/orderman/securities/MapSecurityDB.h" #include "lib/mercury/events/TimerEvent.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/actors/ExecutionGateway.h" #include "lib/orderman/events/ActivateOrderEvent.h" #include "lib/orderman/events/BookUpdateEvent.h" @@ -22,7 +23,6 @@ #include "lib/orderman/events/GatewayStatusChangeEvent.h" #include "lib/orderman/events/MDRequestEvent.h" #include "lib/orderman/events/NewExecutionReportEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderDefinitionUpdateEvent.h" #include "lib/orderman/events/OrderOpResultEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" @@ -77,7 +77,7 @@ struct SessionServicesData { class OrderServicesData { public: OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b, - trace::PerfLog &l); + trace::PerfLog &l, md::SecurityOracle); OrderManager &owner; trace::PerfLog &log; diff --git a/src/lib/orderman/events/NewSecuritiesEvent.h b/src/lib/orderman/events/NewSecuritiesEvent.h index fddcc0e6d..be5f3c505 100644 --- a/src/lib/orderman/events/NewSecuritiesEvent.h +++ b/src/lib/orderman/events/NewSecuritiesEvent.h @@ -7,6 +7,10 @@ namespace bts { namespace orderman { + //todo: + // anywhere that uses this + // get a SecurityOracle + // which keeps a shared_ptr and uses it class NewSecuritiesEvent : public mercury::Event { diff --git a/src/lib/orderman/orders/NativeOrder.cpp b/src/lib/orderman/orders/NativeOrder.cpp index cc49fc8c5..91a527014 100644 --- a/src/lib/orderman/orders/NativeOrder.cpp +++ b/src/lib/orderman/orders/NativeOrder.cpp @@ -4,6 +4,7 @@ #include "lib/jetfix/codec/FIXFragment.h" #include "lib/jetfix/codec/FieldDefinitions.h" #include "lib/logging/logging.h" +#include "lib/domain_model/Security.h" #include "lib/orderman/order_factory/IOrderFactory.h" #include "lib/orderman/order_router/OrderRouter.h" #include "lib/orderman/orderman_utils/ExecutionReport.h" diff --git a/src/lib/orderman/securities/ISecurityDB.h b/src/lib/orderman/securities/ISecurityDB.h index eb7a23fad..427d1171b 100644 --- a/src/lib/orderman/securities/ISecurityDB.h +++ b/src/lib/orderman/securities/ISecurityDB.h @@ -14,7 +14,6 @@ public: virtual ~ISecurityDB() {}; virtual api::Security const *resolve(SecurityKey const&k) const = 0; using SecurityMap = std::unordered_map ; - virtual SecurityMap retrieve_all() const = 0; }; }} diff --git a/src/lib/orderman/securities/MapSecurityDB.h b/src/lib/orderman/securities/MapSecurityDB.h index 2c447d6f9..b2c80252c 100644 --- a/src/lib/orderman/securities/MapSecurityDB.h +++ b/src/lib/orderman/securities/MapSecurityDB.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include "lib/md/SecurityOracle.h" #include "lib/orderman/securities/ISecurityDB.h" namespace bts { @@ -9,15 +9,12 @@ namespace securities { struct MapSecurityDB : public securities::ISecurityDB { + MapSecurityDB(md::SecurityOracle oracle): _oracle(oracle) {} api::Security const *resolve(SecurityKey const&k) const override { - auto it = securities.find(k); - if (it != securities.end()) - return it->second; - else - return nullptr; + return _oracle.get_security(k); } - SecurityMap retrieve_all() const override {return securities;} - SecurityMap securities; + private: + md::SecurityOracle _oracle; }; }} diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index c8279d998..35a91ab41 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -5,6 +5,7 @@ #include "lib/config/protos/jetfix.pb.h" #include "lib/config/protos/orderman_cfg.pb.h" #include "lib/md/MDConsumer.h" +#include "lib/md/MDClient.h" #include "lib/mercury/actors/Timer.h" #include "lib/mercury/executor/Executor.h" #include "lib/metrics/metrics.h" @@ -89,12 +90,12 @@ ActorThread *find_thread_for_actor( struct OrderServiceServer::PImpl { PImpl(bts::api::ServiceContext &ctx, - std::shared_ptr &ss, + md::MDClient& mdc, DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) : service_bridge(), service_ready(false), known_orders(), known_fills(), publish_orders(), publish_fills(), - admin_service(ctx.locate()), md_consumer(ss), + admin_service(ctx.locate()), md_consumer(mdc.current_consumer()), stats(stats), dyn_order_registry(d), log(log), in_flight_order_ops(), current_req_op_id(0) {} @@ -109,7 +110,7 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrdersPublisher publish_orders; OrderServiceServer::FillsPublisher publish_fills; api::admin::AdminService admin_service; - bts::md::MDConsumer md_consumer; + bts::md::MDConsumer& md_consumer; metrics::MetricsContainer &stats; DynamicOrderRegistry const &dyn_order_registry; trace::PerfLog &log; @@ -120,12 +121,10 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrderServiceServer( bts::api::ServiceContext &ctx, - std::shared_ptr &ss, + md::MDClient& mdc, DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) - : api::orderman::OrderServiceBase(ctx), - me(new PImpl(ctx, ss, d, stats, log)) { - + : api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, mdc, d, stats,log)) { me->publish_orders = make_orders_publisher(); me->publish_fills = make_fills_publisher(); @@ -144,6 +143,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for MDSource. See the " "OrdermanCfg key"); + thread->executor.spawn(me->md_consumer, me->stats, log); // OrderManager @@ -151,7 +151,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for OrderManager. See " "the OrdermanCfg key"); - thread->executor.spawn(d, me->log); + thread->executor.spawn(d,me->log, mdc.oracle()); // Timer thread = find_thread_for_actor("Timer", me->actor_threads); @@ -165,21 +165,21 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for AsyncFactory. See " "the OrdermanCfg key"); - thread->executor.spawn(d); + thread->executor.spawn(d, mdc.oracle()); // OrderStore thread = find_thread_for_actor("OrderStore", me->actor_threads); bts_assert_msg(thread, "Could not find a configured thread for OrderStore. See " "the OrdermanCfg key"); - thread->executor.spawn(); + thread->executor.spawn(mdc.oracle()); // FillStore thread = find_thread_for_actor("FillStore", me->actor_threads); bts_assert_msg(thread, "Could not find a configured thread for FilllStore. See " "the OrdermanCfg key"); - thread->executor.spawn(); + thread->executor.spawn(mdc.oracle()); // SecuritySrc thread = find_thread_for_actor("SecuritySrc", me->actor_threads); @@ -203,22 +203,20 @@ OrderServiceServer::OrderServiceServer( "the OrdermanCfg key"); me->service_bridge = new ServiceBridge( [this]() { me->service_ready = true; }, - [this, &ctx](auto const &orders) { ctx.post([this, orders] { this->on_orders(orders); }); }, - [this, &ctx](auto const &fill) { ctx.post([this, fill] { this->on_fills(fill); }); }, - [this, &ctx](auto const &order_op_result) { ctx.post([this, order_op_result] { this->on_order_op_result(order_op_result); }); }, + me->dyn_order_registry, + thread->executor.make_proxy(),mdc.oracle()); - me->dyn_order_registry, thread->executor.make_proxy()); thread->executor.activate(me->service_bridge); diff --git a/src/lib/orderman/server/OrderServiceServer.h b/src/lib/orderman/server/OrderServiceServer.h index 1eca52a46..5e38970cd 100644 --- a/src/lib/orderman/server/OrderServiceServer.h +++ b/src/lib/orderman/server/OrderServiceServer.h @@ -3,13 +3,11 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/api/services/OrderService.server.pb.h" #include "lib/mercury/actors/IPollableActor.h" +#include "lib/md/MDClient.h" #include namespace bts { namespace trace {class PerfLog;} -namespace api { -class ISyntheticSecurityService; -} namespace metrics { class MetricsContainer; } @@ -20,7 +18,7 @@ class OrderOpResultEvent; class OrderServiceServer : public api::orderman::OrderServiceBase { public: OrderServiceServer(bts::api::ServiceContext &context, - std::shared_ptr &, + md::MDClient&, DynamicOrderRegistry const&, metrics::MetricsContainer &, trace::PerfLog &); diff --git a/src/lib/utils/ResultEnum.h b/src/lib/utils/ResultEnum.h new file mode 100644 index 000000000..3509ce0b2 --- /dev/null +++ b/src/lib/utils/ResultEnum.h @@ -0,0 +1,7 @@ +#pragma once +namespace bts { + enum class Result { + ok, + error + }; +} \ No newline at end of file diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index 5e776e159..8c33965e4 100755 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -1,5 +1,6 @@ #include "datetime.h" + #include "bts_assert.h" #include "TimeStamp.h" @@ -317,6 +318,19 @@ Date ll_to_date(LLDateTime time) { return date; } + +bool is_local_time_sunday() { + auto now = local_now(); + return now.date().day_of_week() == boost::date_time::Sunday; +} +bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end) { + using namespace boost::posix_time; + auto now = local_now(); + auto start = ptime(now.date(), hours(hh_start) + minutes(mm_start)); + auto end = ptime(now.date(), hours(hh_end) + minutes(mm_end)); + return (now >= start && now <= end); +} + struct UtcTimeStamp { int hour; int min; diff --git a/src/lib/utils/datetime.h b/src/lib/utils/datetime.h index e6095ad82..133555edc 100755 --- a/src/lib/utils/datetime.h +++ b/src/lib/utils/datetime.h @@ -76,11 +76,15 @@ BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days); BTS_EXPORT LLDateTime ll_from_unix_epoch(long); BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long); + extern "C" BTS_EXPORT void ll_to_parts(LLDateTime, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond); BTS_EXPORT Date ll_to_date(LLDateTime); extern "C" BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond); BTS_EXPORT LLDateTime ll_from_date(const Date& date); +bool is_local_time_sunday(); +bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end); + #ifndef NO_CPP0X extern "C" BTS_EXPORT LLDateTime ll_today(); BTS_EXPORT Date today(); diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 5a7c8c06b..3ecb39e6a 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -11,6 +11,7 @@ #include "lib/utils/Dispatcher.h" #include "lib/config/protos/toggles.pb.h" +#include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" #include "lib/metrics/metrics.h" @@ -58,10 +59,13 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); md::run_market_datapublishers(); - std::shared_ptr synthetic_service = - std::make_shared(sc); + + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service); + + md::QueryServerPtr md_query_server( - md::run_query_server(sc, synthetic_service, stats)); + md::run_query_server(sc, mdclient, stats)); // initialize order plugins orderman::DynamicOrderRegistry dyn_order_registry; @@ -86,7 +90,7 @@ int Run::execute(std::vector const &) { // finished initializing order plugins orderman::OrderServiceServer order_service_server( - sc, synthetic_service, dyn_order_registry, stats, perf_log); + sc, mdclient, dyn_order_registry, stats, perf_log); support::ErrorReportServer error_report_service(sc); admin::AdminServiceServer admin_service(sc); md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc); diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 8cc3ad12e..029ae34bc 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -10,6 +10,7 @@ #include "lib/metrics/system_metrics.h" #include "server/sparkd/http/WebServer.h" #include "lib/md/relay/QueryServer.h" +#include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "server/sparkd/user/AdminServiceServer.h" #include "server/sparkd/user/AuthServiceServer.h" @@ -39,17 +40,14 @@ int Web::execute(const std::vector& args) { md::run_market_datapublishers(); } - std::shared_ptr synthetic_service = - std::make_shared(sc); + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service); md::QueryServerPtr md_query_server( - md::run_query_server(sc, synthetic_service, stats)); + md::run_query_server(sc, mdclient, stats)); admin::AdminServiceServer admin_service(sc); - - - std::this_thread::sleep_until( - std::chrono::time_point::max()); + utils::Dispatcher::current().run(); return 0; } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 82d6aa0de..7c0db2832 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -3,7 +3,6 @@ #include "lib/md/cme/MDP3Utils.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" -#include "lib/md/SecurityDB.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/proto_helper.h" @@ -339,23 +338,11 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - const auto& options = channel_settings_for(channel.id); - unique_ptr listener(new MDP3Listener(_index, _producer, /*_settlement_recorder,*/ producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } - if (options.load_security_db()) { - // Load the list of known securities and add them to the index for fast startup - db::load_securities(channel.symbols, [&](const api::Security& security) { - with_security_like(security, [&](FixedSecurity& fs) { - convert_security(security, fs, true); - listener->add_security(fs); - }); - }); - } - handler.registerListener(*listener); handler.registerMarketDataListener(*listener); handler.registerSecurityListener(*listener); diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 40dbdd748..56e2c2762 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -81,14 +81,35 @@ void run_cleanup() { std::thread* _md_thread; void run_market_data_publishers_forever() { os::set_thread_name("md publisher"); - if (config::get().use_replay()) { + const auto& conf = config::get(); + if (conf.use_replay()) { run_replay(); } - else { + else if (conf.use_onixs()) { run_onixs(); } } +void boot_sleep() { + auto conf = config::get(); + const auto* schedule = &conf.standard(); + if (datetime::is_local_time_sunday()) { + schedule = &conf.sunday(); + } + bts_assert(schedule); + int hh_start = 0, mm_start = 0, hh_end = 0, mm_end = 0; + sscanf(schedule->start().c_str(), "%i:%i", &hh_start, &mm_start); + sscanf(schedule->end().c_str(), "%i:%i", &hh_end, &mm_end); + if (datetime::is_local_time_between(hh_start, mm_start, hh_end, mm_end)) { + std::this_thread::sleep_for( + std::chrono::seconds( + conf.delay_seconds())); + } + else { + std::this_thread::sleep_for(2s); + } +} + void run_market_datapublishers() { bts_assert(_md_thread == nullptr); run_cleanup(); @@ -102,10 +123,10 @@ void run_market_datapublishers() { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) .detach(); } - - std::this_thread::sleep_for(2s); + boot_sleep(); } + void run_record(const std::string& arg){ boost::filesystem::path p(arg); record_everything(p.parent_path().string(), p.filename().string()); @@ -228,9 +249,9 @@ void dump_db(const vector& args) { if (arg == "all") { all = true; break; } - auto key = std::stoi(arg); + auto key = std::stoul(arg); if (key > 0) { - keys.insert(key); + keys.insert((uint)key); } } std::set symbols; diff --git a/testing/manual/ladder/ui_performance.txt b/testing/manual/ladder/ui_performance.txt index 9fb54cc18..ee27662c8 100644 --- a/testing/manual/ladder/ui_performance.txt +++ b/testing/manual/ladder/ui_performance.txt @@ -14,3 +14,4 @@ This test needs to run against production market data or high quality recorded d 4. Repeat 2. +5. If there is *any* doubt that the UI is snappy bring up the previous release and compare. At minimum we need to not regress UI performance.