From dbd8a7d74214b5d3e177757de92ff2f741c474a7 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 14 Dec 2018 15:06:40 -0600 Subject: [PATCH 01/30] Zoom - proto, settings, UI --- .../spark/view/Spark/Ladder/LadderSettings.cs | 22 +++++++++++-- .../view/Spark/Ladder/LadderSettingsVM.cs | 10 ++++-- .../spark/view/Spark/Ladder/LadderVM.cs | 15 ++++++--- .../spark/view/Spark/Ladder/LadderView.xaml | 13 ++++++++ .../view/Spark/Ladder/LadderView.xaml.cs | 33 +++++++++++++++++++ 5 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index 547b910c1..289c4d38f 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -181,6 +181,7 @@ namespace mbts.spark { [ProtoContract] class LadderSettings : IEquatable { const double _default_order_entry_panel_width = 110.0; + const double _default_zoom_factor = 1.0; // Default proto constructor // Added this so that when clients upgrade to a version with new settings, @@ -202,6 +203,7 @@ namespace mbts.spark { cancel_all_visible = true; auto_clear_visible = true; order_entry_panel_initial_width = _default_order_entry_panel_width; + zoom_factor = _default_zoom_factor; } public static LadderSettings Default { @@ -257,7 +259,9 @@ namespace mbts.spark { cancel_sells_visible = true, cancel_all_visible = true, auto_clear_visible = true, - order_entry_panel_initial_width = _default_order_entry_panel_width + order_entry_panel_initial_width = _default_order_entry_panel_width, + zoom_slider_visible = false, + zoom_factor = _default_zoom_factor }; } } @@ -353,6 +357,14 @@ namespace mbts.spark { // [ProtoMember(42), DefaultValue(_default_order_entry_panel_width)] public double order_entry_panel_initial_width { get; set; } + + // the zoom slider is the first new widget on the panel + // since our initial general release, so its enablement + // will be opt-in (default not visible) + [ProtoMember(43)] + public bool zoom_slider_visible { get; set; } + [ProtoMember(44), DefaultValue(_default_zoom_factor)] + public double zoom_factor { get; set; } public bool Equals(LadderSettings other) => @@ -395,7 +407,9 @@ namespace mbts.spark { cancel_sells_visible == other.cancel_sells_visible && cancel_all_visible == other.cancel_all_visible && auto_clear_visible == other.auto_clear_visible && - order_entry_panel_initial_width == other.order_entry_panel_initial_width + order_entry_panel_initial_width == other.order_entry_panel_initial_width && + zoom_slider_visible == other.zoom_slider_visible && + zoom_factor == other.zoom_factor ; public override bool Equals(object obj) => Equals(obj as LadderSettings); @@ -440,7 +454,9 @@ namespace mbts.spark { cancel_sells_visible.GetHashCode(), cancel_all_visible.GetHashCode(), auto_clear_visible.GetHashCode(), - order_entry_panel_initial_width.GetHashCode() + order_entry_panel_initial_width.GetHashCode(), + zoom_slider_visible.GetHashCode(), + zoom_factor.GetHashCode() ); } diff --git a/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs b/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs index 773fc2f15..d2dea4038 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs @@ -17,7 +17,7 @@ namespace mbts.spark { readonly Property _badge_style; readonly Property _alternate_order_size; readonly DelegateProperty _row_height; - Func _get_order_entry_panel_width; + Func _get_order_entry_panel_width, _get_zoom_factor; LadderSettings _current; LadderColumnSettingsVM _orders = new OrdersColumnSettingsVM(), @@ -34,6 +34,7 @@ namespace mbts.spark { _session_high_mkt_line = new LadderMarketBreakLineVM("Session High", 2, 1, false), _session_range_fill = new LadderMarketBreakLineVM("Session Range", 0, 0, false); OrderEntryPanelItemVisibilityVM + _zoom_slider_visbility = new OrderEntryPanelItemVisibilityVM("Zoom Slider"), _day_visibility = new OrderEntryPanelItemVisibilityVM("Day Change"), _high_visibility = new OrderEntryPanelItemVisibilityVM("High"), _low_visibility = new OrderEntryPanelItemVisibilityVM("Low"), @@ -51,7 +52,7 @@ namespace mbts.spark { _cancel_all_visibility = new OrderEntryPanelItemVisibilityVM("Cancel All"); - public LadderSettingsVM(bool is_synthetic, Func get_left_panel_width) { + public LadderSettingsVM(bool is_synthetic, Func get_left_panel_width, Func get_zoom_factor) { this.is_synthetic = is_synthetic; size_presets = new ObservableEditableCollection>(_create_size_preset_vm); size_presets.CollectionChanged += (s, e) => RaisePropertyChanged(nameof(size_presets_text)); @@ -59,6 +60,7 @@ namespace mbts.spark { _badge_style.Changed += _badge_style_Changed; _alternate_order_size = backing(nameof(alternate_order_size)); _get_order_entry_panel_width = get_left_panel_width; + _get_zoom_factor = get_zoom_factor; // be sure to initialize backings before this measure_row_height call _row_height = backing(nameof(row_height), () => LadderRender.measure_row_height(get_result())); _orders.font_size_changed += on_font_size_changed; @@ -130,6 +132,7 @@ namespace mbts.spark { public IEnumerable order_entry_panel_items { get { + yield return _zoom_slider_visbility; yield return _day_visibility; yield return _high_visibility; yield return _low_visibility; @@ -168,6 +171,7 @@ namespace mbts.spark { _session_low_mkt_line.assign_from(_current.session_low_line); _session_high_mkt_line.assign_from(_current.session_high_line); _session_range_fill.assign_from(_current.session_range_fill); + _zoom_slider_visbility.show = settings.zoom_slider_visible; _day_visibility.show = settings.day_visible; _high_visibility.show = settings.high_visible; _low_visibility.show = settings.low_visible; @@ -224,6 +228,8 @@ namespace mbts.spark { clone.cancel_sells_visible = _cancel_sells_visibility.show; clone.cancel_all_visible = _cancel_all_visibility.show; clone.order_entry_panel_initial_width = _get_order_entry_panel_width(); + clone.zoom_slider_visible = _zoom_slider_visbility.show; + clone.zoom_factor = _get_zoom_factor(); return clone; } diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index cd5248fcd..fd099aef9 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -24,12 +24,12 @@ namespace mbts.spark { readonly IBus _bus; readonly LadderModel _model; - readonly Property _left_panel_width; + readonly Property _left_panel_width, _zoom_factor; readonly Property _high, _low, _change, _open, _settlement, _position; readonly Property _volume; readonly Property _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account, _show_tif, _show_order_type, _show_account_position, _show_size_preset, _show_auto_clear, _show_cancel_buys, - _show_cancel_sells, _show_cancel_all; + _show_cancel_sells, _show_cancel_all, _show_zoom; readonly Property _bid_background, _bid_foreground, _ask_background, _ask_foreground; IWindowSettingsManager _settings_manager; ulong _misc_time = 0; @@ -79,12 +79,14 @@ namespace mbts.spark { _show_cancel_buys = bool_prop(nameof(show_cancel_buys), s => s.cancel_buys_visible); _show_cancel_sells = bool_prop(nameof(show_cancel_sells), s => s.cancel_sells_visible); _show_cancel_all = bool_prop(nameof(show_cancel_all), s => s.cancel_all_visible); + _show_zoom = bool_prop(nameof(show_zoom), s => s.zoom_slider_visible); Property bool_prop(string name, Func get_bool) => backing(name, settings_manager.subscribe_current().OnDispatcher(), get_bool); // _left_panel_width = backing(nameof(left_panel_width), settings_manager.subscribe_current().OnDispatcher().Select(s => s.order_entry_panel_initial_width)); + _zoom_factor = backing(nameof(zoom_factor), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), LadderSettings.Default.zoom_factor); position_click_command = new DelegateCommand(() => size_calculator.value = DoubleHelper.safe_convert_to_uint(Math.Abs(position))); send_order_command = new DelegateCommand<(double price, Side? side, bool size_from_alternate)>(send_order); cancel_command = new AsyncCommand(_model.cancel_orders, true); @@ -94,7 +96,7 @@ namespace mbts.spark { var vm = new WindowSettingsDialogVM( settings_manager, s => s.ladder, UserSettings.FileExtensions.LADDER, s => { - var svm = new LadderSettingsVM(security.ContractCase == Security.ContractOneofCase.Synthetic, () => left_panel_width); + var svm = new LadderSettingsVM(security.ContractCase == Security.ContractOneofCase.Synthetic, () => left_panel_width, () => zoom_factor); svm.load(s); return svm; }); @@ -274,7 +276,9 @@ namespace mbts.spark { public bool show_cancel_buys { get => _show_cancel_buys.Value; set => _show_cancel_buys.set(value); } public bool show_cancel_sells { get => _show_cancel_sells.Value; set => _show_cancel_sells.set(value); } public bool show_cancel_all { get => _show_cancel_all.Value; set => _show_cancel_all.set(value); } + public bool show_zoom { get => _show_zoom.Value; set => _show_zoom.set(value); } public double left_panel_width { get => _left_panel_width.Value; set => _left_panel_width.set(value); } + public double zoom_factor { get => _zoom_factor.Value; set => _zoom_factor.set(value); } internal LadderModel model => _model; internal IObservable settings => _settings_manager.subscribe_current(); @@ -283,7 +287,10 @@ namespace mbts.spark { if (_settings_manager.instance_settings == null) return null; var settings_clone = SparkUtils.clone_proto(_settings_manager.instance_settings); - settings_clone.default_order_type = order_properties_vm.selected_order_type?.order_type ?? settings_clone.default_order_type; + settings_clone.default_order_type = order_properties_vm.selected_order_type?.order_type ?? settings_clone.default_order_type; + // we're going to need a way to say we want to always save the zoom factor (when it's different from defaults) without going into Settings to create an instance of instance_settings + // ditto the left panel width + settings_clone.zoom_factor = zoom_factor; return settings_clone; } diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml b/src/client/spark/view/Spark/Ladder/LadderView.xaml index 32847e244..73a817b6e 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml @@ -268,6 +268,12 @@ + + + @@ -289,6 +295,13 @@ + + + diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs index 9fd3e7e37..d6bc08f31 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs @@ -5,6 +5,7 @@ using System.Windows; using System.Windows.Controls; using System.Windows.Controls.Primitives; using System.Windows.Data; +using System.Windows.Input; namespace mbts.spark { public partial class LadderView : UserControl { @@ -16,8 +17,40 @@ namespace mbts.spark { this.vm = vm; DataContext = vm; DataContextChanged += datacontext_changed; + _ui_scaler.MouseDoubleClick += _ui_scaler_MouseDoubleClick; + this.PreviewMouseWheel += LadderView_PreviewMouseWheel; + this.PreviewMouseDown += LadderView_PreviewMouseDown; } + + + // (Left|Right) CTRL + Wheel to zoom + // This is neat b/c the user doesn't even need to have the slider visible to zoom + private void LadderView_PreviewMouseWheel(object sender, System.Windows.Input.MouseWheelEventArgs e) { + base.OnPreviewMouseWheel(e); + if (Keyboard.IsKeyDown(Key.LeftCtrl) || Keyboard.IsKeyDown(Key.RightCtrl)) { + _ui_scaler.Value += (e.Delta > 0) ? 0.1 : -0.1; + } + } + + // (Left|Right) CTRL + Wheel Click to reset + private void LadderView_PreviewMouseDown(object sender, MouseButtonEventArgs e) { + base.OnPreviewMouseDown(e); + if (Keyboard.IsKeyDown(Key.LeftCtrl) || Keyboard.IsKeyDown(Key.RightCtrl)) { + if (e.MiddleButton == MouseButtonState.Pressed) { + _ui_scaler_MouseDoubleClick(_ui_scaler, e); + } + } + } + + // Double click on scaler itself to reset + private void _ui_scaler_MouseDoubleClick(object sender, System.Windows.Input.MouseButtonEventArgs e) { + if (sender is Slider _the_slider) { + _the_slider.Value = 1.0; + } + } + + private void datacontext_changed(object sender, DependencyPropertyChangedEventArgs e) { this.vm = DataContext as LadderVM; } From 6259f8b03fb605f289138a2ce21f0f51f9826298 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 18 Dec 2018 10:53:52 -0600 Subject: [PATCH 02/30] fixing build --- src/client/spark/view/Spark/Ladder/LadderSettings.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index ff783d3e3..75fe933ef 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -261,11 +261,10 @@ namespace mbts.spark { cancel_sells_visible = true, cancel_all_visible = true, auto_clear_visible = true, + clear_visible = true, order_entry_panel_initial_width = _default_order_entry_panel_width, zoom_slider_visible = false, zoom_factor = _default_zoom_factor, - clear_visible = true, - order_entry_panel_initial_width = _default_order_entry_panel_width }; } } From 2ea95e7938dbed479c5d3ae6fccbbb5d0029864b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 20 Dec 2018 16:15:35 -0600 Subject: [PATCH 03/30] SPARK-1903 front end crash when sorting --- .../spark_core/Collections/SortedTree.cs | 126 ++++++++++++------ 1 file changed, 88 insertions(+), 38 deletions(-) diff --git a/src/client/spark_core/Collections/SortedTree.cs b/src/client/spark_core/Collections/SortedTree.cs index 884a2c3a1..00f7e1ef1 100644 --- a/src/client/spark_core/Collections/SortedTree.cs +++ b/src/client/spark_core/Collections/SortedTree.cs @@ -61,7 +61,7 @@ namespace bts.collections { if (_nursery.TryPopValue(k, out SortedList children)) { // remove orphans from _tree (orphan-nodes must reside in the main _tree until parents arrive) foreach (var n in children) - _tree.Remove(n); + _assert_remove(_tree, n); node = new Node(item, children); } else node = new Node(item, _comparer); var pk = parent(item); @@ -84,13 +84,10 @@ namespace bts.collections { if (!_index.TryGetValue(k, out Node node)) { return false; } else { - var newval = replace(node.value); - Log.assert(id(newval).Equals(k)); - _replace(node, newval); + _replace(node, replace); return true; } } - public void Remove(T item) => Remove(id(item)); @@ -185,44 +182,97 @@ namespace bts.collections { } } - void _replace(Node node, T newval) { - TKey pk = parent(newval); - TKey old_pk = parent(node.value); - if (pk.Equals(old_pk)) { - SortedList node_list; - if (pk.Equals(_root_id)) - node_list = _tree; - else if (_index.TryGetValue(pk, out Node pnode)) { - node_list = pnode.children; - } else if (!_nursery.TryGetValue(pk, out node_list)) { - throw new KeyNotFoundException($"Replace was called with a parent-key ({pk}) that was not found in _index or _nursery; item-key was ({id(newval)})."); - } - node_list.Remove(node); - node.value = newval; - node_list.Add(node); - } else { - if (old_pk.Equals(_root_id)) { - _tree.Remove(node); - } else { - if (_index.TryGetValue(old_pk, out Node pnode)) - pnode.children.Remove(node); - else { - _remove_from_nursery(old_pk, node); - } - } - node.value = newval; - if (pk.Equals(_root_id)) { + void _replace(Node node, T new_value) => + _replace(node, _ => new_value); + + void _replace(Node node, Func replace_value) { + /// STEPS (outline) + /// 1) remove node with current value + /// 2) replace node.value + /// 3) re-insert node with new value + + /// Implementation (details): + /// there are 3 scenarios in which step 1 (remove node...) may occur + /// a) node is a root node + /// - remove it from . + /// b) node is a child node who's parent is present + /// - remove it from the parent + /// c) the node is a child node who's parent is NOT present + /// - remove it from AND . + /// In each scenario, after the node is removed, the new parent will be assigned (). + /// If the parent has NOT changed, assign the node back to the same list(s), then return. + /// If the parent HAS changed, determine the list(s) in which the node needs to be inserted, insert the node, then return. + TKey old_pk = parent(node.value), new_pk; + if (old_pk.Equals(_root_id)) { + // a) + _assert_remove(_tree, node); + _assert_replace(ref node, replace_value); + new_pk = parent(node.value); + if (new_pk.Equals(old_pk)) { + // the parent has NOT changed; re-insert the node, then return. _tree.Add(node); - } else if (_index.TryGetValue(pk, out Node pnode)) { - pnode.children.Add(node); - } else { - if (!_nursery.TryGetValue(pk, out SortedList nursery_list)) { - _nursery.Add(pk, nursery_list = new SortedList(_comparer)); - } + return; + } + } else if (_index.TryGetValue(old_pk, out Node old_pnode)) { + // b) + _assert_remove(old_pnode.children, node); + _assert_replace(ref node, replace_value); + new_pk = parent(node.value); + if (new_pk.Equals(old_pk)) { + // the parent has NOT changed; re-insert the node, then return. + old_pnode.children.Add(node); + return; + } + } else if (_nursery.TryGetValue(old_pk, out SortedList nursery_list)) { + // c) + _assert_remove(nursery_list, node); + _assert_remove(_tree, node); + _assert_replace(ref node, replace_value); + new_pk = parent(node.value); + if (new_pk.Equals(old_pk)) { + // the parent has NOT changed; re-insert the node, then return. nursery_list.Add(node); _tree.Add(node); + return; + } else { + // the parent HAS changed; if the old nursery_list is empty, remove it. + if (nursery_list.Count == 0) + _nursery.Remove(old_pk); } + } else { + // error) + throw new InvalidOperationException($"Unknown parent key '{old_pk}'."); } + // the parent HAS changed. Determine the list(s) in which the node needs to inserted. + if (new_pk.Equals(_root_id)) { + _tree.Add(node); + } else if (_index.TryGetValue(new_pk, out Node new_pnode)) { + new_pnode.children.Add(node); + } else { + if (!_nursery.TryGetValue(new_pk, out SortedList new_nursery_list)) { + _nursery.Add(new_pk, new_nursery_list = new SortedList(_comparer)); + } + new_nursery_list.Add(node); + _tree.Add(node); + } + } + + /// + /// Attempts to removes the node from the list and throws an exception if it does not succeed. + /// + static void _assert_remove(SortedList list, Node node) { + if (!list.Remove(node)) + throw new InvalidOperationException("Failed to remove node."); + } + + /// + /// Replaces using the specified delegate, and throws an exception if the new value's id is different from the old value's id. + /// + void _assert_replace(ref Node node, Func replace_value) { + TKey old_key = id(node.value); + node.value = replace_value(node.value); + if (!id(node.value).Equals(old_key)) + throw new InvalidOperationException("Replace op changed Node's id."); } void _remove_from_nursery(TKey pk, Node node) { From e05ce79abf303ea1f7f0ce4364b7a3725b0556c9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 20 Dec 2018 20:30:10 -0500 Subject: [PATCH 04/30] It is not always safe to convert Foo to FooImmutable. It is always safe to convert it to Nullabke --- .../Market/MarketView/MarketViewRow.cs | 24 +++++++------------ .../Market/TimeAndSales/TimeAndSalesModel.cs | 7 ++++-- .../spark/Modules/Spreader/SpreaderModel.cs | 4 ++-- .../Modules/Spreader/VM/SyntheticEditorVM.cs | 6 ++--- .../spark/Modules/Trading/Fills/FillsVM.cs | 4 ++-- .../spark/Orders/OrderPropertiesEditorVM.cs | 10 ++++---- .../Orders/OrderTypeVMs/SpreaderOrderVM.cs | 3 ++- src/client/spark/Orders/OrdersModel.cs | 11 +++++---- .../Dialogs/SupportRequestWindow.xaml.cs | 8 ++++--- src/client/spark/view/Spark/Core/TickUtils.cs | 12 +++++++--- .../view/Spark/Utilities/OrderDataHelper.cs | 3 ++- src/client/spark_api/md/MDSource.cs | 2 +- .../spark_api/security/SecurityFormatter.cs | 3 ++- .../spark_api/security/SecurityHelper.cs | 22 ++++------------- .../spark_test/Security/SecurityTests.cs | 7 +++--- .../templates/c#/immutable.pb.cs | 5 ++-- 16 files changed, 64 insertions(+), 67 deletions(-) diff --git a/src/client/spark/Modules/Market/MarketView/MarketViewRow.cs b/src/client/spark/Modules/Market/MarketView/MarketViewRow.cs index 4dce5ec3a..5eda77cdf 100644 --- a/src/client/spark/Modules/Market/MarketView/MarketViewRow.cs +++ b/src/client/spark/Modules/Market/MarketView/MarketViewRow.cs @@ -210,36 +210,28 @@ namespace mbts.spark { var order = _orders_model.factory.order(security); switch (col) { case MarketViewView.Columns.Bid: - order.Side = Side.Sell; - _bus.post(SparkTags.ShowOrderTicketTag, (order, bid_price, 0u)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Sell), bid_price, 0u)); break; case MarketViewView.Columns.BidMergedSize: - order.Side = Side.Sell; - _bus.post(SparkTags.ShowOrderTicketTag, (order, bid_price, bid_merged_size)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Sell), bid_price, bid_merged_size)); break; case MarketViewView.Columns.BidDirectSize: - order.Side = Side.Sell; - _bus.post(SparkTags.ShowOrderTicketTag, (order, bid_price, bid_direct_size)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Sell), bid_price, bid_direct_size)); break; case MarketViewView.Columns.BidImpliedSize: - order.Side = Side.Sell; - _bus.post(SparkTags.ShowOrderTicketTag, (order, bid_price, bid_implied_size)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Sell), bid_price, bid_implied_size)); break; case MarketViewView.Columns.Ask: - order.Side = Side.Buy; - _bus.post(SparkTags.ShowOrderTicketTag, (order, ask_price, 0u)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Buy), ask_price, 0u)); break; case MarketViewView.Columns.AskMergedSize: - order.Side = Side.Buy; - _bus.post(SparkTags.ShowOrderTicketTag, (order, ask_price, ask_merged_size)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Buy), ask_price, ask_merged_size)); break; case MarketViewView.Columns.AskDirectSize: - order.Side = Side.Sell; - _bus.post(SparkTags.ShowOrderTicketTag, (order, ask_price, ask_direct_size)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Sell), ask_price, ask_direct_size)); break; case MarketViewView.Columns.AskImpliedSize: - order.Side = Side.Sell; - _bus.post(SparkTags.ShowOrderTicketTag, (order, ask_price, ask_implied_size)); + _bus.post(SparkTags.ShowOrderTicketTag, (order.with(side:Side.Sell), ask_price, ask_implied_size)); break; case MarketViewView.Columns.Last: _bus.post(SparkTags.ShowOrderTicketTag, (order, last_price, 0u)); diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs index fb5b92f62..090cd58ad 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs @@ -97,8 +97,11 @@ namespace mbts.spark { public IEnumerable read_trades() { var copy = _securities; foreach (var tm in _channel.read_trades(ref _cursor)) { - if (copy.TryGetValue(tm.Key, out SecurityImmutable sec)) - yield return new TradeInfo(sec, tm); + if (copy.TryGetValue(tm.Key, out SecurityImmutable sec)) { + if (tm != null) { + yield return new TradeInfo(sec, new TradeMessageImmutable(tm)); + } + } } } } diff --git a/src/client/spark/Modules/Spreader/SpreaderModel.cs b/src/client/spark/Modules/Spreader/SpreaderModel.cs index c1f82ab8a..346f74e22 100644 --- a/src/client/spark/Modules/Spreader/SpreaderModel.cs +++ b/src/client/spark/Modules/Spreader/SpreaderModel.cs @@ -53,8 +53,8 @@ namespace mbts.spark { return null; } - public async Task create_synthetic(SecurityImmutable synthetic) { - var result = await _service.create_synthetic_async(synthetic); + public async Task create_synthetic(SecurityImmutable? synthetic) { + var result = await _service.create_synthetic_async(synthetic.Value); return _error.handle_error(result, "Error creating synthetic: {0}", r => r.Error, e => e == CreateResult.Types.Error.Ok, r => new uint?(r.Key), () => null).result; } diff --git a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs index 6381e8f5d..093d9f4d9 100644 --- a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs +++ b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs @@ -294,9 +294,9 @@ namespace mbts.spark { base.Dispose(disposing); } - static string _deduce_symbol(SecurityImmutable security) { - if (!string.IsNullOrWhiteSpace(security.Symbol)) return security.Symbol; - if (security.Synthetic.Legs.Count() > 0) return security.Synthetic.Legs.First().Security.Symbol; + static string _deduce_symbol(SecurityImmutable? security) { + if (!string.IsNullOrWhiteSpace(security?.Symbol)) return security?.Symbol; + if (security?.Synthetic.Legs.Count() > 0) return security?.Synthetic.Legs.First().Security.Symbol; return string.Empty; } diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index 320db04d9..595108167 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -178,7 +178,7 @@ namespace mbts.spark { } else return model.on_fill_tuples; } - static IConnectableObservable> _get_observable(IBus bus, OrdersModel model, IObservable state) => + static IConnectableObservable> _get_observable(IBus bus, OrdersModel model, IObservable state) => state.SelectManyInSeries(s => _get_observable(bus, model, s)); public class Row : SecurityRow, IEquatable, IRowVM { @@ -186,7 +186,7 @@ namespace mbts.spark { readonly FillImmutable _fill; readonly OrderRecordImmutable _order; - public Row(FillTuple fill_tuple, OrdersModel model) : base(fill_tuple.security_md.HasValue ? fill_tuple.security_md.Value.security : SecurityDataHelper.Defaults.Security) { + public Row(FillTuple fill_tuple, OrdersModel model) : base(fill_tuple.security_md.HasValue ? fill_tuple.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable) { _fill = fill_tuple.fill; _order = fill_tuple.order ?? OrderDataHelper.Defaults.OrderRecordImmutable; _model = model; diff --git a/src/client/spark/Orders/OrderPropertiesEditorVM.cs b/src/client/spark/Orders/OrderPropertiesEditorVM.cs index 509c67f35..53cb911f1 100644 --- a/src/client/spark/Orders/OrderPropertiesEditorVM.cs +++ b/src/client/spark/Orders/OrderPropertiesEditorVM.cs @@ -139,13 +139,13 @@ namespace mbts.spark { tif => time_in_forces.Contains(tif)); _dispose = Disposable.Create(() => _order_type.Changed -= on_selected_order_type_changed); // local methods - IOrderTypeVM create_order_type_vm(CustomOrderDefinitionImmutable cd, string id, string display_name) { + IOrderTypeVM create_order_type_vm(CustomOrderDefinitionImmutable? cd, string id, string display_name) { IOrderTypeVM vm = null; - if(cd.CustomCase == CustomOrderDefinition.CustomOneofCase.Dynamic) - vm = OrderTypeVMFactory.create(cd.CustomCase, id, display_name, model, security_md_source, cd.Dynamic.DynamicOrderType); + if(cd?.CustomCase == CustomOrderDefinition.CustomOneofCase.Dynamic) + vm = OrderTypeVMFactory.create(cd.Value.CustomCase, id, display_name, model, security_md_source, cd.Value.Dynamic.DynamicOrderType); else - vm = OrderTypeVMFactory.create(cd.CustomCase, id, display_name, model, security_md_source); - vm.load_from(cd); + vm = OrderTypeVMFactory.create(cd.Value.CustomCase, id, display_name, model, security_md_source); + vm.load_from(cd.Value); return vm; } InlineUserProperty get_order_type_vm_prop(InlineOrderProperties p) { diff --git a/src/client/spark/Orders/OrderTypeVMs/SpreaderOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/SpreaderOrderVM.cs index 7ac53dcc6..f869cc4fd 100644 --- a/src/client/spark/Orders/OrderTypeVMs/SpreaderOrderVM.cs +++ b/src/client/spark/Orders/OrderTypeVMs/SpreaderOrderVM.cs @@ -19,7 +19,8 @@ namespace mbts.spark { var so = model.factory.spreader_order(); so.Security = s.CloneToMutable(); so.ExecutionStrategy = strategy.CloneToMutable(); - return new InlineUserProperty(so, true); + var soi = new SpreaderOrderDefinitionImmutable(so); + return new InlineUserProperty(soi, true); })), id, display_name) { } public SpreaderOrderVM( diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index bf24da63c..eba27fea3 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -93,10 +93,13 @@ namespace mbts.spark { #endregion #region Input - public async Task send_order(OrderDefinitionImmutable order, double price, uint size) { - var result = (await send_order_get_error(order, price, size)); - _activity.on_send_order(order, price, size, result.key, result.error); - return result.key; + public async Task send_order(OrderDefinitionImmutable? order, double price, uint size) { + if (order.HasValue) { + var result = (await send_order_get_error(order.Value, price, size)); + _activity.on_send_order(order.Value, price, size, result.key, result.error); + return result.key; + } + return 0; } async Task<(uint? key, string error)> send_order_get_error(OrderDefinitionImmutable order, double price, uint size) { diff --git a/src/client/spark/view/Controls/Dialogs/SupportRequestWindow.xaml.cs b/src/client/spark/view/Controls/Dialogs/SupportRequestWindow.xaml.cs index 5f4638b6b..89e202b05 100644 --- a/src/client/spark/view/Controls/Dialogs/SupportRequestWindow.xaml.cs +++ b/src/client/spark/view/Controls/Dialogs/SupportRequestWindow.xaml.cs @@ -34,9 +34,11 @@ namespace bts.view.controls { private void any_button_click(object sender, RoutedEventArgs e) { Close(); } - internal static void Show(ErrorReportService srv, ErrorReport report) { - var window = new SupportRequestWindow(new SupportRequestVM(srv, report)); - window.Show(); + internal static void Show(ErrorReportService srv, ErrorReportImmutable? report) { + if (report.HasValue) { + var window = new SupportRequestWindow(new SupportRequestVM(srv, report.Value)); + window.Show(); + } } } diff --git a/src/client/spark/view/Spark/Core/TickUtils.cs b/src/client/spark/view/Spark/Core/TickUtils.cs index 180f1a3b1..e2658ce21 100644 --- a/src/client/spark/view/Spark/Core/TickUtils.cs +++ b/src/client/spark/view/Spark/Core/TickUtils.cs @@ -123,7 +123,7 @@ namespace mbts.spark { return intpart + floatpart; } - public static RegionalTickFactors get_regional_ticks(ContractSpecificationImmutable spec) => + public static RegionalTickFactors get_regional_ticks(ContractSpecificationImmutable? spec) => TickStore.get(spec); public static RegionalTickFactors get_regional_ticks(SecurityImmutable security) => @@ -177,8 +177,14 @@ namespace mbts.spark { public static RegionalTickFactors get(ContractSpecification spec) => _exchange_ticks.GetOrAdd(new SpecKey(spec), _create_rtf); - public static RegionalTickFactors get(ContractSpecificationImmutable spec) => - _exchange_ticks.GetOrAdd(new SpecKey(spec), _create_rtf); + public static RegionalTickFactors get(ContractSpecificationImmutable? spec) { + if (spec.HasValue) { + return _exchange_ticks.GetOrAdd(new SpecKey(spec.Value), _create_rtf); + } + else { + return null; + } + } public static RegionalTickFactors get(SecurityImmutable security) => security.HasContractSpec ? get(security.ContractSpec) : default_rtf; diff --git a/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs b/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs index e9ef09d5b..922765656 100644 --- a/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs +++ b/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs @@ -165,7 +165,8 @@ namespace mbts.spark.utilities { } #endregion - public OrderDefinition order(SecurityImmutable security) => order(security, _admin_model.get_default_account()); + public OrderDefinitionImmutable order(SecurityImmutable security) => + new OrderDefinitionImmutable(order(security, _admin_model.get_default_account())); public OrderDefinition order(SecurityImmutable security, AccountTuple? account) { var rv = new OrderDefinition { SecurityKey = security.Id, diff --git a/src/client/spark_api/md/MDSource.cs b/src/client/spark_api/md/MDSource.cs index c329f6216..9fdebb63b 100644 --- a/src/client/spark_api/md/MDSource.cs +++ b/src/client/spark_api/md/MDSource.cs @@ -54,7 +54,7 @@ namespace bts.md { foreach (var v in values) { - var smd = new SecurityMD(v.security, v.handle); + var smd = new SecurityMD(new SecurityImmutable(v.security), v.handle); _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd); } _version += 1; diff --git a/src/client/spark_api/security/SecurityFormatter.cs b/src/client/spark_api/security/SecurityFormatter.cs index c03482ef1..2c3da3954 100644 --- a/src/client/spark_api/security/SecurityFormatter.cs +++ b/src/client/spark_api/security/SecurityFormatter.cs @@ -22,7 +22,8 @@ namespace bts.api { static readonly ConcurrentDictionary _formatted_names_cache = new ConcurrentDictionary(); public static string format(Security security) => - _formatted_names_cache.GetOrAdd(security.Id, _format(security)); + security != null ? + _formatted_names_cache.GetOrAdd(security.Id, _format(new SecurityImmutable(security))) : ""; public static string format(SecurityImmutable security) => _formatted_names_cache.GetOrAdd(security.Id, _ => _format(security)); diff --git a/src/client/spark_api/security/SecurityHelper.cs b/src/client/spark_api/security/SecurityHelper.cs index af11abc23..7ef75b988 100644 --- a/src/client/spark_api/security/SecurityHelper.cs +++ b/src/client/spark_api/security/SecurityHelper.cs @@ -66,24 +66,10 @@ namespace bts.api { public static readonly IComparer InstanceImmutable = Comparer.Create(Compare); public static int Compare(Security x, Security y) { - if (x.ContractCase == ContractOneofCase.Future) return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1; - else if (y.ContractCase == ContractOneofCase.Future) return 1; - var xlegs = get_flat_future_legs(x).ToList(); - var ylegs = get_flat_future_legs(y).ToList(); - int max = Math.Min(xlegs.Count, ylegs.Count); - for (int i = 0; i < max; i++) { - int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future); - if (rv != 0) return rv; - } - for (int i = 0; i < max; i++) { - int rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity); - if (rv != 0) return rv; - } - for (int i = 0; i < max; i++) { - int rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange); - if (rv != 0) return rv; - } - return xlegs.Count.CompareTo(ylegs.Count); + if (ReferenceEquals(x, y)) return 0; // x ==y + if (x == null) return 1; // y > x + if (y == null) return -1; // x > y + return Compare(new SecurityImmutable(x), new SecurityImmutable(y)); } public static int Compare(SecurityImmutable x, SecurityImmutable y) { diff --git a/src/client/spark_test/Security/SecurityTests.cs b/src/client/spark_test/Security/SecurityTests.cs index 63104702a..f564bd858 100644 --- a/src/client/spark_test/Security/SecurityTests.cs +++ b/src/client/spark_test/Security/SecurityTests.cs @@ -12,9 +12,10 @@ namespace bts.mcore class SecurityTests { [Test] public void test_security_equal() { - SecurityImmutable a = new Security() { Id = 42 }; - SecurityImmutable b = new Security() { Id = 42 }; - SecurityImmutable c = new Security() { Id = 4 }; + + SecurityImmutable a = new SecurityImmutable(new Security() { Id = 42 }); + SecurityImmutable b = new SecurityImmutable(new Security() { Id = 42 }); + SecurityImmutable c = new SecurityImmutable(new Security() { Id = 4 }); Assert.IsTrue(a == b); Assert.IsTrue(a.Equals(b)); diff --git a/src/lib/api/service_compiler/templates/c#/immutable.pb.cs b/src/lib/api/service_compiler/templates/c#/immutable.pb.cs index 4972886db..06c8260c6 100644 --- a/src/lib/api/service_compiler/templates/c#/immutable.pb.cs +++ b/src/lib/api/service_compiler/templates/c#/immutable.pb.cs @@ -31,8 +31,9 @@ namespace {{package}} { public {{name}} CloneToMutable() { return _impl.Clone(); } // Implicit conversion operator - public static implicit operator {{name}}Immutable({{name}} other) { - return new {{name}}Immutable(other, false); + public static implicit operator Nullable<{{name}}Immutable>({{name}} other) { + if (other == null) return null; + else return new {{name}}Immutable(other, false); } public override bool Equals(object other) { From 7cbf0dc6832d5d066cba27811117167f00f9f692 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 21 Dec 2018 10:38:56 -0500 Subject: [PATCH 05/30] CR feedback - immutables --- .../Market/TimeAndSales/TimeAndSalesModel.cs | 4 ++-- .../spark_api/security/SecurityFormatter.cs | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs index 090cd58ad..48a11aff0 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesModel.cs @@ -97,8 +97,8 @@ namespace mbts.spark { public IEnumerable read_trades() { var copy = _securities; foreach (var tm in _channel.read_trades(ref _cursor)) { - if (copy.TryGetValue(tm.Key, out SecurityImmutable sec)) { - if (tm != null) { + if (tm != null) { + if (copy.TryGetValue(tm.Key, out SecurityImmutable sec)) { yield return new TradeInfo(sec, new TradeMessageImmutable(tm)); } } diff --git a/src/client/spark_api/security/SecurityFormatter.cs b/src/client/spark_api/security/SecurityFormatter.cs index 2c3da3954..0a328cdf2 100644 --- a/src/client/spark_api/security/SecurityFormatter.cs +++ b/src/client/spark_api/security/SecurityFormatter.cs @@ -22,8 +22,7 @@ namespace bts.api { static readonly ConcurrentDictionary _formatted_names_cache = new ConcurrentDictionary(); public static string format(Security security) => - security != null ? - _formatted_names_cache.GetOrAdd(security.Id, _format(new SecurityImmutable(security))) : ""; + _formatted_names_cache.GetOrAdd(security.Id, _format(security)); public static string format(SecurityImmutable security) => _formatted_names_cache.GetOrAdd(security.Id, _ => _format(security)); @@ -38,17 +37,17 @@ namespace bts.api { $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}" : exch_desc; - static string _format(SecurityImmutable security) { - switch (security.ContractCase) { - case ContractOneofCase.Future: return format_future(security.Symbol, security.glbx_desc(), security.Future); + static string _format(SecurityImmutable? security) { + switch (security?.ContractCase) { + case ContractOneofCase.Future: return format_future(security?.Symbol, security?.glbx_desc(), security.Value.Future); case ContractOneofCase.Spread: - if (security.HasGlbx) { + if (security?.HasGlbx == true) { // In case we ever bring it back, pretty printed spread names can be found here: // GlbxSecurityFormatter.FormatSpread(security, out string result) - return security.Glbx.SecurityDesc; + return security?.Glbx.SecurityDesc; } - return string.Join("", _format_future_legs(SecurityHelper.get_flat_future_legs(security.Spread.Legs))); - case ContractOneofCase.Synthetic: return security.Synthetic.Name; + return string.Join("", _format_future_legs(SecurityHelper.get_flat_future_legs(security?.Spread.Legs))); + case ContractOneofCase.Synthetic: return security?.Synthetic.Name; default: return "Unknown-security"; } } From 8a4d3d4c97dc92f4693c1b0a4aea87807c2966d8 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 21 Dec 2018 12:24:30 -0600 Subject: [PATCH 06/30] SPARK-1903 CR - unit test for SortedTree.TryReplace, comments in OrderBookRowCollection --- .../OrderBook/OrderBookRowCollection.cs | 2 + .../spark_test/SortedTree/SortedTreeTests.cs | 48 +++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs index 6866c5874..b572530fc 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs @@ -14,6 +14,8 @@ namespace mbts.spark { class OrderBookRowCollection : ObservableCollection { static readonly IEqualityComparer> _node_equality_comparer = EqualityComparer.Create, uint>(n => n.value.order_key); readonly Func _row_factory; + /// Critical: Any and all row mutations must be executed through . + /// Mutations can change the sort-position of a row, and will be broken if its elements aren't properly sorted (it relies heavily on binary-search logic). SortedTree _tree; IComparer _comparer; Func _filter; diff --git a/src/client/spark_test/SortedTree/SortedTreeTests.cs b/src/client/spark_test/SortedTree/SortedTreeTests.cs index 1c9cbfe50..d578f50aa 100644 --- a/src/client/spark_test/SortedTree/SortedTreeTests.cs +++ b/src/client/spark_test/SortedTree/SortedTreeTests.cs @@ -25,9 +25,9 @@ namespace bts.tests { parent_id = parent; } - public char name { get; } + public char name { get; set; } public int id { get; } - public int parent_id { get; } + public int parent_id { get; set; } public override bool Equals(object obj) { Order other = (Order)obj; @@ -133,6 +133,9 @@ namespace bts.tests { /// 2) Insert - all items are properly added via /// 3) Reset - all items are properly added via /// 4) Replace - all items are properly updated with a new value and/or parent via + /// a) delegate returns new value + /// b) delegate mutates existing value + /// - failing to account for this usage led to a bug in the OrderBook ca. 12/20/2018 (https://bluetradesys.atlassian.net/browse/SPARK-1903) // 1) Constructor Initialization var tree_asc = new SortedTree(Order.name_asc, id, pid, input); @@ -154,6 +157,7 @@ namespace bts.tests { // 4) Replace tree_asc.Reset(input); tree_desc.Reset(input); + //// a) delegate returns new value var modified_input = _randomize_values(input); foreach (var v in modified_input) { Assert.IsTrue(tree_asc.TryReplace(id(v), _ => v)); @@ -162,7 +166,31 @@ namespace bts.tests { // verify asc/desc assert_state_then_clear(tree_asc, modified_input, Order.name_asc); assert_state_then_clear(tree_desc, modified_input, Order.name_desc); - + // reset with modified_input since we are mutating next ('input' is essentially a global variable, shared across the test iterations). + tree_asc.Reset(modified_input); + tree_desc.Reset(modified_input); + //// b) delegate mutates existing value + for (int i = 0; i < modified_input.Count; i++) { + var a = modified_input[i]; + var b = modified_input[modified_input.Count - i - 1]; + var swap_name = _swap_name_del(b.name); + var swap_parent = _swap_parent_del(b.parent_id); + var swap_both = _swap_name_and_parent_del(b.name, b.parent_id); + var reset = _swap_name_and_parent_del(a.name, a.parent_id); + foreach (var tree in new[] { tree_asc, tree_desc }) { + Assert.IsTrue(tree.TryReplace(id(a), swap_name)); + Assert.IsTrue(tree.TryReplace(id(a), swap_parent)); + // reset 'a' so swap_both will actually change values (we already swapped name and parent individually above). + Assert.IsTrue(tree.TryReplace(id(a), reset)); + Assert.IsTrue(tree.TryReplace(id(a), swap_both)); + // reset 'a' to finish. Since tree_asc/desc share the same actual orders, this protects the other tree from becoming unsorted as a result of the mutations. + Assert.IsTrue(tree.TryReplace(id(a), reset)); + } + } + // verify asc/desc + assert_state_then_clear(tree_asc, modified_input, Order.name_asc); + assert_state_then_clear(tree_desc, modified_input, Order.name_desc); + // local methods void assert_state_then_clear(SortedTree tree, IEnumerable vals, IComparer comp) { Assert.IsTrue(tree.SequenceEqual(_hierarchically_sorted_list(vals, comp))); clear_action(tree); @@ -265,6 +293,20 @@ namespace bts.tests { return reshuffled; } + Func _swap_name_del(char name) => o => { + o.name = name; + return o; + }; + Func _swap_parent_del(int parent) => o => { + o.parent_id = parent; + return o; + }; + Func _swap_name_and_parent_del(char name, int parent) => o => { + o.name = name; + o.parent_id = parent; + return o; + }; + void _write_tree_to_console(SortedTree tree) { foreach (var str in _fmt_output(tree.RootNodes)) Console.WriteLine(str); From 9147b4605ab763066d39dd75fbfd6689bb9b112f Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 21 Dec 2018 16:21:34 -0600 Subject: [PATCH 07/30] adding zoom_factor to window_specifics --- src/client/spark/view/Spark/Ladder/LadderSettings.cs | 8 +++++--- src/client/spark/view/Spark/Ladder/LadderVM.cs | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderSettings.cs b/src/client/spark/view/Spark/Ladder/LadderSettings.cs index 75fe933ef..7fce37546 100644 --- a/src/client/spark/view/Spark/Ladder/LadderSettings.cs +++ b/src/client/spark/view/Spark/Ladder/LadderSettings.cs @@ -469,13 +469,13 @@ namespace mbts.spark { use_defaults.GetHashCode()); public struct WindowSpecifics { - public readonly double order_entry_panel_width; + public readonly double order_entry_panel_width, zoom_factor; public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width; public WindowSpecifics(LadderSettings s) - : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width) { } + : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor) { } - public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width) { + public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor) { this.order_entry_panel_width = order_entry_panel_width; this.orders_column_width = orders_column_width; this.bid_column_width = bid_column_width; @@ -483,6 +483,7 @@ namespace mbts.spark { this.ask_column_width = ask_column_width; this.last_column_width = last_column_width; this.volume_column_width = volume_column_width; + this.zoom_factor = zoom_factor; } } } @@ -513,6 +514,7 @@ namespace mbts.spark { source.ask_column.width = specifics.ask_column_width; source.last_column.width = specifics.last_column_width; source.volume_column.general.width = specifics.volume_column_width; + source.zoom_factor = specifics.zoom_factor; return source; } } diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index a85f2e02e..7e4ef5000 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -25,6 +25,7 @@ namespace mbts.spark { readonly IBus _bus; readonly LadderModel _model; readonly Property _high, _low, _change, _open, _settlement, _position, _left_panel_width, _zoom_factor; + readonly Property _window_width, _window_height; readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width; readonly Property _volume; readonly Property _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account, @@ -300,7 +301,7 @@ namespace mbts.spark { public LadderSettings get_save_state() { var state = _settings_manager.current_settings(); - state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width)); + state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor)); state.use_defaults = _settings_manager.instance_settings == null; return state; } From 7429c42de2203a8aab4669fd6fde1c6e35c7c833 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 21 Dec 2018 16:24:57 -0600 Subject: [PATCH 08/30] checking this in b/c i've lost things too many times keeping stuff in the stash and switching branches; this is just really testing, no sln yet regarding window borders --- src/client/spark/view/Spark/Ladder/LadderVM.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 7e4ef5000..cf037804f 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -97,6 +97,8 @@ namespace mbts.spark { // _zoom_factor = backing(nameof(zoom_factor), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), LadderSettings.Default.zoom_factor); + _window_height = backing(nameof(window_height), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 600); + _window_width = backing(nameof(window_width), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 420); position_click_command = new DelegateCommand(() => size_calculator.value = DoubleHelper.safe_convert_to_uint(Math.Abs(position))); send_order_command = new DelegateCommand<(double price, Side? side, bool size_from_alternate)>(send_order); cancel_command = new AsyncCommand(_model.cancel_orders, true); @@ -290,6 +292,8 @@ namespace mbts.spark { public bool show_zoom { get => _show_zoom.Value; set => _show_zoom.set(value); } public double left_panel_width { get => _left_panel_width.Value; set => _left_panel_width.set(value); } public double zoom_factor { get => _zoom_factor.Value; set => _zoom_factor.set(value); } + public double window_height { get => _window_height.Value; set => _window_height.set(value); } + public double window_width { get => _window_width.Value; set => _window_width.set(value); } public uint orders_column_width { get => _orders_column_width.Value; set => _orders_column_width.set(value); } public uint bid_column_width { get => _bid_column_width.Value; set => _bid_column_width.set(value); } public uint price_column_width { get => _price_column_width.Value; set => _price_column_width.set(value); } From 5320e16088bc96bc43cbcdac484cb3619a553701 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 26 Dec 2018 11:02:07 -0600 Subject: [PATCH 09/30] commenting out anything having to do w/ window width and height since that interaction is still mysterious; fixing scale factor restortation --- .../spark/view/Spark/Ladder/LadderVM.cs | 12 +++++----- .../spark/view/Spark/Ladder/LadderView.xaml | 7 ++++-- .../view/Spark/Ladder/LadderView.xaml.cs | 22 ++++++++++++++++++- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index cf037804f..4b7528808 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -25,7 +25,7 @@ namespace mbts.spark { readonly IBus _bus; readonly LadderModel _model; readonly Property _high, _low, _change, _open, _settlement, _position, _left_panel_width, _zoom_factor; - readonly Property _window_width, _window_height; + //readonly Property _window_width, _window_height; readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width; readonly Property _volume; readonly Property _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account, @@ -59,6 +59,7 @@ namespace mbts.spark { _last_column_width = backing(nameof(last_column_width), ws.last_column_width); _volume_column_width = backing(nameof(volume_column_width), ws.volume_column_width); _left_panel_width = backing(nameof(left_panel_width), ws.order_entry_panel_width); + _zoom_factor = backing(nameof(zoom_factor), ws.zoom_factor); // remember: account_positions needs to subscribe to selected_account; both the sub and account_positions need to be disposed var account_positions = new KeyDrivenObservableSeries(a => _model.order_gateway.security_total_position(security.Id, a)); _position = backing(nameof(position), account_positions.OnDispatcher()); @@ -96,9 +97,8 @@ namespace mbts.spark { backing(name, settings_manager.subscribe_current().OnDispatcher(), get_bool); // - _zoom_factor = backing(nameof(zoom_factor), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), LadderSettings.Default.zoom_factor); - _window_height = backing(nameof(window_height), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 600); - _window_width = backing(nameof(window_width), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 420); + //_window_height = backing(nameof(window_height), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 600); + //_window_width = backing(nameof(window_width), settings_manager.subscribe_current().OnDispatcher().Select(s => s.zoom_factor), (scale) => scale * 420); position_click_command = new DelegateCommand(() => size_calculator.value = DoubleHelper.safe_convert_to_uint(Math.Abs(position))); send_order_command = new DelegateCommand<(double price, Side? side, bool size_from_alternate)>(send_order); cancel_command = new AsyncCommand(_model.cancel_orders, true); @@ -292,8 +292,8 @@ namespace mbts.spark { public bool show_zoom { get => _show_zoom.Value; set => _show_zoom.set(value); } public double left_panel_width { get => _left_panel_width.Value; set => _left_panel_width.set(value); } public double zoom_factor { get => _zoom_factor.Value; set => _zoom_factor.set(value); } - public double window_height { get => _window_height.Value; set => _window_height.set(value); } - public double window_width { get => _window_width.Value; set => _window_width.set(value); } + //public double window_height { get => _window_height.Value; set => _window_height.set(value); } + //public double window_width { get => _window_width.Value; set => _window_width.set(value); } public uint orders_column_width { get => _orders_column_width.Value; set => _orders_column_width.set(value); } public uint bid_column_width { get => _bid_column_width.Value; set => _bid_column_width.set(value); } public uint price_column_width { get => _price_column_width.Value; set => _price_column_width.set(value); } diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml b/src/client/spark/view/Spark/Ladder/LadderView.xaml index 3303569c9..65b738d80 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml @@ -9,6 +9,7 @@ xmlns:uag="clr-namespace:mbts.ui.spark.algo" xmlns:converters="clr-namespace:mbts.ui.converters" xmlns:txt="clr-namespace:mbts.ui.utility.text" + xmlns:sys="clr-namespace:System;assembly=mscorlib" Height="600" Width="420" Background="{x:Static mbts:BTSColors.common_background_brush}" @@ -273,6 +274,8 @@ + 1.5 + 0.5 @@ -306,8 +309,8 @@ diff --git a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs index 0ea0860ef..273bb4d9a 100644 --- a/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs +++ b/src/client/spark/view/Spark/Ladder/LadderView.xaml.cs @@ -9,6 +9,8 @@ using System.Windows.Input; namespace mbts.spark { partial class LadderView : UserControl { + readonly double _max_zoom, _min_zoom; + internal LadderView(LadderVM vm) { InitializeComponent(); _column_headers_presenter.Columns = new GridViewColumnCollection(); @@ -17,6 +19,19 @@ namespace mbts.spark { this.vm = vm; DataContext = vm; DataContextChanged += datacontext_changed; + if (TryFindResource("MaxZoomFactor") is double found_max) { + _max_zoom = found_max; + } else { + // is there a better way of not specifying the constant twice ? + // once in the resource dictionary here and again in the xaml, seems like there has to be a better way + _max_zoom = 1.5; + } + // ditto + if (TryFindResource("MinZoomFactor") is double found_min) { + _min_zoom = found_min; + } else { + _min_zoom = 0.5; + } _ui_scaler.MouseDoubleClick += _ui_scaler_MouseDoubleClick; this.PreviewMouseWheel += LadderView_PreviewMouseWheel; this.PreviewMouseDown += LadderView_PreviewMouseDown; @@ -29,7 +44,12 @@ namespace mbts.spark { private void LadderView_PreviewMouseWheel(object sender, System.Windows.Input.MouseWheelEventArgs e) { base.OnPreviewMouseWheel(e); if (Keyboard.IsKeyDown(Key.LeftCtrl) || Keyboard.IsKeyDown(Key.RightCtrl)) { - _ui_scaler.Value += (e.Delta > 0) ? 0.1 : -0.1; + var delta = (e.Delta > 0) ? 0.1 : -0.1; + var proposed_scaler_value = _ui_scaler.Value + delta; + // Here we enforce the same contraints that are on the gui slider regarding min and max + if (_min_zoom < proposed_scaler_value && proposed_scaler_value < _max_zoom) { + _ui_scaler.Value = proposed_scaler_value; + } } } From 879963b82809a9f7b43ddc5c40abf4b716d5caaf Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 26 Dec 2018 13:55:38 -0600 Subject: [PATCH 10/30] updating release notes --- doc/user/docs/release_notes.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index d6668be35..6b5c0aa78 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,4 +1,13 @@ # Release Notes +## 1.4.6 + +### Ladder Usability Improvements + - The ladder can now be zoomed in/out according to a user's preference + - The zoom slider can be enabled from Ladder Form -> Settings button -> Left Pane tab -> check Zoom Slider checkbox + - The zoom functionality can also be performed via hotkey and mouse wheel combinations + - Zoom In = CTRL + Mouse wheel up + - Zoom Out = CTRL + Mouse wheel down + - Reset zoom = CTRL + Mouse wheel click or double click on the zoom slider ## 1.4.5 - Fixed entitlement validation for ICS spreads - Tuned performance logging From 7e68dc4ff7697f26ee7724e560253e3750e9d4f0 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 26 Dec 2018 13:58:03 -0600 Subject: [PATCH 11/30] fixing tabs in notes --- doc/user/docs/release_notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index 6b5c0aa78..1650f2388 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -6,8 +6,8 @@ - The zoom slider can be enabled from Ladder Form -> Settings button -> Left Pane tab -> check Zoom Slider checkbox - The zoom functionality can also be performed via hotkey and mouse wheel combinations - Zoom In = CTRL + Mouse wheel up - - Zoom Out = CTRL + Mouse wheel down - - Reset zoom = CTRL + Mouse wheel click or double click on the zoom slider + - Zoom Out = CTRL + Mouse wheel down + - Reset zoom = CTRL + Mouse wheel click or double click on the zoom slider ## 1.4.5 - Fixed entitlement validation for ICS spreads - Tuned performance logging From 95ae7b780e950047bcd70db1256b95ae1a606d39 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 27 Dec 2018 21:35:18 +0000 Subject: [PATCH 12/30] Propagated display status through order state update event. --- src/lib/orderman/events/OrderStateUpdateEvent.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/orderman/events/OrderStateUpdateEvent.h b/src/lib/orderman/events/OrderStateUpdateEvent.h index 678b5f9c9..38ea850b4 100644 --- a/src/lib/orderman/events/OrderStateUpdateEvent.h +++ b/src/lib/orderman/events/OrderStateUpdateEvent.h @@ -25,8 +25,10 @@ class OrderStateUpdateEvent : public mercury::Event { last_price = state.last_price(); avg_price = state.avg_price(); last_msg = state.last_msg(); + display_status = state.display_status(); + custom = state.custom_case(); - + auto native_state = get_native_state(state); if (native_state) { exchange_order_id = native_state->exchange_order_id(); @@ -50,7 +52,8 @@ class OrderStateUpdateEvent : public mercury::Event { double last_price; double avg_price; std::string last_msg; - + api::orderman::OrderState::DisplayStatus display_status; + trace::CorID trace; api::orderman::OrderState::CustomCase custom; @@ -74,7 +77,8 @@ class OrderStateUpdateEvent : public mercury::Event { result.set_last_price(last_price); result.set_avg_price(avg_price); result.set_last_msg(last_msg); - + result.set_display_status(display_status); + switch (custom) { case api::orderman::OrderState::kMarket: rebuild_native_state(result.mutable_market()->mutable_native_state()); From 8075bafb86e55a61712acb66d302ad3ded9745a1 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 31 Dec 2018 13:07:18 -0500 Subject: [PATCH 13/30] Update spark replay defaults --- src/lib/config/protos/MarketData.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 5ad80beea..90b204f74 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -123,8 +123,8 @@ message RecordOptions { // message ReplayOptions { optional string host = 1 [default = "mdreplay"]; - optional string base_path = 2 [default = "/md"]; - optional string file_name = 3 [default = "options.coconut"]; + optional string base_path = 2 [default = "/spark"]; + optional string file_name = 3 [default = "futures.coconut"]; // 2.0 = double speed, 0.5 = half speed optional double speed = 11 [default = 1.0]; From 15e557a209428b02f76b2d2cece4a4ad7a2586ba Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 10:49:41 -0500 Subject: [PATCH 14/30] Update cmakedeps --- src/lib/CMakeDeps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index ec7dd860c..afd768425 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/orders/NativeOrder.cpp;orderman2/orders/Order.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From 6dfe9b3892aeec4f8bed38961f9537d693a0eb25 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 10:50:05 -0500 Subject: [PATCH 15/30] Add CoconutSecurityStatus to md tail --- src/lib/md/Diag/MDPeriodicDumper.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index f1ff98fe7..9fbfdc56c 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -72,6 +72,7 @@ std::string print_book( << e.security->id() << "," << seq << "," << (ambiguous ? "true," : "false,") + << (e.handle.status() == co2::COCONUTSECURITYSTATUS_OK ? "OK," : "GAP,") << twrite.to_uint64() << "," << tread.to_uint64() << ","; for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { @@ -138,7 +139,7 @@ void write_header(PrintFormat format, long seq_mod, const std::vector& en } else { // CSV header - std::cout << "Symbol,Id,Sequence,Ambiguous,TWrite,TRead,BidBook,AskBook"; + std::cout << "Symbol,Id,Sequence,Ambiguous,Status,TWrite,TRead,BidBook,AskBook"; // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { // std::cout << "," << "Bid" << x << ", DirectBidSize" << x << ", ImpliedBidSize" << x; // } From 714dfd5038dc26deb312411e3a1db9b96eacd0e4 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 2 Jan 2019 17:05:38 +0000 Subject: [PATCH 16/30] Fixes sim drift awy from MD source. --- src/lib/CMakeDeps.cmake | 2 +- .../exchange_sim/ExchangeSimApplication.cpp | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index afd768425..ec7dd860c 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/orders/NativeOrder.cpp;orderman2/orders/Order.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp index 2a36f53f6..d1863fbaf 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -95,7 +95,6 @@ convert(const FIX::ManualOrderIndicator &moi) { struct ExchangeSimApplication::MDConsumerRec { api::Security security; utils::TimeStamp last_read; - Book previous_md_book; }; struct ExchangeSimApplication::PImpl { @@ -456,6 +455,8 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) { auto book = matcher.get_book(); + LOG(ExSim, Diag, "Current book {}:\n {}", sec_desc,book); + auto price_factor = pow(10, handle.security().display_factor); const auto cmax = md::COCONUT_MAX_DEPTH; @@ -525,7 +526,7 @@ void ExchangeSimApplication::on_new_security(api::Security const &s) { me->md_producer_handles.insert(make_pair(sec_desc, handle)); me->md_consumer_recs.insert( make_pair(s.glbx().security_desc(), - MDConsumerRec{s2, utils::TimeStamp::now(), Book()})); + MDConsumerRec{s2, utils::TimeStamp::now()})); }); } } @@ -545,6 +546,7 @@ void ExchangeSimApplication::on_depth( md::PriceSize asks[md::COCONUT_MAX_DEPTH], std::array* implied_bids, std::array* implied_asks) { + std::lock_guard l(me->lock); if (sec.has_synthetic()) @@ -556,8 +558,8 @@ void ExchangeSimApplication::on_depth( auto it = me->md_consumer_recs.find(sec.glbx().security_desc()); bts_assert(it != me->md_consumer_recs.end()); - // cache previous Book so we can update it place - Book previous_md_book = it->second.previous_md_book; + auto &matcher = me->matchers[get_sim_sec_desc(sec)]; + Book previous_md_book = matcher.get_book(); // populate Book structure with new bids, asks coming from outside Book current_md_book; @@ -572,9 +574,6 @@ void ExchangeSimApplication::on_depth( current_md_book.asks[a] = Quantity(asks[i].size()); } - // update the indexed Book structure with the latest values - it->second.previous_md_book = current_md_book; - // from the differences between current_md_book and previous_md_book // we generate a set of orders that we apply to our order book merge_with_order_book(it->second.security.glbx().security_desc(), @@ -597,6 +596,9 @@ void ExchangeSimApplication::merge_with_order_book(std::string const &sec_desc, auto src_book = matcher.get_book(); auto transform = compute_book_transform(src, dest); + + LOG(ExSim, Diag, "Book transformation for {}\n source:\n {} dest:\n{}", sec_desc,src,dest); + for (auto const &op : transform) { LOG(ExSim, Diag, "Adding synthetic book operation for {}. {} {} {}@{}", From e173fe55159faa4370bbf834899ac7ff4c74e045 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 2 Jan 2019 19:23:35 +0000 Subject: [PATCH 17/30] Update release notes. --- doc/user/docs/release_notes.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index d6668be35..77eb815cc 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,4 +1,17 @@ # Release Notes +## 1.4.6 +### Ladder Usability Improvements + - The ladder can now be zoomed in/out according to a user's preference + - The zoom slider can be enabled from Ladder Form -> Settings button -> Left Pane tab -> check Zoom Slider checkbox + - The zoom functionality can also be performed via hotkey and mouse wheel combinations + - Zoom In = CTRL + Mouse wheel up + - Zoom Out = CTRL + Mouse wheel down + - Reset zoom = CTRL + Mouse wheel click or double click on the zoom slider +### Exchange simulator + - Improved the exchange simulator to track real market data more closely +### Bug Fixes + - Fixed a UI crash in some cases when attempting to sort the order book by time + - ## 1.4.5 - Fixed entitlement validation for ICS spreads - Tuned performance logging From a833944ec509431d707887f6c766fc1add7f556d Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 3 Jan 2019 19:07:18 -0500 Subject: [PATCH 18/30] Improved perf log latency. --- src/data_files/files/perf_trace/perf_log.sql | 212 ++++++++++++------ src/lib/config/protos/perflog_cfg.proto | 1 - .../jetfix/connection/SocketConnection.cpp | 3 +- src/lib/tracing/PerfLog.cpp | 88 +++++--- 4 files changed, 199 insertions(+), 105 deletions(-) diff --git a/src/data_files/files/perf_trace/perf_log.sql b/src/data_files/files/perf_trace/perf_log.sql index f4c084a43..df7393430 100644 --- a/src/data_files/files/perf_trace/perf_log.sql +++ b/src/data_files/files/perf_trace/perf_log.sql @@ -21,54 +21,37 @@ trace ( uint32_1 INTEGER ); -drop view if exists recent_trace; -create view recent_trace as -select * from trace order by rowid desc limit 140123; - create index if not exists trace_idx on trace(session_start,channel,sn,ts); - -drop view if exists ordered_trace; -create view ordered_trace as -select - recent_trace.*, - row_number() over (order by session_start,ts) entry_rank -from - recent_trace; - -drop view if exists delta_trace; -create view delta_trace as -select - current.*, - (current.ts - previous.ts) / 1000.0 delta -from - ordered_trace current, - ordered_trace previous -where - current.entry_rank = previous.entry_rank+1; +create index if not exists trace_t_idx on trace(session_start,ts); drop view if exists log; create view log as select - delta_trace.*, + trace.*, + trace.rowid entry_id, + (ts-lag(ts) over by_session_ts)/1000.0 delta, case cor_event>0 when 1 then (ts-cor_event)/1000.0 else 0 end scope_delta, case cor_event>0 when 1 then "end of " || entry_kind else entry_kind end scoped_entry from - delta_trace; + trace +window + by_session_ts as (order by session_start,ts); drop view if exists events; create view events as select fix_reads.session_start, "hedge" event_type, + fix_reads.hts hts_start, + fix_reads.entry_id event_id, fix_reads.ts start, fix_writes.ts stop, (fix_writes.ts - fix_reads.ts) / 1000.0 duration, - row_number() over (order by fix_reads.session_start,fix_reads.ts,fix_writes.ts) event_rank , fix_reads.channel, fix_reads.sn from - (select session_start, channel, sn, ts from recent_trace where entry_kind='fix_msg_r') fix_reads, - (select session_start, channel, sn, ts from recent_trace where entry_kind='before_fix_socket_w') fix_writes + (select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='fix_msg_r') fix_reads, + (select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='before_fix_socket_w') fix_writes where fix_reads.session_start = fix_writes.session_start and fix_reads.channel = fix_writes.channel and @@ -77,15 +60,16 @@ union select md_reads.session_start, "requote" event_type, + md_reads.hts hts_start, + md_reads.entry_id event_id, md_reads.ts start, fix_writes.ts stop, (fix_writes.ts - md_reads.ts) / 1000.0 duration, - row_number() over (order by md_reads.session_start,md_reads.ts,fix_writes.ts) event_rank , md_reads.channel, md_reads.sn from - (select session_start, channel, sn, ts from recent_trace where entry_kind='md') md_reads, - (select session_start, channel, sn, ts from recent_trace where entry_kind='before_fix_socket_w') fix_writes + (select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='md') md_reads, + (select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='before_fix_socket_w') fix_writes where md_reads.session_start = fix_writes.session_start and md_reads.channel = fix_writes.channel and @@ -96,8 +80,9 @@ create view event_log as select log.*, event_type, - event_rank, + event_id, events.duration event_duration, + events.hts_start hts_start, events.start event_start, events.stop event_stop from @@ -107,6 +92,32 @@ where log.session_start = events.session_start and log.ts >= events.start and log.ts <= events.stop; +create table if not exists +event_log_archive ( + session_start TIMESTAMP, + ts INTEGER, + hts timestamp, + channel TEXT, + entry_kind TEXT, + txt_1 TEXT, + uint64_1 INTEGER, + thread TEXT, + cor_event INTEGER, + sn INTEGER, + uint32_1 INTEGER, + entry_id INTEGER, + delta float, + scope_delta float, + scoped_entry text, + event_type text, + event_id integer, + event_duration float, + hts_start TIMESTAMP, + event_start integer, + event_stop integer +); +create index if not exists event_log_archive_idx on event_log_archive(session_start,ts); + drop view if exists pretty_log; create view pretty_log as select @@ -121,63 +132,124 @@ select session_start, entry_rank from - event_log; + event_log_archive; drop view if exists last_pretty_log; create view last_pretty_log as select * from pretty_log where session_start=(select max(session_start) from session); --- drop view if exists summary; --- create view summary as --- select session_start,event_type,count(*) count, avg(duration) avg, max(duration) max, min(duration) min, median(duration) median, stdev(duration) stdev from events group by session_start,event_type; +drop view if exists event_summary; +create view event_summary as +select + max(session_start) session_start, + max(event_type) event_type, + max(event_duration) duration, + min(event_start) start, + min(event_stop) stop, + max(hts_start) hts_start +from + event_log_archive +group by + event_id; +-- drop view if exists event_stats; +-- create view event_stats as +-- select +-- session_start, +-- event_type, +-- count(*) count, +-- avg(duration) avg, +-- max(duration) max, +-- min(duration) min, +-- median(duration) median, +-- stdev(duration) stdev +-- from +-- event_summary +-- group by +-- session_start, +-- event_type; -drop view if exists hedge_trace; -create view hedge_trace as -select +drop view if exists hedge_trace_view; +create view hedge_trace_view as +select session.version, session.host, session.build_type, session.session_start, - recent_trace.hts trace_time, - recent_trace.uint64_1 order_key, - hedge_events.duration total, - hedge_events.event_rank rank -from - recent_trace, - (select * from events where event_type='hedge') hedge_events, - session -where - entry_kind='NativeOrder::on_activate' and - recent_trace.cor_event = 0 and - recent_trace.channel = hedge_events.channel and - recent_trace.session_start = hedge_events.session_start and - recent_trace.sn = hedge_events.sn and - recent_trace.session_start = session.session_start; + event_log.hts trace_time, + event_log.uint64_1 order_key, + event_log.event_duration total, + event_log.event_id event_id +from + event_log,session +where + event_log.session_start=session.session_start and + event_type='hedge' and + entry_kind='NativeOrder::on_activate' and + cor_event=0; -drop view if exists requote_trace; -create view requote_trace as +create table if not exists +hedge_trace_archive ( + version TEXT, + host TEXT, + build_type TEXT, + session_start TIMESTAMP, + trace_time TIMESTAMP, + order_key INTEGER, + total float, + event_id integer +); + +drop view if exists hedge_trace; +create view hedge_trace as +select * from hedge_trace_view union select * from hedge_trace_archive; + + +drop view if exists requote_trace_view; +create view requote_trace_view as select session.version, session.host, session.build_type, session.session_start, - recent_trace.hts trace_time, - recent_trace.uint64_1 order_key, - requote_events.duration total, - requote_events.event_rank rank + event_log.hts trace_time, + event_log.uint64_1 order_key, + event_log.event_duration total, + event_log.event_id event_id from - recent_trace, - (select * from events where event_type='requote') requote_events, - session + event_log, session where + event_log.session_start=session.session_start and + event_type='requote' and entry_kind='NativeOrder::on_update' and - recent_trace.cor_event = 0 and - recent_trace.channel = requote_events.channel and - recent_trace.session_start = requote_events.session_start and - recent_trace.sn = requote_events.sn and - recent_trace.session_start = session.session_start ; + event_log.cor_event = 0; -delete from trace where -rowid in (select rowid from trace order by rowid desc limit -1 offset 10000123); +create table if not exists +requote_trace_archive ( + version TEXT, + host TEXT, + build_type TEXT, + session_start TIMESTAMP, + trace_time TIMESTAMP, + order_key INTEGER, + total float, + event_id integer +); + +drop view if exists requote_trace; +create view requote_trace as +select * from requote_trace_view union select * from requote_trace_archive; + +begin transaction; +insert into event_log_archive select * from event_log; +insert into hedge_trace_archive select * from hedge_trace_view; +insert into requote_trace_archive select * from requote_trace_view; +delete from trace; +delete from event_log_archive where +rowid in (select rowid from event_log_archive order by rowid desc limit -1 offset 10000123); +delete from hedge_trace_archive where +rowid in (select rowid from hedge_trace_archive order by rowid desc limit -1 offset 10000123); +delete from requote_trace_archive where +rowid in (select rowid from requote_trace_archive order by rowid desc limit -1 offset 10000123); +commit; diff --git a/src/lib/config/protos/perflog_cfg.proto b/src/lib/config/protos/perflog_cfg.proto index e1b69905c..353f370c5 100644 --- a/src/lib/config/protos/perflog_cfg.proto +++ b/src/lib/config/protos/perflog_cfg.proto @@ -3,7 +3,6 @@ package bts.config; message PerfLogCfg { optional bool active = 1 [default=true]; - optional uint32 recent_traces_size = 2 [default=140123]; optional uint32 max_rows = 3 [default=10000123]; optional uint32 entry_batch_size = 4 [default=200]; } diff --git a/src/lib/jetfix/connection/SocketConnection.cpp b/src/lib/jetfix/connection/SocketConnection.cpp index 9e040e148..40f95c1de 100644 --- a/src/lib/jetfix/connection/SocketConnection.cpp +++ b/src/lib/jetfix/connection/SocketConnection.cpp @@ -109,7 +109,8 @@ int SocketConnection::write(WriteVec const &vec, size_t vec_size, _perf_log.log(trace,"before_fix_socket_w"); auto result = ::writev(_fd, iv, (int)vec_size); _perf_log.log(trace,"after_fix_socket_w"); - _perf_log.flush(); + if (trace.sn) + _perf_log.flush(); if (_log && result > 0) { for (size_t i = 0; i < vec_size; i++) diff --git a/src/lib/tracing/PerfLog.cpp b/src/lib/tracing/PerfLog.cpp index e6cf9a37d..78980b5bb 100644 --- a/src/lib/tracing/PerfLog.cpp +++ b/src/lib/tracing/PerfLog.cpp @@ -3,9 +3,10 @@ #include "lib/config/protos/perflog_cfg.pb.h" #include "lib/config/protos/system.pb.h" #include "lib/persistence/persistence_utils.h" +#include "lib/utils/bts_array_size.h" #include "lib/utils/datetime.h" #include "lib/utils/stats.h" -#include "lib/utils/bts_array_size.h" +#include "lib/utils/thread_manager.h" #include #include @@ -81,6 +82,8 @@ struct PerfLog::PImpl { std::string session_start; boost::circular_buffer entry_batch; std::mutex lock; + std::thread init_thread; + std::atomic init_completed; }; PerfLog::PerfLog() : _pending(), _disabled(false), me() { @@ -98,38 +101,56 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() { try { me = std::make_unique(loc); - init_db(); - - me->insert_trace_statement = - std::make_unique(me->db, insert_trace); - me->insert_session_statement = - std::make_unique(me->db, insert_session); - me->requote_query_statement = - std::make_unique(me->db, requote_query); - me->hedge_query_statement = - std::make_unique(me->db, hedge_query); - me->session_start = datetime::FIX_timestamp(datetime::local_now()); - - me->insert_session_statement->bind( - ":version", config::get().version()); - me->insert_session_statement->bind( - ":host", config::get().host_name()); -#ifdef DEBUG - me->insert_session_statement->bind(":build_type", "debug"); -#else - me->insert_session_statement->bind(":build_type", "release"); -#endif - me->insert_session_statement->bind(":session_start", me->session_start); - me->insert_session_statement->exec(); - me->insert_session_statement->reset(); } catch (SQLite::Exception const &e) { LOG(persist, Error, "Error initializing perf_trace DB {}. Error: {}", loc, std::string(e.getErrorStr())); _disabled = true; + return; } + + me->init_completed = false; + me->init_thread = threading::make_thread("perf_log_ini", [loc,this]() { + try { + LOG(persist,Info, "Started initialization of perf_log db"); + init_db(); + + me->insert_trace_statement = + std::make_unique(me->db, insert_trace); + me->insert_session_statement = + std::make_unique(me->db, insert_session); + me->requote_query_statement = + std::make_unique(me->db, requote_query); + me->hedge_query_statement = + std::make_unique(me->db, hedge_query); + me->session_start = datetime::FIX_timestamp(datetime::local_now()); + + me->insert_session_statement->bind( + ":version", config::get().version()); + me->insert_session_statement->bind( + ":host", config::get().host_name()); +#ifdef DEBUG + me->insert_session_statement->bind(":build_type", "debug"); +#else + me->insert_session_statement->bind(":build_type", "release"); +#endif + me->insert_session_statement->bind(":session_start", + me->session_start); + me->insert_session_statement->exec(); + me->insert_session_statement->reset(); + } catch (SQLite::Exception const &e) { + LOG(persist, Error, + "Error initializing perf_trace DB (2) {}. Error: {}", loc, + std::string(e.getErrorStr())); + _disabled = true; + } + LOG(persist,Info, "Finished initialization of perf_log db"); + me->init_completed = true; + }); } -PerfLog::~PerfLog() {} +PerfLog::~PerfLog() { + me->init_thread.join(); +} std::string PerfLog::init_storage_location() { auto loc = persistence::storage_location("perf_log"); @@ -145,7 +166,7 @@ std::string PerfLog::init_storage_location() { } size_t PerfLog::store_log_entries() { - if (_disabled) { + if (!me->init_completed || _disabled) { _pending.clear(); return 0; } @@ -167,9 +188,6 @@ void PerfLog::init_db() { std::string str((std::istreambuf_iterator(f)), (std::istreambuf_iterator())); config::PerfLogCfg defaults; - boost::replace_all( - str, std::to_string(defaults.recent_traces_size()), - std::to_string(config::get().recent_traces_size())); boost::replace_all( str, std::to_string(defaults.max_rows()), std::to_string(config::get().max_rows())); @@ -187,7 +205,9 @@ std::string encode_number(SQLite::Column const &c) { std::vector> PerfLog::get_latest_requote_traces(size_t n) { std::vector> result; - + if (!me->init_completed) + return result; + std::lock_guard l(me->lock); try { me->requote_query_statement->bind(1, (long long)n); @@ -221,7 +241,9 @@ PerfLog::get_latest_requote_traces(size_t n) { std::vector> PerfLog::get_latest_hedge_traces(size_t n) { std::vector> result; - + if (!me->init_completed) + return result; + std::lock_guard l(me->lock); try { me->hedge_query_statement->bind(1, (long long)n); @@ -276,7 +298,7 @@ size_t PerfLog::flush_current_batch_to_disk() { me->insert_trace_statement->bind(":cor_event", (long long)ev.cor_event.to_uint64()); char tn[17]; - pthread_getname_np(ev.thread_id, tn, BTS_ARRAYSIZE(tn)-1); + pthread_getname_np(ev.thread_id, tn, BTS_ARRAYSIZE(tn) - 1); me->insert_trace_statement->bind(":thread", tn); me->insert_trace_statement->exec(); count++; From 6b84dc433a8e9c9e193cb61eeddefe985bc82c08 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Fri, 4 Jan 2019 09:23:59 -0500 Subject: [PATCH 19/30] Making log analysis faster. --- src/data_files/files/perf_trace/perf_log.sql | 97 ++++++++++++-------- src/lib/orderman/perf_analysis/perf_log.sh | 34 +++++-- 2 files changed, 88 insertions(+), 43 deletions(-) diff --git a/src/data_files/files/perf_trace/perf_log.sql b/src/data_files/files/perf_trace/perf_log.sql index df7393430..5b496274b 100644 --- a/src/data_files/files/perf_trace/perf_log.sql +++ b/src/data_files/files/perf_trace/perf_log.sql @@ -36,7 +36,27 @@ from trace window by_session_ts as (order by session_start,ts); - + +create table if not exists +log_archive( + session_start TIMESTAMP, + ts INTEGER, + hts timestamp, + channel TEXT, + entry_kind TEXT, + txt_1 TEXT, + uint64_1 INTEGER, + thread TEXT, + cor_event INTEGER, + sn INTEGER, + uint32_1 INTEGER, + entry_id INTEGER, + delta float, + scope_delta float, + scoped_entry TEXT +); +create index log_archive_idx on log_archive(session_start,ts); + drop view if exists events; create view events as select @@ -74,49 +94,37 @@ where md_reads.session_start = fix_writes.session_start and md_reads.channel = fix_writes.channel and md_reads.sn = fix_writes.sn; - + +create table if not exists +events_archive ( + session_start TIMESTAMP, + event_type TEXT, + hts_start TIMESTAMP, + event_id INTEGER, + start INTEGER, + stop INTEGER, + duration float, + channel TEXT, + sn INTEGER +); +create index events_archive_idx on events_archive(session_start,start,stop); + drop view if exists event_log; create view event_log as select - log.*, + log_archive.*, event_type, event_id, - events.duration event_duration, - events.hts_start hts_start, - events.start event_start, - events.stop event_stop + events_archive.duration event_duration, + events_archive.hts_start hts_start, + events_archive.start event_start, + events_archive.stop event_stop from - log, - events + log_archive, + events_archive where - log.session_start = events.session_start and - log.ts >= events.start and log.ts <= events.stop; - -create table if not exists -event_log_archive ( - session_start TIMESTAMP, - ts INTEGER, - hts timestamp, - channel TEXT, - entry_kind TEXT, - txt_1 TEXT, - uint64_1 INTEGER, - thread TEXT, - cor_event INTEGER, - sn INTEGER, - uint32_1 INTEGER, - entry_id INTEGER, - delta float, - scope_delta float, - scoped_entry text, - event_type text, - event_id integer, - event_duration float, - hts_start TIMESTAMP, - event_start integer, - event_stop integer -); -create index if not exists event_log_archive_idx on event_log_archive(session_start,ts); + log_archive.session_start = events_archive.session_start and + log_archive.ts >= events_archive.start and log_archive.ts <= events_archive.stop; drop view if exists pretty_log; create view pretty_log as @@ -253,3 +261,18 @@ delete from requote_trace_archive where rowid in (select rowid from requote_trace_archive order by rowid desc limit -1 offset 10000123); commit; +create view event_log_archive_view as +select + log_archive.*, + event_type, + event_id, + events_archive.duration event_duration, + events_archive.hts_start hts_start, + events_archive.start event_start, + events_archive.stop event_stop +from + log_archive, + events_archive +where + log_archive.session_start = events_archive.session_start and + log_archive.ts >= events_archive.start and log_archive.ts <= events_archive.stop; diff --git a/src/lib/orderman/perf_analysis/perf_log.sh b/src/lib/orderman/perf_analysis/perf_log.sh index 0334f2458..4fa4a11b7 100755 --- a/src/lib/orderman/perf_analysis/perf_log.sh +++ b/src/lib/orderman/perf_analysis/perf_log.sh @@ -19,7 +19,7 @@ fi SQLLITE=../../../../externals/linux/bin/sqlite3 SQLITE_EXT=../../../../externals/linux/bin/libsqlitefunctions -DB=file:$HOME/sparkdata/data/databases/perf_log/perf_log.sqlite?mode=ro +DB=file:$HOME/sparkdata/data/databases/perf_log/perf_log.sqlite if [ $# -eq 1 ] then @@ -28,6 +28,17 @@ else SESSION=\'$2\' fi +echo Generating stats... +$SQLLITE $DB< Date: Fri, 4 Jan 2019 11:31:36 -0500 Subject: [PATCH 20/30] More performance improvements on perf_log --- src/data_files/files/perf_trace/perf_log.sql | 355 +++++++++++-------- src/lib/orderman/perf_analysis/perf_log.sh | 31 +- src/web/CMakeDeps.cmake | 2 +- 3 files changed, 216 insertions(+), 172 deletions(-) diff --git a/src/data_files/files/perf_trace/perf_log.sql b/src/data_files/files/perf_trace/perf_log.sql index 5b496274b..882ab5d22 100644 --- a/src/data_files/files/perf_trace/perf_log.sql +++ b/src/data_files/files/perf_trace/perf_log.sql @@ -1,3 +1,6 @@ +/* -------------------------------------------------------------------------------- */ +/* Main tables */ +/* -------------------------------------------------------------------------------- */ create table if not exists session ( version TEXT, @@ -5,6 +8,7 @@ session ( build_type TEXT, session_start TIMESTAMP PRIMARY KEY ); +create index if not exists session_idx on session(session_start); create table if not exists trace ( @@ -20,12 +24,18 @@ trace ( sn INTEGER, uint32_1 INTEGER ); - create index if not exists trace_idx on trace(session_start,channel,sn,ts); create index if not exists trace_t_idx on trace(session_start,ts); -drop view if exists log; -create view log as +/* -------------------------------------------------------------------------------- */ +/* Main Views */ +/* -------------------------------------------------------------------------------- */ + +/* An ordered view of the traces, with extra columns for the delta +betwen consecutive traces as well as the delta between related +(scoped) traces */ +drop view if exists log_view; +create view log_view as select trace.*, trace.rowid entry_id, @@ -37,28 +47,10 @@ from window by_session_ts as (order by session_start,ts); -create table if not exists -log_archive( - session_start TIMESTAMP, - ts INTEGER, - hts timestamp, - channel TEXT, - entry_kind TEXT, - txt_1 TEXT, - uint64_1 INTEGER, - thread TEXT, - cor_event INTEGER, - sn INTEGER, - uint32_1 INTEGER, - entry_id INTEGER, - delta float, - scope_delta float, - scoped_entry TEXT -); -create index log_archive_idx on log_archive(session_start,ts); - -drop view if exists events; -create view events as +/* A view identifying each high level event (requote or hedge) and +annotating it with its latency */ +drop view if exists event_view; +create view event_view as select fix_reads.session_start, "hedge" event_type, @@ -95,8 +87,63 @@ where md_reads.channel = fix_writes.channel and md_reads.sn = fix_writes.sn; +/* A view on hedge events, identifying each order sent as part of the event*/ +drop view if exists hedge_trace_view; +create view hedge_trace_view as +select + session.version, + session.host, + session.build_type, + session.session_start, + trace.hts trace_time, + uint64_1 order_key, + duration total, + event_id +from + events, + trace, + session +where + session.session_start = trace.session_start and + session.session_start = events.session_start and + events.channel=trace.channel and events.sn=trace.sn and + event_type='hedge' and + entry_kind='NativeOrder::on_activate' and + cor_event=0; + +/* A view on requote events, identifying each order sent as part of the event*/ +drop view if exists requote_trace_view; +create view requote_trace_view as +select + session.version, + session.host, + session.build_type, + session.session_start, + trace.hts trace_time, + uint64_1 order_key, + duration total, + event_id +from + events, + trace, + session +where + session.session_start = trace.session_start and + session.session_start = events.session_start and + events.channel=trace.channel and events.sn=trace.sn and + event_type='requote' and + entry_kind='NativeOrder::on_update' and + cor_event=0; + +/* -------------------------------------------------------------------------------- */ +/* Archive tables/view */ +/* -------------------------------------------------------------------------------- */ + +/* Auxiliary tables and views, used to archive the results of the main +views to speed up analysis as data accumulates */ + create table if not exists -events_archive ( +event_archive ( session_start TIMESTAMP, event_type TEXT, hts_start TIMESTAMP, @@ -107,94 +154,27 @@ events_archive ( channel TEXT, sn INTEGER ); -create index events_archive_idx on events_archive(session_start,start,stop); +create index if not exists event_archive_idx on event_archive(session_start,start,stop); -drop view if exists event_log; -create view event_log as -select - log_archive.*, - event_type, - event_id, - events_archive.duration event_duration, - events_archive.hts_start hts_start, - events_archive.start event_start, - events_archive.stop event_stop -from - log_archive, - events_archive -where - log_archive.session_start = events_archive.session_start and - log_archive.ts >= events_archive.start and log_archive.ts <= events_archive.stop; - -drop view if exists pretty_log; -create view pretty_log as -select - scoped_entry entry_kind, - delta, - scope_delta , - event_duration, - event_type, - event_rank, - hts, - ts, - session_start, - entry_rank -from - event_log_archive; - -drop view if exists last_pretty_log; -create view last_pretty_log as -select * from pretty_log where session_start=(select max(session_start) from session); - -drop view if exists event_summary; -create view event_summary as -select - max(session_start) session_start, - max(event_type) event_type, - max(event_duration) duration, - min(event_start) start, - min(event_stop) stop, - max(hts_start) hts_start -from - event_log_archive -group by - event_id; - --- drop view if exists event_stats; --- create view event_stats as --- select --- session_start, --- event_type, --- count(*) count, --- avg(duration) avg, --- max(duration) max, --- min(duration) min, --- median(duration) median, --- stdev(duration) stdev --- from --- event_summary --- group by --- session_start, --- event_type; - -drop view if exists hedge_trace_view; -create view hedge_trace_view as -select - session.version, - session.host, - session.build_type, - session.session_start, - event_log.hts trace_time, - event_log.uint64_1 order_key, - event_log.event_duration total, - event_log.event_id event_id -from - event_log,session -where - event_log.session_start=session.session_start and - event_type='hedge' and - entry_kind='NativeOrder::on_activate' and - cor_event=0; +create table if not exists +log_archive( + session_start TIMESTAMP, + ts INTEGER, + hts timestamp, + channel TEXT, + entry_kind TEXT, + txt_1 TEXT, + uint64_1 INTEGER, + thread TEXT, + cor_event INTEGER, + sn INTEGER, + uint32_1 INTEGER, + entry_id INTEGER, + delta float, + scope_delta float, + scoped_entry TEXT +); +create index if not exists log_archive_idx on log_archive(session_start,ts); create table if not exists hedge_trace_archive ( @@ -207,31 +187,10 @@ hedge_trace_archive ( total float, event_id integer ); - drop view if exists hedge_trace; create view hedge_trace as select * from hedge_trace_view union select * from hedge_trace_archive; - -drop view if exists requote_trace_view; -create view requote_trace_view as -select - session.version, - session.host, - session.build_type, - session.session_start, - event_log.hts trace_time, - event_log.uint64_1 order_key, - event_log.event_duration total, - event_log.event_id event_id -from - event_log, session -where - event_log.session_start=session.session_start and - event_type='requote' and - entry_kind='NativeOrder::on_update' and - event_log.cor_event = 0; - create table if not exists requote_trace_archive ( version TEXT, @@ -243,36 +202,122 @@ requote_trace_archive ( total float, event_id integer ); - drop view if exists requote_trace; create view requote_trace as select * from requote_trace_view union select * from requote_trace_archive; -begin transaction; -insert into event_log_archive select * from event_log; -insert into hedge_trace_archive select * from hedge_trace_view; -insert into requote_trace_archive select * from requote_trace_view; -delete from trace; -delete from event_log_archive where -rowid in (select rowid from event_log_archive order by rowid desc limit -1 offset 10000123); -delete from hedge_trace_archive where -rowid in (select rowid from hedge_trace_archive order by rowid desc limit -1 offset 10000123); -delete from requote_trace_archive where -rowid in (select rowid from requote_trace_archive order by rowid desc limit -1 offset 10000123); -commit; -create view event_log_archive_view as +/* a synthetis of the log and events, focusing only on traces that are +part of hedge/requote events */ +create table if not exists +event_log_archive ( + session_start TIMESTAMP, + ts INTEGER, + hts timestamp, + channel TEXT, + entry_kind TEXT, + txt_1 TEXT, + uint64_1 INTEGER, + thread TEXT, + cor_event INTEGER, + sn INTEGER, + uint32_1 INTEGER, + entry_id INTEGER, + delta float, + scope_delta float, + scoped_entry TEXT, + event_type TEXT, + event_id INTEGER, + event_duration float, + hts_start TIMESTAMP, + event_start INTEGER, + event_stop INTEGER +); +create index if not exists event_log_archive_idx on event_log_archive(session_start); + +drop view if exists pretty_log; +create view pretty_log as +select + scoped_entry entry_kind, + delta, + scope_delta , + event_duration, + event_type, + event_id, + hts, + ts, + session_start, + entry_id +from + event_log_archive; + +drop view if exists last_pretty_log; +create view last_pretty_log as +select + * +from + pretty_log +where + session_start=(select max(session_start) from session); + +/* +drop view if exists event_stats; +create view event_stats as +select + session_start, + event_type, + count(*) count, + avg(duration) avg, + max(duration) max, + min(duration) min, + median(duration) median, + stdev(duration) stdev +from + event_archive +group by + session_start, + event_type; +*/ + +drop view if exists event_log_aux; +create view event_log_aux as select log_archive.*, event_type, event_id, - events_archive.duration event_duration, - events_archive.hts_start hts_start, - events_archive.start event_start, - events_archive.stop event_stop + event_archive.duration event_duration, + event_archive.hts_start hts_start, + event_archive.start event_start, + event_archive.stop event_stop from log_archive, - events_archive + event_archive where - log_archive.session_start = events_archive.session_start and - log_archive.ts >= events_archive.start and log_archive.ts <= events_archive.stop; + log_archive.session_start = event_archive.session_start and + log_archive.ts >= event_archive.start and log_archive.ts <= event_archive.stop; + +begin transaction; +insert into hedge_trace_archive select * from hedge_trace_view; +insert into requote_trace_archive select * from requote_trace_view; +insert into event_archive select * from event_view; +insert into log_archive select * from log_view; +insert into event_log_archive select * from event_log_aux; + +delete from trace; + +delete from event_log_archive where +rowid in (select rowid from event_log_archive order by rowid desc limit -1 offset 10000123); + +delete from hedge_trace_archive where +rowid in (select rowid from hedge_trace_archive order by rowid desc limit -1 offset 10000123); + +delete from requote_trace_archive where +rowid in (select rowid from requote_trace_archive order by rowid desc limit -1 offset 10000123); + +delete from log_archive where +rowid in (select rowid from log_archive order by rowid desc limit -1 offset 10000123); + +delete from event_log_archive where +rowid in (select rowid from event_log_archive order by rowid desc limit -1 offset 10000123); +commit; + diff --git a/src/lib/orderman/perf_analysis/perf_log.sh b/src/lib/orderman/perf_analysis/perf_log.sh index 4fa4a11b7..67b3d4f29 100755 --- a/src/lib/orderman/perf_analysis/perf_log.sh +++ b/src/lib/orderman/perf_analysis/perf_log.sh @@ -31,9 +31,11 @@ fi echo Generating stats... $SQLLITE $DB< Date: Fri, 4 Jan 2019 12:04:10 -0500 Subject: [PATCH 21/30] Fixed perf_log.sql --- src/data_files/files/perf_trace/perf_log.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/data_files/files/perf_trace/perf_log.sql b/src/data_files/files/perf_trace/perf_log.sql index 882ab5d22..309aa5944 100644 --- a/src/data_files/files/perf_trace/perf_log.sql +++ b/src/data_files/files/perf_trace/perf_log.sql @@ -100,13 +100,13 @@ select duration total, event_id from - events, + event_view, trace, session where session.session_start = trace.session_start and - session.session_start = events.session_start and - events.channel=trace.channel and events.sn=trace.sn and + session.session_start = event_view.session_start and + event_view.channel=trace.channel and event_view.sn=trace.sn and event_type='hedge' and entry_kind='NativeOrder::on_activate' and cor_event=0; @@ -124,13 +124,13 @@ select duration total, event_id from - events, + event_view, trace, session where session.session_start = trace.session_start and - session.session_start = events.session_start and - events.channel=trace.channel and events.sn=trace.sn and + session.session_start = event_view.session_start and + event_view.channel=trace.channel and event_view.sn=trace.sn and event_type='requote' and entry_kind='NativeOrder::on_update' and cor_event=0; From 64b94c58e717185de135baee64e40cc5add12c47 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 4 Jan 2019 11:20:21 -0600 Subject: [PATCH 22/30] locking _resources ResourceDictionary prior to indexer access --- src/client/spark/Images/Icons.cs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/client/spark/Images/Icons.cs b/src/client/spark/Images/Icons.cs index f9fee789c..54c88a41b 100644 --- a/src/client/spark/Images/Icons.cs +++ b/src/client/spark/Images/Icons.cs @@ -31,7 +31,9 @@ namespace mbts.ui { } private static ImageSource load_resource_and_freeze(string resource_name) { - return ((DrawingImage)_resources[resource_name]).frozen(); + lock (_resources) { + return ((DrawingImage)_resources[resource_name]).frozen(); + } } #pragma warning disable CS0618 // Type or member is obsolete @@ -152,16 +154,18 @@ namespace mbts.ui { } static IEnumerable _get_geometric_path(string geometric_path_key) { - object obj = _resources[geometric_path_key]; - if (obj is string path) { - if (string.IsNullOrEmpty(path)) { - Log.Api.error($"Resource lookup of '{geometric_path_key}' did not return a valid string.", nameof(geometric_path_key)); - path = string.Empty; - } - return new[] { path }; - } else if (obj is string[] paths) { - return paths.Where(p => !string.IsNullOrEmpty(p)).ToArray(); - } else return null; + lock (_resources) { + object obj = _resources[geometric_path_key]; + if (obj is string path) { + if (string.IsNullOrEmpty(path)) { + Log.Api.error($"Resource lookup of '{geometric_path_key}' did not return a valid string.", nameof(geometric_path_key)); + path = string.Empty; + } + return new[] { path }; + } else if (obj is string[] paths) { + return paths.Where(p => !string.IsNullOrEmpty(p)).ToArray(); + } else return null; + } } } } \ No newline at end of file From ae200b374b9ac2d44fbcb17317ccafa595ff8ad5 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Fri, 4 Jan 2019 12:38:56 -0600 Subject: [PATCH 23/30] PR suggestion --- src/client/spark/Images/Icons.cs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/client/spark/Images/Icons.cs b/src/client/spark/Images/Icons.cs index 54c88a41b..7dc2068aa 100644 --- a/src/client/spark/Images/Icons.cs +++ b/src/client/spark/Images/Icons.cs @@ -154,18 +154,19 @@ namespace mbts.ui { } static IEnumerable _get_geometric_path(string geometric_path_key) { + object obj; lock (_resources) { - object obj = _resources[geometric_path_key]; - if (obj is string path) { - if (string.IsNullOrEmpty(path)) { - Log.Api.error($"Resource lookup of '{geometric_path_key}' did not return a valid string.", nameof(geometric_path_key)); - path = string.Empty; - } - return new[] { path }; - } else if (obj is string[] paths) { - return paths.Where(p => !string.IsNullOrEmpty(p)).ToArray(); - } else return null; + obj = _resources[geometric_path_key]; } + if (obj is string path) { + if (string.IsNullOrEmpty(path)) { + Log.Api.error($"Resource lookup of '{geometric_path_key}' did not return a valid string.", nameof(geometric_path_key)); + path = string.Empty; + } + return new[] { path }; + } else if (obj is string[] paths) { + return paths.Where(p => !string.IsNullOrEmpty(p)).ToArray(); + } else return null; } } } \ No newline at end of file From 9d5b80c9900827956264282a26963d474b1d0b84 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 10 Jan 2019 14:12:27 -0500 Subject: [PATCH 24/30] Forced manual order indicator to false on algos. --- .../orderman/order_factory/OrderFactory.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/lib/orderman/order_factory/OrderFactory.cpp b/src/lib/orderman/order_factory/OrderFactory.cpp index a5eb59f90..3740d9630 100644 --- a/src/lib/orderman/order_factory/OrderFactory.cpp +++ b/src/lib/orderman/order_factory/OrderFactory.cpp @@ -72,8 +72,22 @@ Order &OrderFactory::create(OrderKey parent, auto k = gen_order_id(); auto pk = parent; auto &order = allocate(k, pk, def); - order.on_init_new(def, - PreActivationOrderServices{me->sec_db, *this, me->router}); + + // For non native orders we force the ManalOrderIndicator to be 'N' + // since all non native order are algos and we would rather not + // rely on the order creator (be it shell or another algo) to set + // this correctly for every single algo. + if (dynamic_cast(&order) == nullptr && + def.manual_order_indicator() == + api::orderman::MANUAL_ORDER_INDICATOR_YES) { + auto corrected_def = def; + corrected_def.set_manual_order_indicator( + api::orderman::MANUAL_ORDER_INDICATOR_NO); + order.on_init_new(corrected_def, PreActivationOrderServices{ + me->sec_db, *this, me->router}); + } else + order.on_init_new( + def, PreActivationOrderServices{me->sec_db, *this, me->router}); return order; } From 9838d123c702f0c74fa581745dba58a9be29d891 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 10 Jan 2019 14:15:04 -0500 Subject: [PATCH 25/30] Updated release notes. --- doc/user/docs/release_notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index 10120db8c..af13528d9 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,4 +1,8 @@ # Release Notes +## 1.4.7 +### Bug Fixes + - Corrected FIX tag 1028, ManualOrderIndicator, when orders are triggered by algos + ## 1.4.6 ### Ladder Usability Improvements - The ladder can now be zoomed in/out according to a user's preference From 14012a16da2cafb0b1ac5eee0b31006d23a5c009 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 11 Jan 2019 17:29:51 -0500 Subject: [PATCH 26/30] Add "all" option to sparkd md tail for emumba testing --- src/lib/CMakeDeps.cmake | 2 +- src/lib/md/Diag/MDPeriodicDumper.cpp | 6 ++++-- src/lib/md/Diag/MDPeriodicDumper.h | 3 ++- src/server/sparkd/md/run_md.cpp | 7 +++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index ec7dd860c..afd768425 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/orders/NativeOrder.cpp;orderman2/orders/Order.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 9fbfdc56c..e44d6bb66 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -150,7 +150,7 @@ void write_header(PrintFormat format, long seq_mod, const std::vector& en } } -void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, PrintFormat format) { +void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, PrintFormat format, bool monitor_all) { MDConsumer consumer(symbols, nullptr); @@ -160,7 +160,9 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, entries.push_back({ 0, handle, std::make_unique(sec) }); }); - entries = filter_entries(entries); + if (!monitor_all) { + entries = filter_entries(entries); + } write_header(format, seq_mod, entries); diff --git a/src/lib/md/Diag/MDPeriodicDumper.h b/src/lib/md/Diag/MDPeriodicDumper.h index e59a201f0..483901cd0 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.h +++ b/src/lib/md/Diag/MDPeriodicDumper.h @@ -10,7 +10,8 @@ enum class PrintFormat { CSV }; -void run_periodic_dumper(const std::set&, uint32_t seq_mod, PrintFormat format); + +void run_periodic_dumper(const std::set&, uint32_t seq_mod, PrintFormat format, bool monitor_all); }}//ns \ No newline at end of file diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index c3a9e04d4..75cc6192d 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -252,14 +252,17 @@ void dump_db(const vector& args) { void run_tail(const vector& args) { std::set products; auto conf = parse_md_args(args); + bool monitor_all = false; + for (const auto& b : args) { if (b=="all") monitor_all = true; } + PlatformConfig config; - config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE)); + if (!monitor_all) config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE)); config.set_platform(co2::Platform_Name(conf.platform)); get_system_md_symbols(config, products); auto mod = conf.number > 0 ? conf.number : 1000; PrintFormat fmt = PrintFormat::CSV; for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; } - md::run_periodic_dumper(products, mod, fmt); + md::run_periodic_dumper(products, mod, fmt, monitor_all); } void run_channels() { From 69f38daad7f432404d089361e848ae23617469fa Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 14 Jan 2019 14:37:51 -0500 Subject: [PATCH 27/30] SPARK-1966 Add CME ticker trade details --- src/lib/config/protos/MarketData.proto | 1 + .../sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp | 1 + .../sparkd/md/platforms/mdp3_onixs/MDP3Listener.h | 13 ++++++++++++- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp | 5 ++++- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 90b204f74..ddcebb619 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -110,6 +110,7 @@ message CMEOptions // Prevents parsing trades/rfqs/doing anything not required for quoting optional bool quoter_mode = 6; + optional bool verbose_ticker = 7 [default=false]; } message RecordOptions { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index b788c6a2a..6049e982b 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -19,6 +19,7 @@ MDP3Listener::MDP3Listener( const std::vector channels) : _index(index), _producer(producer), + _verbose_ticker(config::get().verbose_ticker()), _packet_arrived(), _latest_sequence(0), // _settlement_recorder(settles), diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index c29a1d01c..1a9881faf 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -182,6 +182,7 @@ private: CMESecurityIndex& _index; MDProducer& _producer; + bool _verbose_ticker; utils::TimeStamp _packet_arrived; uint32_t _latest_sequence; // co2::SettlementRecorder& _settlement_recorder; @@ -416,7 +417,17 @@ void MDP3Listener::onTradeSummary(const onixs::Security& sec, const TradeSummary onixs::Decimal price = arg.entry().entryPx(); onixs::AggressorSide::Enum agside = onixs::AggressorSide::NoAggressor; arg.entry().aggressorSide(agside); - on_trade(data, price, size, agside, arg.message().transactTime()); + if (_verbose_ticker) { + const auto& details = arg.orderIds(); + auto sz = details.size(); + for (size_t x = 0; x < sz; ++x) { + const auto& d = details[x]; + on_trade(data, price, d.lastQty(), agside, arg.message().transactTime()); + } + } + else { + on_trade(data, price, size, agside, arg.message().transactTime()); + } } } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 425a3e8d3..82d6aa0de 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -269,7 +269,10 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { // Misc settings settings.bookManagement().updateNotification(onixs::BookUpdateNotification::OnEndOfEvent); - settings.tradeProcessing(onixs::TradeProcessing::SummaryLevel); + settings.tradeProcessing( + config::get().verbose_ticker() ? + onixs::TradeProcessing::OrderDetails : + onixs::TradeProcessing::SummaryLevel); // // Logging From 979c6e078ac86454867bd97a85bb9c294397bfb2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 14 Jan 2019 14:39:05 -0500 Subject: [PATCH 28/30] Remove misleading comments --- .../md/platforms/mdp3_onixs/MDP3Listener.h | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index 1a9881faf..fc20add4c 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -453,7 +453,6 @@ void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const Statistic onixs::Timestamp time; args.entry().entryPx(price); args.entry().tradingReferenceDate(time); - // args.entry(). auto display_factor = data->handle.security().display_factor; data->handle.update_settlement( utils::TimeStamp::now(), @@ -465,19 +464,16 @@ void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const Statistic template void MDP3Listener::onSessionOpeningPrice(const onixs::Security& sec, const SessionArgs& args) { tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0); - // auto type = args.entry().settlPriceType(); - // if (type.final() && type.actual()) { - auto* data = _index.get_security(sec.id()); - if (data != nullptr) { - onixs::Decimal price = args.entry().entryPx(); - onixs::Timestamp time = args.message().transactTime(); - auto display_factor = data->handle.security().display_factor; - data->handle.update_opening( - utils::TimeStamp::now(), - double_of_price(price, display_factor), - date_of_timestamp(time)); - } - // + auto* data = _index.get_security(sec.id()); + if (data != nullptr) { + onixs::Decimal price = args.entry().entryPx(); + onixs::Timestamp time = args.message().transactTime(); + auto display_factor = data->handle.security().display_factor; + data->handle.update_opening( + utils::TimeStamp::now(), + double_of_price(price, display_factor), + date_of_timestamp(time)); + } } template void MDP3Listener::onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args) { From 5e3467e3b62bfcf4f67c1bde7885a74fea5b8afc Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 14 Jan 2019 14:41:47 -0500 Subject: [PATCH 29/30] Add release note --- doc/user/docs/release_notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index af13528d9..2ee01defa 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,4 +1,8 @@ # Release Notes +## 1.4.8 +### Marketdata + - Additional trade details for CME ticker + ## 1.4.7 ### Bug Fixes - Corrected FIX tag 1028, ManualOrderIndicator, when orders are triggered by algos From f71ff0e3065c42b87c783a5fded796e7b49844c6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 18 Jan 2019 11:52:42 -0500 Subject: [PATCH 30/30] Add the ability to control affinity for the md tail tool --- src/lib/md/Diag/MDPeriodicDumper.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index e44d6bb66..f751f51df 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -3,6 +3,8 @@ #include "lib/utils/TimeStamp.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/datetime.h" +#include "lib/config/protos/system.pb.h" +#include "lib/utils/thread_manager.h" #include #include @@ -152,6 +154,10 @@ void write_header(PrintFormat format, long seq_mod, const std::vector& en void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, PrintFormat format, bool monitor_all) { + auto qualifier = config::get().qualifier(); + auto thread_name = "tail-" + qualifier; + threading::configure_current_thread(thread_name); + MDConsumer consumer(symbols, nullptr); // read all securities into a vector