diff --git a/.vscode/launch.json b/.vscode/launch.json index fd68a6626..f3c362de0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -215,7 +215,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/server/sparkd", - "args": ["test", "[md]"], + "args": ["test", "[mbo]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/server/", "environment": [], diff --git a/doc/user/content/bts_spark/Algos/_index.md b/doc/user/content/bts_spark/Algos/_index.md new file mode 100644 index 000000000..f3955e979 --- /dev/null +++ b/doc/user/content/bts_spark/Algos/_index.md @@ -0,0 +1,46 @@ ++++ +title = "BTS Basic Algos" +weight = 10 +sort_by = "weight" +insert_anchor_links = "right" ++++ + + +## Lean + +The goal of the lean order is to watch the opposite-inside market size and cross when that size goes below the threshold (this also implies a cross if that size goes to zero and the market trades away from you) If the market trades toward you, you can opt into several different behaviors + +##### Parameters + * Cross Thresh - size thresh in absolute or ratio terms + * Max Chase Ticks - max levels to pay up, in ticks + + +## Join +The goal of the join order is to watch the opposite-inside market and join when the market flips (that is, when the best offer becomes the best bid) and has at least threshold size. + +##### Parameters + * Join Thresh - size thresh in absolute or ratio terms + + +## Top + +The goal of the top order is to not trade by yourself, but the intent at the time of order entry is to get filled.If the order is on the market (say current bid price) AND the quantity on that bid price is below top thresh then cancel the order else remain working. + +##### Parameters + * Top Thresh - size thresh in absolute or ratio terms + + +## Cover +A Cover order’s purpose is to fire a hedge order (of any type) in response to fills from it’s child order on the opposite side of the market N ticks away + +##### Parameters + * Profit Ticks - ticks above or below fill price to place profit / cover order + + +## OCO +An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. + +Composing this order with Cover simulates scalping functionality + +##### Parameters + * Stop Ticks - ticks below or above main order to place stop / other order \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Client/_index.md b/doc/user/content/bts_spark_sdk/Client/_index.md deleted file mode 100644 index dd7a5550e..000000000 --- a/doc/user/content/bts_spark_sdk/Client/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "Client" -weight = 1 -sort_by = "weight" -redirect_to = "bts_spark_sdk/Client/overview/" -+++ \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Server/_index.md b/doc/user/content/bts_spark_sdk/Server/_index.md deleted file mode 100644 index c6b0c27ff..000000000 --- a/doc/user/content/bts_spark_sdk/Server/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "Server" -weight = 2 -sort_by = "weight" -insert_anchor_links = "right" -+++ diff --git a/doc/user/content/bts_spark_sdk/_index.md b/doc/user/content/bts_spark_sdk/_index.md deleted file mode 100644 index 2a698fcda..000000000 --- a/doc/user/content/bts_spark_sdk/_index.md +++ /dev/null @@ -1,6 +0,0 @@ -+++ -title = "BTS Spark SDK" -weight = 2 -sort_by = "weight" -insert_anchor_links = "right" -+++ \ No newline at end of file diff --git a/doc/user/content/release_notes/_index.md b/doc/user/content/release_notes/_index.md index db2b50254..3c158addf 100644 --- a/doc/user/content/release_notes/_index.md +++ b/doc/user/content/release_notes/_index.md @@ -4,6 +4,17 @@ weight = 3 sort_by = "weight" insert_anchor_links = "right" +++ +## 1.6.0 +### User Visible Changes +#### New Features + - You can now view your order's position in queue in the ladder +#### Bug Fixes + - Fix a bug in the order popup in the ladder which could lead to a crash. + +## 1.5.9 +### User Visible Changes + - Fix for stunnel connections on Windows 7 + ## 1.5.8 ### User Visible Changes - Fixed a bug where it was possible to crash the UI by entering an order through the order ticket if Enter was used to dismiss the dialog. diff --git a/externals b/externals index d0e8645b5..67a79263b 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit d0e8645b5ef7380314eddc68114a9f0eaa379318 +Subproject commit 67a79263ba116a7646beb8368cef015530bea027 diff --git a/src/client/spark/App.xaml.cs b/src/client/spark/App.xaml.cs index 25aecff08..ea2b1de78 100644 --- a/src/client/spark/App.xaml.cs +++ b/src/client/spark/App.xaml.cs @@ -75,6 +75,41 @@ namespace mbts.ui.shell Environment.Exit(0); } + void start_stunnel(string path, uint job_error) + { + string config_error; + if (!mbts.config.Configuration.try_initialize(out config_error)) + { + _stunnel_process = Process.Start(path + "bin\\stunnel.exe"); + if (_stunnel_process != null) + { + for (int i = 0; i < 10; i++) + { + if (mbts.config.Configuration.try_initialize(out config_error)) + break; + System.Threading.Thread.Sleep(500); + } + } + else + stunnel_job_error = "Stunnel Job Error code: " + job_error + + " could not start stunnel " + (config_error ?? string.Empty); + } + } + + void kill_stunnel() + { + try + { + Process[] ps = Process.GetProcessesByName("stunnel"); + ps.ForEach(p => + { + try { p.Kill(); } + catch {} + }); + } + catch { } + } + Process _stunnel_process; static internal string stunnel_job_error { get; private set;} void init_stunnel(string[] argv) @@ -85,24 +120,10 @@ namespace mbts.ui.shell if (File.Exists(path + "config\\stunnel.conf")) { uint job_error = BTSUINativeMethods.setup_process_job_object(); - if(job_error == 0) - { - _stunnel_process = Process.Start(path + "bin\\stunnel.exe"); - string config_error; bool success = false; - if(_stunnel_process != null) - { - for(int i = 0; i < 10; i++) - { - if(success = mbts.config.Configuration.try_initialize(out config_error)) - break; - System.Threading.Thread.Sleep(500); - } - } - else - stunnel_job_error = "Stunnel Job Error code: " + job_error + " could not start stunnel"; - } - else - stunnel_job_error = "Stunnel Job Error code: " + job_error; + //if we can't put this process and stunnel into a job, try to stop stunnel then restart + if(job_error != 0) + kill_stunnel(); + start_stunnel(path, job_error); } } catch(Exception ex) diff --git a/src/client/spark/Daemons/ShellServices.cs b/src/client/spark/Daemons/ShellServices.cs index 8fa232571..bcfd91e6e 100644 --- a/src/client/spark/Daemons/ShellServices.cs +++ b/src/client/spark/Daemons/ShellServices.cs @@ -6,6 +6,7 @@ using bts.api.persistence; using bts.api.position; using bts.api.synthetic; using bts.co2; +using bts.md; using bts.metrics; using bts.support; using System.ComponentModel.Composition; @@ -50,6 +51,9 @@ namespace Spark { AlgoServiceWrapper _algo_service_wrapper; [Export] public AlgoServiceWrapper algo_service_wrapper => _algo_service_wrapper ?? (_algo_service_wrapper = new AlgoServiceWrapper(context)); + + [Export] + public MBOService mbo_service => context.locate(); } class AlgoServiceWrapper { diff --git a/src/client/spark/Modules/Trading/Ladder/LadderSettings.cs b/src/client/spark/Modules/Trading/Ladder/LadderSettings.cs index 2b5dc8881..c9c53b193 100644 --- a/src/client/spark/Modules/Trading/Ladder/LadderSettings.cs +++ b/src/client/spark/Modules/Trading/Ladder/LadderSettings.cs @@ -249,7 +249,8 @@ namespace mbts.spark { bid_column = LadderColumnSettings.Default, ask_column = LadderColumnSettings.Default, price_column = LadderColumnSettings.Default, - last_column = LadderColumnSettings.Default; + last_column = LadderColumnSettings.Default, + piq_column = LadderColumnSettings.Default; orders_column.background = BTSColors.LadderSpecial.default_column_background; bid_column.foreground = 0xFFFFFF; bid_column.background = BTSColors.MarketData.bid; @@ -257,6 +258,7 @@ namespace mbts.spark { ask_column.background = BTSColors.MarketData.ask; price_column.background = BTSColors.LadderSpecial.default_column_background; last_column.background = 0xFFFFFF; + piq_column.background = BTSColors.LadderSpecial.default_column_background; var rv = new LadderSettings { row_height = 22, grid_line_color = 0x535353, @@ -265,6 +267,7 @@ namespace mbts.spark { ask_column = ask_column, price_column = price_column, last_column = last_column, + piq_column = piq_column, volume_column = VolumeColumnSettings.Default, calculator_buttons = new[] { 1, 5, 10, 25, 50, 100 }, scroll_bar_visible = false, @@ -400,7 +403,7 @@ namespace mbts.spark { public bool clear_visible { get; set; } // [ProtoMember(43), DefaultValue(_default_order_entry_panel_width)] - public double order_entry_panel_initial_width { get; set; } + public double order_entry_panel_initial_width { get; set; } [ProtoMember(44), DefaultValue(true)] public bool use_defaults { get; set; } @@ -419,6 +422,13 @@ namespace mbts.spark { [ProtoMember(48)] public NetChangeTerms net_change { get; set; } + [ProtoMember(49)] + LadderColumnSettings _piq_column; + public LadderColumnSettings piq_column { + get => _piq_column ?? (_piq_column = LadderSettings.Default.piq_column); + set => _piq_column = value; + } + public bool Equals(LadderSettings other) => other != null && row_height == other.row_height && @@ -428,6 +438,7 @@ namespace mbts.spark { ask_column.safe_equals(other.ask_column) && price_column.safe_equals(other.price_column) && last_column.safe_equals(other.last_column) && + piq_column.safe_equals(other.piq_column) && calculator_buttons.safe_equals(other.calculator_buttons, (a, b) => a.SequenceEqual(b)) && scroll_bar_visible == other.scroll_bar_visible && default_order_type == other.default_order_type && @@ -478,6 +489,7 @@ namespace mbts.spark { EqualityComparer.safe_get_hash_code(ask_column), EqualityComparer.safe_get_hash_code(price_column), EqualityComparer.safe_get_hash_code(last_column), + EqualityComparer.safe_get_hash_code(piq_column), EqualityComparer.safe_get_hash_code(calculator_buttons), scroll_bar_visible.GetHashCode(), default_order_type.GetHashCode(), @@ -519,12 +531,12 @@ namespace mbts.spark { public struct WindowSpecifics { public readonly double order_entry_panel_width, zoom_factor; - public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width; + public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, piq_column_width; public WindowSpecifics(LadderSettings s) - : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor) { } + : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width, s.zoom_factor, s.piq_column.width) { } - public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor) { + public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width, double zoom_factor, uint piq_column_width) { this.order_entry_panel_width = order_entry_panel_width; this.orders_column_width = orders_column_width; this.bid_column_width = bid_column_width; @@ -533,6 +545,7 @@ namespace mbts.spark { this.last_column_width = last_column_width; this.volume_column_width = volume_column_width; this.zoom_factor = zoom_factor; + this.piq_column_width = piq_column_width; } } } @@ -545,6 +558,7 @@ namespace mbts.spark { yield return settings.ask_column; yield return settings.volume_column.general; yield return settings.last_column; + yield return settings.piq_column; } public static IEnumerable visible_columns(this LadderSettings settings) => @@ -562,6 +576,7 @@ namespace mbts.spark { source.price_column.width = specifics.price_column_width; source.ask_column.width = specifics.ask_column_width; source.last_column.width = specifics.last_column_width; + source.piq_column.width = specifics.piq_column_width; source.volume_column.general.width = specifics.volume_column_width; source.zoom_factor = specifics.zoom_factor; return source; diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs index bcecc88a2..2f0b8baa7 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderOrderVM.cs @@ -29,7 +29,7 @@ namespace mbts.spark { } #endregion - readonly LadderVM _parent; + readonly LadderVM _owner; readonly IBus _bus; readonly Property _font_family; readonly Property _font_weight; @@ -37,7 +37,6 @@ namespace mbts.spark { readonly Property _foreground, _background, _hoverbackground, _border; readonly Property _has_multiple_orders_at_same_level, _is_hedge_order; readonly Property _algo_type; - readonly Property _summary; IObservable _bs; LadderSettings.BadgeStyle _current_badge_style; int _n_active_price_change_ops = 0; @@ -45,24 +44,23 @@ namespace mbts.spark { uint filled, quantity, num_root_orders; int working_quantity; - LadderModel model => _parent.model; - Dictionary _orders = new Dictionary(); + LadderModel model => _owner.model; + Dictionary _orders = new Dictionary(); - public LadderOrderVM(OrderRecordImmutable o, double real_price, LadderVM parent, IBus bus, IObservable bs) { - _parent = parent; + public LadderOrderVM((OrderRecordImmutable, uint?) op, double real_price, LadderVM owner, IBus bus, IObservable bs) { + _owner = owner; _bus = bus; _bs = bs; _real_price = real_price; // settings var fs = UserSettings.factory_defaults.ladder; - _font_family = backing(nameof(font_family), parent.settings, get_font_family, get_font_family(fs)); - _font_weight = backing(nameof(font_weight), parent.settings, get_font_weight, get_font_weight(fs)); - _font_size = backing(nameof(font_size), parent.settings, get_font_size, get_font_size(fs)); - _foreground = backing(nameof(foreground), parent.settings, get_foreground, get_foreground(fs)); + _font_family = backing(nameof(font_family), owner.settings, get_font_family, get_font_family(fs)); + _font_weight = backing(nameof(font_weight), owner.settings, get_font_weight, get_font_weight(fs)); + _font_size = backing(nameof(font_size), owner.settings, get_font_size, get_font_size(fs)); + _foreground = backing(nameof(foreground), owner.settings, get_foreground, get_foreground(fs)); _background = backing(nameof(background), _color_bg_brush); _hoverbackground = backing(nameof(hoverbackground), _color_bg_brush); _border = backing(nameof(border), _color_bg_brush); - _summary = backing(nameof(summary), string.Empty); _has_multiple_orders_at_same_level = backing(nameof(has_multiple_orders_at_same_level), false); _is_hedge_order = backing(nameof(is_hedge_order), false); _algo_type = backing(nameof(algo_type), CustomOrderDefinition.CustomOneofCase.None); @@ -71,7 +69,7 @@ namespace mbts.spark { double get_font_size(LadderSettings s) => s.orders_column.font_size; Brush get_foreground(LadderSettings s) => s.orders_column.foreground.rgb_brush(); // - add_order(o); + add_order(op); _dispose = Disposable.Create( _bs.Subscribe(badge => { @@ -111,7 +109,7 @@ namespace mbts.spark { // should be set appropriately if (num_root_orders == 1) { var single_order = _root_nodes().First(); - var single_order_metadata = _parent.model.order_gateway.factory.get_metadata(single_order); + var single_order_metadata = _owner.model.order_gateway.factory.get_metadata(single_order); border = single_order_metadata.Color.rgb_brush(); if (single_order.order_type().is_bts_algo_type()) { algo_type = single_order.order_type(); @@ -125,10 +123,6 @@ namespace mbts.spark { has_multiple_orders_at_same_level = num_root_orders > 1; is_hedge_order = _root_nodes().Any(o => o.sub_type() == OrderSubType.HedgingLeg); - if (_root_nodes().Any()) { - // This is only used in log messages vs the no-longer-existing badge popup, so simplify string - summary = $"Badge containing root orders: {string.Join(",", _root_nodes().Select(o => o.Key))}"; - } _set_badge_text(); } @@ -138,8 +132,8 @@ namespace mbts.spark { RaisePropertyChanged(nameof(text_lines)); } - public void add_order(OrderRecordImmutable new_order) { - _orders[new_order.Key] = new_order; + public void add_order((OrderRecordImmutable order, uint? p) new_op) { + _orders[new_op.order.Key] = new_op; } public void remove_order(uint order_key) { @@ -153,8 +147,9 @@ namespace mbts.spark { // } //} - public uint security_key => _parent.security.Id; + public uint security_key => _owner.security.Id; public double price { get => double.IsNaN(_phantom_price) ? _real_price : _phantom_price; } + public uint? min_piq => _orders.Select(op => op.Value.p).Where(piq => piq.HasValue).Min(); public double real_price => _real_price; public Side side { get { @@ -166,7 +161,7 @@ namespace mbts.spark { } } - IEnumerable _root_nodes() => _orders.Values.Select(_parent.get_root_at_price).DistinctOn(OrderUtils.RecordKeySelector); + IEnumerable _root_nodes() => _orders.Values.Select(_owner.get_root_at_price).DistinctOn(OrderUtils.RecordKeySelector); public IReadOnlyList text_lines { get; private set; } @@ -187,12 +182,13 @@ namespace mbts.spark { public Brush background { get => _background.Value; private set => _background.Value = value; } public Brush hoverbackground { get => _hoverbackground.Value; private set => _hoverbackground.Value = value; } public Brush border { get => _border.Value; private set => _border.Value = value; } - public string summary { get => _summary.Value; private set => _summary.Value = value; } + public string summary => _root_nodes().Any() ? $"Badge containing root orders: {string.Join(",", _root_nodes().Select(o => o.Key))}" : string.Empty; public bool has_multiple_orders_at_same_level { get => _has_multiple_orders_at_same_level.Value; private set => _has_multiple_orders_at_same_level.Value = value; } public bool is_hedge_order { get => _is_hedge_order.Value; private set => _is_hedge_order.Value = value; } public CustomOrderDefinition.CustomOneofCase algo_type { get => _algo_type.Value; private set => _algo_type.Value = value; } - public IEnumerable orders => _orders.Values; + public IReadOnlyList order_keys => _orders.Keys.ToReadOnlyList(); + public IEnumerable root_orders => _root_nodes(); void _cancel_order() { foreach (var order in _root_nodes()) { @@ -207,7 +203,7 @@ namespace mbts.spark { _phantom_price = new_price; RaisePropertyChanged(nameof(price)); ++_n_active_price_change_ops; - if (await _parent.model.order_gateway.update_order(order, new_price, order.State.Quantity)) { + if (await _owner.model.order_gateway.update_order(order, new_price, order.State.Quantity)) { // wait 3 secs, then give up if price change still hasn't taken effect await Task.Delay(3000); } @@ -229,10 +225,10 @@ namespace mbts.spark { price_filter.minimum.value = real_price; price_filter.maximum.value = real_price; - var obvm = new OrderBookVM(null, _parent.bus, null, _parent.model.order_gateway, null, true); + var obvm = new OrderBookVM(null, _owner.bus, null, _owner.model.order_gateway, null, true); var state = new bts.shell.orderbook.OrderBookStartupData( new bts.shell.orderbook.FilterStates { - security_criteria = new SecurityCriteria(_parent.security), + security_criteria = new SecurityCriteria(_owner.security), price = price_filter, statuses = new[] { OrderStatus.Working } }, @@ -278,7 +274,7 @@ namespace mbts.spark { } public override string ToString() { - return $"Price: {this.price} Order Count: {this.orders.Count()}"; + return $"Price: {price} Order Count: {_orders.Values.Count()} Root Order Count: {_root_nodes().Count()} PIQ: {(this.min_piq.HasValue ? this.min_piq.Value.ToString() : "")}"; } } } diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs index 534145cbe..3d6db39c3 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs @@ -30,7 +30,8 @@ namespace mbts.spark { _bid = new LadderColumnSettingsVM("Bid"), _price = new LadderColumnSettingsVM("Price"), _ask = new LadderColumnSettingsVM("Ask"), - _last = new LadderColumnSettingsVM("Last", true); + _last = new LadderColumnSettingsVM("Last", true), + _piq = new LadderColumnSettingsVM("PIQ", true); VolumeColumnVM _volume = new VolumeColumnVM(); LadderMarketBreakLineVM _bid_mkt_line = new LadderMarketBreakLineVM("Bid"), @@ -83,6 +84,7 @@ namespace mbts.spark { _price.font_size_changed += on_font_size_changed; _ask.font_size_changed += on_font_size_changed; _last.font_size_changed += on_font_size_changed; + _piq.font_size_changed += on_font_size_changed; void on_font_size_changed(double oldval, double newval) => _row_height.refresh(); } @@ -121,6 +123,7 @@ namespace mbts.spark { if (!is_synthetic) { yield return _volume.general; yield return _last; + yield return _piq; } } } @@ -131,6 +134,7 @@ namespace mbts.spark { public LadderColumnSettingsVM ask_column => _ask; public LadderColumnSettingsVM volume_column => _volume.general; public LadderColumnSettingsVM last_column => _last; + public LadderColumnSettingsVM piq_column => _piq; public VolumeColumnVM volume => _volume; public LadderMarketBreakLineVM session_range_fill => _session_range_fill; public ObservableEditableCollection> size_presets { get; } @@ -210,6 +214,7 @@ namespace mbts.spark { _price.assign_from(_current.price_column); _volume.assign_from(_current.volume_column); _last.assign_from(_current.last_column); + _piq.assign_from(_current.piq_column); _bid_mkt_line.assign_from(_current.bid_break_line); _ask_mkt_line.assign_from(_current.ask_break_line); _mid_mkt_line.assign_from(_current.mid_break_line); @@ -255,6 +260,7 @@ namespace mbts.spark { clone.ask_column = _ask.get_settings(); clone.volume_column = _volume.get_settings(); clone.last_column = _last.get_settings(); + clone.piq_column = _piq.get_settings(); clone.bid_break_line = _bid_mkt_line.get_settings(); clone.ask_break_line = _ask_mkt_line.get_settings(); clone.mid_break_line = _mid_mkt_line.get_settings(); diff --git a/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs b/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs index 31508a8a3..56e3ff166 100644 --- a/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs +++ b/src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs @@ -27,7 +27,7 @@ namespace mbts.spark { readonly LadderModel _model; readonly Property _high, _low, _change, _open, _settlement, _position, _left_panel_width, _zoom_factor; //readonly Property _window_width, _window_height; - readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width; + readonly Property _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width, _piq_column_width; readonly Property _volume; readonly Property _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account, _show_tif, _show_order_type, _show_account_position, _show_size_preset, _show_auto_clear, _show_clear, _show_cancel_buys, @@ -38,7 +38,7 @@ namespace mbts.spark { ulong _last_trade_time = 0; IEqualityComparer _equality; Dictionary _slot_price_vs_orders; - IReadOnlyDictionary _cache = DictionaryHelper.empty(); + IReadOnlyDictionary _cache = DictionaryHelper.empty(); public static LadderVM create(IBus bus, LadderModel model, IWindowSettingsManager settings_manager, LadderSettings init) { var non_shared_wsm = settings_manager.as_non_shared(); @@ -72,6 +72,7 @@ namespace mbts.spark { _price_column_width = backing(nameof(price_column_width), ws.price_column_width); _ask_column_width = backing(nameof(ask_column_width), ws.ask_column_width); _last_column_width = backing(nameof(last_column_width), ws.last_column_width); + _piq_column_width = backing(nameof(piq_column_width), ws.piq_column_width); _volume_column_width = backing(nameof(volume_column_width), ws.volume_column_width); _left_panel_width = backing(nameof(left_panel_width), ws.order_entry_panel_width); _zoom_factor = backing(nameof(zoom_factor), ws.zoom_factor); @@ -144,13 +145,13 @@ namespace mbts.spark { account_positions, size_calculator = new CalculatorLogic(0, _settings_manager.subscribe_current().Select(s => s.calculator_buttons)), bus.subscribe(SparkTags.LogActivity).OnDispatcher().Subscribe(status_update), - model.orders.OnDispatcher().Subscribe(working => { + model.order_gateway.working_orders_with_piq.OnDispatcher().Subscribe(working => { // Here we manage N orders to 1 price slot in the ladder Dictionary> orders_used_in_this_update = new Dictionary>(_equality); - _cache = working.state; + _cache = working; - - foreach (var wo in _cache.Values) { + foreach (var os in _cache.Values) { + var wo = os.order; var slot_price_target = wo.price(); var order_type = wo.order_type(); var filled = wo.filled(); @@ -164,9 +165,9 @@ namespace mbts.spark { } if (_slot_price_vs_orders.TryGetValue(slot_price_target, out LadderOrderVM slot_vm)) { - slot_vm.add_order(wo); + slot_vm.add_order((wo, os.piq)); } else { - slot_vm = new LadderOrderVM(wo, slot_price_target, this, _bus, _settings_manager.subscribe_current().Select(s => s.badge_style)); + slot_vm = new LadderOrderVM((wo, os.piq), slot_price_target, this, _bus, _settings_manager.subscribe_current().Select(s => s.badge_style)); _slot_price_vs_orders[slot_price_target] = slot_vm; } @@ -183,7 +184,7 @@ namespace mbts.spark { // remove any orders from the lovm that aren't in here foreach(var kvp in orders_used_in_this_update) { var slot_vm = _slot_price_vs_orders[kvp.Key]; - foreach(var existing_order_key in slot_vm.orders.Select(o => o.Key).ToList()) { + foreach(var existing_order_key in slot_vm.order_keys) { if (!kvp.Value.Contains(existing_order_key)) { slot_vm.remove_order(existing_order_key); } @@ -227,6 +228,10 @@ namespace mbts.spark { public event Action on_tick; public MDConsumerHandle md => _model.md; + // we return only entries who have piqs and whose piqs are not uint.Max + // uint.Max happens when the server is restarted but the client is still up + // maybe long term we don't show this data while the exchange connection is down + public IEnumerable<(double, uint)> price_vs_piq => _slot_price_vs_orders.Values.Where(result => result.min_piq.HasValue && result.min_piq.Value != uint.MaxValue).Select(o => (o.price, o.min_piq.Value)); public bool is_synthetic => _model.security.ContractCase == Security.ContractOneofCase.Synthetic; Property _status; @@ -302,10 +307,12 @@ namespace mbts.spark { public uint price_column_width { get => _price_column_width.Value; set => _price_column_width.set(value); } public uint ask_column_width { get => _ask_column_width.Value; set => _ask_column_width.set(value); } public uint last_column_width { get => _last_column_width.Value; set => _last_column_width.set(value); } + public uint piq_column_width { get => _piq_column_width.Value; set => _piq_column_width.set(value); } public uint volume_column_width { get => _volume_column_width.Value; set => _volume_column_width.set(value); } internal LadderModel model => _model; internal IObservable settings => _settings_manager.subscribe_current(); - internal OrderRecordImmutable get_root_at_price(OrderRecordImmutable order) { + internal OrderRecordImmutable get_root_at_price((OrderRecordImmutable order, uint? _) op) { + var order = op.order; double p = order.price(); foreach (var o in lineage(order.ParentKey)) { if (_equality.Equals(p, o.price())) @@ -315,15 +322,15 @@ namespace mbts.spark { } internal IEnumerable lineage(uint key) { - while (key != 0 && _cache.TryGetValue(key, out var order)) { - yield return order; - key = order.ParentKey; + while (key != 0 && _cache.TryGetValue(key, out var op)) { + yield return op.order; + key = op.order.ParentKey; } } public LadderSettings get_save_state() { var state = _settings_manager.current_settings(); - state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor)); + state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width, zoom_factor, piq_column_width)); state.use_defaults = _settings_manager.instance_settings == null; return state; } diff --git a/src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs b/src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs index b5f7b538e..e51c932d5 100644 --- a/src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs +++ b/src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs @@ -43,7 +43,8 @@ namespace mbts.spark { _ask_vis_tup = new VisualGlyphTuple(), _price_vis_tup = new VisualGlyphTuple(), _last_vis_tup = new VisualGlyphTuple(), - _volume_vis_tup = new VisualGlyphTuple(); + _volume_vis_tup = new VisualGlyphTuple(), + _piq_vis_tup = new VisualGlyphTuple(); readonly RefValueContainer> _volume_container = new RefValueContainer>(); @@ -94,7 +95,7 @@ namespace mbts.spark { }; } - public enum ColumnType { Order, Bid, Price, Ask, Last, Volume, None } + public enum ColumnType { Order, Bid, Price, Ask, Last, Volume, None, PIQ } public event Action visible_columns_changed; @@ -194,6 +195,19 @@ namespace mbts.spark { set => SetValue(LastColumnWidthProperty, value); } + public static readonly DependencyProperty PIQColumnWidthProperty = + WPFHelper.create_dp( + nameof(PIQColumnWidth), + FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, + (o, oldval, newval) => o._adjust_column_width(o.columns.FirstOrDefault(c => c.column_type == ColumnType.PIQ), (double)newval - (double)oldval), + 50u); + + public uint PIQColumnWidth { + get => (uint)GetValue(PIQColumnWidthProperty); + set => SetValue(PIQColumnWidthProperty, value); + } + + #region CanUserSendOrders public static readonly DependencyProperty CanUserSendOrdersProperty = DependencyProperty.Register( @@ -445,7 +459,8 @@ namespace mbts.spark { double x = 0; var col_list = new List { new Column(_settings.orders_column, ColumnType.Order, null, x), - new Column(_settings.bid_column, ColumnType.Bid, _bid_vis_tup, (x += _settings.orders_column.show ? _settings.orders_column.width : 0)), + new Column(_settings.piq_column, ColumnType.PIQ, _piq_vis_tup, (x += _settings.orders_column.show ? _settings.orders_column.width : 0)), + new Column(_settings.bid_column, ColumnType.Bid, _bid_vis_tup, (x += _settings.piq_column.show ? _settings.piq_column.width : 0)), (_price_column = new Column(_settings.price_column, ColumnType.Price, _price_vis_tup, (x += _settings.bid_column.show ? _settings.bid_column.width : 0))), new Column(_settings.ask_column, ColumnType.Ask, _ask_vis_tup, (x += _settings.price_column.show ? _settings.price_column.width : 0)) }; @@ -564,6 +579,8 @@ namespace mbts.spark { if (depth_changed) _refresh_sizes(); } + // TODO - detect changes in PIQ to selectively redraw + _refresh_piq(); _refresh_volume_at_price(); _last_origin_price = _mkt_break_lines.origin; _price_dirty = false; @@ -600,6 +617,14 @@ namespace mbts.spark { LadderRender.render_last(_last_vis_tup.visual, _last_vis_tup.glyph_context, _last_trade, last_index, _settings.row_height, _settings.last_column); } + void _refresh_piq() { + if (!_settings.piq_column.show) + return; + int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); + var piq_list = _vm.price_vs_piq; + LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, origin_index, _price_list, piq_list.ToArray(), _settings.row_height, security.ContractSpec, _settings.piq_column); + } + void _refresh_sizes() { Log.assert_d(_price_list != null); int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer); @@ -768,6 +793,7 @@ namespace mbts.spark { _ask_vis_tup.glyph_context = glyph_context(s.ask_column); _price_vis_tup.glyph_context = glyph_context(s.price_column); _last_vis_tup.glyph_context = glyph_context(s.last_column); + _piq_vis_tup.glyph_context = glyph_context(s.piq_column); _volume_vis_tup.glyph_context = glyph_context(s.volume_column.general); GlyphContext glyph_context(LadderColumnSettings cs) => @@ -795,6 +821,7 @@ namespace mbts.spark { case ColumnType.Price: SetCurrentValue(price_column_widthProperty, col.settings.width); break; case ColumnType.Ask: SetCurrentValue(ask_column_widthProperty, col.settings.width); break; case ColumnType.Last: SetCurrentValue(LastColumnWidthProperty, col.settings.width); break; + case ColumnType.PIQ: SetCurrentValue(PIQColumnWidthProperty, col.settings.width); break; case ColumnType.Volume: SetCurrentValue(VolumeColumnWidthProperty, col.settings.width); _volume_container.mark_dirty(); @@ -962,4 +989,4 @@ namespace mbts.spark { } } } -} +} \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs b/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs index 05d66a2c5..51bda9a32 100644 --- a/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs +++ b/src/client/spark/Modules/Trading/Ladder/View/LadderOrdersHost.cs @@ -219,7 +219,7 @@ namespace mbts.spark { Log.Ui.info("Drop - " + vm.summary); double price = _parent.price_for_point(e.GetPosition(_parent)); if (price != vm.price) { - foreach (var order in vm.orders) { + foreach (var order in vm.root_orders) { vm.try_set_price(order, price); } } diff --git a/src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs b/src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs index 3c830df3f..6873c3bd4 100644 --- a/src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs +++ b/src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs @@ -287,6 +287,30 @@ namespace mbts.spark { } } + // this is basically a copy of redner_sizes up above + internal static void render_piqs(DrawingVisual dv, GlyphContext glyph_context, int origin_index, double[] prices, (double price, uint piq)[] piqs, double row_height, ContractSpecificationImmutable spec, LadderColumnSettings settings) { + using (var dc = dv.RenderOpen()) { + using (_draw_with_clip(dc, settings, row_height, prices.Length)) { + if (!prices.within_range(origin_index) || piqs.is_null_or_empty()) + return; + var foreground = settings.foreground.rgb_brush(); + int curr_index = origin_index; + double curr_price = prices[curr_index]; + for (int i = 0; i < piqs.Length; i++) { + double price = piqs[i].price; + // prices are in descending order, so get diff from curr_price - price + int tick_diff = spec.tickdiff_unchecked(curr_price, price); + curr_index += tick_diff; + curr_price = price; + if (curr_index >= 0 && curr_index < prices.Length) { + var gr = format_text(piqs[i].piq.ToString(), curr_index, row_height, settings, glyph_context); + dc.DrawGlyphRun(foreground, gr); + } else break; + } + } + } + } + public static void render_mkt_break(DrawingVisual dv, int index, double row_height, uint color) => render_single_grid_line(dv, index, row_height, color, 0, SystemParameters.FullPrimaryScreenWidth); diff --git a/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml b/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml index 88aab21f8..f5355fd6b 100644 --- a/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml +++ b/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml @@ -541,6 +541,7 @@ price_column_width="{Binding price_column_width, Mode=TwoWay}" ask_column_width="{Binding ask_column_width, Mode=TwoWay}" LastColumnWidth="{Binding last_column_width, Mode=TwoWay}" + PIQColumnWidth="{Binding piq_column_width, Mode=TwoWay}" VolumeColumnWidth="{Binding volume_column_width, Mode=TwoWay}" > diff --git a/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml.cs b/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml.cs index ce8797c49..af05ba764 100644 --- a/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml.cs +++ b/src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml.cs @@ -140,6 +140,10 @@ namespace mbts.spark { header = "Last"; width_prop = nameof(LadderVM.last_column_width); break; + case LadderGridHost.ColumnType.PIQ: + header = "PIQ"; + width_prop = nameof(LadderVM.piq_column_width); + break; default: throw ExceptionBuilder.Argument.unexpected_case(type, nameof(type)); } return _create(host, header, $"DataContext.{width_prop}"); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 94b685ada..a326e1efa 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -28,12 +28,6 @@ using System.Reactive.Linq; using System.Threading.Tasks; using System.Windows.Controls; using System.Windows.Input; -using _GridSettingsManager = - mbts.ui.GridSettingsManager< - mbts.spark.OrderBookRowVM, - bts.api.orderman.OrderSecurity, - mbts.OrderRecordTuplePredicate, - bts.shell.orderbook.OrderBookGridSettings>; using Columns = mbts.spark.OrderBookGridView.ColumnKeys; namespace mbts.spark { @@ -68,6 +62,7 @@ namespace mbts.spark { HashSet _attachable_selected_orders = new HashSet(), _detachable_selected_orders = new HashSet(); + FilterData _cached_filter_data; public OrderBookVM(IObservable owner_title, IBus bus, INotificationService notification_service, OrdersModel model, IWindowSettingsManager settings_manager, bool rows_are_initially_expanded = false) { _bus = bus; @@ -140,6 +135,7 @@ namespace mbts.spark { return dialog; })); grid_settings_manager = new OrderBookGridSettingsManager(owner_title, bus, notification_service, model, rows, columns); + grid_settings_manager.algo_tree_display_mode_changed += _on_filter_changed; IDisposable attach_vm_hooks = null; begin_attaching += on_begin_attaching; end_attaching += on_end_attaching; @@ -214,6 +210,7 @@ namespace mbts.spark { }), Disposable.Create(() => { _columns.filter_changed -= _on_filter_changed; + grid_settings_manager.algo_tree_display_mode_changed -= _on_filter_changed; _row_collection.CollectionChanged -= _row_collection_CollectionChanged; begin_attaching -= on_begin_attaching; end_attaching -= on_end_attaching; @@ -468,16 +465,28 @@ namespace mbts.spark { (var old_filter, var old_sort) = _columns.create(); // suspend column filter updates _columns.filter_changed -= _on_filter_changed; + grid_settings_manager.algo_tree_display_mode_changed -= _on_filter_changed; #pragma warning disable CS0612 // Type or member is obsolete _columns.load_from((state.filters, (state.orderbook_grid?.general ?? OrderBookStartupData.DefaultGrid.general).sorting_criterion)); #pragma warning restore CS0612 // Type or member is obsolete // resume column filter updates _columns.filter_changed += _on_filter_changed; - (var new_filter, var new_sort) = _columns.create(); - if (!new_sort.Equals(old_sort)) - _row_collection.comparer = _columns.get_sorting_data().comparer; - if (!new_filter.Equals(old_filter)) - _on_filter_changed(); + grid_settings_manager.algo_tree_display_mode_changed += _on_filter_changed; + var new_filter = new FilterData(grid_settings_manager.algo_tree_display_mode, _columns.create_filter_states()); + var new_sort = _columns.get_sorting_data(); + if (!new_filter.Equals(_cached_filter_data)) { + _cached_filter_data = new_filter; + if (!new_sort.Equals(old_sort)) { + /// Both filter and sort have changed, use to avoid syncing twice. + using (_row_collection.defer_refresh()) { + _row_collection.filter = new_filter.build_predicate(); + _row_collection.comparer = new_sort.comparer; + } + } else { + _row_collection.filter = new_filter.build_predicate(); + } + } else if (!new_sort.Equals(old_sort)) + _row_collection.comparer = new_sort.comparer; } public void on_sorting(object sender, DIYGridSortingEventArgs e) { @@ -527,8 +536,13 @@ namespace mbts.spark { } } - void _on_filter_changed() => - _row_collection.filter = _columns.create_filter_states().build_predicate(); + void _on_filter_changed() { + var fd = new FilterData(grid_settings_manager.algo_tree_display_mode, _columns.create_filter_states()); + if (!fd.Equals(_cached_filter_data)) { + _cached_filter_data = fd; + _row_collection.filter = fd.build_predicate(); + } + } async Task _attach_orders(IReadOnlyList rows, OrderBookRowVM target) { Log.Ui.info($"{nameof(OrderBookVM)} begin attaching orders ({rows.Count} total)..."); @@ -818,6 +832,43 @@ namespace mbts.spark { } } + struct FilterData : IEquatable { + public readonly AlgoTreeDisplayMode algo_tree_display_mode; + public readonly FilterStates column_filters; + + public FilterData(AlgoTreeDisplayMode algo_tree_display_mode, FilterStates column_filters) { + this.algo_tree_display_mode = algo_tree_display_mode; + this.column_filters = column_filters; + } + + public Func build_predicate() { + var pred = column_filters.build_predicate(); + if (algo_tree_display_mode == AlgoTreeDisplayMode.RootAndLeafNodes) { + if (pred == null) { + pred = tree_disp_pred; + } else { + var col_pred = pred; + pred = o => tree_disp_pred(o) && col_pred(o); + } + bool tree_disp_pred(OrderSecurityAlgoMeta o) => + !o.order.has_parent() || + o.order.is_exchange_order() || + o.order.order_type() == CustomOrderDefinition.CustomOneofCase.Spreader; + } + return pred; + } + + public bool Equals(FilterData other) => + algo_tree_display_mode == other.algo_tree_display_mode && + column_filters.safe_equals(other.column_filters); + + public override bool Equals(object obj) => + obj is FilterData fd && Equals(fd); + + public override int GetHashCode() => + EqualityComparer.aggregate_hash((int)algo_tree_display_mode, column_filters?.GetHashCode() ?? 0); + } + /// /// Monitors selected rows, and determines whether or not they are all active-orders, with common parameters. /// @@ -998,4 +1049,4 @@ namespace mbts.spark { } } } -} +} \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs new file mode 100644 index 000000000..96659633e --- /dev/null +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs @@ -0,0 +1,127 @@ +using bts.api; +using bts.api.orderman; +using mbts; +using mbts.spark; +using mbts.ui; +using mbts.ui.controls; +using mbts.ui.settings; +using mbts.ui.utility; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using Columns = mbts.spark.OrderBookGridView.ColumnKeys; + +namespace bts.shell.orderbook { + class OrderBookGridSettingsManager : IDisposable, IObserver { + readonly IObserver _core_man_as_observer; + AlgoTreeDisplayMode _algo_tree_display_mode; + + public OrderBookGridSettingsManager( + IObservable owner_title, + IBus bus, + INotificationService notification_service, + OrdersModel model, + IEnumerable rows, + ObservableCollection columns) { + // make font-weight immutable for all columns (the font-weight rule is hard-coded: Regular for root-orders, Light for all others) + // TODO: Remove the immutable font-weight flags when cell animation is ready + var col_settings_flags = EnumHelper.get_values().ToDictionary( + c => c.ToString(), c => GridColumnSettingsFeatureFlags.ImmutableFontWeight); + // make font-color immutable for Side, Size, and Filled columns (the font-color rule is hard-coded: MarketData.bid for Buy orders, MarketData.ask for Sells) + // TODO: Remove the immutable font-color flags when cell animation is ready + foreach (var c in new[] { Columns.Side, Columns.Size, Columns.Filled }) { + col_settings_flags[c.ToString()] = GridColumnSettingsFeatureFlags.ImmutableFontColor | GridColumnSettingsFeatureFlags.ImmutableFontWeight; + } + core_manager = new GridSettingsManager( + owner_title, bus, model, rows, columns, + () => new OrderRecordTuplePredicate(), + () => { + var pvm = PredicateVMWrapper.create( + OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); + pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); + return pvm; + }, + col_id => { + var pvm = PredicateVMWrapper.create( + OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); + pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); + return pvm; + }, + (gen, anims) => new OrderBookGridSettings(gen, anims.Select(a => new OrderRecordTuplePredicateAnimations(a.predicate_data, a.cell_animations, a.row_animations, a.sound_notifications))) { algo_tree_display_mode = _algo_tree_display_mode }, + p => p.create_predicate(notification_service.data_cache), + p => p.create_notification_predicate(notification_service.data_cache), + notification_service == null ? new Func, SoundNotification, IDisposable>((p, sn) => Disposable.empty) : notification_service.link_to_order, + notification_service == null ? new Func, IDisposable>(t => Disposable.empty) : notification_service.link_to_order, + col_settings_flags); + _core_man_as_observer = core_manager; + } + + public event Action algo_tree_display_mode_changed; + + public GridSettingsManager core_manager { get; } + + public AlgoTreeDisplayMode algo_tree_display_mode { + get => _algo_tree_display_mode; + set { + if (_algo_tree_display_mode != value) { + _algo_tree_display_mode = value; + algo_tree_display_mode_changed?.Invoke(); + } + } + } + + public void load_from(OrderBookGridSettings gs) { + core_manager.load_from(gs); + algo_tree_display_mode = gs.algo_tree_display_mode; + } + + public OrderBookGridSettings create(IEnumerable sorting_criterion) { + var gs = core_manager.create(); + gs.general = gs.general.with(sorting_criterion: sorting_criterion); + gs.algo_tree_display_mode = _algo_tree_display_mode; + return gs; + } + + public OrderBookSettingsVM create_vm(OrderBookGridSettings gs, IObservable owner_title) => + new OrderBookSettingsVM(core_manager.create_vm(gs.general, owner_title)) { + algo_tree_display_mode = gs.algo_tree_display_mode + }; + + public void Dispose() => core_manager.Dispose(); + + static OrderRecordTuplePredicate.PredicateTypeOneofCase _corresponding_predicate_type(Columns col) { + switch (col) { + case Columns.Security: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Security; + case Columns.Time: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Time; + case Columns.LastChange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastChange; + case Columns.User: return OrderRecordTuplePredicate.PredicateTypeOneofCase.User; + case Columns.Account: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Account; + case Columns.Exchange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Exchange; + case Columns.TimeInForce: return OrderRecordTuplePredicate.PredicateTypeOneofCase.TimeInForce; + case Columns.OrderType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderType; + case Columns.SubType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SubType; + case Columns.Side: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Side; + case Columns.Size: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Size; + case Columns.Price: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Price; + case Columns.Filled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Filled; + case Columns.AvgFilled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AvgFilled; + case Columns.LastPrice: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastPrice; + case Columns.Status: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Status; + case Columns.LastMessage: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastMessage; + case Columns.SecurityKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SecurityKey; + case Columns.OrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderKey; + case Columns.ParentKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.ParentKey; + case Columns.AttachedTargetOrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AttachedTargetOrderKey; + default: return default; + } + } + + void IObserver.OnNext(OrderBookGridSettings value) { + _core_man_as_observer.OnNext(value); + algo_tree_display_mode = value.algo_tree_display_mode; + } + void IObserver.OnError(Exception error) => _core_man_as_observer.OnError(error); + void IObserver.OnCompleted() => _core_man_as_observer.OnCompleted(); + } +} \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs index 70c1a4b3e..6791e07c4 100644 --- a/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettingsVM.cs @@ -3,118 +3,13 @@ using bts.api.orderman; using mbts; using mbts.spark; using mbts.ui; -using mbts.ui.controls; using mbts.ui.settings; using mbts.ui.shell; using mbts.ui.spark; -using mbts.ui.utility; using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; -using Columns = mbts.spark.OrderBookGridView.ColumnKeys; namespace bts.shell.orderbook { - class OrderBookGridSettingsManager : IDisposable, IObserver { - readonly IObserver _core_man_as_observer; - AlgoTreeDisplayMode _algo_tree_display_mode; - - public OrderBookGridSettingsManager( - IObservable owner_title, - IBus bus, - INotificationService notification_service, - OrdersModel model, - IEnumerable rows, - ObservableCollection columns) { - // make font-weight immutable for all columns (the font-weight rule is hard-coded: Regular for root-orders, Light for all others) - // TODO: Remove the immutable font-weight flags when cell animation is ready - var col_settings_flags = EnumHelper.get_values().ToDictionary( - c => c.ToString(), c => GridColumnSettingsFeatureFlags.ImmutableFontWeight); - // make font-color immutable for Side, Size, and Filled columns (the font-color rule is hard-coded: MarketData.bid for Buy orders, MarketData.ask for Sells) - // TODO: Remove the immutable font-color flags when cell animation is ready - foreach (var c in new[] { Columns.Side, Columns.Size, Columns.Filled }) { - col_settings_flags[c.ToString()] = GridColumnSettingsFeatureFlags.ImmutableFontColor | GridColumnSettingsFeatureFlags.ImmutableFontWeight; - } - core_manager = new GridSettingsManager( - owner_title, bus, model, rows, columns, - () => new OrderRecordTuplePredicate(), - () => { - var pvm = PredicateVMWrapper.create( - OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); - pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); - return pvm; - }, - col_id => { - var pvm = PredicateVMWrapper.create( - OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); - pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); - return pvm; - }, - (gen, anims) => new OrderBookGridSettings(gen, anims.Select(a => new OrderRecordTuplePredicateAnimations(a.predicate_data, a.cell_animations, a.row_animations, a.sound_notifications))) { algo_tree_display_mode = _algo_tree_display_mode }, - p => p.create_predicate(notification_service.data_cache), - p => p.create_notification_predicate(notification_service.data_cache), - notification_service == null ? new Func, SoundNotification, IDisposable>((p, sn) => Disposable.empty) : notification_service.link_to_order, - notification_service == null ? new Func, IDisposable>(t => Disposable.empty) : notification_service.link_to_order, - col_settings_flags); - _core_man_as_observer = core_manager; - } - - public GridSettingsManager core_manager { get; } - - public void load_from(OrderBookGridSettings gs) { - core_manager.load_from(gs); - _algo_tree_display_mode = gs.algo_tree_display_mode; - } - - public OrderBookGridSettings create(IEnumerable sorting_criterion) { - var gs = core_manager.create(); - gs.general = gs.general.with(sorting_criterion: sorting_criterion); - gs.algo_tree_display_mode = _algo_tree_display_mode; - return gs; - } - - public OrderBookSettingsVM create_vm(OrderBookGridSettings gs, IObservable owner_title) => - new OrderBookSettingsVM(core_manager.create_vm(gs.general, owner_title)) { - algo_tree_display_mode = gs.algo_tree_display_mode - }; - - public void Dispose() => core_manager.Dispose(); - - static OrderRecordTuplePredicate.PredicateTypeOneofCase _corresponding_predicate_type(Columns col) { - switch (col) { - case Columns.Security: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Security; - case Columns.Time: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Time; - case Columns.LastChange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastChange; - case Columns.User: return OrderRecordTuplePredicate.PredicateTypeOneofCase.User; - case Columns.Account: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Account; - case Columns.Exchange: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Exchange; - case Columns.TimeInForce: return OrderRecordTuplePredicate.PredicateTypeOneofCase.TimeInForce; - case Columns.OrderType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderType; - case Columns.SubType: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SubType; - case Columns.Side: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Side; - case Columns.Size: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Size; - case Columns.Price: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Price; - case Columns.Filled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Filled; - case Columns.AvgFilled: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AvgFilled; - case Columns.LastPrice: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastPrice; - case Columns.Status: return OrderRecordTuplePredicate.PredicateTypeOneofCase.Status; - case Columns.LastMessage: return OrderRecordTuplePredicate.PredicateTypeOneofCase.LastMessage; - case Columns.SecurityKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.SecurityKey; - case Columns.OrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.OrderKey; - case Columns.ParentKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.ParentKey; - case Columns.AttachedTargetOrderKey: return OrderRecordTuplePredicate.PredicateTypeOneofCase.AttachedTargetOrderKey; - default: return default; - } - } - - void IObserver.OnNext(OrderBookGridSettings value) { - _core_man_as_observer.OnNext(value); - _algo_tree_display_mode = value.algo_tree_display_mode; - } - void IObserver.OnError(Exception error) => _core_man_as_observer.OnError(error); - void IObserver.OnCompleted() => _core_man_as_observer.OnCompleted(); - } - class OrderBookSettingsVM : DisposableViewModelBase, IWindowSettingsVM, ITemplateSelectorKey { AlgoTreeDisplayMode _algo_tree_display_mode; @@ -147,29 +42,4 @@ namespace bts.shell.orderbook { return gs; } } - //class OrderBookSettingsVM : GridSettingsVM { - // AlgoTreeDisplayMode _algo_tree_display_mode; - - // public OrderBookSettingsVM( - // IObservable owner_title, IBus bus, OrdersModel orders_model, - // Func predicate_data_factory, - // Func> notification_predicate_factory, - // Func> create_predicate_vm, - // Func> create_column_predicate_vm, - // Func>, OrderBookGridSettings> create_settings_obj, - // IReadOnlyDictionary column_flags = null) - // : base(owner_title, bus, orders_model, predicate_data_factory, notification_predicate_factory, create_predicate_vm, create_column_predicate_vm, create_settings_obj, column_flags) { } - - // public AlgoTreeDisplayMode algo_tree_display_mode { get => _algo_tree_display_mode; set => Set(ref _algo_tree_display_mode, value); } - - // public override void load(OrderBookGridSettings grid_settings) { - // base.load(grid_settings); - // algo_tree_display_mode = grid_settings.algo_tree_display_mode; - // } - // public override OrderBookGridSettings get_result() { - // var gs = base.get_result(); - // gs.algo_tree_display_mode = algo_tree_display_mode; - // return gs; - // } - //} } \ No newline at end of file diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index b86d552f5..da474d92a 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -56,7 +56,7 @@ namespace bts.shell { void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable),uint> update) { + void _on_orders(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history _data_cache.update(update.as_iincremental()); foreach (var o in update.added.Concat(update.changed)) diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index d1c156139..b24ffd2fb 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -28,7 +28,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index 174d83344..7045f2fe4 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -7,18 +7,31 @@ namespace bts.api.orderman { public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { this.security = sec; this.order = order; + piq = null; } public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { security = t.sec; order = t.order; + piq = null; } + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order, uint? piq) { + this.security = sec; + this.order = order; + this.piq = piq; + } + //public OrderSecurity(OrderSecurity t) { + // security = t.security; + // order = t.order; + // piq = t.piq; + //} public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); public SecurityImmutable? security { get; } public OrderRecordImmutable order { get; } + public uint? piq { get; } public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); - public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security; - public override int GetHashCode() => (security, order).GetHashCode(); + public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security && piq == other.piq; + public override int GetHashCode() => (security, order, piq).GetHashCode(); public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); public static bool operator != (OrderSecurity left, OrderSecurity right) => !left.Equals(right); diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index ee5c9409c..5b17779b8 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -4,8 +4,10 @@ using bts.api.core; using bts.api.orderman; using bts.api.position; using bts.collections; +using bts.md; using bts.spark; using bts.utils; +using mbts.config; using mbts.spark.utilities; using mbts.utils; using System; @@ -16,8 +18,10 @@ using System.ComponentModel.Composition; using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Threading.Tasks; using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; +using OrderUpdatePiq = bts.api.core.TableUpdate; namespace mbts.spark { [Export] @@ -28,21 +32,26 @@ namespace mbts.spark { readonly OrderFactory _factory; readonly OrderService _service; readonly PositionService _position_service; + readonly MBOService _mbo_service; readonly SpreaderModel _spreader; readonly ErrorReporter _error; readonly DynamicOrderHelper _dynamic_helper; readonly OrderActivitySummarizer _activity; readonly TableUpdateDemuxer _security_orders_demuxer, _security_working_orders_demuxer; + // we need a new one of these for the time being since the ladder is subscribing to orders with piq + readonly TableUpdateDemuxer _security_orders_piq_demuxer; readonly TableUpdateDemuxer _security_positions_demuxer; IDisposable _dispose; + static readonly bool _piq_enabled = Configuration.get().enabled; [ImportingConstructor] - OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, + OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, MBOService mbo_service, SpreaderModel spreader, DynamicOrderHelper dynamic_helper, AdminModel admin_model, IAlgoStudioModel algo_model, MarketDataAuxiliaryModel md_auxiliary_model, SecurityStore sstore, ErrorReporter error) { _bus = bus; _factory = factory; _service = service; _position_service = position_service; + _mbo_service = mbo_service; _spreader = spreader; _error = error; _dynamic_helper = dynamic_helper; @@ -51,13 +60,33 @@ namespace mbts.spark { this.md_auxiliary_model = md_auxiliary_model; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = + on_orders = _service .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) .join(sstore, order => order.security_key()) - .Table().Cache(); + .Table() + .Cache(); + + if (_piq_enabled) { + on_orders_with_piq = on_orders + .LeftJoin< + (SecurityImmutable?, OrderRecordImmutable), + OrderPIQImmutable, + OrderSecurity, + uint + >(mbo_service.subscribei_piq().Table(), + si_ort => si_ort.Item2.Key, + p => p.OrderId, + (si_ort, r, p) => new OrderSecurity(si_ort.Item1, si_ort.Item2, p.Position), + (si_ort, r) => new OrderSecurity(si_ort.Item1, si_ort.Item2, null), + osp => osp.order.Key); + } else { + on_orders_with_piq = on_orders + .Select((ft) => new OrderSecurity(ft.Item1, ft.Item2, null), os => os.order.Key); + } + on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( @@ -65,6 +94,13 @@ namespace mbts.spark { (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); + working_orders_with_piq = + on_orders_with_piq.Scan( + ImmutableDictionary.Empty, + (workers, table) => _update_working_orders(workers, table)) + .Replay(1).RefCount(); + + on_order_securities = on_orders .as_iincremental() @@ -73,6 +109,7 @@ namespace mbts.spark { _activity = new OrderActivitySummarizer(bus, on_orders); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); + _security_orders_piq_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_working_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); var algo_meta_join = algo_model.definitions.LeftJoin( @@ -122,6 +159,7 @@ namespace mbts.spark { .Cache(); _dispose = Disposable.Create( on_orders.Select(t => t.order, o => o.Key).Subscribe(_security_orders_demuxer), + on_orders_with_piq.Select(t => t.order, o => o.Key).Subscribe(_security_orders_piq_demuxer), on_orders .Filter(order => order.order.is_active() && order.order.HasState && order.order.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible, key => key.order.Key) .Select(t => t.order, o => o.Key) @@ -140,13 +178,16 @@ namespace mbts.spark { public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; public IObservable on_orders { get; } + public IObservable on_orders_with_piq { get; } public IObservable> on_order_tuples { get; } public IObservable, uint>> on_orders_can_detach { get; } public IObservable, uint>> on_orders_can_attach { get; } public IObservable> on_order_securities { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } + public IObservable> working_orders_with_piq { get; } public IObservable> on_positions { get; } + public AdminModel admin_model { get; } public IAlgoStudioModel algo_model { get; } @@ -231,8 +272,30 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + var clone = target.ToBuilder(); + foreach (var o in table.removed) { + if (o.order.is_active()) { + clone.Remove(o.order.Key); + } + } + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); + } + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); + } + } + foreach (var o in table.added) { + if (o.order.is_active()) { + clone.Add(o.order.Key, o); + } + } + return clone.ToImmutableDictionary(); + } + + static ImmutableDictionary _update_working_orders(ImmutableDictionary target, OrderUpdatePiq table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { @@ -256,6 +319,7 @@ namespace mbts.spark { } } + // usinig new OrdersModel extensions to reduce the amount of code change needed to switch uis over to the new OrdersModel, // without changing any code in the actual new OrdersModel class. static class OrdersModelExtensions { diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index f8ca8f2ab..8f542fd5a 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -82,7 +82,7 @@ namespace mbts { // old order will be a copy of new order, except with filledQuantity set to 0 var old_state = ort.order.State.CloneToMutable(); old_state.FilledQuantity = 0; - return (new OrderSecurity(ort.security, ort.order.with(state: old_state)), ort); + return (new OrderSecurity(ort.security, ort.order.with(state: old_state), ort.piq), ort); } } return null; diff --git a/src/client/spark/Spark.csproj b/src/client/spark/Spark.csproj index c6eb7e982..dbf62a939 100644 --- a/src/client/spark/Spark.csproj +++ b/src/client/spark/Spark.csproj @@ -4,7 +4,8 @@ Blue Trading Systems Blue Trading Systems Spark - The game has changed. Make your Move. + Advanced Derivatives Trading + spark.manifest true Debug;Release x64 diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs index 1d60d7638..d0d2c98f7 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs @@ -115,7 +115,8 @@ namespace mbts.ui.controls { ProductDescription = "Product Description", HubAlias = "Hub Alias", StripName = "Strip Name", - MicCode = "Mic Code" + MicCode = "Mic Code", + PositionInQueue = "PIQ" ; } @@ -266,4 +267,4 @@ namespace mbts.ui.controls { public TColKey key { get; } } } -} +} \ No newline at end of file diff --git a/src/client/spark_api/Co2/Co2NativeMethods.cs b/src/client/spark_api/Co2/Co2NativeMethods.cs index 495230306..f744b9963 100644 --- a/src/client/spark_api/Co2/Co2NativeMethods.cs +++ b/src/client/spark_api/Co2/Co2NativeMethods.cs @@ -35,6 +35,7 @@ namespace mbts.co2 public readonly IntPtr hot; public readonly IntPtr cold_data; public readonly IntPtr cold_header; + private readonly IntPtr unused_mbo; } [DllImport("libspark")] diff --git a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs index ac054c2c8..593edf9f7 100644 --- a/src/client/spark_api/api/Channels/QuickBusClientChannel.cs +++ b/src/client/spark_api/api/Channels/QuickBusClientChannel.cs @@ -110,19 +110,21 @@ namespace bts.api.core { the_update.Clear(); try { if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) { - foreach (var change in changes) { - switch (change.type) { - case ApiNativeMethods.ChangeType.REMOVE: - the_update.removed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.REPLACE: - the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.changed_keys.Add(change.key); - break; - case ApiNativeMethods.ChangeType.CREATE: - the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); - the_update.added_keys.Add(change.key); - break; + if (changes != null) { + foreach (var change in changes) { + switch (change.type) { + case ApiNativeMethods.ChangeType.REMOVE: + the_update.removed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.REPLACE: + the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.changed_keys.Add(change.key); + break; + case ApiNativeMethods.ChangeType.CREATE: + the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len)))); + the_update.added_keys.Add(change.key); + break; + } } } observer.OnNext(the_update); diff --git a/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs b/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs index 6b0adeda3..e986b732c 100644 --- a/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs +++ b/src/client/spark_core/Collections/Operable/Hierarchical/OperableHierarchicalCollectionModel.cs @@ -47,7 +47,7 @@ namespace bts.collections { public int total_row_count { get; private set; } IEnumerable> _root_nodes => - _filter == null ? _tree.RootNodes : _cached_filtered ?? (_cached_filtered = _tree.RootNodes.filtered(_filter, _comparer, true).ToList()); + _filter == null ? _tree.RootNodes : _cached_filtered ?? (_cached_filtered = _tree.RootNodes.filtered(_filter).sorted_hard(_comparer)); // collapse public void collapse_row(TKey key) => _toggle_is_expanded(key, false); diff --git a/src/client/spark_core/Collections/SortedTree.cs b/src/client/spark_core/Collections/SortedTree.cs index f8467669b..b09a7cd65 100644 --- a/src/client/spark_core/Collections/SortedTree.cs +++ b/src/client/spark_core/Collections/SortedTree.cs @@ -5,7 +5,7 @@ using System.Collections.Generic; using System.Linq; namespace bts.collections { - public interface INode { + public interface INode { T value { get; } IEnumerable> inner { get; } } @@ -19,32 +19,12 @@ namespace bts.collections { } } - public static IEnumerable> filtered(this IEnumerable> root_nodes, Func filter, IComparer comparer, bool check_already_sorted = false) { - var node_comp = comparer.wrap>(n => n.value); + public static IEnumerable> filtered(this IEnumerable> root_nodes, Func filter) { return impl(root_nodes); IEnumerable> impl(IEnumerable> nodes) { foreach (var n in nodes) { if (filter(n.value)) { - // TODO: (better) leverage the fact that the node.inner is already sorted - // (impl may require additional params, custom implementation of INode, etc.) - var inner = new List>(); - if (check_already_sorted) { - using (var en = impl(n.inner).GetEnumerator()) { - if (en.MoveNext()) - inner.Add(en.Current); - var max = en.Current; - while (en.MoveNext()) { - if (node_comp.Compare(en.Current, max) >= 0) { - inner.Add(en.Current); - max = en.Current; - } else inner.InsertSorted(en.Current, node_comp); - } - } - } else { - foreach (var _n in impl(n.inner)) - inner.InsertSorted(_n, node_comp); - } - yield return new Node(n.value, inner); + yield return new Node(n.value, impl(n.inner)); } else { foreach (var _n in impl(n.inner)) yield return _n; @@ -53,6 +33,20 @@ namespace bts.collections { } } + public static List> sorted_hard(this IEnumerable> root_nodes, IComparer comparer) { + var node_comp = comparer.wrap>(n => n.value); + var result = new List>(impl(root_nodes)); + result.Sort(node_comp); + return result; + IEnumerable> impl(IEnumerable> nodes) { + foreach (var n in nodes) { + var inner = new List>(impl(n.inner)); + inner.Sort(node_comp); + yield return new Node(n.value, inner); + } + } + } + struct Node : INode { public Node(T value, IEnumerable> inner) { this.value = value; diff --git a/src/client/spark_core/utils/OrderUtilities.cs b/src/client/spark_core/utils/OrderUtilities.cs new file mode 100644 index 000000000..b2042ad14 --- /dev/null +++ b/src/client/spark_core/utils/OrderUtilities.cs @@ -0,0 +1,19 @@ +namespace bts.api.orderman { + + public static class OrderUtilities { + public static NativeOrderStateImmutable? native_order_state(this OrderStateImmutable order) { + switch(order.CustomCase) { + case OrderState.CustomOneofCase.Market: + return order.Market.NativeState; + case OrderState.CustomOneofCase.Limit: + return order.Limit.NativeState; + case OrderState.CustomOneofCase.Stop: + return order.Stop.NativeState; + case OrderState.CustomOneofCase.StopLimit: + return order.StopLimit.NativeState; + default: + return null; + } + } + } +} \ No newline at end of file diff --git a/doc/user/content/bts_spark_sdk/Client/Overview.md b/src/client/spark_sdk/doc/Overview.md similarity index 100% rename from doc/user/content/bts_spark_sdk/Client/Overview.md rename to src/client/spark_sdk/doc/Overview.md diff --git a/doc/user/content/bts_spark_sdk/Client/Threading.md b/src/client/spark_sdk/doc/Threading.md similarity index 100% rename from doc/user/content/bts_spark_sdk/Client/Threading.md rename to src/client/spark_sdk/doc/Threading.md diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index 562234dd8..c5f6807cc 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -78,7 +78,7 @@ namespace bts.tests return securities; } - [Test(Timeout = 30)] + [Test(Timeout = 30)] public async Task test_position_limits() { string test_symbol = "CL"; diff --git a/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs b/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs index 432fce36b..7284e48db 100644 --- a/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs +++ b/src/client/sparkcmd/Cmd/ConfigCheckCommand.cs @@ -1,14 +1,19 @@ -using ManyConsole; using System.Linq; +using System; +using System.Collections.Generic; +using CommandLine; namespace bts { - public class Configcheck : ConsoleCommand { - public Configcheck() { - IsCommand("configcheck", "Ye olde Configcheck"); - HasAdditionalArguments(null, " [Keys to check]"); - } - public override int Run(string[] args) { - mbts.config.Configuration.configcheck(args.Length, args.ToArray()); + + [Verb("configcheck", HelpText="Ye olde Configcheck")] + public class ConfigcheckOptions { + [Value(0, MetaName="keys", HelpText="Keys to print")] + public IEnumerable keys { get; set; } + } + public class Configcheck { + public static int Run(ConfigcheckOptions options) { + var keys = options.keys.ToArray(); + mbts.config.Configuration.configcheck(keys.Length, keys); return 0; } } diff --git a/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs b/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs deleted file mode 100644 index 18c73075d..000000000 --- a/src/client/sparkcmd/Cmd/ConsoleCommandEx.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ManyConsole { - public static class ConsoleCommandEx { - public static string get_argument(this ConsoleCommand _, string arg_name) { - var arg = Environment.GetCommandLineArgs().FirstOrDefault(str => str.StartsWith("--"+arg_name)); - if (arg != null) { - var parts = arg.Split(new[] { '=' }, 2); - if (parts.Length > 1) { - return parts[1]; - } - } - return ""; - } - } -} diff --git a/src/client/sparkcmd/Cmd/OrderStress.cs b/src/client/sparkcmd/Cmd/OrderStress.cs new file mode 100644 index 000000000..43f8d9765 --- /dev/null +++ b/src/client/sparkcmd/Cmd/OrderStress.cs @@ -0,0 +1,157 @@ +using bts.api; +using bts.api.core; +using bts.api.orderman; +using bts.co2; +using bts.test; +using CommandLine; +using mbts.co2; +using mbts.config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace bts { + + [Verb("orderstress", HelpText = "Create a bunch of orders")] + public class OrderStressOptions { + // Maybe options to create fewer orders or sleep + + [Option('s', "sleep", HelpText = "Sleep ms between order submission", Default = 100)] + public int sleep { get; set; } + + [Option('d', "depth", HelpText = "How many orders per side", Default = 10u)] + public uint depth { get; set; } + + [Option('q', "quanityt", HelpText = "Order quantity", Default = 1u)] + public uint quantity { get; set; } + }; + + public struct Market { + public SecurityImmutable security; + public IntPrice? bid; + public IntPrice? ask; + } + + public static class OrderStress { + static IEnumerable all_markets(ServiceContext ctx) { + var channels = + UserSymbol.get_all() + .Select(u => + new SecurityChannel() { + Time = 0, + Flags = (int)(MDRequest.Types.Flags.Direct | MDRequest.Types.Flags.Implied), + Type = u.type, + Symbol = u.symbol, + Exchange = u.exchange + } + ); + var request = new MDRequest() { Host = "system_test" }; + request.Channels.AddRange(channels); + + var relay = ctx.locate(); + System.Threading.Thread.Sleep(100); + var result = relay.requestmd(request); + + foreach (var ch in result.Channels) { + foreach (var update in ch.Updates) { + if (update.Security != null) { + var m = new Market() { + security = new SecurityImmutable(update.Security), + bid = best( + (a,b) => a > b ? a : b, + update.Direct?.Bids?.FirstOrDefault(), update.Implied?.Bids?.FirstOrDefault()), + ask = best( + (a,b) => a < b ? a : b, + update.Direct?.Asks?.FirstOrDefault(), update.Implied?.Asks?.FirstOrDefault()) + }; + if (m.bid.HasValue || m.ask.HasValue) { + yield return m; + } + } + } + } + } + + private static IntPrice? best(Func comp, params PriceSizeMessage[] ps) { + IntPrice? result = null; + foreach (var p in ps) { + if (p != null) { + var ip = new IntPrice(p.Mantissa, (sbyte)p.Exponent); + if (result == null) { + result = ip; + } + else { + result = comp(result.Value, ip); + } + } + } + return result; + } + + static Create mkorder(uint security, double price, uint size, Side side) { + return new Create { + Price = price, + Quantity = size, + NewOrder = new OrderDefinition { + SecurityKey = security, + Side = side, + TimeInForce = TimeInForce.Day, + ManualOrderIndicator = ManualOrderIndicator.Yes, + CtiCode = CTICode.CtiCode1, + CustomerOrFirm = CustomerOrFirm.Firm, + AccountId = 1, + UserId = 1, + CustomDef = new CustomOrderDefinition { + Limit = new LimitOrderDefinition() + } + } + }; + } + + public static int Run(OrderStressOptions o) { + + var ctx = new ServiceContext(); + ConsoleEx.WriteLine("Please log in to Spark", ConsoleColor.Yellow); + ConsoleEx.Write("Username: ", ConsoleColor.Yellow); + var name = Console.ReadLine(); + ConsoleEx.Write("Password: ", ConsoleColor.Yellow); + var pw = Console.ReadLine(); + + var login_result = ctx.login(name, pw); + if (!login_result.Result.success) { + ConsoleEx.WriteLine("Invalid Credentials", ConsoleColor.Red); + } + + if (Configuration.get().type.ToUpper() == "PROD") { + ConsoleEx.WriteLine("Refusing to run on production system", ConsoleColor.Red); + return -1; + } + + var orders = ctx.locate(); + foreach (var m in all_markets(ctx)) { + ConsoleEx.WriteLine(m.security.security_desc(), ConsoleColor.White); + var tick = m.security.ContractSpec.LowTick; + if (m.bid.HasValue) { + // start at bid and place 1 order at 10 levels + for (var x=0; x < o.depth; ++x) { + var price = m.bid.Value.price() - (tick * x); + ConsoleEx.WriteLine($"Buy {o.quantity} at {price}", ConsoleColor.Blue); + orders.create_order_async(mkorder(m.security.Id, price, o.quantity, Side.Buy)); + System.Threading.Thread.Sleep(o.sleep); + } + } + if (m.ask.HasValue) { + for (var x = 0; x < o.depth; ++x) { + var price = m.ask.Value.price() + (tick * x); + ConsoleEx.WriteLine($"Sell {o.quantity} at {price}", ConsoleColor.Red); + orders.create_order_async(mkorder(m.security.Id, price, o.quantity, Side.Sell)); + System.Threading.Thread.Sleep(o.sleep); + } + } + } + return 3; + } + } + +} \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs index 18f0c18c9..57cf26389 100644 --- a/src/client/sparkcmd/Cmd/ScriptClientCommand.cs +++ b/src/client/sparkcmd/Cmd/ScriptClientCommand.cs @@ -6,32 +6,33 @@ using System.Reflection; using System.Threading.Tasks; using System.Reactive.Linq; using System.Reactive; -using ManyConsole; using System.Linq; +using CommandLine; namespace sparkcmd { - public class ScriptClientCommand : ConsoleCommand { - bool _server = false; - string _host; - string _logfile; + [Verb("script", HelpText="Remote control for shell diagnostic host")] + public class ScriptClientOptions { + [Option('s', "server", HelpText="Run a script server for testing.")] + public bool server { get; set; } - public ScriptClientCommand() { - IsCommand("script", "Remote control for shell diagnostic host"); - HasOption("s|server", "Run a script server for testing.", x => _server = true); - HasOption("h|host", "Connect to {host}", h => _host = h); - HasOption("f|fromfile", "Read commands from file instead of stdin", f => _logfile = f); - } - public override int Run(string[] args) { - if (_server) { + [Option('h', "host", HelpText="Connect to {host}")] + public string host { get; set; } + + [Option('f', "fromfile", HelpText="Read commands from file instead of stdin")] + public string file { get; set; } + + } + public class ScriptClientCommand { + public static int Run(ScriptClientOptions o) { + if (o.server) { ScriptClient.runserver(); } else { - var host = string.IsNullOrEmpty(_host) ? args.First() : _host; - if (string.IsNullOrEmpty(_logfile) && File.Exists(_logfile)) { - ScriptClient.start(host, File.OpenRead(_logfile)); + if (string.IsNullOrEmpty(o.file) && File.Exists(o.file)) { + ScriptClient.start(o.host, File.OpenRead(o.file)); } else { - ScriptClient.start(host, Console.OpenStandardInput()); + ScriptClient.start(o.host, Console.OpenStandardInput()); } } return 0; diff --git a/src/client/sparkcmd/Cmd/ServiceTail.cs b/src/client/sparkcmd/Cmd/ServiceTail.cs index 7ebbb0c92..6549e1509 100644 --- a/src/client/sparkcmd/Cmd/ServiceTail.cs +++ b/src/client/sparkcmd/Cmd/ServiceTail.cs @@ -1,6 +1,6 @@ using bts.api.core; using bts.test; -using ManyConsole; +using CommandLine; using System; using System.Collections.Generic; using System.Linq; @@ -8,26 +8,44 @@ using System.Text; using System.Threading.Tasks; namespace bts { - class TailService : ConsoleCommand { + + enum PrintFormat { + ToString, + Json + } + [Verb("tail", HelpText="'console dsqa'")] + class TailServiceOptions { + [Option('m', "method", Required=true, HelpText = "'[ServiceName].[Method]' to call")] + public string method { get;set; } + + [Option('t', "time", HelpText="Include timestamp")] + public bool time { get; set; } + + [Option('p', "path", HelpText="Path inside the row to use for display. Ex: 'Id' or 'Name' ")] + public string path { get; set; } + + [Option('a', "all", HelpText="Include 'all' rows on every print")] + public bool all { get; set; } + + [Option('j', "json", HelpText="Format rows as json")] + public bool json { get; set; } + } + class TailService { static bool _all = false; + static PrintFormat _format = PrintFormat.ToString; + static bool _timestamp = false; - public TailService() { - void ignore(string _) { } - IsCommand("tailservice", "Tail updates to named service api. Ex: tail --method=SyntheticSecurityService.metadata"); - HasRequiredOption("method", "Service.method to tail", ignore); - HasOption("time", "Include timestamp", _ => _timestamp = true); - // The following are not implemented. If you need them get to work. - HasOption("path", "Path inside the row to use for display. Ex: 'Id' or 'Name' ", ignore); - HasOption("all", "Invlude 'all' rows on every print", _ => _all = true); - HasAdditionalArguments(null, " [nothing]"); - } - public override int Run(string[] remainingArguments) { - var arg = this.get_argument("method"); - var split = arg.Split('.'); + public static int Run(TailServiceOptions o) { + var split = o.method.Split('.'); if (split.Length != 2) { - Console.WriteLine("Try --method=[ServiceName].[Method]"); + Console.WriteLine("Try --method [ServiceName].[Method]"); return -1; } + + if (o.json) { + _format = PrintFormat.Json; + } + tail(split[0], split[1]); return 0; } @@ -46,7 +64,17 @@ namespace bts { Console.Write(" "); } if (_all) { } - Console.WriteLine(value); + + switch(_format) { + case PrintFormat.Json: + dynamic dv = value; + var json = Newtonsoft.Json.JsonConvert.SerializeObject(new { dv.added, dv.removed, dv.changed }); + Console.WriteLine(json); + break; + case PrintFormat.ToString: + Console.WriteLine(value); + break; + } } } private static void tail(string service_name, string method_name) { diff --git a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs index 2b2d1acd0..90ff496e1 100644 --- a/src/client/sparkcmd/Cmd/ServiceTestCommand.cs +++ b/src/client/sparkcmd/Cmd/ServiceTestCommand.cs @@ -2,35 +2,74 @@ using bts.api; using bts.api.core; using bts.api.synthetic; using bts.test; -using ManyConsole; +using bts.md; +using CommandLine; using System; using System.Threading; namespace bts { - public class ServiceTest : ConsoleCommand { - bool _synthetic; - bool _user; - bool _admin; - public ServiceTest() { - IsCommand("service", "Put code here for testing during service development"); - HasOption("synthetic", "Run synthetic test", _ => _synthetic = true); - HasOption("user", "user test", _ => _user = true); - HasOption("admin", "admin test", _ => _admin = true); - } - public override int Run(string[] remainingArguments) { - if (_synthetic) { + public class ServiceTestOptions { + [Option(nameof(synthetic))] + public bool synthetic { get; set; } + [Option(nameof(user))] + public bool user { get; set; } + [Option(nameof(admin))] + public bool admin { get; set; } + + [Option(nameof(mbo))] + public bool mbo { get; set; } + + [Option("key")] + public UInt32 key { get; set; } + + [Option("side")] + public api.Side side { get; set; } + } + + public class ServiceTest { + public static int Run(ServiceTestOptions o) { + if (o.synthetic) { synthetic_test(); } - if (_user) { + if (o.user) { user_test(); } - if (_admin) { + if (o.admin) { admin_test(); } + if (o.mbo) { + mbo_test(o); + } return 0; } + static string try_get(string[] args, int index) { + if (args.Length < index) { + return args[index]; + } + return ""; + } + + private static void mbo_test(ServiceTestOptions o) { + var key = o.key; + var side = o.side; + using (var ctx = new ServiceContext()) { + var mbo = ctx.locate(); + var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key }); + + Console.WriteLine($"Order data for {data.SecurityId}"); + var vector = side == api.Side.Buy ? data.Bid : data.Ask; + var price = 0.0; + foreach(var e in vector) { + if (e.Price != price) { + price = e.Price; + Console.WriteLine("================"); + } + Console.WriteLine(e); + } + } + } private static void synthetic_test() { using (var ctx = new ServiceContext()) { diff --git a/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs b/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs index 3c74b9c87..7d01ad08d 100644 --- a/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs +++ b/src/client/sparkcmd/Cmd/SettingsManagerCommand.cs @@ -1,14 +1,13 @@ -using ManyConsole; using System.Reflection; +using CommandLine; namespace bts { - public class SettinsManagerCommand : ConsoleCommand { - public SettinsManagerCommand() { - IsCommand("SettingsManager", "Copy workspaces around"); - } + [Verb("settingsmanager", HelpText="Copy workspaces around")] + public class SettingsManagerOptions { } - public override int Run(string[] remainingArguments) { + public class SettingsManagerCommand { + public static int Run(SettingsManagerOptions _) { var asm = Assembly.Load("spark"); var type = asm.GetType("bts.settings.SettingsManagerTool"); var run = type.GetMethod("run", BindingFlags.Static | BindingFlags.Public); diff --git a/src/client/sparkcmd/Cmd/TestCommand.cs b/src/client/sparkcmd/Cmd/TestCommand.cs index b8a0f790c..b5b919f0a 100644 --- a/src/client/sparkcmd/Cmd/TestCommand.cs +++ b/src/client/sparkcmd/Cmd/TestCommand.cs @@ -1,6 +1,5 @@ using bts.test; -using ManyConsole; using System; using mbts; using System.Collections.Generic; @@ -8,64 +7,73 @@ using bts.api.test; using System.Threading.Tasks; using mbts.config; using System.Linq; +using CommandLine; +using Newtonsoft.Json; namespace sparkcmd { - public class TestCommand : ConsoleCommand { - uint _timeout = 0; - uint _parallel = 1; - bool _list; - TestType test_type = TestType.Unit; + [Verb("test", HelpText="Run Tests")] + public class TestCommandOptions { - bool self_contained = false; + [Option('t', "timeout", HelpText="Timeout unit tets", Default=0u)] + public uint timeout { get; set; } + [Option('l', "list", HelpText="List tests")] + public bool list { get; set; } + [Option('i', "insular", HelpText="Tests run sparkd and use their own config (if specified)")] + public bool self_contained { get; set; } - public TestCommand() { - IsCommand("test", "Run managed unit tests"); - // i comes thorugh as "t" which seems daft. - HasOption("t|timeout", "Timeout unit tests", i => _timeout = 10); - HasOption("n|parallel", "How many tests to run in parallel", i => _parallel = i); - HasOption("l|list", "List all tests", _ => _list = true); - HasOption("unit","Run unit tests", _ => test_type = TestType.Unit); - HasOption("system","Run system tests", _ => test_type = TestType.System); - HasOption("shell","Run shell tests", _ => test_type = TestType.Shell); - HasOption("i|insular|selfcontained", "Tests run sparkd and use their own config (if specified)", _ => self_contained = true); - HasAdditionalArguments(null, " [Test List]"); + [Option("type", HelpText ="Run all tests of type [Unit|System|Shell]")] + public TestType type { get; set; } + + + [Value(1, MetaName="Tests", HelpText="Test list")] + public IEnumerable tests { get; set; } + + } + public static class TestCommand { + static TestType test_type(string t) { + switch(t.ToLower()) { + case "unit": return TestType.Unit; + case "system": return TestType.System; + case "shell": return TestType.Shell; + } + return TestType.Unit; } - public override int Run(string[] tests) { - if (_list) { - list_tests(); + public static int Run(TestCommandOptions o) { + Console.WriteLine(JsonConvert.SerializeObject(o)); + + if (o.list) { + list_tests((o.type)); return 0; } - //Configuration.initialize_for_test(); - - if (_timeout == 0 && tests.Length == 1 && !self_contained) { + if (o.timeout == 0 && o.tests.Count() == 1 && !o.self_contained) { //System.Threading.Thread.Sleep(5000); using (var trr = new TestResultReceiver(true)) { - InProcessTestRunner.run_test_synchronously(tests[0], Enumerator, trr.get_reporter()); + InProcessTestRunner.run_test_synchronously(o.tests.First(), Enumerator, trr.get_reporter()); trr.print_summary(); return trr.ErrorCount; } } - else if (tests.Length > 0) { - return run_tests(Enumerator.match(tests)); + else if (o.tests.Count() > 0) { + return run_tests(o, Enumerator.match(o.tests)); } else { - return run_tests(Enumerator.get_of_type(test_type)); + return run_tests(o, Enumerator.get_of_type(o.type)); } } - Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); - TestEnumerator Enumerator => _enumerator.Value; + static Lazy _enumerator = new Lazy(() => new TestEnumerator("spark_test")); + static TestEnumerator Enumerator => _enumerator.Value; - int run_tests(IEnumerable tests) { + static int run_tests(TestCommandOptions o, IEnumerable tests) { var runner = new TestRunner(); // waits for all child processes to exit or time out - return runner.run_tests(tests, _timeout, self_contained); + return runner.run_tests(tests, o.timeout, o.self_contained); } - void list_tests() { + static void list_tests(TestType test_type) { Console.WriteLine("All tests: "); foreach (var t in Enumerator.get_groups_of_type(test_type)) { ConsoleEx.WriteLine(t.name, ConsoleColor.Green); diff --git a/src/client/sparkcmd/Program.cs b/src/client/sparkcmd/Program.cs index 790308e27..a91fed7f2 100644 --- a/src/client/sparkcmd/Program.cs +++ b/src/client/sparkcmd/Program.cs @@ -2,16 +2,40 @@ using bts.api.core; using bts.api.synthetic; using bts.api.test; -using ManyConsole; using System; using System.Linq; +using CommandLine; +using bts; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Reflection; namespace sparkcmd { class Program { + + // static object option(Func run) {return run;} + // static Type argType(object arg) { + // return arg.GetType().GetGenericArguments().First(); + // } + // static int Run(Func,int> on_error, params object[] args) { + // var types = args.Select(argType).ToArray(); + // var parse = CommandLine.Parser.Default.GetType().GetMethod("ParseArguments", BindingFlags.Public).MakeGenericMethod(types); + // return 3; + // } static int Main(string[] args) { - mbts.config.Configuration.initialize(); - var commands = ConsoleCommandDispatcher.FindCommandsInSameAssemblyAs(typeof(Program)); - return ConsoleCommandDispatcher.DispatchCommand(commands, args, Console.Out); + mbts.utils.BTSNativeMethods.provide_command_line(args, args.Length); + // note: this can be made to infer the options type list from the list of delegates + return Parser.Default + .ParseArguments(args) + .MapResult( + Configcheck.Run, + ScriptClientCommand.Run, + TailService.Run, + ServiceTest.Run, + SettingsManagerCommand.Run, + TestCommand.Run, + OrderStress.Run, + errors => 1); } } } \ No newline at end of file diff --git a/src/client/sparkcmd/sparkcmd.csproj b/src/client/sparkcmd/sparkcmd.csproj index 9046c4610..52d9db007 100644 --- a/src/client/sparkcmd/sparkcmd.csproj +++ b/src/client/sparkcmd/sparkcmd.csproj @@ -1,27 +1,30 @@ - - - - Exe - net471;netcoreapp2.1 - win7-x64;ubuntu.16.04-x64 - Blue Trading Systems - Blue Trading Systems - Spark - x64 - x64 - 1.0.0 - 7.2 - - - - - - - - - - - - - - + + + + Exe + net471;netcoreapp2.1 + win7-x64;ubuntu.16.04-x64 + Blue Trading Systems + Blue Trading Systems + Spark + x64 + x64 + 1.0.0 + 7.2 + + + + + + + + + + + + + + + + + diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index b77880cc0..03ccc1f50 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orderman_utils/orderman_utils.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/algo/AlgoOrder.cpp;orderman/orders/algo/AlgoWrapperOrder.cpp;orderman/orders/algo/IAlgoOrder.cpp;orderman/orders/algo/IAlgoServices.cpp;orderman/orders/algo/IBeforeLaunchAlgoServices.cpp;orderman/orders/algo/algo_orders/CoverAlgo.cpp;orderman/orders/algo/algo_orders/IcebergAlgo.cpp;orderman/orders/algo/algo_orders/JoinAlgo.cpp;orderman/orders/algo/algo_orders/LeanAlgo.cpp;orderman/orders/algo/algo_orders/TopAlgo.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/BookOrderBase.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/core/IOrderServices.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orderman_utils/orderman_utils.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/algo/AlgoWrapperOrder.cpp;orderman/orders/algo/IAlgoOrder.cpp;orderman/orders/algo/IAlgoServices.cpp;orderman/orders/algo/IBeforeLaunchAlgoServices.cpp;orderman/orders/algo/algo_orders/CoverAlgo.cpp;orderman/orders/algo/algo_orders/IcebergAlgo.cpp;orderman/orders/algo/algo_orders/JoinAlgo.cpp;orderman/orders/algo/algo_orders/LeanAlgo.cpp;orderman/orders/algo/algo_orders/TopAlgo.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/BookOrderBase.cpp;orderman/orders/book_algos/OCOOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/PricerOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegFillState.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index 67cdb8a9f..c332cbc98 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/api/core/ApiClientInterop.cpp b/src/lib/api/core/ApiClientInterop.cpp index 3927bbc3e..487b2eabb 100644 --- a/src/lib/api/core/ApiClientInterop.cpp +++ b/src/lib/api/core/ApiClientInterop.cpp @@ -176,7 +176,7 @@ void safe_call_subscription_callback(raw_subscription_callback cb, EventType typ if (updates.size() > 0) cb(type, &updates[0], updates.size(), extra); else - cb(type, nullptr, updates.size(), extra); + cb(type, nullptr, 0, extra); }); } diff --git a/src/lib/api/public/ClientChannel.h b/src/lib/api/public/ClientChannel.h index 3918f3f05..25cbd019f 100644 --- a/src/lib/api/public/ClientChannel.h +++ b/src/lib/api/public/ClientChannel.h @@ -5,6 +5,8 @@ #include "ApiException.h" #include "RawUpdate.h" #include "ServiceContext.h" +#include "lib/logging/logging.h" + #include #include #include @@ -12,6 +14,17 @@ namespace bts { namespace api { +template +std::shared_ptr deserialize(const char* buffer, size_t length) { + google::protobuf::io::ArrayInputStream stream(buffer, length); + auto item = std::make_shared(); + if (item->ParseFromZeroCopyStream(&stream)) { + return item; + } + LOG(Api,Error,"Failed to deserialize proto from api stream. This is likely fatal."); + return nullptr; +} + class ClientChannel { int _id; @@ -80,29 +93,29 @@ public: } else { bool first_update = true; - auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable { + auto cb = [observer, first_update, method, qualifier](EventType event_type, const StreamUpdate* changes, int change_count) mutable { if (event_type == EventType::TABLE_UPDATED) { - std::vector> added_items; + std::vector> added_items, changed_items; std::vector added_keys, - removed_keys; + changed_keys, + removed_keys; for (size_t x = 0; x < change_count; ++x) { const StreamUpdate& change = changes[x]; - if (change.type != StreamUpdate::REMOVE) { - added_keys.reserve(change_count); - added_items.reserve(change_count); - added_keys.push_back(change.key); - google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len); - auto item = std::make_shared(); - if (item->ParseFromZeroCopyStream(&stream)) { - added_items.push_back(move(item)); - } - } - if (change.type != StreamUpdate::CREATE) { - removed_keys.reserve(change_count); - removed_keys.push_back(change.key); + switch(change.type) { + case StreamUpdate::REMOVE: + removed_keys.push_back(change.key); + break; + case StreamUpdate::REPLACE: + changed_items.push_back(deserialize(change.buffer, change.buffer_len)); + changed_keys.push_back(change.key); + break; + case StreamUpdate::CREATE: + added_items.push_back(deserialize(change.buffer, change.buffer_len)); + added_keys.push_back(change.key); + break; } } - observer.on_next(RawUpdate>(added_items, added_keys, removed_keys)); + observer.on_next(RawUpdate>(added_items, added_keys, changed_items, changed_keys, removed_keys)); } else if (event_type == EventType::TABLE_CREATED) { if (!first_update) { diff --git a/src/lib/api/public/RawUpdate.h b/src/lib/api/public/RawUpdate.h index be79fa432..93e7cabe3 100644 --- a/src/lib/api/public/RawUpdate.h +++ b/src/lib/api/public/RawUpdate.h @@ -6,10 +6,16 @@ namespace api { template struct RawUpdate { - RawUpdate(std::vector& added_items, std::vector& added_keys, std::vector& removed_keys) : + RawUpdate(std::vector& added_items, + std::vector& added_keys, + std::vector& changed_items, + std::vector& changed_keys, + std::vector& removed_keys) : is_reset(false), added_items(std::move(added_items)), added_keys(std::move(added_keys)), + changed_items(std::move(changed_items)), + changed_keys(std::move(changed_keys)), removed_keys(std::move(removed_keys)) {} @@ -20,12 +26,16 @@ namespace api { bool is_reset; std::vector added_items; std::vector added_keys; + + std::vector changed_items; + std::vector changed_keys; + std::vector removed_keys; private: RawUpdate() : is_reset(true), - added_items(), added_keys(), removed_keys() + added_items(), added_keys(), changed_items(), changed_keys(), removed_keys() {} }; diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h index d37345679..bc2885e2f 100644 --- a/src/lib/api/public/Update.h +++ b/src/lib/api/public/Update.h @@ -9,12 +9,21 @@ namespace bts { namespace api { + template + class Change { + T _old, _new; + public: + Change(const T& old_value, const T& new_value) : _old(old_value), _new(new_value) { } + const T& previous() const { return _old; } + const T& current() const { return _new; } + }; template struct Update { - Update(std::vector& all, std::vector& added, std::vector& removed) : + Update(std::vector& all, std::vector& added, std::vector> changed, std::vector& removed) : _all_items(std::move(all)), _added_items(std::move(added)), + _changed_items(std::move(changed)), _removed_items(std::move(removed)) {} @@ -22,17 +31,20 @@ namespace api { Update(Update&& other) : _all_items(std::move(other._all_items)), _added_items(std::move(other._added_items)), + _changed_items(std::move(other._changed_items)), _removed_items(std::move(other._removed_items)) {} TKey key(const T& t) const { return ServiceType::key(t); } const std::vector& all() const { return _all_items; } const std::vector& added() const { return _added_items; } + const std::vector>& changed() const { return _changed_items; } const std::vector& removed() const { return _removed_items; } private: std::vector _all_items; std::vector _added_items; + std::vector> _changed_items; std::vector _removed_items; }; @@ -49,9 +61,12 @@ namespace api { } state->clear(); std::vector all, added; - return Update(all, added, removed); + std::vector> changed; + return Update(all, added, changed, removed); } else { + // + // Remove std::vector removed; removed.reserve(raw.removed_keys.size()); for (int key : raw.removed_keys) { @@ -61,15 +76,28 @@ namespace api { state->erase(iter); } } + // + // Add for (size_t x = 0; x < raw.added_keys.size(); ++x) { (*state)[raw.added_keys[x]] = raw.added_items[x]; } + // + // Change + std::vector> changed; + changed.reserve(raw.changed_keys.size()); + for (size_t x = 0; x < raw.changed_keys.size(); ++x) { + changed.push_back(Change{(*state)[raw.changed_keys[x]], raw.changed_items[x]}); + (*state)[raw.changed_keys[x]] = raw.changed_items[x]; + } + // + // All std::vector all; all.reserve(state->size()); for (const auto& item : *state) { all.push_back(item.second); } - return Update(all, raw.added_items, removed); + + return Update(all, raw.added_items, changed, removed); } }); }; diff --git a/src/lib/api/services/MBOService.proto b/src/lib/api/services/MBOService.proto new file mode 100644 index 000000000..7ceabb12c --- /dev/null +++ b/src/lib/api/services/MBOService.proto @@ -0,0 +1,32 @@ +syntax="proto3"; +package bts.md; +option csharp_namespace="bts.md"; +option cc_enable_arenas=true; +import "lib/api/apioptions.proto"; +import "lib/api/apicore.proto"; + +message OrderPIQ { + uint32 order_id = 1 [(bts.api.key)=true]; + uint32 position = 2; +} + +message OrderPosition { + uint64 exchange_id = 1; + uint32 order_id = 2; + float price = 10; + int32 size = 11; +} +message PositionData { + uint32 security_id = 1; + repeated OrderPosition bid = 10; + repeated OrderPosition ask = 20; +} +message MBOSecurityQuery { + uint32 security_id = 1; +} + +service MBOService { + rpc piq (bts.api.VoidMessage) returns (stream OrderPIQ); + rpc get_orders_for_security (MBOSecurityQuery) returns (PositionData); + +} \ No newline at end of file diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d83daf33b..197eecae0 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -17,7 +17,6 @@ message ExpirationNameList { message ExpirationNameListReply { repeated ExpirationNameList securities = 2; } - message SecurityKeyList { repeated uint32 keys = 1; } @@ -28,6 +27,5 @@ service MarketDataRelay { rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); - rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); } diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index be967e9d4..250936649 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -123,7 +123,8 @@ message CMEOptions optional bool load_security_db = 33 [default = true]; reserved 34; - // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; + // optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED]; + optional bool validate_mbo = 35 [default = false]; } optional string onixs_license = 1 [default = "{default}"]; diff --git a/src/lib/config/protos/toggles.proto b/src/lib/config/protos/toggles.proto index c66a3fe8d..31e9f7129 100644 --- a/src/lib/config/protos/toggles.proto +++ b/src/lib/config/protos/toggles.proto @@ -32,6 +32,12 @@ message VolumeAtPrice{ optional bool enabled = 1 [default = true]; } +message MBOStats { + optional bool enabled = 1 [default = false]; +} +message PlacementInQueue { + optional bool enabled = 1 [default = true]; +} message FixNestedScrolling { optional bool enabled = 1 [default = true]; } diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 46e5bfbcd..d25ab3d5f 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -160,7 +160,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, threading::configure_current_thread(thread_name); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/ExchangeOrderKey.h b/src/lib/md/ExchangeOrderKey.h new file mode 100644 index 000000000..b3d59e459 --- /dev/null +++ b/src/lib/md/ExchangeOrderKey.h @@ -0,0 +1,7 @@ +#pragma once +#include "lib/utils/ValueWrapper.h" +namespace bts { +namespace md { +using ExchangeOrderKey = utils::ValueWrapper; +using SparkOrderKey = utils::ValueWrapper; +} } \ No newline at end of file diff --git a/src/lib/md/IntPrice.h b/src/lib/md/IntPrice.h index a66806eb3..dfca4e73f 100644 --- a/src/lib/md/IntPrice.h +++ b/src/lib/md/IntPrice.h @@ -54,9 +54,18 @@ class IntPrice { return _mantissa < other._mantissa; } else { - return price() < other.price(); + // avoiding conversion to double is 4-5% faster + // return price() < other.price(); + auto delta = *this - other; + return delta.mantissa() < 0; } } + bool operator <=(const IntPrice& other) { + return *this == other || *this < other; + } + bool operator >=(const IntPrice& other) { + return !(*this < other); + } IntPrice operator - (const IntPrice& o) const { return op(o, [](int64_t m1, int64_t m2) { return m1 - m2; }); } @@ -169,7 +178,6 @@ inline md::IntPrice round_to_quote_price(const IntPrice& tick, const IntPrice& p return price; } - inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) { os << p.price(); return os; diff --git a/src/lib/md/MBOProcessor.cpp b/src/lib/md/MBOProcessor.cpp new file mode 100644 index 000000000..88cc1f3f2 --- /dev/null +++ b/src/lib/md/MBOProcessor.cpp @@ -0,0 +1,270 @@ +#include "MBOProcessor.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/ConcurrentPool.h" +#include "lib/config/config.h" +#include "lib/config/protos/toggles.pb.h" + +#include +#include +#include +#include + +#include + + +namespace bts { +namespace md { + +struct SecurityKeyHashCompare { + static size_t hash(const SecurityKey& k) { + return k.value(); + } + static bool equal(const SecurityKey& a, const SecurityKey& b) { + return a.value() == b.value(); + } +}; + +void print_pool_stats(const utils::ConcurrentPool& pool) { + size_t count = 0; + std::vector bid_sz,ask_sz; + pool.unsafe_iterate([&](const MBOSnapshot& s) { + if (s.bids.size() > 0 || s.asks.size() > 0) { + count++; + bid_sz.push_back(s.bids.size()); + ask_sz.push_back(s.asks.size()); + } + }); + + if (count == 0) return; + + std::sort(bid_sz.begin(), bid_sz.end()); + std::sort(ask_sz.begin(), ask_sz.end()); + size_t bid_tot =0, ask_tot = 0; + for(size_t i : bid_sz) { bid_tot += i; } + for(size_t i : ask_sz) { ask_tot += i; } + + std::stringstream bid_percentiles, ask_percentiles; + for (double x : { 0.5, 0.75, 0.9, 0.95, 0.99 }) { + size_t index = count * x; + bid_percentiles << bid_sz[index] << " "; + ask_percentiles << ask_sz[index] << " "; + } + bid_percentiles << bid_sz[count-1] << " "; + ask_percentiles << ask_sz[count-1] << " "; + LOG(md,Info, + "MBO statistics: count: {} - %[50,75,90,95,99,100] bid: [{}] - ask: [{}]", + count, bid_percentiles.str(), ask_percentiles.str()); +} + +typedef tbb::concurrent_hash_map MapType; +struct MBOProcessorImpl { + utils::ConcurrentPool pool; + MapType snapshots; + utils::Dispatcher& dispatcher; + utils::DispatcherTimer timer; + MBOProcessorImpl(utils::Dispatcher& d) : + pool(2048,512), + snapshots(), + dispatcher(d), + timer(d, 600*1000, [&]() { print_pool_stats(pool); }) + { + if (config::get().enabled()) { + timer.start(); + } + } +}; + +// +// Processor + +MBOProcessor::MBOProcessor(utils::Dispatcher* d) : + _me(std::make_shared(d == nullptr ? utils::Dispatcher::background() : *d)), + _producer(_me), + _consumer(_me) +{} + +MBOWriter& MBOProcessor::writer() { return _producer;} +MBOReader& MBOProcessor::reader() { return _consumer; } + +// +// Writer + +MBOWriter::MBOWriter(std::shared_ptr& me): _me(me) { } +MBOWriter::~MBOWriter() { } +MBOSnapshot* MBOWriter::allocate_snapshot(size_t, size_t) { + return _me->pool.allocate(); +} +void MBOWriter::release_snapshot(MBOSnapshot** s) { + _me->pool.release(s); +} +void MBOWriter::publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) { + bts_assert(*snapshot != nullptr); + MapType::accessor accessor; + _me->snapshots.insert(accessor, key); + std::swap(accessor->second, *snapshot); + if (*snapshot != nullptr) { + // Note: relase() must happen inside accessor lifetime to ensure correctness + _me->pool.release(snapshot); + } +} + +bool MBOWriter::validate_snapshot(uint32_t exchid, SecurityKey key, MBOSnapshot* control) { + // bts_assert_msg(false, "TO USE THIS VALIDATION YOU MUST MAKE enqueue_incremental SYNCRONOUS"); + #define REQUIRE(a,b,y) \ + if ((a) != (b)) { \ + LOG(md,Error, "Validation error exch:{} sec:{} {}!={} other:{}", exchid, key.value(), a, b, y); \ + return false; \ + } + + // retrieve current snapshot and check they are equal + _me->dispatcher.post([this,exchid,key,control]() { + MBOReader reader(_me); + auto snap = reader.acquire_snapshot(key); + if (snap) { + REQUIRE (snap->bids.size(), control->bids.size(), '!'); + REQUIRE (snap->asks.size(), control->asks.size(), '!'); + for (size_t x = 0; x < snap->bids.size(); ++x) { + REQUIRE (snap->bids[x].id, control->bids[x].id, x); + REQUIRE (snap->bids[x].size, control->bids[x].size, x); + } + for (size_t x = 0; x < snap->asks.size(); ++x) { + REQUIRE (snap->asks[x].id, control->asks[x].id, x); + REQUIRE (snap->asks[x].size, control->asks[x].size, x); + } + } + return true; + }); + #undef REQUIRE + return true; +} + +void MBOWriter::enqueue_incremental(MBOWorkingState& state, const inc::Update& update) { + LOG(md,Diag, "sec:{} : {}", state.handle.key(), update); + + _me->dispatcher.post([this,&state,update]() { + // fast part + apply_incremental(state,update); + // slow part + publish_incremental(state); + }); +} +void MBOWriter::publish_incremental(MBOWorkingState& state) { + // determine if we need to publish + auto key = SecurityKey(state.handle.key()); + if (key.value() != invalid_security_key) { + auto snap = allocate_snapshot(state.bid.size(), state.ask.size()); + copy_to_snapshot(state, snap); + publish_snapshot(key, &snap); + } +} + +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* dest) { + if (dest == nullptr) return; + // copy working state to snapshot + dest->updated = state.updated; + auto type_map = [](auto pair) { + return SnapshotOrder { pair.second.id, std::get(pair.first), pair.second.size }; + }; + std::transform(state.bid.begin(), state.bid.end(), std::back_inserter(dest->bids), type_map); + std::transform(state.ask.begin(), state.ask.end(), std::back_inserter(dest->asks), type_map); +} + +template +void apply_incremental(uint32_t security_key, C& collection, std::unordered_map& index, const inc::Update& update) { + auto key = std::make_tuple(update.price, update.priority); + switch(update.action) { + case inc::UpdateAction::New: + collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + index[update.order_id] = key; + break; + case inc::UpdateAction::Change: { + // look up old key + auto r = index.find(update.order_id); + if (r != index.end()) { + auto oldkey = r->second; + collection.erase(oldkey); + index.erase(r); + } + else { + LOG(md,Warn, "MBO Update w/out Add for: sec:{} {}", security_key, update); + } + collection.insert(std::make_pair(key, MBOWorkingState::Data { update.order_id, update.size })); + index[update.order_id] = key; + } + break; + case inc::UpdateAction::Delete: + collection.erase(key); + index.erase(update.order_id); + break; + } +} +void apply_incremental(MBOWorkingState& state, const inc::Update& update) { + // apply the update to the working state + if (update.type == inc::UpdateType::Bid) { + apply_incremental(state.handle.key(), state.bid, state.orderid_index, update); + } + else { + apply_incremental(state.handle.key(),state.ask, state.orderid_index, update); + } +} + +// +// Reader + +MBOReader::MBOReader(std::shared_ptr& me): _me(me) { } +MBOReader::~MBOReader() { } +MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const { + // get a result eagerly so we don't have to do it while accessor has the entry locked + auto result = _me->pool.allocate(); + MapType::const_accessor accessor; + if (_me->snapshots.find(accessor, key) && accessor->second != nullptr) { + result->copy_from(*accessor->second); + } + else { + release_snapshot(&result); + } + return result; +} +void MBOReader::release_snapshot(MBOSnapshot**s) const { + _me->pool.release(s); +} + +}} + +namespace std { + using namespace bts; + using namespace bts::md; + +ostream& operator<<(ostream& os, const inc::Update& update) { + os << "Update: { " + << "id:" << update.order_id.value() << " " + << "pri:" << update.priority << " " + << update.price.price() << " " + << update.size << " "; + + switch(update.type) { + case inc::UpdateType::Bid: + os << "B "; + break; + case inc::UpdateType::Ask: + os << "A "; + break; + } + + switch(update.action) { + case inc::UpdateAction::New: + os << "+"; + break; + case inc::UpdateAction::Change: + os << "C"; + break; + case inc::UpdateAction::Delete: + os << "-"; + break; + } + + os << "}"; + return os; +} + +} \ No newline at end of file diff --git a/src/lib/md/MBOProcessor.h b/src/lib/md/MBOProcessor.h new file mode 100644 index 000000000..f3a92b59b --- /dev/null +++ b/src/lib/md/MBOProcessor.h @@ -0,0 +1,186 @@ +#pragma once +#include "lib/domain_model/SecurityKey.h" +#include "lib/domain_model/Security.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/Dispatcher.h" +#include "IntPrice.h" +#include "ExchangeOrderKey.h" +#include "MDProducerHandle.h" + +#ifndef _WIN32 +#include "google/btree/btree_map.h" +#endif + +#include +#include +#include +#include + +namespace bts { +namespace md { + +struct SnapshotOrder { + ExchangeOrderKey id; + IntPrice price; + int32_t size; +}; +struct MBOSnapshot { + MBOSnapshot() = default; + MBOSnapshot(const MBOSnapshot&) = delete; + MBOSnapshot(MBOSnapshot&& other) = delete; + + utils::TimeStamp updated; + std::vector bids; + std::vector asks; + + void copy_from(const MBOSnapshot& src) { + bids = src.bids; + asks = src.asks; + updated = src.updated; + } + + void reset() { + updated = utils::TimeStamp(); + constexpr int max = 256; + if (bids.size() > max) { + bids.resize(max); + } + if (asks.size() > max) { + asks.resize(max); + } + bids.clear(); + asks.clear(); + } +}; + +namespace inc { + + // according to this link MBO book only has these updates types + // https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Market+Data+Snapshot+Full+Refresh#MDP3.0-MarketDataSnapshotFullRefresh-MarketbyOrder(MBO)DataBlock + enum class UpdateType { + Bid = 0, + Ask = 1, + // ImpliedBid, // 'E' + // ImpliedAsk, // 'F' + // BookReset // 'J' + }; + + // OrderUpdateAction=37708 + enum class UpdateAction { + New = 0, + Change = 1, + Delete = 2 + }; + + typedef uint64_t Priority; + struct Update { + ExchangeOrderKey order_id; + Priority priority; + IntPrice price; + int32_t size; + UpdateType type; + UpdateAction action; + }; +} + + +struct MBOWorkingState { + typedef std::tuple key_t; + struct Data { + ExchangeOrderKey id; + int32_t size; + }; + struct LessBid { + bool operator()(const key_t& left, const key_t& right) const { + const auto& left_price = std::get(left); + const auto& right_price = std::get(right); + if (left_price == right_price) { + return + std::get(left) < + std::get(right); + } + return !(left_price < right_price); + } + }; + + MBOWorkingState(MDProducerHandle& h) : handle(h), updated(), bid(),ask() {} + + MDProducerHandle handle; + utils::TimeStamp updated; + // Used for 'change' messages which update price + std::unordered_map orderid_index; + +#ifdef _WIN32 + std::map bid; + std::map ask; +#else + btree::btree_map bid; + btree::btree_map ask; +#endif +}; + + +void copy_to_snapshot(MBOWorkingState& state, MBOSnapshot* destination); +void apply_incremental(MBOWorkingState&, const inc::Update&); + +template +void apply_incremental(MBOWorkingState& state, const V& vec) { + for (const auto& update : vec) { + apply_incremental(state, update); + } +} + +struct MBOProcessorImpl; +class MBOProcessor; + +class MBOWriter { + std::shared_ptr _me; + friend class MBOProcessor; + MBOWriter(std::shared_ptr&); + ~MBOWriter(); + + public: + // + // Snapshot api + MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0); + void release_snapshot(MBOSnapshot**); + void publish_snapshot(SecurityKey, MBOSnapshot**); + bool validate_snapshot(uint32_t, SecurityKey, MBOSnapshot*); + + // + // Incremental api + + // Enqueue the update to the mbo work queue. + void enqueue_incremental(MBOWorkingState&, const inc::Update&); + // Update working state and publish directly. + void publish_incremental(MBOWorkingState&); +}; + +class MBOReader { + std::shared_ptr _me; + friend class MBOProcessor; + friend class MBOWriter; + MBOReader(std::shared_ptr&); + ~MBOReader(); + public: + MBOSnapshot* acquire_snapshot(SecurityKey key) const; + void release_snapshot(MBOSnapshot**) const; +}; + +class MBOProcessor { + std::shared_ptr _me; + MBOWriter _producer; + MBOReader _consumer; + +public: + MBOProcessor(utils::Dispatcher* _=nullptr); + MBOWriter& writer(); + MBOReader& reader(); +}; + + +}} + +namespace std { +std::ostream& operator<<(std::ostream& os, const bts::md::inc::Update&); +} diff --git a/src/lib/md/MBOServiceServer.cpp b/src/lib/md/MBOServiceServer.cpp new file mode 100644 index 000000000..6cabb35a0 --- /dev/null +++ b/src/lib/md/MBOServiceServer.cpp @@ -0,0 +1,340 @@ +#ifndef _WIN32 +#include "MBOServiceServer.h" + +#include "lib/api/services/OrderService.client.pb.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/TimeStamp.h" +#include "MDClient.h" +#include "ExchangeOrderKey.h" +#include "MBOProcessor.h" + +#include "lib/utils/LockBox.h" + +#include +#include +#include + + +using namespace bts::api::orderman; + +namespace bts { +namespace md { + +// +// Simple class to handle MD recovery and maintain an index of SecurityKey=>MDConsumerHandle + +class HandleIndex { + MDClient& _client; + int _sub; + std::optional _consumer; + +public: + utils::LockBox> index; + + HandleIndex(md::MDClient& client): + _client(client), + _sub(0), + _consumer(std::nullopt), + index() { + _sub = _client.on_consumer().subscribe([=](const MDConsumer& c) { on_consumer(c); }); + on_consumer(_client.current_consumer()); + } + + ~HandleIndex() { _client.on_consumer().unsubscribe(_sub); } + + +private: + void on_consumer(const MDConsumer& c) { + MDConsumer temp(c); + if (temp.all_heartbeats_ok()) { + auto existing = _consumer ? _consumer->id() : 0; + LOG(md,Info, "Updateing MBOService MDConsumer. Old {} new {}", existing, temp.id()); + // clear the index + index.lock([](auto& index) { index.clear(); }); + // Swap to new consumer + if (_consumer) { + _consumer->swap(temp); + } + else { _consumer = temp; } + // Subscribe to on_security + auto id = _consumer->id(); + _consumer->on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { + if (_consumer->id() == id) { + index.lock([handle](auto& index) { + index[SecurityKey(handle.key())] = handle; + }); + } + }); + } + } +}; + +class ActiveOrder { + SecurityKey _security; + SparkOrderKey _id; + ExchangeOrderKey _ex_id; + int32_t _position; + +public: + // empty constructor needed for unordered map operator[] + // ActiveOrder(): _security(), _id(), _ex_id(), _position(-1) { bts_assert(false); } + ActiveOrder(SecurityKey sec, SparkOrderKey s, ExchangeOrderKey ex): _security(sec), _id(s), _ex_id(ex), _position(-1) {} + ActiveOrder(SecurityKey sec, SparkOrderKey s, const std::string& ex): _security(sec), _id(s), _ex_id(), _position(-1) { + _ex_id = ExchangeOrderKey(std::stoul(ex)); + } + + SecurityKey security_id() const { return _security; } + SparkOrderKey id() const { return _id; } + ExchangeOrderKey exchange_id() const { return _ex_id; } + int32_t& position() { return _position; }; + int32_t position() const { return _position; }; + bool set_position(int32_t pos) { + if (pos != _position) { + _position = pos; + return true; + } + return false; + } + OrderPIQ to_piq() const { + OrderPIQ result; + result.set_order_id(_id.value()); + result.set_position(_position); + return result; + } +}; + +// +// MBOService + +struct MBOServiceImpl { + MBOServiceImpl( + api::ServiceContext& ctx, + utils::Dispatcher& d, + md::MDClient& c, + MBOServiceBase::PiqPublisher&& pub + ): + dispatcher(d), + client(c), + orders(ctx.locate()), + handle_index(c), + publish(pub), + timer(d, 200, [&]() {timer_tick();}), + orders_sub() + {} + + utils::Dispatcher& dispatcher; + md::MDClient& client; + OrderService orders; + + HandleIndex handle_index; + MBOServiceBase::PiqPublisher publish; + utils::DispatcherTimer timer; + + rxcpp::composite_subscription orders_sub; + std::unordered_map active; + + void timer_tick(); + void flush(); +}; + + + +/////////////////////// + +std::optional search_orders(const ExchangeOrderKey oid, const std::vector& orders) { + IntPrice price = {}; + size_t position = 0; + for (size_t x = 0; x < orders.size(); ++x) { + const auto& oo = orders[x]; + if (oo.id == oid) { + if (oo.price == price) return { position }; + else return 0; + } + else if (price == oo.price) { + position += oo.size; + } + else { + price = oo.price; + position = oo.size; + } + } + return std::nullopt; +} + +bool update_position(const HandleIndex& handles, ActiveOrder& order) { + const auto& index = handles.index; + auto handle = index.lock([id=order.security_id()](const auto& index) { + auto iter = index.find(id); + return iter != index.end() ? iter->second : MDConsumerHandle {}; + }); + + if (handle.is_initialized()) { + auto mbo = handle.get_mbo(); + if (mbo) { + auto search_result = search_orders(order.exchange_id(), mbo->bids); + if (!search_result) { + search_result = search_orders(order.exchange_id(), mbo->asks); + } + if (search_result) { + return order.set_position(search_result.value()); + } + } + } + return false; +} + +void MBOServiceImpl::timer_tick() { + //Dispatcher thread + std::vector rows; + for (auto& pair : active) { + if (update_position(handle_index, pair.second)) { + rows.push_back(pair.second.to_piq()); + } + } + if (!rows.empty()) { + publish(nullptr, &rows, nullptr); + } +} + +const NativeOrderState* native_order_state(const OrderRecord& record) { + switch(record.state().custom_case()) { + case OrderState::kMarket: + return &record.state().market().native_state(); + case OrderState::kLimit: + return &record.state().limit().native_state(); + case OrderState::kStop: + return &record.state().stop().native_state(); + case OrderState::kStopLimit: + return &record.state().stop_limit().native_state(); + default: + return nullptr; + } +} + +bool is_active_exchange_order(const OrderRecord& record) { + return record.state().status() == ORDER_STATUS_WORKING && + native_order_state(record) != nullptr; +} + +void try_remove(MBOServiceImpl& me, std::set& to_remove, const OrderRecord& record) { + auto key = SparkOrderKey(record.key()); + auto iter = me.active.find(key); + if (iter != me.active.end()) { + me.active.erase(key); + to_remove.insert(key.value()); + } +} +void try_add(MBOServiceImpl& me, std::vector& to_add, const OrderRecord& record) { + if (is_active_exchange_order(record)) { + auto native = native_order_state(record); + bts_assert(native); + ActiveOrder order( + SecurityKey(record.def().security_key()), + SparkOrderKey(record.key()), + native->exchange_order_id()); + update_position(me.handle_index, order); + to_add.push_back(order.to_piq()); + me.active.insert(std::make_pair(order.id(), order)); + } +} + +MBOServiceServer::MBOServiceServer( + api::ServiceContext& ctx, + utils::Dispatcher& d, + md::MDClient& c + ) : + MBOServiceBase(ctx), + _me(new MBOServiceImpl(ctx,d,c,make_piq_publisher())) + { + // subscribe to mdclient ✔ + // make publishers ✔ + // subscribe to orders => stash active orders dict + _me->orders_sub = + _me->orders.subscribe_orders().subscribe([=](const auto& update) { + // api thread + const auto& added = update.added(); + const auto& changed = update.changed(); + const auto& removed = update.removed(); + _me->dispatcher.post([this,added,changed,removed]() { + // dispatcher thread + std::vector to_add, to_change; + std::set to_remove; + + // Remove + for(const auto& r : removed) { + // If it was active remove it and add to to_remove + try_remove(*_me, to_remove, *r); + } + // Added + for(const auto& add : added) { + // If active add look up the position and add to the state + try_add(*_me, to_add, *add); + } + // Changed + for(const auto& ch : changed) { + // new order is active and old is not look up piq and add to state + bool is_active = is_active_exchange_order(*ch.current()); + bool was_active = is_active_exchange_order(*ch.previous()); + if (is_active && !was_active) { + try_add(*_me, to_add, *ch.current()); + } + else if (was_active && !is_active) { + try_remove(*_me, to_remove, *ch.current()); + } + // is_active && was_active => don't care + // !is_active && !was_active => don't care + } + + _me->publish(&to_add, &to_change, &to_remove); + if (!_me->timer.running()) _me->timer.start(); + }); + }); + } + +MBOServiceServer::~MBOServiceServer() { } + + +void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request, const api::Response& reply) { + PositionData response; + response.set_security_id(request.security_id()); + auto handle = _me->handle_index.index.lock([id=request.security_id()](const auto& index) { + auto iter = index.find(SecurityKey(id)); + return iter != index.end() ? iter->second : MDConsumerHandle {}; + }); + + std::unordered_map reverse; + for (const auto& pair : _me->active) { + reverse.insert(std::make_pair(pair.second.exchange_id(), pair.first)); + } + + if(handle.is_initialized()) { + auto mbo = handle.get_mbo(); + if (mbo) { + for (const auto& o : mbo->bids) { + auto* entry = response.add_bid(); + entry->set_exchange_id(o.id.value()); + entry->set_price(o.price.price()); + entry->set_size(o.size); + auto iter = reverse.find(o.id); + if (iter != reverse.end()) { + entry->set_order_id(iter->second.value()); + } + } + for (const auto& o : mbo->asks) { + auto* entry = response.add_ask(); + entry->set_exchange_id(o.id.value()); + entry->set_price(o.price.price()); + entry->set_size(o.size); + auto iter = reverse.find(o.id); + if (iter != reverse.end()) { + entry->set_order_id(iter->second.value()); + } + } + } + } + + reply(response); +} + +}} //namespace +#endif \ No newline at end of file diff --git a/src/lib/md/MBOServiceServer.h b/src/lib/md/MBOServiceServer.h new file mode 100644 index 000000000..6eccab38c --- /dev/null +++ b/src/lib/md/MBOServiceServer.h @@ -0,0 +1,29 @@ +#pragma once +#include "lib/api/public/ServiceContext.h" +#include "lib/api/services/MBOService.server.pb.h" +#include +#include + +namespace bts { +namespace utils { class Dispatcher; } +namespace api { namespace orderman { class OrderService; } } +namespace md { + +class MDClient; + +struct MBOServiceImpl; +class MBOServiceServer : public MBOServiceBase { + std::unique_ptr _me; + public: + MBOServiceServer( + api::ServiceContext& ctx, + utils::Dispatcher&, + md::MDClient&); + ~MBOServiceServer(); + + void get_orders_for_security(const MBOSecurityQuery&, const api::Response&) override; + +}; + + +}} //namespace diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 1cb454971..3695640ba 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -3,6 +3,7 @@ #include "SecurityCache.h" #include "SecurityOracle.h" #include "lib/utils/TimeStamp.h" +#include "MBOProcessor.h" #include namespace bts { @@ -15,19 +16,22 @@ struct MDClientImpl{ std::unique_ptr _security_timer; std::unique_ptr _recovery_timer; std::set _symbols; + MBOReader* _mbo; MDConsumer _consumer; utils::Event _event; MDClientImpl( utils::Dispatcher* d, api::ISyntheticSecurityService* s, + MBOReader* mbo, const std::set* symbols): _dispatcher(d), _ss(s), _security_timer(), _recovery_timer(), _symbols(symbols ? *symbols : get_system_md_symbols()), - _consumer(_cache, _ss, &_symbols), + _mbo(mbo), + _consumer(_cache, _ss, mbo, &_symbols), _event() { if (d) { @@ -42,7 +46,7 @@ struct MDClientImpl{ void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_cache,_ss); + MDConsumer temp(_cache,_ss,_mbo,&_symbols); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); @@ -71,11 +75,12 @@ struct MDClientImpl{ MDClient::MDClient( utils::Dispatcher* d, api::ISyntheticSecurityService* s, + MBOReader* mbo, const std::set* symbols) : - _self(new MDClientImpl(d,s, symbols)) + _self(new MDClientImpl(d,s,mbo,symbols)) { } -MDClient::~MDClient() { -} + +MDClient::~MDClient() { } size_t MDClient::scan_securities() { return _self->scan_securities(); } void MDClient::check_heartbeats() { return _self->check_heartbeats(); } diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index 47925bc88..51b03a777 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -11,7 +11,9 @@ namespace bts { namespace md { class SecurityOracle; +class MBOReader; struct MDClientImpl; + class MDClient { std::unique_ptr _self; @@ -20,6 +22,7 @@ public: MDClient( utils::Dispatcher* d=nullptr, api::ISyntheticSecurityService* ss=nullptr, + MBOReader* mbo=nullptr, const std::set* s=nullptr); ~MDClient(); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 20a4f73da..a303c34ad 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -3,6 +3,7 @@ #include "MemorySegmentUtils.h" #include "PlatformLoader.h" #include "SecurityCache.h" +#include "MBOProcessor.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -535,6 +536,7 @@ class MDConsumerImpl { Mutex _workmutex; vector> _instances; std::shared_ptr _sci; + MBOReader* _mbo; map> _deferred_handles; vector _callbacks; mutable SecurityCache _cache; @@ -545,10 +547,12 @@ public: const set& symbols, SecurityCache& cache, api::ISyntheticSecurityService* ss, + MBOReader* mbo, bool shared, co2::Platform platform) : _workmutex(), _instances(), _sci(), + _mbo(mbo), _deferred_handles(), _callbacks(), _cache(cache) @@ -625,7 +629,7 @@ public: for (auto& inst : _instances) { if (inst->find_handle(key, &cold, &hot)) { MDConsumerHandle handle; - handle.init(inst->cold_header(), *cold, hot); + handle.init(inst->cold_header(), *cold, hot, _mbo); return handle; } } @@ -657,9 +661,15 @@ public: } // Erase anything that wasn't found from the resulting map + // Set the mbo pointer on anything else for (auto it = result.begin(); it != result.end(); ) { - if (!it->second.is_initialized()) it = result.erase(it); - else ++it; + if (!it->second.is_initialized()) { + it = result.erase(it); + } + else { + it->second.set_mbo(_mbo); + ++it; + } } return result; @@ -910,6 +920,7 @@ public: MDConsumer::MDConsumer( SecurityCache& cache, api::ISyntheticSecurityService* ss, + MBOReader* mbo, const std::set* symbols, bool shared, co2::Platform platform) : _pimpl() { @@ -920,7 +931,7 @@ MDConsumer::MDConsumer( symbols = &all_symbols; } - _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, shared, platform)); + _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, mbo, shared, platform)); } MDConsumer::~MDConsumer() {} diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 02c762a8f..103e8bb19 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -26,6 +26,7 @@ class MDConsumerImpl; class MDConsumerInst; class MDConsumerChannel; class SecurityCache; +class MBOReader; class MDConsumer { @@ -33,6 +34,7 @@ public: MDConsumer( SecurityCache&, api::ISyntheticSecurityService* ss, + MBOReader* mbo=nullptr, const std::set* sym = nullptr, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 0218be0eb..40efeded5 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -1,5 +1,6 @@ #ifndef NO_CPP0X #include "MDConsumerHandle.h" +#include "MBOProcessor.h" #include "lib/domain_model/Security.h" #include @@ -14,7 +15,7 @@ namespace md { MDConsumerHandle::MDConsumerHandle() { memset(this, 0, sizeof(MDConsumerHandle)); _type = HANDLETYPE_VOID; - static_assert(sizeof(MDConsumerHandle) <= 32, ""); + static_assert(sizeof(MDConsumerHandle) <= 40, ""); } MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { memset(this, 0, sizeof(MDConsumerHandle)); @@ -33,9 +34,10 @@ MDConsumerHandle::MDConsumerHandle(const MDConsumerHandle& other) { MDConsumerHandle::~MDConsumerHandle() {} -void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot) { +void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot, const MBOReader* mbo) { set_cold(&cheader, ¢ry); set_hot(pvhot); + set_mbo(mbo); } TimeStamp copy_bbo(const HotEntry& hot, PriceSize& out_bid, PriceSize& out_ask) { @@ -371,6 +373,16 @@ CoconutSecurityStatus MDConsumerHandle::status() const { never; } +std::unique_ptr> MDConsumerHandle::get_mbo() const { + if (_type == HANDLETYPE_DEPTH) { + MBOSnapshot* snapshot = book.mbo->acquire_snapshot(SecurityKey(key())); + if (snapshot != nullptr) { + return {snapshot,[mbo=book.mbo](auto* s) { mbo->release_snapshot(&s); }}; + } + } + return nullptr; + } + }}//namespace diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 38a7772eb..506e64f7b 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -13,6 +13,9 @@ namespace bts { namespace md { class MDConsumerHandle; +class MBOReader; +struct MBOSnapshot; + struct SyntheticStrategy { std::unique_ptr strategy; api::Security security; @@ -141,6 +144,9 @@ TimeStamp optimistic_copy( return t; } +// +// MDConsumerHandle +// // Can be an invalid handle if you request a security which has not been discovered yet. class BTS_EXPORT MDConsumerHandle @@ -151,7 +157,7 @@ public: ~MDConsumerHandle(); MDConsumerHandle(const MDConsumerHandle& other); - void init(const ColdHeader&, const ColdEntry&, const void* hot); + void init(const ColdHeader&, const ColdEntry&, const void* hot, const MBOReader*); bool is_initialized() const { return _type != HANDLETYPE_VOID; } bool is_synthetic() const { return _type == HANDLETYPE_SYNTHETIC; } @@ -267,8 +273,11 @@ public: return *this; } + std::unique_ptr> get_mbo() const; + private: friend class MDConsumerInst; + friend class MDConsumerImpl; template HandleReadResult get_book(Read read, TimeStamp& time, uint32_t copydepth, PriceSize* bids, PriceSize* asks) const { @@ -326,6 +335,19 @@ private: _type = HANDLETYPE_BBO; } } + void set_mbo(const MBOReader* mbo) { + switch(_type) { + case HANDLETYPE_DEPTH: + book.mbo = mbo; + break; + case HANDLETYPE_BBO: + bbo.mbo = mbo; + break; + default: + //ok + break; + } + } enum HandleType : uint8_t { HANDLETYPE_VOID, @@ -337,11 +359,13 @@ private: const HotDepthEntry* hot; const ColdEntry* cdata; const ColdHeader* cheader; + const MBOReader* mbo; }; struct BBOHandle { const HotEntry* hot; const ColdEntry* cdata; const ColdHeader* cheader; + const MBOReader* mbo; }; struct SyntheticHandle { std::shared_ptr def; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 5027f3d75..17a36b072 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -32,7 +32,7 @@ CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, for (size_t x = 0; x < count; ++x) { symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } - return new MDConsumer(cache, nullptr/*synthetics*/, &symbols, shared); + return new MDConsumer(cache, nullptr/*synthetics*/, nullptr /*mbo*/, &symbols, shared); } CAPI void destroy_md_consumer(MDConsumer* consumer) { diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 7b409662f..248491942 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -1,4 +1,4 @@ -#ifndef NO_CPP0X +#ifndef _WIN32 #include "QueryServer.h" #include "lib/config/config.h" #include "lib/utils/Dispatcher.h" @@ -9,6 +9,7 @@ #include "../MDConsumer.h" #include "../MDClient.h" #include "../PlatformLoader.h" +#include "../MBOProcessor.h" #include "lib/logging/logging.h" @@ -30,6 +31,7 @@ #include #include #include +#include #include #include @@ -129,7 +131,6 @@ struct MDConsumerGroup { }; - void convert(PriceSize ps, PriceSizeMessage* pps) { pps->set_mantissa(ps.mantissa()); pps->set_exponent(ps.exponent()); @@ -211,6 +212,7 @@ public: for (auto& pair : _groups) { pair.second->set_consumer(_consumer); } + // _handle_index.clear(); subscribe_securities(); utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } @@ -380,10 +382,7 @@ public: google::protobuf::MapPair v(h.handle.key(), security.ice().security_desc()); x->insert(v); } - else - { - LOG(md,Error, "Invalid ExchangeData_case {}", security.ExchangeData_case()); - } + // There is currently no ExchangeData for synthetics } } } @@ -399,7 +398,6 @@ public: } } } - }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -497,6 +495,7 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { } }; + struct QueryServer { QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, diff --git a/src/lib/md/test/Co2ManagedTestHelper.cpp b/src/lib/md/test/Co2ManagedTestHelper.cpp index 4d69543ac..30caf3bde 100644 --- a/src/lib/md/test/Co2ManagedTestHelper.cpp +++ b/src/lib/md/test/Co2ManagedTestHelper.cpp @@ -26,7 +26,7 @@ extern "C" BTS_EXPORT void co2_get_1234_handle(size_t& size, MDConsumerHandle& auto pheader = reinterpret_cast(&buffer[0]); pheader->has_depth = false; size = sizeof(MDConsumerHandle); - handle.init(*pheader, *pcold, phot.get()); + handle.init(*pheader, *pcold, phot.get(), nullptr); // Copy the addresses for comparison *hotaddr = (uintptr_t)phot.get(); *coldaddr = (uintptr_t)pcold.get(); diff --git a/src/lib/md/test/Co2TestInstrument.cpp b/src/lib/md/test/Co2TestInstrument.cpp index 06d037b1c..0c2c2274a 100644 --- a/src/lib/md/test/Co2TestInstrument.cpp +++ b/src/lib/md/test/Co2TestInstrument.cpp @@ -43,7 +43,7 @@ MDProducerHandle Co2TestInstrument::producer() { MDConsumerHandle Co2TestInstrument::consumer() { MDConsumerHandle handle; - handle.init(*_cheader, *_cold, _hot.get()); + handle.init(*_cheader, *_cold, _hot.get(), nullptr/*mbo*/); return handle; } diff --git a/src/lib/md/test/mbo_tests.cpp b/src/lib/md/test/mbo_tests.cpp new file mode 100644 index 000000000..69abb194e --- /dev/null +++ b/src/lib/md/test/mbo_tests.cpp @@ -0,0 +1,320 @@ +#ifndef _WIN32 +#include "catch/catch.hpp" +#include "../IntPrice.h" +#include "../MBOProcessor.h" +#include "Co2TestInstrument.h" + +#include +#include + +namespace bts { +namespace md { + +// fwd declare from MBOServiceServer.cpp +std::optional search_orders(const ExchangeOrderKey oid, const std::vector& orders); + +namespace test { +int mbo_tests = 0; + +TEST_CASE("MBO publish snapshot", "[md][mbo]") { + MBOProcessor p; + // publish a snapshot + auto sec = SecurityKey(99); + { + auto s = p.writer().allocate_snapshot(); + s->bids.push_back({ ExchangeOrderKey(1), { 100, 0 }, 42 }); + p.writer().publish_snapshot(sec, &s); + } + + auto s = p.reader().acquire_snapshot(sec); + REQUIRE(s); + REQUIRE(s->bids.size() == 1); + REQUIRE(s->bids[0].id.value() == 1); + REQUIRE(s->asks.size() == 0); +} + +std::vector gen_test_orders(uint64_t id=100, inc::UpdateType type=inc::UpdateType::Bid) { + std::vector result; + inc::Priority priority = 10000; + for (int p = 100; p < 501; p+=100) { + // 6 sizes + for (int32_t s = 1; s <= 5; ++s) { + IntPrice price { p + 5 , -2}; + auto order_id = ExchangeOrderKey(++id); + auto size = s; + priority++; + result.push_back( inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); + // std::cout << std::endl << order_id.value() << ": " << price.price() << "-" << size; + } + } + std::random_shuffle(result.begin(), result.end()); + return result; +} + +TEST_CASE("MBO state to snapshot", "[md][mbo]") { + // convert state to snapshot + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + state.updated = utils::TimeStamp::now(); + + auto bids = gen_test_orders(1000, inc::UpdateType::Bid); + apply_incremental(state, bids); + auto asks = gen_test_orders(2000, inc::UpdateType::Ask); + apply_incremental(state, asks); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + REQUIRE(result.updated == state.updated); + REQUIRE(result.bids.size() == state.bid.size()); + REQUIRE(result.asks.size() == state.ask.size()); + REQUIRE(bids.size() == result.bids.size()); + REQUIRE(asks.size() == result.asks.size()); + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + REQUIRE(result.bids[x].price >= result.bids[x+1].price); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + REQUIRE(result.asks[x].price <= result.asks[x+1].price); + } +} + +template +std::vector concat(const std::vector& a, const std::vector& b) { + std::vector res; + std::copy(a.begin(), a.end(), std::back_inserter(res)); + std::copy(b.begin(), b.end(), std::back_inserter(res)); + return res; +} + +TEST_CASE("MBO incremental", "[md][mbo]") { + // basic incremental test + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + auto orders = + concat( gen_test_orders(128, inc::UpdateType::Bid), + gen_test_orders(256, inc::UpdateType::Ask)); + + std::random_shuffle(orders.begin(), orders.end()); + apply_incremental(state, orders); + + REQUIRE(state.bid.size() + state.ask.size() == orders.size()); + + // generate some updates/deletes + std::vector updates; + // - change all size=2 to size=6 + // - delete all size=3 + size_t removed = 0; + for (const auto& e : orders) { + if (e.size == 2) { + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + 6, + e.type, + inc::UpdateAction::Change + }); + } + else if (e.size == 3) { + ++removed; + updates.push_back( inc::Update { + e.order_id, + e.priority, + e.price, + e.size, + e.type, + inc::UpdateAction::Delete + }); + } + } + apply_incremental(state,updates); + + REQUIRE(state.bid.size() + state.ask.size() == (orders.size() - removed)); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto all = concat (result.bids, result.asks); + std::map counts; + for (const auto& order : all) { + REQUIRE(order.size != 3); + REQUIRE(order.size != 2); + counts[order.size] ++; + } + REQUIRE(counts[6] > 0); + REQUIRE(counts[6] == counts[1]); + REQUIRE(counts[5] == counts[4]); + REQUIRE(counts[6] == counts[5]); +} + +TEST_CASE("MBO sort order", "[md][mbo]") { + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + std::vector updates; + // multiple updates at the same price + for (unsigned x=0; x < 100; ++x) { + // bid + auto p = 1000 + x; + updates.push_back( inc::Update { + ExchangeOrderKey(1000 + x), + 1000 + x, + { 1000, -1 }, + (int)x+1, + inc::UpdateType::Bid, + inc::UpdateAction::New + }); + // ask + if (x < 75) { // ensure bid.size != ask.size + updates.push_back( inc::Update { + ExchangeOrderKey(2000 + x), + 2000 + x, + { 2000, -1 }, + (int)x+1, + inc::UpdateType::Ask, + inc::UpdateAction::New + }); + } + } + std::random_shuffle(updates.begin(), updates.end()); + + apply_incremental(state, updates); + + MBOSnapshot result; + copy_to_snapshot(state, &result); + + // Since all orders are at the same price we can assert they are in priority order + // Since priority is not explicit in the shapshot we set order_id to the same value + + for (size_t x = 0; x < result.bids.size() - 1; ++x) { + REQUIRE(result.bids[x].id.value() < result.bids[x+1].id.value()); + } + for (size_t x = 0; x < result.asks.size() - 1; ++x) { + REQUIRE(result.asks[x].id.value() < result.asks[x+1].id.value()); + } +} + + +TEST_CASE("MBO change price and size", "[md][mbo]") { + // When an order's size goes up it gets a new priority + Co2TestInstrument instr { "MBO", api::SECURITYTYPE_FUTURE }; + auto handle = instr.producer(); + MBOWorkingState state(handle); + + constexpr auto bid = inc::UpdateType::Bid; + constexpr auto ask = inc::UpdateType::Ask; + constexpr auto add = inc::UpdateAction::New; + constexpr auto remove = inc::UpdateAction::Delete; + constexpr auto change = inc::UpdateAction::Change; + auto p = [](double price) -> IntPrice { return IntPrice(price * 1000, -3); }; + + // snapshot + std::vector snapshot = { + { ExchangeOrderKey(701267154552), 1160179309, p(2.975), 1, ask, add}, + { ExchangeOrderKey(701267991871), 1161068569, p(2.925), 9, bid, add}, + { ExchangeOrderKey(701267963307), 1161037482, p(2.975), 1, ask, add}, + { ExchangeOrderKey(701267991967), 1161068670, p(2.900), 1, bid, add}, + { ExchangeOrderKey(701267928346), 1160999982, p(3.000), 1, ask, add}, + { ExchangeOrderKey(701267993125), 1161069878, p(2.900), 3, bid, add}, + { ExchangeOrderKey(701267037972), 1160052467, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267099137), 1160119020, p(2.875), 5, bid, add}, + { ExchangeOrderKey(701267299079), 1160330214, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267308619), 1160340068, p(2.875), 5, bid, add}, + { ExchangeOrderKey(701267310436), 1160341952, p(2.875), 3, bid, add}, + { ExchangeOrderKey(701267872642), 1160939720, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267895533), 1160964252, p(2.875), 1, bid, add}, + { ExchangeOrderKey(701267953989), 1161027476, p(2.875), 3, bid, add}, + { ExchangeOrderKey(701267993135), 1161069890, p(2.875), 2, bid, add}, + { ExchangeOrderKey(701266410447), 1159379621, p(2.850), 3, bid, add}, + { ExchangeOrderKey(701267324626), 1160356597, p(2.850), 3, bid, add}, + { ExchangeOrderKey(701267992564), 1161069297, p(2.850), 1, bid, add}, + { ExchangeOrderKey(701266420294), 1159389827, p(2.775), 5, bid, add}, + { ExchangeOrderKey(701267334462), 1160366796, p(2.775), 5, bid, add}, + { ExchangeOrderKey(701266411103), 1159380297, p(2.625), 5, bid, add}, + { ExchangeOrderKey(701267325284), 1160357275, p(2.625), 5, bid, add}, + }; + + apply_incremental(state, snapshot); + REQUIRE(state.bid.size() == 19); + REQUIRE(state.ask.size() == 3); + + std::vector incremental = { + { ExchangeOrderKey(701267993738), 1161070510, p(2.925), 2, bid, add}, + { ExchangeOrderKey(701267993759), 1161070532, p(2.975), 3, ask, add}, + { ExchangeOrderKey(701267993884), 1161070662, p(2.950), 3, ask, add}, + { ExchangeOrderKey(701267994228), 1161071029, p(2.925), 2, bid, add}, + { ExchangeOrderKey(701267994243), 1161071044, p(2.925), 4, bid, add}, + { ExchangeOrderKey(701267994228), 1161071029, p(2.925), 2, bid, remove}, + { ExchangeOrderKey(701267993135), 1161071281, p(2.875), 3, bid, change}, + }; + apply_incremental(state, incremental); + REQUIRE(state.bid.size() == 21); + REQUIRE(state.ask.size() == 5); + + { + auto iter = state.bid.begin(); + auto next = state.bid.begin(); + next++; + for (; next != state.bid.end(); ++iter, ++next) { + auto cur_price = std::get(iter->first); + auto cur_prio = std::get<1>(iter->first); + auto next_price = std::get(next->first); + auto next_prio = std::get<1>(next->first); + REQUIRE(cur_price >= next_price); + if (cur_price == next_price) { + REQUIRE(cur_prio < next_prio); + } + } + } + { + auto iter = state.ask.begin(); + auto next = state.ask.begin(); + next++; + for (; next != state.ask.end(); ++iter, ++next) { + auto cur_price = std::get(iter->first); + auto cur_prio = std::get<1>(iter->first); + auto next_price = std::get(next->first); + auto next_prio = std::get<1>(next->first); + REQUIRE(cur_price <= next_price); + if (cur_price == next_price) { + REQUIRE(cur_prio < next_prio); + } + } + } + + // Do some tests on the snapshot + MBOSnapshot result; + copy_to_snapshot(state, &result); + + auto ask30 = search_orders(ExchangeOrderKey(701267928346), result.asks); + REQUIRE(ask30); + REQUIRE(*ask30 == 0); + + auto bid2875 = search_orders(ExchangeOrderKey(701267993135), result.bids); + REQUIRE(bid2875); + REQUIRE(*bid2875 == 20); + + bid2875 = search_orders(ExchangeOrderKey(701267299079), result.bids); + REQUIRE(bid2875); + REQUIRE(*bid2875 == 6); + + auto not_bid = search_orders(ExchangeOrderKey(701967299879), result.bids); + auto not_ask = search_orders(ExchangeOrderKey(701967299879), result.asks); + REQUIRE(!not_bid); + REQUIRE(!not_ask); +} + +}}} +#endif \ No newline at end of file diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 3a9a5dcdd..d9da74ddd 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1,3 +1,4 @@ +#ifndef _WIN32 #include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" @@ -15,6 +16,7 @@ #include "lib/config/protos/MarketData.pb.h" #include "lib/utils/Sqlite_utils.h" +#include "md_test_utils.h" #include #include @@ -55,29 +57,8 @@ Platform TEST_PLATFORM = PLATFORM_SIM; // // Helper functions for Producer/Consumer tests // -vector create_test_securities(MDProducer& producer, const char* postfix ="") { - FixedFutureSecurity ffs; - FixedOptionSecurity fos; - ffs.symbol("FOO"+string(postfix)); - fos.symbol("FOOX"+string(postfix)); - ffs.exchange = VENUE_BTS; - fos.exchange = VENUE_BTS; - ffs.key = invalid_security_key; - fos.key = invalid_security_key; - ffs.expiration_year = fos.expiration_year = 2015; - ffs.expiration_day = fos.expiration_day = 27; - fos.strike = 1000; - fos.call_put = PUTORCALL_CALL; - // Add 12 expirations to 2 different shared memory regions - vector result; - for (uint32_t x = 1; x <= 12; ++x) { - ffs.expiration_month = fos.expiration_month = x; - result.push_back(producer.add_security(ffs)); - result.push_back(producer.add_security(fos)); - // In a real app these securities would now be passed to the SecurityIDRecorder - } - return result; -} + +std::vector create_test_securities(MDProducer& producer, const char* postfix = ""); set get_test_symbols(const char* postfix="") { set symbols; @@ -146,12 +127,6 @@ Security mkspread(string symbol, security_key key, Security leg1, Security leg2, return mkspread(symbol, key, vlegs); } -void assign_ids(vector::iterator begin, vector::iterator end, security_key start_key = 1000) { - for (; begin != end; ++begin) { - (*begin).set_key(++start_key); - } -} - class Co2Tests { @@ -370,7 +345,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -441,7 +416,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback vector securities; @@ -481,7 +456,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -539,7 +514,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -600,7 +575,7 @@ public: // Symbol list for the consumer auto symbols = get_test_symbols(); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback - callback fires for existing securities vector securities; @@ -628,7 +603,7 @@ public: // Symbol list for the consumer SecurityCache cache; auto symbols = get_test_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); MDConsumer consumer = client.current_consumer(); map products; consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { @@ -761,7 +736,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &symbols, true, TEST_PLATFORM); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -806,7 +781,7 @@ public: csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -853,7 +828,7 @@ public: csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -905,7 +880,7 @@ public: consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &consumer_symbols, true, TEST_PLATFORM); size_t security_count = 0; // Indirection for check function @@ -974,7 +949,7 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); + MDConsumer consumer(cache, nullptr, nullptr, &consumer_symbols, true, TEST_PLATFORM); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1260,7 +1235,7 @@ struct Co2RelayTests { UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); return consumer; } @@ -1327,7 +1302,7 @@ struct Co2RelayTests { // todo: setup_query_server should use client.symbols() instead of it's own list // const auto symbols = setup_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices @@ -1460,7 +1435,7 @@ struct Co2RelayTests { vector producers; Entitlements entitlements; const auto symbols = setup_symbols(); - MDClient client(nullptr, nullptr, &symbols); + MDClient client(nullptr, nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels @@ -1638,7 +1613,7 @@ namespace ImpliedTests { UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(cache, nullptr, &symbols); + MDConsumer consumer(cache, nullptr, nullptr, &symbols); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); @@ -2417,3 +2392,4 @@ TEST_CASE("IntPrice", "[md]") { }}} +#endif \ No newline at end of file diff --git a/src/lib/md/test/md_test_utils.cpp b/src/lib/md/test/md_test_utils.cpp new file mode 100644 index 000000000..bc2848c16 --- /dev/null +++ b/src/lib/md/test/md_test_utils.cpp @@ -0,0 +1,41 @@ +#include "md_test_utils.h" +#include "../MDProducer.h" + +#include +#include + +namespace bts { +namespace md { +namespace test { + +std::vector create_test_securities(MDProducer& producer, const char* postfix ="") { + FixedFutureSecurity ffs; + FixedOptionSecurity fos; + ffs.symbol("FOO"+std::string(postfix)); + fos.symbol("FOOX"+std::string(postfix)); + ffs.exchange = api::VENUE_BTS; + fos.exchange = api::VENUE_BTS; + ffs.key = invalid_security_key; + fos.key = invalid_security_key; + ffs.expiration_year = fos.expiration_year = 2015; + ffs.expiration_day = fos.expiration_day = 27; + fos.strike = 1000; + fos.call_put = api::PUTORCALL_CALL; + // Add 12 expirations to 2 different shared memory regions + std::vector result; + for (uint32_t x = 1; x <= 12; ++x) { + ffs.expiration_month = fos.expiration_month = x; + result.push_back(producer.add_security(ffs)); + result.push_back(producer.add_security(fos)); + // In a real app these securities would now be passed to the SecurityIDRecorder + } + return result; +} + +void assign_ids(std::vector::iterator begin, std::vector::iterator end, security_key start_key) { + for (; begin != end; ++begin) { + (*begin).set_key(++start_key); + } +} + +}}}//namespace \ No newline at end of file diff --git a/src/lib/md/test/md_test_utils.h b/src/lib/md/test/md_test_utils.h index 23a82070f..7eccddc3c 100644 --- a/src/lib/md/test/md_test_utils.h +++ b/src/lib/md/test/md_test_utils.h @@ -2,6 +2,7 @@ #include "lib/md/IntPrice.h" #include "lib/md/PriceSize.h" +#include "../MDProducerHandle.h" #include namespace bts { @@ -28,5 +29,6 @@ Book compute_synthetic_book(const api::Security& security, const std::vector::iterator begin, std::vector::iterator end, security_key start_key = 1000); }}}//namespace \ No newline at end of file diff --git a/src/lib/orderman/orderman_utils/orderman_utils.h b/src/lib/orderman/orderman_utils/orderman_utils.h index 32d5d1939..50b97ce43 100644 --- a/src/lib/orderman/orderman_utils/orderman_utils.h +++ b/src/lib/orderman/orderman_utils/orderman_utils.h @@ -28,6 +28,7 @@ inline bool is_parent_order(const bts::api::orderman::OrderRecord &ord) { } bool should_forward_attachment(const api::orderman::CustomOrderDefinition &def); +bool can_consume_attachment(api::orderman::CustomOrderDefinition::CustomCase) ; } // namespace orderman } // namespace bts diff --git a/src/lib/orderman/orders/algo/AlgoOrder.cpp b/src/lib/orderman/orders/algo/AlgoOrder.cpp deleted file mode 100644 index 94dbb7f2b..000000000 --- a/src/lib/orderman/orders/algo/AlgoOrder.cpp +++ /dev/null @@ -1,780 +0,0 @@ -#include "lib/orderman/orders/algo/AlgoOrder.h" -#include "lib/orderman/actors/order_manager/ChildOrder.h" -#include "lib/orderman/actors/order_manager/OrderBuilder.h" -#include "lib/orderman/actors/order_manager/OrderServices.h" -#include "lib/orderman/orderman_utils/orderman_utils.h" -#include "lib/orderman/orders/PreActivationOrderServices.h" -#include "lib/utils/bts_assert.h" - -#include -#include - -namespace bts { -namespace orderman { - -using namespace api::orderman; -template using Map = boost::container::flat_map; -enum class AlgoOrder::Status { INACTIVE, W_ATTACH, WORKING, COMPLETED }; - -class ChildTemplate { - public: - ChildTemplate() {} - - ChildTemplate(OrderKey parent_key, api::orderman::OrderDefinition const &def, - IOrderFactory *factory, size_t reserve_sz) - : builder(parent_key, def, *factory, reserve_sz) {} - - OrderBuilder builder; -}; - -struct AlgoOrder::ChildRecord { - ChildOrder order; - std::optional state; - bool is_subscribing = false; -}; - -struct AlgoOrder::PImpl { - using Edge = std::pair; - using Action = std::function; - - Status status = Status::INACTIVE; - OrderServices *services = nullptr; - OrderDefinition definition; - const api::Security *security; - OrderState const &state() const { return _state; } - - Map book_subs; - Map children; - bool has_timer_sub = false; - - bool is_state_dirty = false; - - Map transition_actions; - Map invariants; - securities::ISecurityDB const *security_db = nullptr; - IOrderFactory *order_factory = nullptr; - std::vector> child_templates; - OrderKey attach_target = null_order_key(); - - std::optional pending_update; - OrderStatus external_status() const { return _state.status(); } - - void set_external_status(OrderStatus s) { - is_state_dirty = true; - _state.set_status(s); - } - - void set_state(OrderState const &new_state) { - is_state_dirty = true; - _state = new_state; - } - - void set_last_msg(std::string const &msg) { - is_state_dirty = true; - _state.set_last_msg(msg); - } - - void set_price_quantity(Price p, Quantity q) { - is_state_dirty = true; - _state.set_price(p.value()); - _state.set_quantity(q.value()); - } - - void set_fill_quantities(Quantity total_fills, Price last_price, - Price avg_price) { - is_state_dirty = true; - _state.set_filled_quantity(total_fills.value()); - _state.set_last_price(last_price.value()); - _state.set_avg_price(avg_price.value()); - } - - private: - OrderState _state; -}; - -struct AlgoOrder::ReportStateGuard { - ReportStateGuard(AlgoOrder &owner) : owner(owner) {} - ~ReportStateGuard() { - if (owner.self->is_state_dirty) { - owner.self->is_state_dirty = false; - owner.self->services->report_state(owner.self->state()); - } - } - AlgoOrder &owner; -}; - -AlgoOrder::AlgoOrder() : self(new PImpl()) { setup_transitions(); } -AlgoOrder::~AlgoOrder() {} - -// -------------------------------------------------------------------------------- -// Queries -// -------------------------------------------------------------------------------- -api::orderman::OrderDefinition const &AlgoOrder::definition() const { - return *def(); -} - -OrderKey AlgoOrder::order_key() const { return key(); } - -OrderKey AlgoOrder::parent_order_key() const { return parent_key(); } - -api::Security const *AlgoOrder::lookup_security(SecurityKey k) const { - return self->security_db->resolve(k); -} - -Price AlgoOrder::price() const { return Price(self->state().price()); } - -Quantity AlgoOrder::quantity() const { - return Quantity(self->state().quantity()); -} - -Quantity AlgoOrder::total_fills() const { - return Quantity(self->state().filled_quantity()); -} - -bool AlgoOrder::is_attaching() const { - return self->attach_target != null_order_key(); -} - -// -------------------------------------------------------------------------------- -// Commands -// -------------------------------------------------------------------------------- -ChildTemplate & -AlgoOrder::make_child_template(api::orderman::OrderDefinition const &child_def, - size_t reserve) { - auto t = std::make_unique(key(), child_def, - self->order_factory, reserve); - if (self->services) - t->builder.activate(*self->services); - self->child_templates.emplace_back(std::move(t)); - return *(self->child_templates.back()); -} - -OrderKey AlgoOrder::launch_child(ChildTemplate &t, Price p, Quantity q, - bool subscribe_to_changes) { - if (!is_working()) - return null_order_key(); - return index_and_launch_child(t.builder.build(), p, q, subscribe_to_changes); -} - -OrderKey AlgoOrder::launch_child(api::orderman::OrderDefinition const &def, - Price p, Quantity q, - bool subscribe_to_changes) { - if (!is_working()) - return null_order_key(); - return index_and_launch_child(ChildOrder{def, *self->services}, p, q, - subscribe_to_changes); -} - -void AlgoOrder::update_child(OrderKey child, - api::orderman::Update const &update) { - if (!is_working()) - return; - - bts_assert(self->status == Status::WORKING); - if (auto it = self->children.find(child); it != self->children.end()) - it->second.order.update(update); - else - terminate_error(fmt::format("Algo {} tried to update unknown child: {}", - key().value(), child.value())); -} - -void AlgoOrder::cancel_child(OrderKey child) { - if (!is_working()) - return; - - bts_assert(self->status == Status::WORKING); - if (auto it = self->children.find(child); it != self->children.end()) - it->second.order.cancel(); - else - terminate_error(fmt::format("Algo {} tried to cancel unknown child: {}", - key().value(), child.value())); -} - -void AlgoOrder::detach_child(OrderKey child) { - if (!is_working()) - return; - - if (auto it = self->children.find(child); it != self->children.end()) { - it->second.order.mark_detached_from_parent(); - self->children.erase(it); - } else - terminate_error(fmt::format("Algo {} tried to cancel unknown child: {}", - key().value(), child.value())); -} - -void AlgoOrder::update_fill_state(Quantity q, Price p) { - if (!is_working()) - return; - - auto previous_qty = self->state().filled_quantity(); - if (previous_qty > q.value()) { - LOG(OrderMan, Error, - "AlgoOrder {} is reporting a decrease in fill quantity " - "from {} to {}. This should never happen. Canceling {}", - key().value(), previous_qty, q.value(), key().value()); - terminate_error("Internal algo error. Report fill size decreased"); - } else { - auto new_avg_price = Price(0); - if (q.value() > 0) - new_avg_price.value() = - (self->state().filled_quantity() * self->state().avg_price() + - (q.value() - previous_qty) * p.value()) / - q.value(); - self->set_fill_quantities(q, p, new_avg_price); - } -} - -void AlgoOrder::cancel(std::string const &msg) { - if (!is_working()) - return; - terminate(ORDER_STATUS_CANCELED, msg); -} - -void AlgoOrder::terminate_error(std::string const &msg) { - if (!is_working() && self->status != Status::W_ATTACH) - return; - - // If we attached and an error was signaled we do not want to - // cancel our child because that child might be a long running - // order with a good queue position that we want to re-use. - if (is_attaching()) - self->children.erase(self->attach_target); - terminate(ORDER_STATUS_ERROR, msg); -} - -void AlgoOrder::detach_all() { - if (!is_working()) - return; - for (auto &[_, c] : self->children) { - (void)_; - c.order.mark_detached_from_parent(); - } - self->children.clear(); -} - -void AlgoOrder::complete() { - if (!is_working()) - return; - terminate(ORDER_STATUS_COMPLETED); -} - -// -------------------------------------------------------------------------------- -// Subscriptions -// -------------------------------------------------------------------------------- - -void AlgoOrder::subscribe_book(SecurityKey k) { - if (!is_working()) - return; - bts_assert(self->book_subs.find(k) == self->book_subs.end()); - - self->book_subs[k] = self->services->subscribe_to_book( - k, [this, k](auto const &ev) { on_book(k, ev.book); }); -} - -void AlgoOrder::unsubscribe_book(SecurityKey k) { - if (!is_working()) - return; - bts_assert(self->book_subs.find(k) != self->book_subs.end()); - if (auto it = self->book_subs.find(k); it != self->book_subs.end()) - self->services->unsubscribe_from_book(it->second); -} - -void AlgoOrder::subscribe_child_change(OrderKey k) { - if (!is_working()) - return; - - auto it = self->children.find(k); - bts_assert(it != self->children.end()); - auto &child_record = it->second; - bts_assert(!child_record.is_subscribing); - child_record.is_subscribing = true; -} - -void AlgoOrder::unsubscribe_child_change(OrderKey k) { - if (!is_working()) - return; - - auto it = self->children.find(k); - bts_assert(it != self->children.end()); - auto &child_record = it->second; - bts_assert(child_record.is_subscribing); - child_record.is_subscribing = false; -} - -void AlgoOrder::subscribe_timer() { - if (!is_working()) - return; - bts_assert(!self->has_timer_sub); - self->has_timer_sub = true; - self->services->subscribe_to_time([this](auto const &) { on_timer(); }); -} - -void AlgoOrder::unsubscribe_timer() { - if (!is_working()) - return; - bts_assert(self->has_timer_sub); - self->services->unsubscribe_from_time(); - self->has_timer_sub = false; -} - -// Private -void AlgoOrder::on_init_new(OrderDefinition const &def, - const PreActivationOrderServices &serv) { - bts_assert(def.has_custom_def()); - - // cache these so we can resolve securities and create order builders - self->security_db = &serv.sec_db; - self->order_factory = &serv.order_factory; - - // Save our order definition - self->definition = def; - - // If we are consuming the attachment we remove it from our definition so - // that derived orders can use our definition to create their childre - // without triggering an attachment. - if (_is_attaching(self->definition)) { - self->attach_target = - OrderKey(self->definition.attach_target_order_key()); - self->definition.clear_attach_target_order_key(); - } - - self->security = serv.sec_db.resolve(SecurityKey(def.security_key())); - if (self->security == nullptr) - terminate(ORDER_STATUS_REJECTED_BY_SPARK, "Unknown security"); - - auto result = do_before_launch(); - if (result.fail()) - terminate(ORDER_STATUS_REJECTED_BY_SPARK, result.error_msg); -} - -void AlgoOrder::on_init_recovered(OrderRecord const &rec, - const PreActivationOrderServices &serv) { - auto const &def = rec.def(); - self->definition = def; - self->set_state(rec.state()); - self->security_db = &serv.sec_db; - self->order_factory = &serv.order_factory; - - // Infer internal status from persisted external status - switch (self->external_status()) { - - case ORDER_STATUS_WORKING: - case ORDER_STATUS_PENDING_WORKING: - terminate(ORDER_STATUS_ERROR, "System restarted with active algo order"); - return; - - // Terminal states - case ORDER_STATUS_ERROR: - case ORDER_STATUS_REJECTED_BY_SPARK: - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_CANCELED: - terminate(rec.state().status(), rec.state().last_msg()); - break; - - // It should not be possible to reach these - case ORDER_STATUS_DISCONNECTED: - case ORDER_STATUS_INACTIVE: - case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: - case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - break; - } -} - -OrderOpResult AlgoOrder::on_activate(Price p, Quantity q, - OrderServices &services) { - ReportStateGuard _(*this); - - bts_assert(!self->services); - self->services = &services; - self->security_db = &services.sec_db(); - self->order_factory = &services.order_factory(); - - switch (self->status) { - - case Status::INACTIVE: { - self->services->subscribe_to_child_updates([this](auto const &u) { - this->on_child_order_update(u.key, u.quick_rebuild_state()); - }); - self->set_external_status(ORDER_STATUS_WORKING); - self->set_price_quantity(p, q); - for (auto &ct : self->child_templates) - ct->builder.activate(*self->services); - if (q.value() == 0) - terminate(ORDER_STATUS_COMPLETED); - else if (is_attaching()) - transition(Status::W_ATTACH); - else - transition(Status::WORKING); - return {}; - } - case Status::COMPLETED: - return {}; - - case Status::W_ATTACH: - case Status::WORKING: - bts_assert(false); - } - - bts_assert(false); - return {}; -} - -OrderOpResult AlgoOrder::on_update(Update const &update) { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::W_ATTACH: - self->pending_update = update; - return {}; - case Status::COMPLETED: - return {}; - case Status::WORKING: { - self->set_price_quantity(Price(update.price()), - Quantity(update.quantity())); - if (update.has_custom()) { - *self->definition.mutable_custom_def() = update.custom(); - self->services->report_definition(self->definition); - } - do_update(update); - return {}; - } - case Status::INACTIVE: - bts_assert(false); - return {}; - } - bts_assert(false); - return {}; -} - -OrderOpResult AlgoOrder::on_update(Price p, Quantity q) { - ReportStateGuard _(*this); - - Update up; - up.set_price(p.value()); - up.set_quantity(q.value()); - return on_update(up); -} - -orderman::OrderOpResult AlgoOrder::on_detach() { - ReportStateGuard _(*this); - - LOG(OrderMan, Info, "AlgoOrder order with key {} received detach", - key().value()); - - switch (self->status) { - case Status::W_ATTACH: - case Status::WORKING: - for (auto &[_, c] : self->children) { - (void)_; - c.order.mark_detached_from_parent(); - } - self->children.clear(); - terminate(ORDER_STATUS_CANCELED); - break; - case Status::COMPLETED: - break; - case Status::INACTIVE: - bts_assert(false); - break; - } - return {}; -} - -orderman::OrderOpResult AlgoOrder::on_cancel() { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::W_ATTACH: - case Status::WORKING: - terminate(ORDER_STATUS_CANCELED); - break; - case Status::COMPLETED: - break; - case Status::INACTIVE: - bts_assert(false); - break; - } - return {}; -} - -void AlgoOrder::on_attach_complete(OrderOpResult r, ChildOrder const &child, - api::orderman::OrderDefinition const &, - api::orderman::OrderState const &state) { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::W_ATTACH: - if (r.ok()) { - bts_assert(state.status() == ORDER_STATUS_WORKING); - - LOG(OrderMan, Info, - "AlgoOrder order with key {} successfully attached to {}", - key().value(), child.key().value()); - - // Store the order - auto &child_record = (self->children)[child.key()]; - child_record.is_subscribing = true; - child_record.order = child; - update_child_record(state, &child_record); - transition(Status::WORKING); - } else { - terminate_error(fmt::format( - "AlgoOrder order with key {} failed to attach to {}: {}", - key().value(), self->attach_target.value(), r.error_msg)); - } - break; - case Status::COMPLETED: - break; - case Status::INACTIVE: - case Status::WORKING: - bts_assert(false); - break; - } -} - -void AlgoOrder::on_child_order_update( - OrderKey child_key, api::orderman::OrderState const &new_child_state) { - ReportStateGuard _(*this); - - switch (self->status) { - case Status::WORKING: - case Status::W_ATTACH: - if (auto it = self->children.find(child_key); - it != self->children.end()) { - auto &child_record = it->second; - switch (new_child_state.status()) { - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_WORKING: - case ORDER_STATUS_PENDING_WORKING: - case ORDER_STATUS_CANCELED: - if (child_record.state.has_value() && - child_record.state.value().total_fills.value() > - new_child_state.filled_quantity()) { - LOG(OrderMan, Error, - "AlgoOrder {} observed child {} decrease its total fills " - "from {} to {}. This should never happen. Canceling {}", - key().value(), child_key.value(), - child_record.state.value().total_fills.value(), - new_child_state.filled_quantity(), key().value()); - terminate_error("Child order filled quantity decreased"); - } else { - update_child_record(new_child_state, &child_record); - if (child_record.is_subscribing) - on_child_change(child_key, child_record.state.value()); - } - break; - - case ORDER_STATUS_DISCONNECTED: - case ORDER_STATUS_REJECTED_BY_SPARK: - case ORDER_STATUS_ERROR: - terminate_error("Unexpected child order state"); - break; - - case ORDER_STATUS_INACTIVE: - case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: - case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - break; - } - } else - terminate_error( - fmt::format("Unknown child order update: {}", child_key.value())); - return; - case Status::COMPLETED: - return; - case Status::INACTIVE: - bts_assert(false); - return; - } - bts_assert(false); -} - -void AlgoOrder::setup_transitions() { - // From INACTIVE - self->transition_actions[{Status::INACTIVE, Status::COMPLETED}] = [this]() { - _transition(Status::COMPLETED); - }; - self->transition_actions[{Status::INACTIVE, Status::W_ATTACH}] = [this]() { - self->services->attach( - self->attach_target, - [this](auto result, auto const &child_order, auto const &child_def, - auto const &child_state) { - this->on_attach_complete(result, child_order, child_def, - child_state); - }); - _transition(Status::W_ATTACH); - }; - self->transition_actions[{Status::INACTIVE, Status::WORKING}] = [this]() { - _transition(Status::WORKING); - do_launch(); - }; - - // From W_ATTACH - self->transition_actions[{Status::W_ATTACH, Status::COMPLETED}] = [this]() { - _transition(Status::COMPLETED); - }; - self->transition_actions[{Status::W_ATTACH, Status::WORKING}] = [this]() { - _transition(Status::WORKING); - auto &maybe_state = self->children[self->attach_target].state; - bts_assert(maybe_state.has_value()); - do_launch_attached(self->attach_target, maybe_state.value()); - if (self->status == Status::WORKING && self->pending_update.has_value()) { - auto update = self->pending_update.value(); - self->pending_update.reset(); - do_update(self->pending_update.value()); - } - }; - - // From WORKING - self->transition_actions[{Status::WORKING, Status::COMPLETED}] = [this]() { - cleanup(); - _transition(Status::COMPLETED); - }; -} - -void AlgoOrder::check_invariants() { - bts_assert(self->security_db); - bts_assert(self->order_factory); - - switch (self->status) { - case Status::INACTIVE: - bts_assert(false); - return; - - case Status::W_ATTACH: - bts_assert(self->services); - bts_assert(self->security); - bts_assert(is_attaching()); - bts_assert(self->external_status() == ORDER_STATUS_WORKING); - bts_assert(self->book_subs.empty()); - bts_assert(self->children.empty()); - bts_assert(!self->has_timer_sub); - break; - - case Status::WORKING: - bts_assert(self->services); - bts_assert(self->security); - bts_assert(self->external_status() == ORDER_STATUS_WORKING); - bts_assert(!self->pending_update.has_value()); - break; - - case Status::COMPLETED: - bts_assert(self->children.empty()); - bts_assert(self->external_status() == ORDER_STATUS_CANCELED || - self->external_status() == ORDER_STATUS_COMPLETED || - self->external_status() == ORDER_STATUS_ERROR || - self->external_status() == ORDER_STATUS_REJECTED_BY_SPARK); - bts_assert(self->book_subs.empty()); - bts_assert(!self->has_timer_sub); - bts_assert(!self->pending_update.has_value()); - break; - } -} - -void AlgoOrder::transition(Status new_status) { - if (auto it = self->transition_actions.find({self->status, new_status}); - it != self->transition_actions.end()) { - it->second(); - } else { - LOG(OrderMan, Fatal, "Illegal transition from {} to {}", - state_name(self->status), state_name(new_status)); - bts_assert(false); - } -} - -void AlgoOrder::_transition(Status st) { - LOG(OrderMan, Info, "Transitioninig AlgoOrder with key {} from {} to {}", - key().value(), state_name(self->status), state_name(st)); - self->status = st; - check_invariants(); -} - -void AlgoOrder::terminate(api::orderman::OrderStatus st, - std::string const &error) { - self->set_external_status(st); - if (error.size()) - self->set_last_msg(error); - transition(Status::COMPLETED); -} - -void AlgoOrder::cleanup() { - for (auto const &s : self->book_subs) { - self->services->unsubscribe_from_book(s.second); - self->book_subs.clear(); - } - for (auto &c : self->children) { - c.second.order.cancel(); - self->children.clear(); - } - if (self->has_timer_sub) - unsubscribe_timer(); - self->pending_update.reset(); -} - -void AlgoOrder::update_child_record(api::orderman::OrderState const &new_state, - ChildRecord *child_record) { - if (!child_record->state.has_value()) - child_record->state = ChildState{}; - auto &old_state = child_record->state.value(); - bts_assert(old_state.total_fills.value() <= new_state.filled_quantity()); - - child_record->state = ChildState{ - Price(new_state.price()), - Quantity(new_state.quantity()), - Quantity(new_state.filled_quantity()), - Price(new_state.avg_price()), - Quantity(new_state.filled_quantity() - old_state.total_fills.value()), - Price(new_state.last_price()), - map_child_status(new_state.status())}; -} - -OrderKey AlgoOrder::index_and_launch_child(ChildOrder &&child, Price p, - Quantity q, bool subscribe) { - child.activate(p, q); - auto k = child.key(); - auto &cr = self->children[k]; - cr = {std::move(child), {}, subscribe}; - return k; -} - -bool AlgoOrder::is_working() const { return self->status == Status::WORKING; } - -log::CharPtrWrapper AlgoOrder::state_name(Status st) { - switch (st) { - case Status::COMPLETED: - return {"COMPLETED"}; - case Status::W_ATTACH: - return {"W_ATTACH"}; - case Status::INACTIVE: - return {"INACTIVE"}; - case Status::WORKING: - return {"WORKING"}; - } - bts_assert(false); - return {""}; -} - -ChildStatus AlgoOrder::map_child_status(api::orderman::OrderStatus st) { - switch (st) { - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_PENDING_WORKING: - return ChildStatus::COMPLETED; - case ORDER_STATUS_WORKING: - return ChildStatus::WORKING; - case ORDER_STATUS_CANCELED: - return ChildStatus::CANCELED; - default: - bts_assert(false); - return ChildStatus::CANCELED; - } -} - -bool AlgoOrder::_is_attaching(api::orderman::OrderDefinition const &def) const { - auto ak = OrderKey(def.attach_target_order_key()); - return (ak != null_order_key() && - !should_forward_attachment(def.custom_def())); -} - -} // namespace orderman -} // namespace bts diff --git a/src/lib/orderman/orders/algo/AlgoOrder.h b/src/lib/orderman/orders/algo/AlgoOrder.h deleted file mode 100644 index 4cc5443eb..000000000 --- a/src/lib/orderman/orders/algo/AlgoOrder.h +++ /dev/null @@ -1,208 +0,0 @@ -#pragma once - -#include "lib/orderman/orders/Order.h" -#include -#include "lib/orderman/orders/algo/IAlgoOrder.h" - -namespace bts { -namespace orderman { - -class Book; -class ChildTemplate; - -class AlgoOrder : private Order { - public: - AlgoOrder(); - ~AlgoOrder(); - - // -------------------------------------------------------------- - // Queries - // -------------------------------------------------------------- - - // Returns the OrderDefinition for this order - api::orderman::OrderDefinition const &definition() const; - - // Returns the key of the order() - OrderKey order_key() const; - - // Returns the parent key of the order() - OrderKey parent_order_key() const; - - // Resolves the security identified by k, returning nullptr if no - // such security exists - api::Security const *lookup_security(SecurityKey k) const; - - // Returns the price for this order. This value is not available - // from do_before_launch() - Price price() const; - - // Returns the quantity for this order. This value is not available - // from do_before_launch(). - Quantity quantity() const; - - // Returns the filled quantity for this order. - Quantity total_fills() const; - - // Returns wheter this order is attaching to an existing order - bool is_attaching() const; - - // -------------------------------------------------------------- - // Commands - // -------------------------------------------------------------- - - // Build a child template that can later can be used to quickly - // launch child orders. The reserve parameter indicates how many - // orders should be created ahead of time, before they are - // launched. This is usefull to remove the relatively expensive - // order creation operation to a background thread. This is the - // only command that can be invoked from the do_before_launch() - // call. - ChildTemplate & - make_child_template(api::orderman::OrderDefinition const &child_def, - size_t reserve = 1); - - // Launch a new child order, based on a previously created template - // at the specified price and quantity. This method can not be - // called from do_before_launch(). By default launching a child - // results in an automatic subscription to its state changes. - OrderKey launch_child(ChildTemplate &, Price, Quantity, - bool subscribe_to_changes = true); - - // Launch a new child order, using the speficied definition, price - // and quantity. This method cannot be called from - // do_before_launch(). By default launching a child results in an - // automatic subscription to its state changes. - OrderKey launch_child(api::orderman::OrderDefinition const &, Price, - Quantity, bool subscribe_to_changes = true); - - // Update previously launched child order. This method cannot be - // called from do_before_launch(). - void update_child(OrderKey child, api::orderman::Update const &); - - // Cancel previously launched child order. This method cannot be - // called from do_before_launch(). - void cancel_child(OrderKey child); - - // Deatch from previously launched child order. This method cannot - // be called from do_before_launch(). - void detach_child(OrderKey child); - - // Update this order's filled quantity and last price. This method - // cannot be called from do_before_launch(). - void update_fill_state(Quantity total_fills, Price last_price); - - // Cancel this order, optionally reporting an explanation. This - // method cannot be called from do_before_launch(). - void cancel(std::string const &msg = ""); - - // Terminate this order, canceling all children, signaling an error - // condition. This method cannot be called from do_before_launch(). - void terminate_error(std::string const &msg); - - // Signal this order completed successfully and cancel any active - // children. This method cannot be called from do_before_launch(). - void complete(); - - // Detach from all children. This method cannot be called from - // do_before_launch(). - void detach_all(); - - // -------------------------------------------------------------- - // Subscriptions - // -------------------------------------------------------------- - - // Subscribe to book updates for the specified security - void subscribe_book(SecurityKey); - void unsubscribe_book(SecurityKey); - - // Subscribe to child order updates for the specified child - // order. This is rarely needed since launch_child() by default - // automatically subscribes to new orders. - void subscribe_child_change(OrderKey); - void unsubscribe_child_change(OrderKey); - - // Subscribe to a periodic timer - void subscribe_timer(); - void unsubscribe_timer(); - - // -------------------------------------------------------------- - // Events - // -------------------------------------------------------------- - - // The child order specified by the order key has changed - virtual void on_child_change(OrderKey, ChildState const &) {} - - // The book for the specified security has changed - virtual void on_book(SecurityKey, Book const &) {} - - // -------------------------------------------------------------- - // Hooks - // -------------------------------------------------------------- - - // Override this to setup and validate an order prior to it being - // launched. This call occurs outside of the latency sensitive path - // so it can be used for any expensive initialization - // operations. Note that at this point the order is not fully - // initialized only a subset of the order functionality is - // available. Note as well that at this point there is no - // guarantee the order will execute at all, since the order might - // simply have been created as part of a pool of orders used to - // speed up order launch. - virtual OrderOpResult do_before_launch() { return {}; } - - // Implement this method to start your order - virtual void do_launch() = 0; - - // Implement this method to start your order as the parent of an - // existing order. Attached child orders are automatically - // subscribed to, so there is no need to call - // subscribe_child_change() for the attached order. - virtual void do_launch_attached(OrderKey /* attached_child */, - ChildState /* attached_child_state */) {} - - // Implement this method to respond to updates to you order. The - // implementation should validate the update and either apply it, - // in case the update is valid or invoke terminate_error() - // otherwise. - virtual void do_update(api::orderman::Update const &) {} - - // Implement this method to receive a periodical call - virtual void on_timer() {} - - private: - friend class OrderFactory; - struct ReportStateGuard; - struct ChildRecord; - enum class Status; - struct PImpl; - std::unique_ptr self; - void on_init_new(api::orderman::OrderDefinition const &, - const PreActivationOrderServices &) override; - void on_init_recovered(api::orderman::OrderRecord const &, - const PreActivationOrderServices &) override; - OrderOpResult on_activate(Price, Quantity, OrderServices &) override; - OrderOpResult on_update(api::orderman::Update const &) override; - OrderOpResult on_update(Price, Quantity) override; - OrderOpResult on_detach() override; - OrderOpResult on_cancel() override; - void on_attach_complete(OrderOpResult, ChildOrder const &, - api::orderman::OrderDefinition const &, - api::orderman::OrderState const &); - void on_child_order_update(OrderKey, api::orderman::OrderState const &); - void setup_transitions(); - void check_invariants(); - void transition(Status); - void _transition(Status); - void terminate(api::orderman::OrderStatus, std::string const &error = ""); - void cleanup(); - void update_child_record(api::orderman::OrderState const &, ChildRecord *); - OrderKey index_and_launch_child(ChildOrder &&child, Price p, Quantity q, - bool subscribe_child_change); - bool is_working() const; - log::CharPtrWrapper state_name(Status); - ChildStatus map_child_status(api::orderman::OrderStatus); - bool _is_attaching(api::orderman::OrderDefinition const &) const; -}; - -} // namespace orderman -} // namespace bts diff --git a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp index bb884b1de..5522bbcf5 100644 --- a/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp +++ b/src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp @@ -129,11 +129,17 @@ void AlgoWrapperOrder::on_init_new(OrderDefinition const &def, me->definition = def; // If we are consuming the attachment we remove it from our definition so - // that derived orders can use our definition to create their childre + // that derived orders can use our definition to create their children // without triggering an attachment. if (_is_attaching(me->definition)) { - me->attach_target = OrderKey(me->definition.attach_target_order_key()); - me->definition.clear_attach_target_order_key(); + if (can_consume_attachment(me->definition.custom_def().custom_case())) { + me->attach_target = OrderKey(me->definition.attach_target_order_key()); + me->definition.clear_attach_target_order_key(); + } + else { + terminate(ORDER_STATUS_REJECTED_BY_SPARK, "Algo does not support attachment"); + return; + } } me->security = serv.sec_db.resolve(SecurityKey(def.security_key())); @@ -574,7 +580,8 @@ void AlgoWrapperOrder::on_child_order_update( } else { update_child_record(new_child_state, &child_record); if (child_record.is_subscribing) - me->algo->on_child_change(child_key, child_record.state.value()); + me->algo->on_child_change(child_key, + child_record.state.value()); } break; @@ -630,7 +637,8 @@ void AlgoWrapperOrder::setup_transitions() { _transition(Status::WORKING); auto &maybe_state = me->children[me->attach_target].state; bts_assert(maybe_state.has_value()); - me->algo->do_launch_attached(*this,me->attach_target, maybe_state.value()); + me->algo->do_launch_attached(*this, me->attach_target, + maybe_state.value()); if (me->status == Status::WORKING && me->pending_update.has_value()) { auto update = me->pending_update.value(); me->pending_update.reset(); diff --git a/src/lib/orderman/orders/algo/IAlgoOrder.cpp b/src/lib/orderman/orders/algo/IAlgoOrder.cpp index c2de7ee7e..11240c91b 100644 --- a/src/lib/orderman/orders/algo/IAlgoOrder.cpp +++ b/src/lib/orderman/orders/algo/IAlgoOrder.cpp @@ -1,4 +1,5 @@ #include "lib/orderman/orders/algo/IAlgoOrder.h" +#include "lib/orderman/orders/algo/IAlgoServices.h" namespace bts { namespace orderman { @@ -8,6 +9,9 @@ OrderOpResult IAlgoOrder::do_before_launch(IBeforeLaunchAlgoServices &) { return {}; } -void IAlgoOrder::do_launch_attached(IAlgoServices &, OrderKey, ChildState) {} +void IAlgoOrder::do_launch_attached(IAlgoServices &serv, OrderKey, ChildState) { + serv.terminate_error("Algo does not implement attachment functionality"); +} + } // namespace orderman } // namespace bts diff --git a/src/lib/utils/ConcurrentPool.h b/src/lib/utils/ConcurrentPool.h new file mode 100644 index 000000000..92d86e7ea --- /dev/null +++ b/src/lib/utils/ConcurrentPool.h @@ -0,0 +1,69 @@ +#pragma once +#include +#include +#include + +namespace bts { +namespace utils { + +template +class ConcurrentPool { + tbb::concurrent_vector> _all; + tbb::concurrent_queue _free; + size_t _initial_size; + size_t _grow_size; + size_t _outstanding; + public: + ConcurrentPool(size_t initial_size=256, size_t grow_size=32): + _all(), _free(), _initial_size(initial_size), _grow_size(grow_size), _outstanding(0) { + reserve(initial_size); + } + + ~ConcurrentPool() { } + + T* allocate() { + T* result = nullptr; + ++_outstanding; + if (_free.try_pop(result)) { + return result; + } + else { + reserve(_grow_size); + return allocate(); + } + } + + void reserve(size_t count) { + std::unique_ptr buffer {new T[count]}; + _outstanding += count; + for (size_t x = 0; x < count; ++x) { + T* temp = &buffer[x]; + release(&temp); + } + _all.push_back(std::move(buffer)); + } + + void release(T** item) { + if (*item) { + (*item)->reset(); + _free.push(*item); + *item = nullptr; + --_outstanding; + } + } + + template + void unsafe_iterate(CB cb) const { + for (auto iter = _all.begin(); iter != _all.end(); ++iter) { + size_t size = (iter == _all.begin()) ? _initial_size : _grow_size; + for (size_t x = 0; x < size; ++x) { + const std::unique_ptr& bucket = *iter; + const T* start = bucket.get(); + cb(start[x]); + } + } + } + +}; + +}} //namespace diff --git a/src/lib/utils/Dispatcher.cpp b/src/lib/utils/Dispatcher.cpp index 9c504ee8c..1d7ab64d3 100755 --- a/src/lib/utils/Dispatcher.cpp +++ b/src/lib/utils/Dispatcher.cpp @@ -200,7 +200,7 @@ void Dispatcher::invoke(Action action) } } -void Dispatcher::post(Action action, int64_t delay_ms) +void Dispatcher::post(Action&& action, int64_t delay_ms) { if (delay_ms > 0) { diff --git a/src/lib/utils/Dispatcher.h b/src/lib/utils/Dispatcher.h index 4a4470fb8..2adc656af 100755 --- a/src/lib/utils/Dispatcher.h +++ b/src/lib/utils/Dispatcher.h @@ -87,7 +87,7 @@ public: // Perform an action on the dispatcher thread. Block until the action is complete. void invoke(Action action); // Queue an action on the dispatcher thread and return immediately. - void post(Action action, int64_t delay_ms=0); + void post(Action&& action, int64_t delay_ms=0); boost::asio::io_service* io_service(); diff --git a/src/lib/utils/LockBox.h b/src/lib/utils/LockBox.h new file mode 100644 index 000000000..4e37b8024 --- /dev/null +++ b/src/lib/utils/LockBox.h @@ -0,0 +1,24 @@ +#pragma once +#include + +namespace bts { +namespace utils { + +template +class LockBox { + mutable std::mutex _mutex; + T _data; +public: + template + auto lock(CB cb) { + std::lock_guard guard(_mutex); + return cb(_data); + } + template + auto lock(CB cb) const { + std::lock_guard guard(_mutex); + return cb(_data); + } +}; + +}} //namespace diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index a4aceb09a..e12c685e9 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;algo/AlgoServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/algo/AlgoServiceServer.cpp b/src/server/sparkd/algo/AlgoServiceServer.cpp deleted file mode 100644 index e444a82a9..000000000 --- a/src/server/sparkd/algo/AlgoServiceServer.cpp +++ /dev/null @@ -1,244 +0,0 @@ -//#include "AlgoServiceServer.h" -//#include "lib/api/service_helpers/AlgoServiceUtils.h" -//#include "lib/api/public/ServiceContext.h" -// -//#include "lib/persistence/SqliteStore.h" -//#include "lib/logging/logging.h" -//#include "lib/utils/bts_assert.h" -//#include "lib/utils/PBUtils.h" -//#include "lib/utils/TimeStamp.h" -// -//#include -//#include -//#include -//#include -//#include -// -//namespace bts { -//namespace orderman { -// -//using namespace std; -//using namespace bts::api::orderman; -// -//const string& algo_key(const CompositeAlgoRecord& row) { -// return row.key(); -//} -//const string& algo_template_key(const AlgoTemplateRecord& row) { -// return row.key(); -//} -// -//struct AlgoServiceServer::PImpl{ -// const std::string template_table_name, -// algo_table_name; -// std::unique_ptr store; -// bts::api::ServiceContext& context; -// AlgoServiceBase::AlgosStatefulPublisher algo_publisher; -// AlgoServiceBase::Algo_templatesStatefulPublisher template_publisher; -// PImpl(bts::api::ServiceContext& context, -// const AlgoServiceBase::AlgosStatefulPublisher& algo_publisher, -// const AlgoServiceBase::Algo_templatesStatefulPublisher& template_publisher) -// : -// template_table_name("templates"), -// algo_table_name("composite_algos"), -// store(new bts::persistence::SqliteStore("algo")), -// context(context), -// algo_publisher(algo_publisher), -// template_publisher(template_publisher){ -// } -//}; -// -//AlgoServiceServer::AlgoServiceServer(bts::api::ServiceContext& context) -//: -//AlgoServiceBase (context), -//me(new PImpl(context, -// bts::api::StatefulPublisher(make_algos_publisher(), algo_key, true), -// bts::api::StatefulPublisher(make_algo_templates_publisher(), algo_template_key, true))) -//{ -// _init_publish_algos(); -// _init_publish_templates(); -//} -//AlgoServiceServer::~AlgoServiceServer(){} -// -//void AlgoServiceServer::create_composite_algo( -// const CompositeAlgoCreateRequest& request, -// const api::Response& response){ -// CompositeAlgoCreateResult result; -// result.set_success(true); -// if (!_has_algo(request.new_algo().name())){ -// CompositeAlgoRecord new_algo_rec; -// new_algo_rec.set_key(request.new_algo().name()); -// new_algo_rec.mutable_common()->CopyFrom(_new_common(request.user_id())); -// new_algo_rec.mutable_algo_definition()->CopyFrom(request.new_algo()); -// if(_store_and_publish_algo(new_algo_rec)) -// result.mutable_record()->CopyFrom(new_algo_rec); -// else{ -// result.set_success(false); -// result.set_err_msg("Could not store algo"); -// } -// } -// else{ -// result.set_success(false); -// result.set_err_msg("Algo with this name already exists"); -// } -// response(result); -//} -// -//void AlgoServiceServer::create_template( -// const TemplateCreateRequest& request, -// const api::Response& response){ -// TemplateCreateResult result; -// result.set_success(true); -// string key = template_key(request.new_template()); -// const AlgoTemplateRecord* ex_rec = _find_template(key); -// if(ex_rec == nullptr){ -// AlgoTemplateRecord new_record; -// new_record.set_key(key); -// new_record.mutable_common()->CopyFrom(_new_common(request.user_id())); -// new_record.mutable_algo_template()->CopyFrom(request.new_template()); -// if(_store_and_publish_template(true,new_record)) -// result.mutable_record()->CopyFrom(new_record); -// else{ -// result.set_success(false); -// result.set_err_msg("Could not store algo"); -// } -// } -// else{ -// result.set_success(false); -// result.set_err_msg("Template already exists"); -// result.mutable_record()->CopyFrom(*ex_rec); -// } -// response(result); -//} -// -//void AlgoServiceServer::update_template( -// const TemplateUpdateRequest& request, -// const api::Response& response){ -// AlgoServiceReply result; -// result.set_success(true); -// const AlgoTemplateRecord* ex_rec = _find_template(request.key()); -// if(ex_rec == nullptr){ -// result.set_success(false); -// result.set_err_msg("Could not locate template"); -// } -// else{ -// AlgoTemplateRecord mrec = AlgoTemplateRecord(*ex_rec); -// mrec.mutable_algo_template()->CopyFrom(request.updated_template()); -// const AlgoTemplateRecord updated_rec = mrec; -// if(!_store_and_publish_template(false,updated_rec)){ -// result.set_success(false); -// result.set_err_msg("Could not serialize template"); -// } -// // TODO: need to make sure -// // a. template's algo is not changed -// // b. template is consistent with the algo -// } -// response(result); -//} -// -//void AlgoServiceServer::remove_template( -// const AlgoServiceRemoveRequest& request, -// const api::Response& response){ -// AlgoServiceReply result; -// result.set_success(true); -// const AlgoTemplateRecord* ex_rec = _find_template(request.key()); -// if(ex_rec == nullptr){ -// result.set_success(false); -// result.set_err_msg("Could not locate template"); -// } -// else{ -// set keys; -// keys.insert(request.key()); -// me->store->remove(me->template_table_name,keys); -// me->template_publisher.erase(request.key()); -// me->template_publisher.flush(); -// } -// response(result); -//} -// -//PersistenceRecordCommon AlgoServiceServer::_new_common(uint32_t user_id){ -// PersistenceRecordCommon rv; -// rv.set_is_active(true); -// rv.set_creator_user_id(user_id); -// return rv; -//} -// -//// algo helper functions -//void AlgoServiceServer::_init_publish_algos(){ -// map> all_data = me->store->retrieve_all(); -// auto table=all_data.find(me->algo_table_name); -// if(table!=all_data.end()){ -// for (const auto& p : table->second){ -// CompositeAlgoRecord algo; -// if(algo.ParseFromString(p.second)) -// me->algo_publisher.add_or_update(algo); -// else -// LOG(Gen, Error, "AlgoServiceServer._load_algos " -// "could not load {}", table->first); -// } -// me->algo_publisher.flush(); -// } -//} -// -//bool AlgoServiceServer::_has_algo(const std::string& key){ -// const auto& algo_map = me->algo_publisher.state_data(); -// return algo_map.find(key) != algo_map.end(); -//} -// -//bool AlgoServiceServer::_store_and_publish_algo ( -// const CompositeAlgoRecord& rec){ -// string ser; -// if(rec.SerializeToString(&ser)){ -// me->store->insert(me->algo_table_name, -// rec.algo_definition().name(), -// ser); -// me->algo_publisher.add_or_update(rec); -// me->algo_publisher.flush(); -// return true; -// } -// else -// return false; -//} -// -//void AlgoServiceServer::_init_publish_templates(){ -// map> all_data = me->store->retrieve_all(); -// auto table=all_data.find(me->template_table_name); -// if(table!=all_data.end()){ -// for (const auto& p : table->second){ -// AlgoTemplateRecord t_record; -// if(t_record.ParseFromString(p.second)) -// me->template_publisher.add_or_update(t_record); -// else -// LOG(Gen, Error, "AlgoServiceServer._load_templates " -// "could not load template {}", table->first); -// } -// me->template_publisher.flush(); -// } -//} -// -//const AlgoTemplateRecord* AlgoServiceServer::_find_template( -// const std::string& key){ -// const auto& template_map = me->template_publisher.state_data(); -// const auto it = template_map.find(key); -// if(it != template_map.end()) -// return &(it->second); -// return nullptr; -//} -// -//bool AlgoServiceServer::_store_and_publish_template ( -// bool insert, -// const AlgoTemplateRecord& rec){ -// string ser; -// if(rec.SerializeToString(&ser)){ -// if(insert) -// me->store->insert(me->template_table_name,rec.key(),ser); -// else -// me->store->update(me->template_table_name,rec.key(),ser); -// me->template_publisher.add_or_update(rec); -// me->template_publisher.flush(); -// return true; -// } -// else -// return false; -//} -// -//}} diff --git a/src/server/sparkd/algo/AlgoServiceServer.h b/src/server/sparkd/algo/AlgoServiceServer.h deleted file mode 100644 index 079a7a36d..000000000 --- a/src/server/sparkd/algo/AlgoServiceServer.h +++ /dev/null @@ -1,54 +0,0 @@ -//#pragma once -// -//#include -//#include "lib/api/services/AlgoService.server.pb.h" -//#include "lib/utils/bts_no_copy.h" -//#include "lib/persistence/IStore.h" -//#include "lib/api/public/ServiceContext.h" -//#include "lib/api/services/OrderService.pb.h" -//#include "lib/api/services/OrderService.client.pb.h" -// -//namespace bts { -//namespace orderman { -// -//class AlgoServiceServer : public api::orderman::AlgoServiceBase -//{ -//private: -// struct PImpl; -// std::unique_ptr me; -//public: -// AlgoServiceServer(bts::api::ServiceContext& context); -// ~AlgoServiceServer(); -// -// void create_composite_algo( -// const bts::api::orderman::CompositeAlgoCreateRequest& request, -// const api::Response& reply) override; -// -// void create_template( -// const bts::api::orderman::TemplateCreateRequest& request, -// const api::Response& reply) override; -// -// void update_template( -// const bts::api::orderman::TemplateUpdateRequest& request, -// const api::Response& reply) override; -// -// void remove_template( -// const bts::api::orderman::AlgoServiceRemoveRequest& request, -// const api::Response& reply) override; -//private: -// bts::api::orderman::PersistenceRecordCommon _new_common(uint32_t user_id); -// // loads algos from the db and publises them -// void _init_publish_algos(); -// bool _has_algo(const std::string& name); -// bool _store_and_publish_algo ( -// const bts::api::orderman::CompositeAlgoRecord& rec); -// // template helpers -// // loads templates from the db and publises them -// void _init_publish_templates(); -// const bts::api::orderman::AlgoTemplateRecord* _find_template( -// const std::string& key); -// bool _store_and_publish_template ( -// bool insert, -// const bts::api::orderman::AlgoTemplateRecord& rec); -//}; -//}} diff --git a/src/server/sparkd/cmds/Benchmark.cpp b/src/server/sparkd/cmds/Benchmark.cpp index 360f29f89..d62e01f9e 100644 --- a/src/server/sparkd/cmds/Benchmark.cpp +++ b/src/server/sparkd/cmds/Benchmark.cpp @@ -640,6 +640,21 @@ static void perf_log(benchmark::State &state) { } BENCHMARK(perf_log); +void int_price_less(benchmark::State& state) { + for (auto _ : state) { + state.PauseTiming(); + int64_t count = state.range(0); + std::vector prices; + prices.reserve((int)count); + for (int64_t x = 0; x < count; ++x) { + prices.push_back({x % 100, (int8_t)-(x%5)}); + } + state.ResumeTiming(); + std::sort(prices.begin(), prices.end()); + } +} +BENCHMARK(int_price_less)->Range(8, 8 << 11); + std::string Benchmark::help() { return "Run micro-benchmarks. Use --benchmark_filter=regex to run only " "specific benchmarks."; diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index d32604920..bbb1422ca 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -14,6 +14,8 @@ #include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" +#include "lib/md/MBOProcessor.h" +#include "lib/md/MBOServiceServer.h" #include "lib/metrics/metrics.h" #include "lib/metrics/system_metrics.h" #include "lib/orderman/order_factory/DynamicOrderRegistry.h" @@ -31,7 +33,6 @@ #include "server/sparkd/support/ShellLatencyServer.h" #include "server/sparkd/user/AdminServiceServer.h" #include "server/sparkd/user/AuthServiceServer.h" -#include "server/sparkd/algo/AlgoServiceServer.h" #include "server/sparkd/algo/AlgoDefinitionServiceServer.h" #include @@ -57,15 +58,19 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); - md::run_market_datapublishers(); + + md::MBOProcessor mbo(&utils::Dispatcher::get_named("MBO")); + auto* mbowriter = config::get().enabled() ? &mbo.writer() : nullptr; + md::run_market_datapublishers(mbowriter); api::synthetic::SyntheticSecurityService synthetic_service(sc); - md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service); - + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service, &mbo.reader()); md::QueryServerPtr md_query_server( md::run_query_server(sc, mdclient, stats)); + md::MBOServiceServer mboserver(sc, utils::Dispatcher::background(), mdclient); + // initialize order plugins orderman::DynamicOrderRegistry dyn_order_registry; SparkServices plugin_services(dyn_order_registry); diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 029ae34bc..abbe80ed6 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -37,7 +37,7 @@ int Web::execute(const std::vector& args) { user::AuthServiceServer login_server(sc); if (std::find(args.begin(), args.end(), "-md") != args.end()) { - md::run_market_datapublishers(); + md::run_market_datapublishers(nullptr); } api::synthetic::SyntheticSecurityService synthetic_service(sc); diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h index e0e9d3b56..d1915f85d 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h @@ -1,15 +1,17 @@ #pragma once #include "lib/md/SecurityIndex.h" +#include "lib/md/MBOProcessor.h" #include namespace bts { namespace md { struct SecurityData { - SecurityData(uint32_t id, MDProducerHandle& h) : id(id), handle(h), electronic_volume(0) {} + SecurityData(uint32_t id, MDProducerHandle& h) : id(id), handle(h), mbo_state(h), electronic_volume(0) { } uint32_t id; // exchange id MDProducerHandle handle; + MBOWorkingState mbo_state; uint32_t electronic_volume; }; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index 6049e982b..b06b185f7 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -15,7 +15,7 @@ namespace md { MDP3Listener::MDP3Listener( CMESecurityIndex& index, MDProducer& producer, - // co2::SettlementRecorder& , + MBOWriter* mbo, const std::vector channels) : _index(index), _producer(producer), @@ -27,7 +27,8 @@ MDP3Listener::MDP3Listener( _symbol_filter(), _channels(channels), _concurrency_monitor(0), - _book_updates() + _book_updates(), + _mbo(mbo) { _book_updates.reserve(32); } @@ -109,7 +110,7 @@ void MDP3Listener::onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) { } // -// Security discovery +// Book updates void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) { onBookUpdate(sec, book, book_type_direct); @@ -119,6 +120,24 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con onBookUpdate(sec, book, book_type_implied); } +void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::MboBook& book) { + if (_mbo) { + auto data = _index.get_security(sec.id()); + if (data == nullptr) return; + auto key = data->handle.key(); + if (key == invalid_security_key) return; + auto* snapshot = _mbo->allocate_snapshot(); + // copy from onixs book + for (const auto& bid : book.bids()) { + snapshot->bids.push_back({ ExchangeOrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() }); + } + for (const auto& ask : book.offers()) { + snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() }); + } + _mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot); + } +} + // // Trades diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index fc20add4c..1fdb93393 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -2,6 +2,7 @@ #include "lib/utils/bts_no_copy.h" #include "lib/md/coconut2.h" +#include "lib/md/MBOProcessor.h" #include "CMESecurityIndex.h" #include "lib/tracing/bts_trace.h" @@ -13,11 +14,57 @@ #include #include #include +#include namespace bts { namespace md { - +inline inc::UpdateType mbo_update_type(onixs::EntryTypeBook::Enum otype) { + switch(otype) { + case onixs::EntryTypeBook::Bid: + case onixs::EntryTypeBook::ImpliedBid: + return inc::UpdateType::Bid; + break; + case onixs::EntryTypeBook::Offer: + case onixs::EntryTypeBook::ImpliedOffer: + return inc::UpdateType::Ask; + break; + default: + bts_assert_msg(false, "Unexpected update type"); + } +} + +inline inc::UpdateAction mbo_update_action(onixs::OrderUpdateAction::Enum a) { + switch(a) { + case onixs::OrderUpdateAction::New: + return inc::UpdateAction::New; + case onixs::OrderUpdateAction::Update: + return inc::UpdateAction::Change; + case onixs::OrderUpdateAction::Delete: + return inc::UpdateAction::Delete; + default: + bts_assert_msg(false, "Unexpected update action"); + } +} + +inline inc::UpdateAction mbo_update_action(onixs::UpdateAction::Enum a) { + switch(a) { + case onixs::UpdateAction::New: + return inc::UpdateAction::New; + case onixs::UpdateAction::Change: + return inc::UpdateAction::Change; + case onixs::UpdateAction::Delete: + return inc::UpdateAction::Delete; + case onixs::UpdateAction::DeleteThru: + case onixs::UpdateAction::DeleteFrom: + case onixs::UpdateAction::Overlay: + return inc::UpdateAction::Delete; + default: + bts_assert_msg(false, "Unexpected update action"); + } +} + + class MDP3Listener : public onixs::HandlerListener, public onixs::MarketDataListener, @@ -31,7 +78,7 @@ class MDP3Listener : public: MDP3Listener(CMESecurityIndex& index, MDProducer&, - // co2::SettlementRecorder&, + MBOWriter* mbo, const std::vector); virtual ~MDP3Listener(); @@ -54,6 +101,139 @@ public: void onPacket(onixs::Handler&, const onixs::PacketArgs&) override; void onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) override; + // + // Handle MBO incremental updates directly + + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook32Args& args) override { + onIncrementalRefreshBook(args); + }; + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshBook46Args& args) override { + onIncrementalRefreshBook(args); + } + template + void onIncrementalRefreshBook(const Args& args) { + if (!_mbo) return; + + for(const auto& entry : args.message().orderIdEntries()) { + int qty; + entry.displayQty(qty); + uint8_t ref; + if (entry.referenceId(ref)) { + // entries is indexed by 0 but ref is indexed by 1 :facepalm: + const auto& ref_entry = args.message().entries()[(int)ref-1]; + uint32_t secid = ref_entry.securityId(); + auto data = _index.get_security(secid); + if (!data) continue; + + auto order_id = ExchangeOrderKey(entry.orderId()); + inc::Priority priority; + bts_assert(entry.orderPriority(priority)); + + onixs::Decimal oprice; + ref_entry.entryPx(oprice); + auto price = convert_price(oprice, data->handle.security().display_factor); + + int32_t size; + bts_assert(entry.displayQty(size)); + auto type = mbo_update_type(ref_entry.entryType()); + + auto action = mbo_update_action(entry.orderUpdateAction()); + + _mbo->enqueue_incremental( + data->mbo_state, + inc::Update { + order_id, + priority, + price, + size, + type, + action + }); + } + } + } + + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook43Args& args) override { + onIncrementalRefreshOrderBook(args); + } + void onMessage(onixs::Handler&, const onixs::IncrementalRefreshOrderBook47Args& args) override { + onIncrementalRefreshOrderBook(args); + } + template + void onIncrementalRefreshOrderBook(const Args& args) { + if (!_mbo) return; + for (const auto& entry : args.message().entries()) { + auto data = _index.get_security(entry.securityId()); + if (!data) continue; + + // order id + uint64_t temp; + bts_assert( + entry.orderId(temp)); + auto order_id = ExchangeOrderKey(temp); + + inc::Priority priority; + bts_assert( + entry.orderPriority(priority)); + + onixs::Decimal onxprice; + bts_assert( + entry.entryPx(onxprice)); + auto price = convert_price(onxprice, data->handle.security().display_factor); + + int32_t size; + bts_assert( + entry.displayQty(size)); + + auto type = mbo_update_type(entry.entryType()); + auto action = mbo_update_action(entry.updateAction()); + + _mbo->enqueue_incremental( + data->mbo_state, + inc:: Update { + order_id, + priority, + price, + size, + type, + action + }); + } + } + + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook44Args& args) override { + onSnapshotFullRefreshOrderBook(args); + } + void onMessage(onixs::Handler&, const onixs::SnapshotFullRefreshOrderBook53Args& args) override { + onSnapshotFullRefreshOrderBook(args); + } + template + void onSnapshotFullRefreshOrderBook(const Args& args) { + if (!_mbo) return; + auto data = _index.get_security(args.message().securityId()); + if (!data) return; + for (const auto& entry : args.message().entries()) { + auto order_id = ExchangeOrderKey(entry.orderId()); + inc::Priority priority; + bts_assert(entry.orderPriority(priority)); + auto price = convert_price(entry.entryPx(), data->handle.security().display_factor); + auto size = entry.displayQty(); + inc::UpdateType type = mbo_update_type(entry.entryType()); + _mbo->enqueue_incremental( + data->mbo_state, + inc::Update { + order_id, + priority, + price, + size, + type, + inc::UpdateAction::New + }); + } + } + + + // // Security discovery template @@ -74,9 +254,9 @@ public: // // Book - // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override; void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override; + void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override; // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override; template @@ -199,6 +379,7 @@ private: std::pair, std::array> asks; }; std::vector _book_updates; + MBOWriter* _mbo; }; ////////////////////////////////////////// @@ -228,9 +409,7 @@ void MDP3Listener::onFuture(onixs::Handler&, const onixs::Security& osec, const tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = def.message(); auto exchange_id = sec.securityId(); - // LOG(md, Info, "onDefinition future {} {}", - // std::string(sec.asset().items(), sec.asset().size()), - // std::string(sec.symbol().items(), sec.symbol().size())); + // LOG(md, Info, "onDefinition future exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; bts_assert(exchange_id == osec.id()); @@ -258,7 +437,7 @@ void MDP3Listener::onOption(onixs::Handler&, const onixs::Security&, const Instr tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = def.message(); auto exchange_id = sec.securityId(); - // LOG(Info, MarketData) << "onDefinition option " << exchange_id; + // LOG(md, Info, "onDefinition option exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; FixedOptionSecurity fsec; @@ -302,9 +481,7 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0); const auto& sec = spread_args.message(); auto exchange_id = sec.securityId(); - // LOG(md, Info, "onDefinition spread {} {}", - // std::string(sec.asset().items(), sec.asset().size()), - // std::string(sec.symbol().items(), sec.symbol().size())); + // LOG(md, Info, "onDefinition spread exch:{}", exchange_id); if (filter_security(exchange_id, sec) != ADD) return; // Filter out any already expired spreads diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 0ea409588..e1880efc4 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -1,6 +1,7 @@ #include "MDP3Main.h" #include "lib/md/cme/MDP3Utils.h" +#include "lib/md/MBOProcessor.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" #include "lib/os/os.h" @@ -116,12 +117,23 @@ map> create_feed_engines(const vector& symbols) { +void MDP3Main::start(std::vector& symbols, MBOWriter* mbo) { await_cme_config(); initialize(); const auto& channels = parse_channels(symbols); _engines = create_feed_engines(channels); for (const auto& channel : channels) { - unique_ptr phandler = prepare_handler(channel.id); - _listeners.push_back(prepare_listener(*phandler, channel)); + unique_ptr phandler = prepare_handler(channel.id, mbo); + _listeners.push_back(prepare_listener(*phandler, channel, mbo)); _handlers.insert(std::make_pair(channel.id, move(phandler))); std::stringstream symstr; for (const auto& sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " "; } @@ -145,7 +157,7 @@ void MDP3Main::start(std::vector& symbols) { } for (auto& pair : _handlers) { - onixs::SessionSettings settings = session_settings_for_channel(pair.first); + onixs::SessionSettings settings = session_settings_for_channel(pair.first, mbo); pair.second->start(settings); } LOG(md,Info, "** CME MDP3 Publisher started **"); @@ -185,7 +197,7 @@ void MDP3Main::initialize() { // // // Initialize HandlerSettings and allocate the channel Handler -unique_ptr MDP3Main::prepare_handler(const string& channel_id) { +unique_ptr MDP3Main::prepare_handler(const string& channel_id, MBOWriter* mbo) { const auto& pub_options = config::get(); const auto& options = channel_settings_for(channel_id); LOG(md,Info,"Loaded channel options for channel {}:\n{}",channel_id ,options.DebugString()); @@ -198,6 +210,9 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { settings.bookManagement().directBooks().maintain(true); settings.bookManagement().impliedBooks().maintain(true); settings.bookManagement().consolidatedBooks().maintain(false); + if (options.validate_mbo() && mbo != nullptr) { + settings.bookManagement().mboBooks().maintain(true); + } // 10k default is *far* too low settings.recoveryQueueMaxSize(options.recovery_queue_size()); @@ -320,14 +335,14 @@ unique_ptr MDP3Main::prepare_handler(const string& channel_id) { // // Create a listener for this channel and register callbacks -unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel) { +unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter* mbo) { // We need to get a handler to the producer channels so we can produce heartbeats vector producer_channels; for (auto& symbol : channel.symbols) { producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); } - unique_ptr listener(new MDP3Listener(_index, _producer, /*_settlement_recorder,*/ producer_channels)); + unique_ptr listener(new MDP3Listener(_index, _producer, mbo, producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 974d01bc8..bda70e4c6 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -16,6 +16,8 @@ namespace md { // void run_onixs_mdp3_server(UserSymbol* sym, size_t symcount); +class MBOWriter; + struct CMEChannel; /* @@ -30,12 +32,12 @@ public: MDP3Main(); ~MDP3Main(); - void start(std::vector&); + void start(std::vector&, MBOWriter*); void send_heartbeats() override; void initialize(); - std::unique_ptr prepare_handler(const std::string& channel); - std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel); + std::unique_ptr prepare_handler(const std::string& channel, MBOWriter*); + std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel, MBOWriter*); private: BTS_NO_COPY(MDP3Main); diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 89994f057..8b58d9b16 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -23,6 +23,7 @@ #include #include #include +#include using namespace bts; using namespace std; @@ -68,8 +69,7 @@ void run_replay() { replay_recording(file, naggle_ms, fallback_path); } -void run_onixs() -{ +void run_onixs(MBOWriter* mbo) { std::vector> platforms; // get all glbx symbols std::vector cme_symbols; @@ -78,7 +78,7 @@ void run_onixs() { MDP3Main* main = new MDP3Main(); std::shared_ptr ptr(main); - main->start(cme_symbols); + main->start(cme_symbols, mbo); platforms.push_back(ptr); } @@ -110,15 +110,14 @@ void run_cleanup() { } -std::thread* _md_thread; -void run_market_data_publishers_forever() { - os::set_thread_name("md publisher"); +std::optional _md_thread; +void run_market_data_publishers_forever(MBOWriter* mbo) { const auto& conf = config::get(); if (conf.use_replay()) { run_replay(); } else if (conf.use_onixs()) { - run_onixs(); + run_onixs(mbo); } } @@ -142,14 +141,15 @@ void boot_sleep() { } } -void run_market_datapublishers() { - bts_assert(_md_thread == nullptr); +void run_market_datapublishers(MBOWriter* mbo) { + bts_assert(_md_thread == std::nullopt); run_cleanup(); if (!config::get().use_md()) return; // start a thread for the md - _md_thread = new std::thread([]() { - run_market_data_publishers_forever(); - }); + _md_thread = std::optional( + threading::make_thread("md publisher", [mbo]() { + run_market_data_publishers_forever(mbo); + })); if (config::get().use_sim()) { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) @@ -359,7 +359,7 @@ void md_tool(std::vector const &args) { else if (cmd == "cleanup") run_cleanup(); else if (cmd == "dump") run_dump(vector(args.begin()+1, args.end())); else if (cmd == "replay") run_replay(); - else if (cmd == "run") run_market_data_publishers_forever(); + else if (cmd == "run") run_market_data_publishers_forever(nullptr); else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); diff --git a/src/server/sparkd/md/run_md.h b/src/server/sparkd/md/run_md.h index 587a6e0c1..dc771b0bf 100644 --- a/src/server/sparkd/md/run_md.h +++ b/src/server/sparkd/md/run_md.h @@ -5,7 +5,9 @@ namespace bts { namespace md { -void run_market_datapublishers(); +class MBOWriter; + +void run_market_datapublishers(MBOWriter*); void md_tool(std::vector const& args); }} diff --git a/src/server/sparkd/position/PositionServiceServer.cpp b/src/server/sparkd/position/PositionServiceServer.cpp index 274ae22b9..49706a39c 100644 --- a/src/server/sparkd/position/PositionServiceServer.cpp +++ b/src/server/sparkd/position/PositionServiceServer.cpp @@ -48,6 +48,10 @@ me(new PImpl(context, me->publisher.add_or_update(_on_fill(*f,false)); for (const auto& f : update.added()) me->publisher.add_or_update(_on_fill(*f,true)); + for (const auto& f : update.changed()) { + me->publisher.add_or_update(_on_fill(*f.previous(),false)); + me->publisher.add_or_update(_on_fill(*f.current(),true)); + } me->publisher.flush(); }); } diff --git a/src/server/sparkd/user/AdminServiceServer.cpp b/src/server/sparkd/user/AdminServiceServer.cpp index 09b33068d..9e317e7fc 100644 --- a/src/server/sparkd/user/AdminServiceServer.cpp +++ b/src/server/sparkd/user/AdminServiceServer.cpp @@ -66,6 +66,10 @@ AdminServiceServer::AdminServiceServer(bts::api::ServiceContext& context) if(it != _order_map.end()) _order_map.erase(it); } + for (const auto& ch : update.changed()) { + const auto& o = ch.current(); + _order_map[o->key()] = o; + } for (const auto& o : update.added()){ _order_map[o->key()]=o; }